@ -31,6 +31,7 @@ var defaultSites = {
'Inc.com' : 'inc.com' ,
'Investors Chronicle' : 'investorschronicle.co.uk' ,
'La Repubblica' : 'repubblica.it' ,
'Le Monde' : 'lemonde.fr' ,
'Le Temps' : 'letemps.ch' ,
'Los Angeles Times' : 'latimes.com' ,
'Medium' : 'medium.com' ,
@ -74,7 +75,7 @@ var defaultSites = {
'The Spectator' : 'spectator.co.uk' ,
'The Seattle Times' : 'seattletimes.com' ,
'The Sydney Morning Herald' : 'smh.com.au' ,
'The Telegraph' : 'telegraph.co.uk' ,
'The Telegraph' : 'telegraph.co.uk' ,
'The Times' : 'thetimes.co.uk' ,
'The Toronto Star' : 'thestar.com' ,
'The Washington Post' : 'washingtonpost.com' ,
@ -92,78 +93,65 @@ const restrictions = {
// Don't remove cookies before page load
const allow _cookies = [
'ad.nl' ,
'asia.nikkei.com' ,
'nytimes.com' ,
'wsj.com' ,
'ft.com' ,
'letemps.ch' ,
'mercurynews.com' ,
'economist.com' ,
'bostonglobe.com' ,
'denverpost.com' ,
'hacked.com' ,
'ocregister.com' ,
'newstatesman.com' ,
'spectator.co.uk' ,
'towardsdatascience.com' ,
'medium.com' ,
'theadvocate.com.au' ,
'economist.com' ,
'ed.nl' ,
'examiner.com.au' ,
' thestar .com',
' washingtonpost .com',
'ft.com' ,
'hacked.com' ,
'hbr.org' ,
'lemonde.fr' ,
'letemps.ch' ,
'medium.com' ,
'mercurynews.com' ,
'newstatesman.com' ,
'nymag.com' ,
'theaustralian.com.au' ,
'ad.nl' ,
'ed.nl' ,
'nytimes.com' ,
'ocregister.com' ,
'parool.nl' ,
'spectator.co.uk' ,
'telegraaf.nl' ,
'theadvocate.com.au' ,
'theaustralian.com.au' ,
'thestar.com' ,
'towardsdatascience.com' ,
'trouw.nl' ,
'vn.nl' ,
'volkskrant.nl'
'volkskrant.nl' ,
'washingtonpost.com' ,
'wsj.com' ,
]
// Removes cookies after page load
const remove _cookies = [
'ad.nl' ,
'asia.nikkei.com' ,
'nytimes.com' ,
'ft.com' ,
'letemps.ch' ,
'mercurynews.com' ,
'theage.com.au' ,
'economist.com' ,
'bostonglobe.com' ,
'denverpost.com' ,
'economist.com' ,
'ed.nl' ,
'examiner.com.au' ,
'ft.com' ,
'hacked.com' ,
'ocregister.com' ,
'hbr.org' ,
'letemps.ch' ,
'medium.com' ,
'mercurynews.com' ,
'newstatesman.com' ,
'nymag.com' ,
'nytimes.com' ,
'ocregister.com' ,
'spectator.co.uk' ,
'towardsdatascience.com' ,
'medium.com' ,
'telegraaf.nl' ,
'theadvocate.com.au' ,
'examiner.com.au' ,
'thestar.com' ,
'centralwesterndaily.com.au' ,
'theherald.com.au' ,
'foreignpolicy.com' ,
'wsj.com' ,
'glassdoor.com' ,
'cen.acs.org' ,
'hbr.org' ,
'thenewsrep.com' ,
'towardsdatascience.com' ,
'vn.nl' ,
'washingtonpost.com' ,
'nymag.com' ,
'nationalpost.com' ,
'handelsblatt.com' ,
'thediplomat.com' ,
'latimes.com' ,
'theatlantic.com' ,
'chicagobusiness.com' ,
'qz.com' ,
'ad.nl' ,
'ed.nl' ,
'telegraaf.nl' ,
'vn.nl'
'wsj.com' ,
]
// Override User-Agent with Googlebot