diff --git a/background.js b/background.js index e444de59..d44c73b5 100644 --- a/background.js +++ b/background.js @@ -60,7 +60,7 @@ var remove_cookies = []; var remove_cookies_select_hold, remove_cookies_select_drop; // Set User-Agent -var use_google_bot, use_bing_bot, use_facebook_bot; +var use_google_bot, use_bing_bot, use_facebook_bot, use_semrush_bot; // Set Referer var use_drudgereport_referer, use_facebook_referer, use_google_referer, use_twitter_referer; // Set random IP-address @@ -107,6 +107,7 @@ function initSetRules() { use_google_bot = []; use_bing_bot = []; use_facebook_bot = []; + use_semrush_bot = []; use_drudgereport_referer = []; use_facebook_referer = []; use_google_referer = []; @@ -140,6 +141,8 @@ const userAgentMobileB = "Chrome/115.0.5790.171 Mobile Safari/537.36 (compatible const userAgentDesktopF = 'facebookexternalhit/1.1 (+http://www.facebook.com/externalhit_uatext.php)'; +const userAgentDesktopS = "Mozilla/5.0 (compatible; SemrushBot; +http://www.semrush.com/bot.html)"; + var enabledSites = []; var disabledSites = []; var optionSites = {}; @@ -260,6 +263,10 @@ function addRules(domain, rule) { if (!use_facebook_bot.includes(domain)) use_facebook_bot.push(domain); break; + case 'semrushbot': + if (!use_semrush_bot.includes(domain)) + use_semrush_bot.push(domain); + break; } } if (rule.referer) { @@ -414,7 +421,7 @@ function set_rules(sites, sites_updated, sites_custom) { blockedJsInlineDomains = Object.keys(blockedJsInline); disableJavascriptInline(); use_random_ip = Object.keys(random_ip); - change_headers = use_google_bot.concat(use_bing_bot, use_facebook_bot, use_drudgereport_referer, use_facebook_referer, use_google_referer, use_twitter_referer, use_random_ip); + change_headers = use_google_bot.concat(use_bing_bot, use_facebook_bot, use_semrush_bot, use_drudgereport_referer, use_facebook_referer, use_google_referer, use_twitter_referer, use_random_ip); } // add grouped sites to en/disabledSites (and exclude sites) @@ -1042,6 +1049,7 @@ if (matchUrlDomain(change_headers, details.url) && !ignore_types.includes(detail !(matchUrlDomain('www.wsj.com', details.url)); var bingbotEnabled = matchUrlDomain(use_bing_bot, details.url); var facebookbotEnabled = matchUrlDomain(use_facebook_bot, details.url); + var semrushbotEnabled = matchUrlDomain(use_semrush_bot, details.url); // if referer exists, set it requestHeaders = requestHeaders.map(function (requestHeader) { @@ -1116,6 +1124,14 @@ if (matchUrlDomain(change_headers, details.url) && !ignore_types.includes(detail }) } + // override User-Agent to use Semrushbot + if (semrushbotEnabled) { + requestHeaders.push({ + "name": "User-Agent", + "value": userAgentDesktopS + }) + } + // random IP for sites in use_random_ip let domain_random; if (domain_random = matchUrlDomain(use_random_ip, details.url)) { diff --git a/changelog.txt b/changelog.txt index cb274db8..23811393 100644 --- a/changelog.txt +++ b/changelog.txt @@ -3,7 +3,7 @@ Changelog Bypass Paywalls Clean - Firefox Updates (install signed xpi-file): https://gitlab.com/magnolia1234/bypass-paywalls-firefox-clean/-/releases Post-release -Fix Haaretz.co.il (fetch from archive.is) +Fix Haaretz Group (Semrushbot) Fix The Diplomat (magazine) * v3.4.9.0 (2023-12-31) diff --git a/custom/manifest.json b/custom/manifest.json index c40c7774..508e7c89 100644 --- a/custom/manifest.json +++ b/custom/manifest.json @@ -51,5 +51,5 @@ "webRequestBlocking", "*://*/*" ], - "version": "3.4.9.2" + "version": "3.4.9.3" } diff --git a/manifest.json b/manifest.json index 881e5a9a..f49f1365 100644 --- a/manifest.json +++ b/manifest.json @@ -827,5 +827,5 @@ "*://archive.vn/*", "*://webcache.googleusercontent.com/*" ], - "version": "3.4.9.2" + "version": "3.4.9.3" } diff --git a/sites.js b/sites.js index 38f6efa1..b34007e8 100644 --- a/sites.js +++ b/sites.js @@ -1053,6 +1053,7 @@ var defaultSites = { "themarker.com" ], allow_cookies: 1, + useragent: "semrushbot", cs_dompurify: 1 }, "Haas Mediengruppe (opt-in to custom sites)": {