Fix Haaretz Group (Semrushbot)

background.js

@@ -60,7 +60,7 @@ var remove_cookies = [];
 var remove_cookies_select_hold, remove_cookies_select_drop;
 
 // Set User-Agent
-var use_google_bot, use_bing_bot, use_facebook_bot;
+var use_google_bot, use_bing_bot, use_facebook_bot, use_semrush_bot;
 // Set Referer
 var use_drudgereport_referer, use_facebook_referer, use_google_referer, use_twitter_referer;
 // Set random IP-address
@@ -107,6 +107,7 @@ function initSetRules() {
   use_google_bot = [];
   use_bing_bot = [];
   use_facebook_bot = [];
+  use_semrush_bot = [];
   use_drudgereport_referer = [];
   use_facebook_referer = [];
   use_google_referer = [];
@@ -140,6 +141,8 @@ const userAgentMobileB = "Chrome/115.0.5790.171 Mobile Safari/537.36 (compatible
 
 const userAgentDesktopF = 'facebookexternalhit/1.1 (+http://www.facebook.com/externalhit_uatext.php)';
 
+const userAgentDesktopS = "Mozilla/5.0 (compatible; SemrushBot; +http://www.semrush.com/bot.html)";
+
 var enabledSites = [];
 var disabledSites = [];
 var optionSites = {};
@@ -260,6 +263,10 @@ function addRules(domain, rule) {
       if (!use_facebook_bot.includes(domain))
         use_facebook_bot.push(domain);
       break;
+    case 'semrushbot':
+      if (!use_semrush_bot.includes(domain))
+        use_semrush_bot.push(domain);
+      break;
     }
   }
   if (rule.referer) {
@@ -414,7 +421,7 @@ function set_rules(sites, sites_updated, sites_custom) {
   blockedJsInlineDomains = Object.keys(blockedJsInline);
   disableJavascriptInline();
   use_random_ip = Object.keys(random_ip);
-  change_headers = use_google_bot.concat(use_bing_bot, use_facebook_bot, use_drudgereport_referer, use_facebook_referer, use_google_referer, use_twitter_referer, use_random_ip);
+  change_headers = use_google_bot.concat(use_bing_bot, use_facebook_bot, use_semrush_bot, use_drudgereport_referer, use_facebook_referer, use_google_referer, use_twitter_referer, use_random_ip);
 }
 
 // add grouped sites to en/disabledSites (and exclude sites)
@@ -1042,6 +1049,7 @@ if (matchUrlDomain(change_headers, details.url) && !ignore_types.includes(detail
     !(matchUrlDomain('www.wsj.com', details.url));
   var bingbotEnabled = matchUrlDomain(use_bing_bot, details.url);
   var facebookbotEnabled = matchUrlDomain(use_facebook_bot, details.url);
+  var semrushbotEnabled = matchUrlDomain(use_semrush_bot, details.url);
 
   // if referer exists, set it
   requestHeaders = requestHeaders.map(function (requestHeader) {
@@ -1116,6 +1124,14 @@ if (matchUrlDomain(change_headers, details.url) && !ignore_types.includes(detail
     })
   }
 
+  // override User-Agent to use Semrushbot
+  if (semrushbotEnabled) {
+    requestHeaders.push({
+      "name": "User-Agent",
+      "value": userAgentDesktopS
+    })
+  }
+
   // random IP for sites in use_random_ip
   let domain_random;
   if (domain_random = matchUrlDomain(use_random_ip, details.url)) {

changelog.txt

@@ -3,7 +3,7 @@ Changelog Bypass Paywalls Clean - Firefox
 Updates (install signed xpi-file): https://gitlab.com/magnolia1234/bypass-paywalls-firefox-clean/-/releases
 
 Post-release
-Fix Haaretz.co.il (fetch from archive.is)
+Fix Haaretz Group (Semrushbot)
 Fix The Diplomat (magazine)
 
 * v3.4.9.0 (2023-12-31)

custom/manifest.json

@@ -51,5 +51,5 @@
     "webRequestBlocking",
     "*://*/*"
   ],
-  "version": "3.4.9.2"
+  "version": "3.4.9.3"
 }

manifest.json

@@ -827,5 +827,5 @@
     "*://archive.vn/*",
     "*://webcache.googleusercontent.com/*"
   ],
-  "version": "3.4.9.2"
+  "version": "3.4.9.3"
 }

sites.js

@@ -1053,6 +1053,7 @@ var defaultSites = {
       "themarker.com"
     ],
     allow_cookies: 1,
+    useragent: "semrushbot",
     cs_dompurify: 1
   },
   "Haas Mediengruppe (opt-in to custom sites)": {