Update custom sites (custom useragent/referer)

background.js

@@ -16,6 +16,7 @@ var dompurify_sites = [];
 var optin_setcookie = false;
 var optin_update = true;
 var blocked_referer = false;
+var domain;
 
 // defaultSites are loaded from sites.js at installation extension
 
@@ -59,9 +60,9 @@ var remove_cookies = [];
 var remove_cookies_select_hold, remove_cookies_select_drop;
 
 // Set User-Agent
-var use_google_bot, use_bing_bot, use_facebook_bot, use_semrush_bot;
+var use_google_bot, use_bing_bot, use_facebook_bot, use_semrush_bot, use_useragent_custom, use_useragent_custom_obj;
 // Set Referer
-var use_drudgereport_referer, use_facebook_referer, use_google_referer, use_twitter_referer;
+var use_drudgereport_referer, use_facebook_referer, use_google_referer, use_twitter_referer, use_referer_custom, use_referer_custom_obj;
 // Set random IP-address
 var random_ip = {};
 var use_random_ip = [];
@@ -107,10 +108,14 @@ function initSetRules() {
   use_bing_bot = [];
   use_facebook_bot = [];
   use_semrush_bot = [];
+  use_useragent_custom = [];
+  use_useragent_custom_obj = {};
   use_drudgereport_referer = [];
   use_facebook_referer = [];
   use_google_referer = [];
   use_twitter_referer = [];
+  use_referer_custom = [];
+  use_referer_custom_obj = {};
   random_ip = {};
   change_headers = [];
   amp_unhide = [];
@@ -165,7 +170,7 @@ function setDefaultOptions() {
   });
 }
 
-function check_sites_updated(sites_updated_json) {
+function check_sites_updated(sites_updated_json, check_sites_updated_ext_version = false) {
   fetch(sites_updated_json)
   .then(response => {
     if (response.ok) {
@@ -178,6 +183,11 @@ function check_sites_updated(sites_updated_json) {
         ext_api.storage.local.set({
           sites_updated: json
         });
+        if (check_sites_updated_ext_version) {
+          let updated_ext_version_new = Object.values(json).map(x => x.upd_version || '').sort().pop();
+          if (updated_ext_version_new)
+            setExtVersionNew(updated_ext_version_new);
+        }
       })
     }
   }).catch(function (err) {
@@ -267,6 +277,11 @@ function addRules(domain, rule) {
         use_semrush_bot.push(domain);
       break;
     }
+  } else if (rule.useragent_custom) {
+    if (!use_useragent_custom.includes(domain)) {
+      use_useragent_custom.push(domain);
+      use_useragent_custom_obj[domain] = rule.useragent_custom;
+    }
   }
   if (rule.referer) {
     switch (rule.referer) {
@@ -287,6 +302,11 @@ function addRules(domain, rule) {
         use_twitter_referer.push(domain);
       break;
     }
+  } else if (rule.referer_custom) {
+    if (!use_referer_custom.includes(domain)) {
+      use_referer_custom.push(domain);
+      use_referer_custom_obj[domain] = rule.referer_custom;
+    }
   }
   if (rule.random_ip) {
     random_ip[domain] = rule.random_ip;
@@ -420,7 +440,7 @@ function set_rules(sites, sites_updated, sites_custom) {
   blockedJsInlineDomains = Object.keys(blockedJsInline);
   disableJavascriptInline();
   use_random_ip = Object.keys(random_ip);
-  change_headers = use_google_bot.concat(use_bing_bot, use_facebook_bot, use_semrush_bot, use_drudgereport_referer, use_facebook_referer, use_google_referer, use_twitter_referer, use_random_ip);
+  change_headers = use_google_bot.concat(use_bing_bot, use_facebook_bot, use_semrush_bot, use_useragent_custom, use_drudgereport_referer, use_facebook_referer, use_google_referer, use_twitter_referer, use_referer_custom, use_random_ip);
 }
 
 // add grouped sites to en/disabledSites (and exclude sites)
@@ -526,14 +546,17 @@ ext_api.storage.local.get({
   disabledSites = defaultSites_grouped_domains.concat(customSites_domains, updatedSites_domains_new).filter(x => !enabledSites.includes(x));
   add_grouped_enabled_domains(grouped_sites);
   set_rules(sites, updatedSites, customSites);
+  let check_sites_updated_ext_version;
+  if (optin_update)
+    check_update();
+  else
+    check_sites_updated_ext_version = true;
   if (enabledSites.includes('#options_optin_update_rules')) {
     sites_updated_json = sites_updated_json_online;
     sites_custom_ext_json = ext_path + 'custom/sites_custom.json';
   }
-  check_sites_updated(sites_updated_json);
+  check_sites_updated(sites_updated_json, check_sites_updated_ext_version);
   check_sites_custom_ext();
-  if (optin_update)
-    check_update();
   if (!Object.keys(sites).length)
     ext_api.runtime.openOptionsPage();
 });
@@ -1049,6 +1072,7 @@ if (matchUrlDomain(change_headers, details.url) && !ignore_types.includes(detail
   var bingbotEnabled = matchUrlDomain(use_bing_bot, details.url);
   var facebookbotEnabled = matchUrlDomain(use_facebook_bot, details.url);
   var semrushbotEnabled = matchUrlDomain(use_semrush_bot, details.url);
+  var useragent_customEnabled = matchUrlDomain(use_useragent_custom, details.url);
 
   // if referer exists, set it
   requestHeaders = requestHeaders.map(function (requestHeader) {
@@ -1061,6 +1085,8 @@ if (matchUrlDomain(change_headers, details.url) && !ignore_types.includes(detail
         requestHeader.value = 'https://www.facebook.com/';
       } else if (matchUrlDomain(use_twitter_referer, details.url)) {
         requestHeader.value = 'https://t.co/';
+      } else if (domain = matchUrlDomain(use_referer_custom, details.url)) {
+        requestHeader.value = use_referer_custom_obj[domain];
       }
       setReferer = true;
     }
@@ -1092,6 +1118,11 @@ if (matchUrlDomain(change_headers, details.url) && !ignore_types.includes(detail
         name: 'Referer',
         value: 'https://t.co/'
       });
+    } else if (domain = matchUrlDomain(use_referer_custom, details.url)) {
+      requestHeaders.push({
+        name: 'Referer',
+        value: use_referer_custom_obj[domain]
+      });
     }
   }
 
@@ -1108,7 +1139,7 @@ if (matchUrlDomain(change_headers, details.url) && !ignore_types.includes(detail
   }
 
   // override User-Agent to use Bingbot
-  if (bingbotEnabled) {
+  else if (bingbotEnabled) {
     requestHeaders.push({
       "name": "User-Agent",
       "value": useUserAgentMobile ? userAgentMobileB : userAgentDesktopB
@@ -1116,7 +1147,7 @@ if (matchUrlDomain(change_headers, details.url) && !ignore_types.includes(detail
   }
 
   // override User-Agent to use Facebookbot
-  if (facebookbotEnabled) {
+  else if (facebookbotEnabled) {
     requestHeaders.push({
       "name": "User-Agent",
       "value": userAgentDesktopF
@@ -1124,13 +1155,21 @@ if (matchUrlDomain(change_headers, details.url) && !ignore_types.includes(detail
   }
 
   // override User-Agent to use Semrushbot
-  if (semrushbotEnabled) {
+  else if (semrushbotEnabled) {
     requestHeaders.push({
       "name": "User-Agent",
       "value": userAgentDesktopS
     })
   }
 
+  // override User-Agent to custom
+  else if (domain = useragent_customEnabled) {
+    requestHeaders.push({
+      "name": "User-Agent",
+      "value": use_useragent_custom_obj[domain]
+    })
+  }
+
   // random IP for sites in use_random_ip
   let domain_random;
   if (domain_random = matchUrlDomain(use_random_ip, details.url)) {
@@ -1263,6 +1302,19 @@ function updateBadge(activeTab) {
       ext_api.action.setBadgeText({text: badgeText});
 }
 
+function setExtVersionNew(check_ext_version_new) {
+  ext_api.management.getSelf(function (result) {
+    var installType = result.installType;
+    var ext_version_len = (installType === 'development') ? 7 : 5;
+    ext_version_new = check_ext_version_new;
+    if (ext_version_new.substring(0, ext_version_len) <= ext_version.substring(0, ext_version_len))
+      ext_version_new = '';
+    ext_api.storage.local.set({
+      ext_version_new: ext_version_new
+    });
+  });
+}
+
 var ext_version_new;
 function check_update() {
   let manifest_new = 'https://gitlab.com/magnolia1234/bypass-paywalls-' + url_loc + '-clean/raw/master/manifest.json';
@@ -1270,16 +1322,8 @@ function check_update() {
   .then(response => {
     if (response.ok) {
       response.json().then(json => {
-        ext_api.management.getSelf(function (result) {
-          var installType = result.installType;
-          var ext_version_len = (installType === 'development') ? 7 : 5;
-          ext_version_new = json['version'];
-          if (ext_version_new.substring(0, ext_version_len) <= ext_version.substring(0, ext_version_len))
-            ext_version_new = '';
-          ext_api.storage.local.set({
-            ext_version_new: ext_version_new
-          });
-        });
+        let json_ext_version_new = json['version'];
+        setExtVersionNew(json_ext_version_new);
       })
     }
   }).catch(function (err) {

changelog.txt

@@ -8,6 +8,7 @@ Remove Westfalische Mediengruppe (fix obsolete)
 Fix Haaretz Group (Semrushbot)
 Fix The Diplomat (magazine)
 Fix Times of India (epaper)
+Update custom sites (custom useragent/referer)
 
 * v3.4.9.0 (2023-12-31)
 Add Business Insider Nederland

contentScript.js

@@ -335,7 +335,7 @@ if (matchDomain('medium.com') || matchDomain(medium_custom_domains) || (!matchDo
   if (paywall) {
     paywall.removeAttribute('class');
     paywall.firstChild.before(freediumLink(url));
-    paywall.firstChild.before(googleWebcacheLink(url, 'BPC > Try for full article text (articles before 2023-12-10)'));
+    paywall.firstChild.before(googleWebcacheLink(url));
   }
   window.setTimeout(function () {
     let banner = pageContains('div > div > p', /author made this story available to/);

custom/manifest.json

@@ -51,5 +51,5 @@
     "webRequestBlocking",
     "*://*/*"
   ],
-  "version": "3.4.9.4"
+  "version": "3.4.9.5"
 }

manifest.json

@@ -822,5 +822,5 @@
     "*://archive.vn/*",
     "*://webcache.googleusercontent.com/*"
   ],
-  "version": "3.4.9.4"
+  "version": "3.4.9.5"
 }

options/options_custom.js

@@ -269,25 +269,27 @@ function edit_options() {
     var edit_site = sites_custom[title];
     document.querySelector('input[data-key="title"]').value = title;
     document.querySelector('input[data-key="domain"]').value = edit_site.domain;
-    document.querySelector('textarea[data-key="group"]').value = edit_site.group ? edit_site.group : '';
+    document.querySelector('textarea[data-key="group"]').value = edit_site.group || '';
     document.querySelector('input[data-key="allow_cookies"]').checked = (edit_site.allow_cookies > 0);
     document.querySelector('input[data-key="remove_cookies"]').checked = (edit_site.remove_cookies > 0);
     document.querySelector('select[data-key="useragent"]').selectedIndex = (edit_site.googlebot > 0) ? 1 : useragent_options.indexOf(edit_site.useragent);
+    document.querySelector('textarea[data-key="useragent_custom"]').value = edit_site.useragent_custom || '';
     document.querySelector('select[data-key="referer"]').selectedIndex = referer_options.indexOf(edit_site.referer);
+    document.querySelector('textarea[data-key="referer_custom"]').value = edit_site.referer_custom || '';
     document.querySelector('select[data-key="random_ip"]').selectedIndex = random_ip_options.indexOf(edit_site.random_ip);
     document.querySelector('input[data-key="block_js"]').checked = (edit_site.block_js > 0 || edit_site.block_javascript > 0);
     document.querySelector('input[data-key="block_js_ext"]').checked = (edit_site.block_js_ext > 0 || edit_site.block_javascript_ext > 0);
-    document.querySelector('input[data-key="block_js_inline"]').value = edit_site.block_js_inline ? edit_site.block_js_inline : '';
-    document.querySelector('input[data-key="block_regex"]').value = edit_site.block_regex ? edit_site.block_regex : '';
+    document.querySelector('input[data-key="block_js_inline"]').value = edit_site.block_js_inline || '';
+    document.querySelector('input[data-key="block_regex"]').value = edit_site.block_regex || '';
     document.querySelector('input[data-key="amp_unhide"]').checked = (edit_site.amp_unhide > 0);
-    document.querySelector('input[data-key="amp_redirect"]').value = edit_site.amp_redirect ? edit_site.amp_redirect : '';
-    document.querySelector('input[data-key="ld_json"]').value = edit_site.ld_json ? edit_site.ld_json : '';
-    document.querySelector('input[data-key="ld_json_next"]').value = edit_site.ld_json_next ? edit_site.ld_json_next : '';
-    document.querySelector('input[data-key="ld_json_url"]').value = edit_site.ld_json_url ? edit_site.ld_json_url : '';
-    document.querySelector('input[data-key="ld_google_webcache"]').value = edit_site.ld_google_webcache ? edit_site.ld_google_webcache : '';
-    document.querySelector('input[data-key="add_ext_link"]').value = edit_site.add_ext_link ? edit_site.add_ext_link : '';
+    document.querySelector('input[data-key="amp_redirect"]').value = edit_site.amp_redirect || '';
+    document.querySelector('input[data-key="ld_json"]').value = edit_site.ld_json || '';
+    document.querySelector('input[data-key="ld_json_next"]').value = edit_site.ld_json_next || '';
+    document.querySelector('input[data-key="ld_json_url"]').value = edit_site.ld_json_url || '';
+    document.querySelector('input[data-key="ld_google_webcache"]').value = edit_site.ld_google_webcache || '';
+    document.querySelector('input[data-key="add_ext_link"]').value = edit_site.add_ext_link || '';
     document.querySelector('select[data-key="add_ext_link_type"]').selectedIndex = add_ext_link_type_options.indexOf(edit_site.add_ext_link_type);
-    document.querySelector('textarea[data-key="cs_code"]').value = edit_site.cs_code ? edit_site.cs_code : '';
+    document.querySelector('textarea[data-key="cs_code"]').value = edit_site.cs_code || '';
   });
 }
 
@@ -349,7 +351,9 @@ function renderOptions() {
       'allow_cookies': 1,
       'remove_cookies': 1,
       'useragent': 0,
+      'useragent_custom': 0,
       'referer': 0,
+      'referer_custom': 0,
       'random_ip': 0,
       'block_js (domain)': 1,
       'block_js_ext': 1,
@@ -386,7 +390,7 @@ function renderOptions() {
             inputEl.appendChild(option);
           }
         } else {
-          if (!['cs_code', 'group'].includes(key)) {
+          if (!['cs_code', 'group', 'referer_custom', 'useragent_custom'].includes(key)) {
             inputEl = document.createElement('input');
             inputEl.size = 25;
           } else {
@@ -440,8 +444,10 @@ function renderOptions() {
       (sites_custom[key]['allow_cookies'] > 0 ? ' | allow_cookies' : '') +
       (sites_custom[key]['remove_cookies'] > 0 ? ' | remove_cookies' : '') +
       (sites_custom[key]['useragent'] ? ' | useragent: ' + sites_custom[key]['useragent'] : '') +
+      (sites_custom[key]['useragent_custom'] ? ' | useragent_custom' : '') +
       (sites_custom[key]['googlebot'] > 0 ? ' | googlebot' : '') +
       (sites_custom[key]['referer'] ? ' | referer: ' + sites_custom[key]['referer'] : '') +
+      (sites_custom[key]['referer_custom'] ? ' | referer_custom' : '') +
       (sites_custom[key]['random_ip'] ? ' | random_ip: ' + sites_custom[key]['random_ip'] : '') +
       ((sites_custom[key]['block_js'] > 0 || sites_custom[key]['block_javascript'] > 0) ? ' | block_js' : '') +
       ((sites_custom[key]['block_js_ext'] > 0 || sites_custom[key]['block_javascript_ext'] > 0) ? ' | block_js_ext' : '') +

sites_updated.json

@@ -83,12 +83,6 @@
     "cs_code": "[{\"cond\":\"div.single-post-content\", \"rm_attrib\":\"class\"}]",
     "upd_version": "3.4.8.5"
   },
-  "Literary Review (UK)": {
-    "domain": "literaryreview.co.uk",
-    "allow_cookies": 1,
-    "ld_json_url": "p.subscribe-for-more|div#_articlereview",
-    "upd_version": "3.4.6.3"
-  },
   "London Review of Books": {
     "domain": "lrb.co.uk",
     "allow_cookies": 1,
@@ -168,11 +162,5 @@
     "block_regex_general": "\\/tncms\\/api\\/access(\\..+)?\\.js",
     "cs_dompurify": 1,
     "upd_version": "3.4.1.6"
-  },
-  "Weser-Kurier": {
-    "domain": "weser-kurier.de",
-    "allow_cookies": 1,
-    "block_regex": "\\.piano\\.io\\/",
-    "upd_version": "3.4.6.4"
   }
 }