Fix-update WSJ (option to disable Googlebot; http error 500)

background.js

@@ -383,7 +383,9 @@ var excludedSites = [];
 
 function setDefaultOptions() {
   ext_api.storage.local.set({
-    sites: defaultSites
+    sites: filterObject(defaultSites, function (val, key) {
+      return !val.includes('#options_disable_')
+    })
   }, function () {
     ext_api.runtime.openOptionsPage();
   });
@@ -949,7 +951,7 @@ ext_api.webRequest.onBeforeSendHeaders.addListener(function(details) {
   var useUserAgentMobile = false;
   var setReferer = false;
 
-if ((['main_frame', 'sub_frame', 'xmlhttprequest'].includes(details.type) || matchUrlDomain(['thetimes.co.uk'], details.url)) && matchUrlDomain(change_headers, details.url)){
+if (matchUrlDomain(change_headers, details.url) && (['main_frame', 'sub_frame', 'xmlhttprequest'].includes(details.type) || matchUrlDomain(['thetimes.co.uk'], details.url)) && !(matchUrlDomain(['wsj.com'], details.url) && enabledSites.includes('#options_disable_gb_wsj'))){
   // if referer exists, set it to google
   requestHeaders = requestHeaders.map(function (requestHeader) {
     if (requestHeader.name === 'Referer') {
@@ -1313,6 +1315,11 @@ ext_api.storage.local.get(["optInShown", "customShown"], function (result) {
   }
 });
 
+function filterObject(obj, callback) {
+  return Object.fromEntries(Object.entries(obj).
+    filter(([key, val]) => callback(val, key)));
+}
+
 function isSiteEnabled(details) {
   var enabledSite = matchUrlDomain(enabledSites, details.url);
   if (!ext_name.startsWith('Bypass Paywalls Clean'))

changelog.txt

@@ -3,6 +3,7 @@ Changelog Bypass Paywalls Clean - Firefox
 
 Post-release
 Fix-update Wired (Googlebot)
+Fix-update WSJ (option to disable Googlebot; http error 500)
 
 * v2.1.3.0 (2021-03-21)
 Add Berliner Zeitung (Germany)

contentScript.js

@@ -392,7 +392,7 @@ else if (matchDomain("wsj.com") && !matchDomain("cn.wsj.com")) {
         let snippet = document.querySelector('.snippet-promotion');
         let wsj_pro = document.querySelector('meta[name="page.site"][content="wsjpro"]');
         if (snippet || wsj_pro) {
-            window.location.href = window.location.href.replace('wsj.com', 'wsj.com/amp');
+            window.location.href = url.replace('wsj.com', 'wsj.com/amp');
         }
     });
 }

manifest.json

@@ -492,5 +492,5 @@
         "*://*.wallkit.net/*",
         "*://*.wsj.net/*"
     ],
-    "version": "2.1.3.1"
+    "version": "2.1.3.2"
 }

options.js

@@ -98,6 +98,9 @@ function renderOptions() {
 
 function selectAll() {
   var inputEls = Array.from(document.querySelectorAll('input'));
+  inputEls = inputEls.filter(function (input) {
+      return (!input.dataset.value.includes('#options_disable_'));
+    });
   inputEls.forEach(function (inputEl) {
     inputEl.checked = true;
   });

sites.js

@@ -289,7 +289,8 @@ var defaultSites =
   "TinyPass": "tinypass.com",
   "TribDss": "tribdss.com",
   "* BPC settings": "###",
-  "Show options on update": "#options_on_update"
+  "Show options on update": "#options_on_update",
+  "The Wall Street Journal - no Googlebot (http error 500)": "#options_disable_gb_wsj"
 }
 
 const au_comm_media_domains = ['bendigoadvertiser.com.au', 'bordermail.com.au', 'canberratimes.com.au', 'centralwesterndaily.com.au', 'dailyadvertiser.com.au', 'dailyliberal.com.au', 'examiner.com.au', 'illawarramercury.com.au', 'newcastleherald.com.au', 'northerndailyleader.com.au', 'portnews.com.au', 'standard.net.au', 'theadvocate.com.au', 'thecourier.com.au', 'westernadvocate.com.au'];