Fix WaPo (disable Googlebot)

background.js

@@ -235,7 +235,6 @@ var use_google_bot_default = [
   'thetimes.co.uk',
   'usatoday.com',
   'usinenouvelle.com',
-  'washingtonpost.com',
   'wired.com',
   'wiwo.de',
   'worldpoliticsreview.com',
@@ -1151,7 +1150,6 @@ ext_api.webRequest.onBeforeSendHeaders.addListener(function(details) {
   var setReferer = false;
 
 if (matchUrlDomain(change_headers, details.url) && (['main_frame', 'sub_frame', 'xmlhttprequest'].includes(details.type) || matchUrlDomain('thetimes.co.uk', details.url)) &&
-  !(matchUrlDomain('washingtonpost.com', details.url) && (details.url.includes('/interactive/') || (details.url.includes('/talk/api/') && ['xmlhttprequest'].includes(details.type)))) &&
   !(matchUrlDomain('barrons.com', details.url) && enabledSites.includes('#options_disable_gb_barrons')) &&
   !(matchUrlDomain('wsj.com', details.url) && enabledSites.includes('#options_disable_gb_wsj'))) {
   // if referer exists, set it to google
@@ -1240,7 +1238,7 @@ if (matchUrlDomain(change_headers, details.url) && (['main_frame', 'sub_frame',
         }
         if ((!['font', 'stylesheet'].includes(details.type) || matchUrlDomain(cs_limit_except, currentTabUrl)) && !csDone) {
           let lib_file = 'lib/empty.js';
-          if (matchUrlDomain(['cicero.de', 'economictimes.com', 'gva.be', 'lesechos.fr', 'newleftreview.org', 'newyorker.com', 'nzherald.co.nz', 'prospectmagazine.co.uk', 'sudouest.fr', 'techinasia.com', 'valor.globo.com'].concat(nl_mediahuis_region_domains), currentTabUrl))
+          if (matchUrlDomain(['cicero.de', 'economictimes.com', 'gva.be', 'lesechos.fr', 'newleftreview.org', 'newyorker.com', 'nzherald.co.nz', 'prospectmagazine.co.uk', 'sudouest.fr', 'techinasia.com', 'valor.globo.com', 'washingtonpost.com'].concat(nl_mediahuis_region_domains), currentTabUrl))
             lib_file = 'lib/purify.min.js';
           ext_api.tabs.executeScript(tabId, {
             file: lib_file,

changelog.txt

@@ -5,7 +5,7 @@ Post-release
 Fix Il Secolo XIX (amp-redirect)
 Fix Miami Herald (group McClatchy)
 Fix New Left Review (timing)
-Fix WaPo (amp/interactive)
+Fix WaPo (disable Googlebot & fix amp/interactive)
 Maintenance contentScript (update sanitize html-input)
 
 * v2.2.8.0 (2021-07-04)

contentScript.js

@@ -211,10 +211,14 @@ else {
                   if (par.items) {
                     par_elem = document.createElement('ul');
                     for (let item of par.items)
-                      if (item.text && item.intentions[0].href) {
+                      if (item.text) {
                         par_sub1 = document.createElement('li');
-                        par_sub2 = document.createElement('a');
-                        par_sub2.href = item.intentions[0].href;
+                        if (item.intentions[0] && item.intentions[0].href) {
+                          par_sub2 = document.createElement('a');
+                          par_sub2.href = item.intentions[0].href;
+                        } else {
+                          par_sub2 = document.createElement('span');
+                        }
                         par_sub2.innerText = item.text;
                         par_sub1.appendChild(par_sub2);
                         par_elem.appendChild(par_sub1);
@@ -2519,12 +2523,20 @@ else if (matchDomain('washingtonpost.com')) {
   } else {
     function wapo_main(node) {
       removeDOMElement(node);
-      window.location.href = url.split('?')[0] + '?outputType=amp';
+      let url_amp = url.split('?')[0] + '?outputType=amp';
+      replaceDomElementExt(url_amp, false, false, 'div.article-body', 'Failed to load from amp-page: ');
+    }
+    function wapo_overlay(node) {
+      node.removeAttribute('style');
     }
     let url = window.location.href;
     if (!url.includes('outputType=amp')) {
       waitDOMElement('div[id^="paywall-"]', 'DIV', wapo_main, false);
-      csDoneOnce = true;
+      waitDOMElement('div[data-qa*="wall"]', 'DIV', removeDOMElement, true);
+      waitDOMAttribute('body', 'BODY', 'style', wapo_overlay, true);
+      waitDOMAttribute('html', 'HTML', 'style', wapo_overlay, false);
+      if (!url.includes('/interactive/'))
+        csDoneOnce = true;
     } else {
       let subscr_sections = document.querySelectorAll('[subscriptions-section="content"]');
       for (let subscr_section of subscr_sections)
@@ -2616,6 +2628,24 @@ function waitDOMElement(selector, tagName = '', callback, multiple = false) {
   });
 }
 
+function waitDOMAttribute(selector, tagName = '', attributeName = '', callback, multiple = false) {
+  let targetNode = document.querySelector(selector);
+  if (!targetNode)
+	  return;
+  new window.MutationObserver(function (mutations) {
+    for (let mutation of mutations) {
+      if (mutation.target.attributes[attributeName]) {
+        callback(mutation.target);
+        if (!multiple)
+          this.disconnect();
+      }
+    }
+  }).observe(targetNode, {
+    attributes: true,
+    attributeFilter: [attributeName]
+  });
+}
+
 function addDivBpcDone() {
   let div_bpc_new = document.createElement('div');
   div_bpc_new.setAttribute('id', 'bpc_done');
@@ -2646,6 +2676,9 @@ function replaceDomElementExt(url, proxy, base64, selector, text_fail = '') {
           html = decode_utf8(atob(html));
           selector = 'body';
         }
+        if (matchDomain(['washingtonpost.com']) && html.includes('<amp-')) {
+          html = html.replace(/<amp-/g, '<').replace(/<\/amp-/g, '</');
+        }
         let parser = new DOMParser();
         let doc = parser.parseFromString(DOMPurify.sanitize(html, {ADD_ATTR: ['layout'], ADD_TAGS: ['amp-img']}), 'text/html');
         //console.log(DOMPurify.removed);

manifest.json

@@ -534,5 +534,5 @@
         "*://*.wallkit.net/*",
         "*://*.wsj.net/*"
     ],
-    "version": "2.2.8.5"
+    "version": "2.2.8.6"
 }