Update custom sites (load text from json)

README.md

@@ -54,7 +54,7 @@ Use adblocker with custom (content)filter: https://gitlab.com/magnolia1234/bypas
 
 ### List of supported websites
 
-_* free articles only._
+_* free/cached articles only._
 
 ##### World news
 [Foreign Affairs](https://www.foreignaffairs.com) -
@@ -606,7 +606,7 @@ Grouped in options:\
 [Central Western Daily](https://www.centralwesterndaily.com.au) -
 [Daily Liberal](https://www.dailyliberal.com.au) -
 [Illawarra Mercury](https://www.illawarramercury.com.au) -
-[Newcastle Herald](https://www.newcastleherald.com.au) -
+[Newcastle Herald](https://www.newcastleherald.com.au)* -
 [The Advocate](https://www.theadvocate.com.au) -
 [The Border Mail](https://www.bordermail.com.au) -
 [The Canberra Times](https://www.canberratimes.com.au) -
@@ -666,7 +666,7 @@ Grouped in options:\
 [Grupo Abril](https://grupoabril.com.br) -
 [O Estado de S. Paulo](https://estadao.com.br) -
 [O Globo](https://oglobo.globo.com) -
-[Valor Econômico](https://valor.globo.com)
+[Valor Econômico](https://valor.globo.com)*
 ##### Chile
 [Diario Financiero](https://www.df.cl) -
 [El Mercurio](https://digital.elmercurio.com) -
@@ -684,7 +684,7 @@ Grouped in options:\
 [El Comercio](https://elcomercio.pe) -
 [Gestión](https://gestion.pe)
 
-_* free articles only._
+_* free/cached articles only._
 
 ### Sites with limited number of free articles
 The free article limit can normally be bypassed by removing cookies for the site.  
@@ -707,7 +707,7 @@ Check 'Options'-link in popup-menu and go to custom sites.
 \* by default BPC has limited permissions, but you can opt-in to enable custom sites (and also clear cookies/block general paywall-scripts for non-listed sites). You can also just request permissions for the custom sites you added yourself (or `clear cookies` to ask for permission for current site).
 
 By default sites' cookies/local storage are removed after page loads (to bypass article limit).  
-Also you can enable Googlebot/Bingbot user-agent, set referer (to Facebook, Google or Twitter; ignored when Googlebot is set), set random ip-address, disable Javascript for (sub)domain(s) and/or external domains, block regular expression and/or unhide text on (or when paywall(selector) redirect to) amp-page.
+Also you can enable Googlebot/Bingbot user-agent, set referer (to Facebook, Google or Twitter; ignored when Googlebot is set), set random ip-address, disable Javascript for (sub)domain(s) and/or external domains, block regular expression, unhide text on (or when paywall(selector) redirect to) amp-page and/or load text from json (paywall|article selector).
 
 [Example list of custom sites](https://gitlab.com/magnolia1234/bypass-paywalls-firefox-clean/-/blob/master/custom/sites_custom.json) or [download list (json)](https://gitlab.com/magnolia1234/bypass-paywalls-firefox-clean/-/raw/master/custom/sites_custom.json)
 

background.js

@@ -8,6 +8,7 @@ var ext_name = manifestData.name;
 var ext_version = manifestData.version;
 
 const cs_limit_except = ['elespanol.com', 'faz.net', 'inkl.com', 'nation.africa', 'nationalgeographic.com', 'thetimes.co.uk'];
+const dompurify_sites = ['asiatimes.com', 'bloomberg.com', 'cicero.de', 'economictimes.com', 'hs.fi', 'lesechos.fr', 'marianne.net', 'newcastleherald.com.au', 'newleftreview.org', 'nzherald.co.nz', 'prospectmagazine.co.uk', 'stratfor.com', 'techinasia.com', 'timesofindia.com', 'valor.globo.com', 'vn.nl'].concat(fr_groupe_sud_ouest_domains, nl_mediahuis_region_domains, no_nhst_media_domains, usa_theathletic_domains);
 var currentTabUrl = '';
 var csDone = false;
 var optin_setcookie = false;
@@ -68,6 +69,8 @@ var amp_unhide;
 var amp_redirect;
 // code for contentScript
 var cs_code;
+// load text from json
+var ld_json;
 
 // custom: block javascript
 var block_js_custom = [];
@@ -88,6 +91,7 @@ function initSetRules() {
   amp_unhide = [];
   amp_redirect = {};
   cs_code = {};
+  ld_json = {};
   block_js_custom = [];
   block_js_custom_ext = [];
   blockedRegexes = {};
@@ -269,6 +273,11 @@ function set_rules(sites, sites_updated, sites_custom) {
           block_js_custom_ext.push(domain);
         if (rule.amp_unhide > 0)
           amp_unhide.push(domain);
+        if (rule.ld_json) {
+          ld_json[domain] = rule.ld_json;
+          if (!dompurify_sites.includes(domain))
+            dompurify_sites.push(domain);
+        }
       }
     }
   }
@@ -1016,7 +1025,7 @@ if (matchUrlDomain(change_headers, details.url) && !['font', 'image', 'styleshee
         }
         if ((!['font', 'stylesheet'].includes(details.type) || matchUrlDomain(cs_limit_except, currentTabUrl)) && !csDone) {
           let lib_file = 'lib/empty.js';
-          if (matchUrlDomain(['asiatimes.com', 'bloomberg.com', 'cicero.de', 'economictimes.com', 'hs.fi', 'lesechos.fr', 'marianne.net', 'newcastleherald.com.au', 'newleftreview.org', 'nzherald.co.nz', 'prospectmagazine.co.uk', 'stratfor.com', 'techinasia.com', 'timesofindia.com', 'valor.globo.com', 'vn.nl'].concat(fr_groupe_sud_ouest_domains, nl_mediahuis_region_domains, no_nhst_media_domains, usa_theathletic_domains), currentTabUrl))
+          if (matchUrlDomain(dompurify_sites, currentTabUrl))
             lib_file = 'lib/purify.min.js';
           var bg2csData = {
             optin_setcookie: optin_setcookie,
@@ -1028,6 +1037,9 @@ if (matchUrlDomain(change_headers, details.url) && !['font', 'image', 'styleshee
           let cs_code_domain = '';
           if (cs_code_domain = matchUrlDomain(Object.keys(cs_code), currentTabUrl))
             bg2csData.cs_code = cs_code[cs_code_domain];
+          let ld_json_domain = '';
+          if (ld_json_domain = matchUrlDomain(Object.keys(ld_json), currentTabUrl))
+            bg2csData.ld_json = ld_json[ld_json_domain];
           ext_api.tabs.executeScript(tabId, {
             code: 'var bg2csData = ' + JSON.stringify(bg2csData) + ';'
           }, function () {

changelog.txt

@@ -3,6 +3,7 @@ Changelog Bypass Paywalls Clean - Firefox
 
 Post-release
 Fix update check (GitLab CORS issue or BitBucket http error 429)
+Update custom sites (load text from json)
 
 * v2.5.7.0 (2022-02-20)
 Remove Ruhr Nachrichten (fix obsolete)

contentScript.js

@@ -38,10 +38,40 @@ if (!matchDomain(arr_localstorage_hold)) {
   window.localStorage.clear();
 }
 
+var bg2csData;
+
+// custom/updated sites: load text from json
+if ((bg2csData !== undefined) && bg2csData.ld_json && dompurify_loaded) {
+  if (bg2csData.ld_json.includes('|')) {
+    let ld_json_split = bg2csData.ld_json.split('|');
+    let paywall_sel = ld_json_split[0];
+    let article_sel = ld_json_split[1];
+    let paywall = document.querySelector(paywall_sel);
+    if (paywall) {
+      removeDOMElement(paywall);
+      let scripts = document.querySelectorAll('script[type="application/ld+json"]');
+      let json;
+      for (let script of scripts) {
+        if (script.innerText.includes('articleBody'))
+          json = script;
+      }
+      if (json) {
+        let json_text = JSON.parse(json.text).articleBody;
+        let content = document.querySelector(article_sel);
+        if (json_text && content) {
+          let parser = new DOMParser();
+          let doc = parser.parseFromString('<div>' + DOMPurify.sanitize(json_text) + '</div>', 'text/html');
+          let content_new = doc.querySelector('div');
+          content.parentNode.replaceChild(content_new, content);
+        }
+      }
+    }
+  }
+}
+
 var div_bpc_done = document.querySelector('div#bpc_done');
 if (!div_bpc_done) {
 
-var bg2csData;
 // check for opt-in confirmation (from background.js)
 if ((bg2csData !== undefined) && bg2csData.optin_setcookie) {
   if (domain = matchDomain(['belfasttelegraph.co.uk', 'independent.ie'])) {
@@ -106,7 +136,7 @@ function amp_unhide_access_hide(amp_access = '', amp_access_not = '', amp_ads_se
     amp_iframes_replace(amp_iframe_link, source);
 }
 
-// custom sites: try to unhide text on amp-page
+// custom/updated sites: try to unhide text on amp-page
 if ((bg2csData !== undefined) && bg2csData.amp_unhide) {
   window.setTimeout(function () {
     let amp_page_hide = document.querySelector('script[src*="/amp-access-"], script[src*="/amp-subscriptions-"]');
@@ -118,7 +148,7 @@ if ((bg2csData !== undefined) && bg2csData.amp_unhide) {
   }, 100); // Delay (in milliseconds)
 }
 
-// updated sites: amp-redirect
+// custom/updated sites: amp-redirect
 if ((bg2csData !== undefined) && bg2csData.amp_redirect) {
   window.setTimeout(function () {
     let amp_script = document.querySelector('script[src^="https://cdn.ampproject.org/"]');
@@ -206,14 +236,14 @@ else {
       if (paywall) {
         paywall.removeAttribute('style');
         let url_src = window.location.href.replace('newcastleherald.com.au', 'canberratimes.com.au');
-        replaceDomElementExt(url_src, true, false, 'p[class^="Paragraph_wrapper__"]', '', 'div.article__body');
+        replaceDomElementExt(url_src, true, false, 'p[class^="Paragraph_wrapper__"]', 'BPC > text not available on other group site: ', 'div.article__body');
       }
       let hidden_images = document.querySelectorAll('img[src^="data:image/"][data-src]');
       for (let hidden_image of hidden_images)
         hidden_image.setAttribute('src', hidden_image.getAttribute('data-src'));
       let lead_image = document.querySelector('div#story-body img.w-full[src]');
       if (lead_image) {
-        let lead_src = lead_image.src.split('.jpg')[0];
+        let lead_src = lead_image.src.toLowerCase().split(/\.(png|jpg)/)[0];
         let body_image = document.querySelector('div.article__body img[src]');
         if (lead_src && body_image && body_image.src.includes(lead_src))
           removeDOMElement(body_image);

custom/sites_custom.json

@@ -76,6 +76,11 @@
     "block_regex": "loader-cdn\\.azureedge\\.net",
     "domain": "gazette.com"
   },
+  "Gplanet.co.il": {
+    "allow_cookies": 1,
+    "domain": "gplanet.co.il",
+    "ld_json": "div.subscription-form|div.content-inner"
+  },
   "Handelszeitung.ch": {
     "allow_cookies": 1,
     "domain": "handelszeitung.ch",

manifest.json

@@ -620,5 +620,5 @@
         "*://*.wallkit.net/*",
         "*://*.wsj.net/*"
     ],
-    "version": "2.5.7.1"
+    "version": "2.5.7.2"
 }

options/options_custom.html

@@ -20,7 +20,7 @@
   <h2>Custom Sites</h2>
   <div style="width:90%;">
     To add a new site, enter an unique title/domain (without www.).<br>
-    Select options for useragent (like Googlebot), set referer (ignored when Googlebot is set), set random ip-address, block Javascript (on (sub)domain(s) of site and/or external domains), block regular expression and/or unhide text on (or when paywall(selector) redirect to) amp-page.<br>
+    Select options for useragent (like Googlebot), set referer (ignored when Googlebot is set), set random ip-address, block Javascript, block regular expression, unhide text on (or when paywall(selector) redirect to) amp-page and/or load text from json (paywall|article selector).<br>
     Custom sites (new) are enabled automatically in <small><button><a href="options.html" style="text-decoration:none;color:inherit">Options</a></button></small> (cookies will be removed by default unless you enable allow_cookies).<br>
     If you want to use custom sites (for non-listed sites) enable it in <small><button><a href="optin/opt-in.html" style="text-decoration:none;color:inherit">Opt-in</a></button></small>
     <strong>Custom sites enabled: <span id="custom-enabled"></span></strong><br>

options/options_custom.js

@@ -192,6 +192,7 @@ function edit_options() {
     document.querySelector('input[data-key="block_regex"]').value = edit_site.block_regex ? edit_site.block_regex : '';
     document.querySelector('input[data-key="amp_unhide"]').checked = (edit_site.amp_unhide > 0);
     document.querySelector('input[data-key="amp_redirect"]').value = edit_site.amp_redirect ? edit_site.amp_redirect : '';
+    document.querySelector('input[data-key="ld_json"]').value = edit_site.ld_json ? edit_site.ld_json : '';
     document.querySelector('select[data-key="referer"]').selectedIndex = referer_options.indexOf(edit_site.referer);
     document.querySelector('select[data-key="random_ip"]').selectedIndex = random_ip_options.indexOf(edit_site.random_ip);
   });
@@ -251,16 +252,17 @@ function renderOptions() {
       'title': 0,
       'domain': 0,
       'allow_cookies': 1,
-      'block_javascript': 1,
+      'block_javascript (domain)': 1,
       'block_javascript_ext': 1,
       'block_regex': 0,
       'amp_unhide': 1,
-      'amp_redirect': 0
+      'amp_redirect': 0,
+      'ld_json': 0
     };
     for (var key in add_checkboxes) {
       labelEl = document.createElement('label');
       inputEl = document.createElement('input');
-      inputEl.dataset.key = key;
+      inputEl.dataset.key = key.split(' (')[0];
       labelEl.appendChild(inputEl);
       if (add_checkboxes[key]) {
         inputEl.type = 'checkbox';
@@ -270,7 +272,8 @@ function renderOptions() {
           title: 'Example',
           domain: 'example.com',
           block_regex: '\\.example\\.com\\/js\\/',
-          amp_redirect: 'div.paywall'
+          amp_redirect: 'div.paywall',
+          ld_json: 'div.paywall|div.article'
         };
         if (placeholders[key])
           inputEl.placeholder = placeholders[key];
@@ -288,7 +291,7 @@ function renderOptions() {
       labelEl = document.createElement('label');
       labelEl.appendChild(document.createTextNode(key + ' '));
       inputEl = document.createElement('select');
-      inputEl.dataset.key = key;
+      inputEl.dataset.key = key.split(' (')[0];
       labelEl.appendChild(inputEl);
       
       for (let elem of add_options[key]) {
@@ -325,7 +328,8 @@ function renderOptions() {
       (sites_custom[key]['referer'] ? ' | referer: ' + sites_custom[key]['referer'] : '') +
       (sites_custom[key]['random_ip'] ? ' | random_ip: ' + sites_custom[key]['random_ip'] : '') +
       (sites_custom[key]['amp_unhide'] > 0 ? ' | amp_unhide' : '') +
-      (sites_custom[key]['amp_redirect'] ? ' | amp_redirect' : '');
+      (sites_custom[key]['amp_redirect'] ? ' | amp_redirect' : '') +
+      (sites_custom[key]['ld_json'] ? ' | ld_json' : '');
       optionEl.value = key;
       selectEl.add(optionEl);
     }

sites_updated.json

@@ -1,104 +1,13 @@
 {
-  "Australian Community Media (daily)": {
-    "domain": "###_au_comm_media",
-    "group": [
-      "bendigoadvertiser.com.au",
-      "bordermail.com.au",
-      "canberratimes.com.au",
-      "centralwesterndaily.com.au",
-      "dailyadvertiser.com.au",
-      "dailyliberal.com.au",
-      "examiner.com.au",
-      "illawarramercury.com.au",
-      "newcastleherald.com.au",
-      "northerndailyleader.com.au",
-      "portnews.com.au",
-      "standard.net.au",
-      "theadvocate.com.au",
-      "thecourier.com.au",
-      "westernadvocate.com.au"
-    ],
-    "allow_cookies": 1,
-    "block_regex": "cdn-au\\.piano\\.io\\/api\\/tinypass.+\\.js"
-  },
-  "Bon Appétit": {
-    "domain": "bonappetit.com",
-    "remove_cookies_select_drop": ["pay_ent_smp"],
-    "block_regex": "(journey\\.bonappetit\\.com\\/build-.+\\.js|cdn\\.ampproject\\.org\\/v\\d\\/amp-(ad|subscriptions)-.+\\.js)",
-    "useragent": "googlebot"
-  },
-  "Diario Correo": {
-    "domain": "diariocorreo.pe",
-    "allow_cookies": 1,
-    "block_regex": "\\/diariocorreo\\.pe\\/pf\\/dist\\/engine\\/react\\.js",
-    "cs_code": [{
-        "cond": ".story-content__nota-premium",
-        "rm_attrib": "style",
-        "rm_class": "story-content__nota-premium"
-      }
-    ]
-  },
-  "Eastwest.eu": {
-    "domain": "eastwest.eu",
-    "allow_cookies": 1,
-    "cs_code": [{
-        "cond": ".paywall",
-        "rm_attrib": "style",
-        "rm_class": "paywall",
-        "elems": [{
-            "cond": "div#testo_articolo > p, div#testo_articolo > h3",
-            "rm_elem": 1
-          }, {
-            "cond": "div.offerta_abbonamenti",
-            "rm_elem": 1
-          }
-        ]
-      }
-    ]
-  },
-  "GQ": {
-    "domain": "gq.com",
-    "remove_cookies_select_drop": ["pay_ent_smp"],
-    "block_regex": "(journey\\.gq\\.com\\/build-.+\\.js|cdn\\.ampproject\\.org\\/v\\d\\/amp-(ad|subscriptions)-.+\\.js)",
-    "useragent": "googlebot"
-  },
   "Neue Osnabrücker Zeitung": {
     "domain": "noz.de",
     "allow_cookies": 1,
     "block_regex": "cdn\\.ampproject\\.org\\/v\\d\\/amp-(access|(sticky-)?ad|fx-flying-carpet)-.+\\.js",
     "amp_unhide": 1
   },
-  "New York Magazine (+ Curbed, Grub Street, The Cut & Vulture)": {
-    "domain": "###_usa_nymag",
-    "group": [
-      "curbed.com",
-      "grubstreet.com",
-      "nymag.com",
-      "thecut.com",
-      "vulture.com"
-    ],
-    "remove_cookies_select_drop": ["nymcid", "first-nymcid"],
-    "block_regex": "fosse\\.nymag\\.com\\/fosse\\/.+\\/scripts\\/.+\\.js"
-  },
-  "Seeking Alpha": {
-    "domain": "seekingalpha.com",
-    "allow_cookies": 1,
-    "block_regex": "(\\.tinypass\\.com\\/|cdn\\.ampproject\\.org(\\/.+)?\\/v\\d\\/amp-(access|ad|loader)-.+\\.js)"
-  },
   "The Washington Post": {
     "domain": "washingtonpost.com",
     "allow_cookies": 1,
     "block_regex": "\\.washingtonpost\\.com\\/tetro\\/evaluate"
-  },
-  "Vogue USA": {
-    "domain": "vogue.com",
-    "remove_cookies_select_drop": ["pay_ent_smp"],
-    "block_regex": "(journey\\.vogue\\.com\\/build-.+\\.js|cdn\\.ampproject\\.org\\/v\\d\\/amp-(ad|subscriptions)-.+\\.js)",
-    "useragent": "googlebot"
-  },
-  "Vogue Business": {
-    "allow_cookies": 1,
-    "domain": "voguebusiness.com",
-    "useragent": "googlebot"
   }
 }