diff --git a/README.md b/README.md index c7bb8d59..9f6b5a55 100644 --- a/README.md +++ b/README.md @@ -54,7 +54,7 @@ Use adblocker with custom (content)filter: https://gitlab.com/magnolia1234/bypas ### List of supported websites -_* free articles only._ +_* free/cached articles only._ ##### World news [Foreign Affairs](https://www.foreignaffairs.com) - @@ -606,7 +606,7 @@ Grouped in options:\ [Central Western Daily](https://www.centralwesterndaily.com.au) - [Daily Liberal](https://www.dailyliberal.com.au) - [Illawarra Mercury](https://www.illawarramercury.com.au) - -[Newcastle Herald](https://www.newcastleherald.com.au) - +[Newcastle Herald](https://www.newcastleherald.com.au)* - [The Advocate](https://www.theadvocate.com.au) - [The Border Mail](https://www.bordermail.com.au) - [The Canberra Times](https://www.canberratimes.com.au) - @@ -666,7 +666,7 @@ Grouped in options:\ [Grupo Abril](https://grupoabril.com.br) - [O Estado de S. Paulo](https://estadao.com.br) - [O Globo](https://oglobo.globo.com) - -[Valor Econômico](https://valor.globo.com) +[Valor Econômico](https://valor.globo.com)* ##### Chile [Diario Financiero](https://www.df.cl) - [El Mercurio](https://digital.elmercurio.com) - @@ -684,7 +684,7 @@ Grouped in options:\ [El Comercio](https://elcomercio.pe) - [Gestión](https://gestion.pe) -_* free articles only._ +_* free/cached articles only._ ### Sites with limited number of free articles The free article limit can normally be bypassed by removing cookies for the site. @@ -707,7 +707,7 @@ Check 'Options'-link in popup-menu and go to custom sites. \* by default BPC has limited permissions, but you can opt-in to enable custom sites (and also clear cookies/block general paywall-scripts for non-listed sites). You can also just request permissions for the custom sites you added yourself (or `clear cookies` to ask for permission for current site). By default sites' cookies/local storage are removed after page loads (to bypass article limit). -Also you can enable Googlebot/Bingbot user-agent, set referer (to Facebook, Google or Twitter; ignored when Googlebot is set), set random ip-address, disable Javascript for (sub)domain(s) and/or external domains, block regular expression and/or unhide text on (or when paywall(selector) redirect to) amp-page. +Also you can enable Googlebot/Bingbot user-agent, set referer (to Facebook, Google or Twitter; ignored when Googlebot is set), set random ip-address, disable Javascript for (sub)domain(s) and/or external domains, block regular expression, unhide text on (or when paywall(selector) redirect to) amp-page and/or load text from json (paywall|article selector). [Example list of custom sites](https://gitlab.com/magnolia1234/bypass-paywalls-firefox-clean/-/blob/master/custom/sites_custom.json) or [download list (json)](https://gitlab.com/magnolia1234/bypass-paywalls-firefox-clean/-/raw/master/custom/sites_custom.json) diff --git a/background.js b/background.js index 09c7c60f..8e0bd184 100644 --- a/background.js +++ b/background.js @@ -8,6 +8,7 @@ var ext_name = manifestData.name; var ext_version = manifestData.version; const cs_limit_except = ['elespanol.com', 'faz.net', 'inkl.com', 'nation.africa', 'nationalgeographic.com', 'thetimes.co.uk']; +const dompurify_sites = ['asiatimes.com', 'bloomberg.com', 'cicero.de', 'economictimes.com', 'hs.fi', 'lesechos.fr', 'marianne.net', 'newcastleherald.com.au', 'newleftreview.org', 'nzherald.co.nz', 'prospectmagazine.co.uk', 'stratfor.com', 'techinasia.com', 'timesofindia.com', 'valor.globo.com', 'vn.nl'].concat(fr_groupe_sud_ouest_domains, nl_mediahuis_region_domains, no_nhst_media_domains, usa_theathletic_domains); var currentTabUrl = ''; var csDone = false; var optin_setcookie = false; @@ -68,6 +69,8 @@ var amp_unhide; var amp_redirect; // code for contentScript var cs_code; +// load text from json +var ld_json; // custom: block javascript var block_js_custom = []; @@ -88,6 +91,7 @@ function initSetRules() { amp_unhide = []; amp_redirect = {}; cs_code = {}; + ld_json = {}; block_js_custom = []; block_js_custom_ext = []; blockedRegexes = {}; @@ -269,6 +273,11 @@ function set_rules(sites, sites_updated, sites_custom) { block_js_custom_ext.push(domain); if (rule.amp_unhide > 0) amp_unhide.push(domain); + if (rule.ld_json) { + ld_json[domain] = rule.ld_json; + if (!dompurify_sites.includes(domain)) + dompurify_sites.push(domain); + } } } } @@ -1016,7 +1025,7 @@ if (matchUrlDomain(change_headers, details.url) && !['font', 'image', 'styleshee } if ((!['font', 'stylesheet'].includes(details.type) || matchUrlDomain(cs_limit_except, currentTabUrl)) && !csDone) { let lib_file = 'lib/empty.js'; - if (matchUrlDomain(['asiatimes.com', 'bloomberg.com', 'cicero.de', 'economictimes.com', 'hs.fi', 'lesechos.fr', 'marianne.net', 'newcastleherald.com.au', 'newleftreview.org', 'nzherald.co.nz', 'prospectmagazine.co.uk', 'stratfor.com', 'techinasia.com', 'timesofindia.com', 'valor.globo.com', 'vn.nl'].concat(fr_groupe_sud_ouest_domains, nl_mediahuis_region_domains, no_nhst_media_domains, usa_theathletic_domains), currentTabUrl)) + if (matchUrlDomain(dompurify_sites, currentTabUrl)) lib_file = 'lib/purify.min.js'; var bg2csData = { optin_setcookie: optin_setcookie, @@ -1028,6 +1037,9 @@ if (matchUrlDomain(change_headers, details.url) && !['font', 'image', 'styleshee let cs_code_domain = ''; if (cs_code_domain = matchUrlDomain(Object.keys(cs_code), currentTabUrl)) bg2csData.cs_code = cs_code[cs_code_domain]; + let ld_json_domain = ''; + if (ld_json_domain = matchUrlDomain(Object.keys(ld_json), currentTabUrl)) + bg2csData.ld_json = ld_json[ld_json_domain]; ext_api.tabs.executeScript(tabId, { code: 'var bg2csData = ' + JSON.stringify(bg2csData) + ';' }, function () { diff --git a/changelog.txt b/changelog.txt index d9f62b67..cd22daf6 100644 --- a/changelog.txt +++ b/changelog.txt @@ -3,6 +3,7 @@ Changelog Bypass Paywalls Clean - Firefox Post-release Fix update check (GitLab CORS issue or BitBucket http error 429) +Update custom sites (load text from json) * v2.5.7.0 (2022-02-20) Remove Ruhr Nachrichten (fix obsolete) diff --git a/contentScript.js b/contentScript.js index 363a7f39..d8e1fb6a 100644 --- a/contentScript.js +++ b/contentScript.js @@ -38,10 +38,40 @@ if (!matchDomain(arr_localstorage_hold)) { window.localStorage.clear(); } +var bg2csData; + +// custom/updated sites: load text from json +if ((bg2csData !== undefined) && bg2csData.ld_json && dompurify_loaded) { + if (bg2csData.ld_json.includes('|')) { + let ld_json_split = bg2csData.ld_json.split('|'); + let paywall_sel = ld_json_split[0]; + let article_sel = ld_json_split[1]; + let paywall = document.querySelector(paywall_sel); + if (paywall) { + removeDOMElement(paywall); + let scripts = document.querySelectorAll('script[type="application/ld+json"]'); + let json; + for (let script of scripts) { + if (script.innerText.includes('articleBody')) + json = script; + } + if (json) { + let json_text = JSON.parse(json.text).articleBody; + let content = document.querySelector(article_sel); + if (json_text && content) { + let parser = new DOMParser(); + let doc = parser.parseFromString('
' + DOMPurify.sanitize(json_text) + '
', 'text/html'); + let content_new = doc.querySelector('div'); + content.parentNode.replaceChild(content_new, content); + } + } + } + } +} + var div_bpc_done = document.querySelector('div#bpc_done'); if (!div_bpc_done) { -var bg2csData; // check for opt-in confirmation (from background.js) if ((bg2csData !== undefined) && bg2csData.optin_setcookie) { if (domain = matchDomain(['belfasttelegraph.co.uk', 'independent.ie'])) { @@ -106,7 +136,7 @@ function amp_unhide_access_hide(amp_access = '', amp_access_not = '', amp_ads_se amp_iframes_replace(amp_iframe_link, source); } -// custom sites: try to unhide text on amp-page +// custom/updated sites: try to unhide text on amp-page if ((bg2csData !== undefined) && bg2csData.amp_unhide) { window.setTimeout(function () { let amp_page_hide = document.querySelector('script[src*="/amp-access-"], script[src*="/amp-subscriptions-"]'); @@ -118,7 +148,7 @@ if ((bg2csData !== undefined) && bg2csData.amp_unhide) { }, 100); // Delay (in milliseconds) } -// updated sites: amp-redirect +// custom/updated sites: amp-redirect if ((bg2csData !== undefined) && bg2csData.amp_redirect) { window.setTimeout(function () { let amp_script = document.querySelector('script[src^="https://cdn.ampproject.org/"]'); @@ -206,14 +236,14 @@ else { if (paywall) { paywall.removeAttribute('style'); let url_src = window.location.href.replace('newcastleherald.com.au', 'canberratimes.com.au'); - replaceDomElementExt(url_src, true, false, 'p[class^="Paragraph_wrapper__"]', '', 'div.article__body'); + replaceDomElementExt(url_src, true, false, 'p[class^="Paragraph_wrapper__"]', 'BPC > text not available on other group site: ', 'div.article__body'); } let hidden_images = document.querySelectorAll('img[src^="data:image/"][data-src]'); for (let hidden_image of hidden_images) hidden_image.setAttribute('src', hidden_image.getAttribute('data-src')); let lead_image = document.querySelector('div#story-body img.w-full[src]'); if (lead_image) { - let lead_src = lead_image.src.split('.jpg')[0]; + let lead_src = lead_image.src.toLowerCase().split(/\.(png|jpg)/)[0]; let body_image = document.querySelector('div.article__body img[src]'); if (lead_src && body_image && body_image.src.includes(lead_src)) removeDOMElement(body_image); diff --git a/custom/sites_custom.json b/custom/sites_custom.json index c09c5ebe..1148399c 100644 --- a/custom/sites_custom.json +++ b/custom/sites_custom.json @@ -76,6 +76,11 @@ "block_regex": "loader-cdn\\.azureedge\\.net", "domain": "gazette.com" }, + "Gplanet.co.il": { + "allow_cookies": 1, + "domain": "gplanet.co.il", + "ld_json": "div.subscription-form|div.content-inner" + }, "Handelszeitung.ch": { "allow_cookies": 1, "domain": "handelszeitung.ch", diff --git a/manifest.json b/manifest.json index 1bba4e52..995bd0c9 100644 --- a/manifest.json +++ b/manifest.json @@ -620,5 +620,5 @@ "*://*.wallkit.net/*", "*://*.wsj.net/*" ], - "version": "2.5.7.1" + "version": "2.5.7.2" } diff --git a/options/options_custom.html b/options/options_custom.html index 7497a3a0..3bc1c681 100644 --- a/options/options_custom.html +++ b/options/options_custom.html @@ -20,7 +20,7 @@

Custom Sites

To add a new site, enter an unique title/domain (without www.).
- Select options for useragent (like Googlebot), set referer (ignored when Googlebot is set), set random ip-address, block Javascript (on (sub)domain(s) of site and/or external domains), block regular expression and/or unhide text on (or when paywall(selector) redirect to) amp-page.
+ Select options for useragent (like Googlebot), set referer (ignored when Googlebot is set), set random ip-address, block Javascript, block regular expression, unhide text on (or when paywall(selector) redirect to) amp-page and/or load text from json (paywall|article selector).
Custom sites (new) are enabled automatically in (cookies will be removed by default unless you enable allow_cookies).
If you want to use custom sites (for non-listed sites) enable it in Custom sites enabled:
diff --git a/options/options_custom.js b/options/options_custom.js index 1636338a..4b5a2118 100644 --- a/options/options_custom.js +++ b/options/options_custom.js @@ -192,6 +192,7 @@ function edit_options() { document.querySelector('input[data-key="block_regex"]').value = edit_site.block_regex ? edit_site.block_regex : ''; document.querySelector('input[data-key="amp_unhide"]').checked = (edit_site.amp_unhide > 0); document.querySelector('input[data-key="amp_redirect"]').value = edit_site.amp_redirect ? edit_site.amp_redirect : ''; + document.querySelector('input[data-key="ld_json"]').value = edit_site.ld_json ? edit_site.ld_json : ''; document.querySelector('select[data-key="referer"]').selectedIndex = referer_options.indexOf(edit_site.referer); document.querySelector('select[data-key="random_ip"]').selectedIndex = random_ip_options.indexOf(edit_site.random_ip); }); @@ -251,16 +252,17 @@ function renderOptions() { 'title': 0, 'domain': 0, 'allow_cookies': 1, - 'block_javascript': 1, + 'block_javascript (domain)': 1, 'block_javascript_ext': 1, 'block_regex': 0, 'amp_unhide': 1, - 'amp_redirect': 0 + 'amp_redirect': 0, + 'ld_json': 0 }; for (var key in add_checkboxes) { labelEl = document.createElement('label'); inputEl = document.createElement('input'); - inputEl.dataset.key = key; + inputEl.dataset.key = key.split(' (')[0]; labelEl.appendChild(inputEl); if (add_checkboxes[key]) { inputEl.type = 'checkbox'; @@ -270,7 +272,8 @@ function renderOptions() { title: 'Example', domain: 'example.com', block_regex: '\\.example\\.com\\/js\\/', - amp_redirect: 'div.paywall' + amp_redirect: 'div.paywall', + ld_json: 'div.paywall|div.article' }; if (placeholders[key]) inputEl.placeholder = placeholders[key]; @@ -288,7 +291,7 @@ function renderOptions() { labelEl = document.createElement('label'); labelEl.appendChild(document.createTextNode(key + ' ')); inputEl = document.createElement('select'); - inputEl.dataset.key = key; + inputEl.dataset.key = key.split(' (')[0]; labelEl.appendChild(inputEl); for (let elem of add_options[key]) { @@ -325,7 +328,8 @@ function renderOptions() { (sites_custom[key]['referer'] ? ' | referer: ' + sites_custom[key]['referer'] : '') + (sites_custom[key]['random_ip'] ? ' | random_ip: ' + sites_custom[key]['random_ip'] : '') + (sites_custom[key]['amp_unhide'] > 0 ? ' | amp_unhide' : '') + - (sites_custom[key]['amp_redirect'] ? ' | amp_redirect' : ''); + (sites_custom[key]['amp_redirect'] ? ' | amp_redirect' : '') + + (sites_custom[key]['ld_json'] ? ' | ld_json' : ''); optionEl.value = key; selectEl.add(optionEl); } diff --git a/sites_updated.json b/sites_updated.json index 43607732..72f90d67 100644 --- a/sites_updated.json +++ b/sites_updated.json @@ -1,104 +1,13 @@ { - "Australian Community Media (daily)": { - "domain": "###_au_comm_media", - "group": [ - "bendigoadvertiser.com.au", - "bordermail.com.au", - "canberratimes.com.au", - "centralwesterndaily.com.au", - "dailyadvertiser.com.au", - "dailyliberal.com.au", - "examiner.com.au", - "illawarramercury.com.au", - "newcastleherald.com.au", - "northerndailyleader.com.au", - "portnews.com.au", - "standard.net.au", - "theadvocate.com.au", - "thecourier.com.au", - "westernadvocate.com.au" - ], - "allow_cookies": 1, - "block_regex": "cdn-au\\.piano\\.io\\/api\\/tinypass.+\\.js" - }, - "Bon Appétit": { - "domain": "bonappetit.com", - "remove_cookies_select_drop": ["pay_ent_smp"], - "block_regex": "(journey\\.bonappetit\\.com\\/build-.+\\.js|cdn\\.ampproject\\.org\\/v\\d\\/amp-(ad|subscriptions)-.+\\.js)", - "useragent": "googlebot" - }, - "Diario Correo": { - "domain": "diariocorreo.pe", - "allow_cookies": 1, - "block_regex": "\\/diariocorreo\\.pe\\/pf\\/dist\\/engine\\/react\\.js", - "cs_code": [{ - "cond": ".story-content__nota-premium", - "rm_attrib": "style", - "rm_class": "story-content__nota-premium" - } - ] - }, - "Eastwest.eu": { - "domain": "eastwest.eu", - "allow_cookies": 1, - "cs_code": [{ - "cond": ".paywall", - "rm_attrib": "style", - "rm_class": "paywall", - "elems": [{ - "cond": "div#testo_articolo > p, div#testo_articolo > h3", - "rm_elem": 1 - }, { - "cond": "div.offerta_abbonamenti", - "rm_elem": 1 - } - ] - } - ] - }, - "GQ": { - "domain": "gq.com", - "remove_cookies_select_drop": ["pay_ent_smp"], - "block_regex": "(journey\\.gq\\.com\\/build-.+\\.js|cdn\\.ampproject\\.org\\/v\\d\\/amp-(ad|subscriptions)-.+\\.js)", - "useragent": "googlebot" - }, "Neue Osnabrücker Zeitung": { "domain": "noz.de", "allow_cookies": 1, "block_regex": "cdn\\.ampproject\\.org\\/v\\d\\/amp-(access|(sticky-)?ad|fx-flying-carpet)-.+\\.js", "amp_unhide": 1 }, - "New York Magazine (+ Curbed, Grub Street, The Cut & Vulture)": { - "domain": "###_usa_nymag", - "group": [ - "curbed.com", - "grubstreet.com", - "nymag.com", - "thecut.com", - "vulture.com" - ], - "remove_cookies_select_drop": ["nymcid", "first-nymcid"], - "block_regex": "fosse\\.nymag\\.com\\/fosse\\/.+\\/scripts\\/.+\\.js" - }, - "Seeking Alpha": { - "domain": "seekingalpha.com", - "allow_cookies": 1, - "block_regex": "(\\.tinypass\\.com\\/|cdn\\.ampproject\\.org(\\/.+)?\\/v\\d\\/amp-(access|ad|loader)-.+\\.js)" - }, "The Washington Post": { "domain": "washingtonpost.com", "allow_cookies": 1, "block_regex": "\\.washingtonpost\\.com\\/tetro\\/evaluate" - }, - "Vogue USA": { - "domain": "vogue.com", - "remove_cookies_select_drop": ["pay_ent_smp"], - "block_regex": "(journey\\.vogue\\.com\\/build-.+\\.js|cdn\\.ampproject\\.org\\/v\\d\\/amp-(ad|subscriptions)-.+\\.js)", - "useragent": "googlebot" - }, - "Vogue Business": { - "allow_cookies": 1, - "domain": "voguebusiness.com", - "useragent": "googlebot" } }