Update custom sites (load text from json_next)

1 year ago · 953b0cf9cf
parent 3ebe54f466
commit 953b0cf9cf
7 changed files with 86 additions and 9 deletions
--- a/background.js
+++ b/background.js
@ -77,8 +77,10 @@ var amp_unhide;
 var amp_redirect;
 // code for contentScript
 var cs_code;
-// load text from json
+// load text from json (script[type="application/ld+json"])
 var ld_json;
+// load text from json (script#__NEXT_DATA__)
+var ld_json_next;
 // load text from Google webcache
 var ld_google_webcache;
 // add external link to article
@ -105,6 +107,7 @@ function initSetRules() {
  amp_redirect = {};
  cs_code = {};
  ld_json = {};
+  ld_json_next = {};
  ld_google_webcache = {};
  add_ext_link = {};
  block_js_custom = [];
@ -340,6 +343,8 @@ function set_rules(sites, sites_updated, sites_custom) {
          amp_unhide.push(domain);
        if (rule.ld_json)
          ld_json[domain] = rule.ld_json;
+        if (rule.ld_json_next)
+          ld_json_next[domain] = rule.ld_json_next;
        if (rule.ld_google_webcache)
          ld_google_webcache[domain] = rule.ld_google_webcache;
        if (rule.add_ext_link && rule.add_ext_link_type)
@ -744,6 +749,9 @@ if (typeof browser !== 'object') {
    let ld_json_domain = '';
    if (ld_json_domain = matchUrlDomain(Object.keys(ld_json), url))
      bg2csData.ld_json = ld_json[ld_json_domain];
+    let ld_json_next_domain = '';
+    if (ld_json_next_domain = matchUrlDomain(Object.keys(ld_json_next), url))
+      bg2csData.ld_json_next = ld_json_next[ld_json_next_domain];
    let ld_google_webcache_domain = '';
    if (ld_google_webcache_domain = matchUrlDomain(Object.keys(ld_google_webcache), url))
      bg2csData.ld_google_webcache = ld_google_webcache[ld_google_webcache_domain];
@ -1303,6 +1311,8 @@ function clear_cookies() {
    }
    if (rules.ld_json)
      ld_json[custom_domain] = rules.ld_json;
+    if (rules.ld_json_next)
+      ld_json_next[custom_domain] = rules.ld_json_next;
    if (rules.add_ext_link && rules.add_ext_link_type)
      add_ext_link[custom_domain] = {css: rules.add_ext_link, type: rules.add_ext_link_type};
    ext_api.tabs.reload({bypassCache: true});
--- a/changelog.txt
+++ b/changelog.txt
@ -11,6 +11,7 @@ Fix ilMessaggero.it (+ regional sites)
 Fix Modern Healthcare (sponsored)
 Fix Sportico (amp)
 Update custom (flex) sites (no fix group)
+Update custom sites (load text from json_next)

 * v3.1.6.0 (2023-04-30)
 Remove Zeit.de (fix obsolete)
--- a/contentScript.js
+++ b/contentScript.js
@ -61,7 +61,9 @@ if (bg2csData.ld_json) {
      let paywall_sel = ld_json_split[0];
      let article_sel = ld_json_split[1];
      let paywall = document.querySelectorAll(paywall_sel);
-      if (paywall.length) {
+      let article = document.querySelector(article_sel);
+      let article_append = ld_json_split[2];// optional
+      if (paywall.length && article) {
        removeDOMElement(...paywall);
        let json_script = getArticleJsonScript();
        if (json_script) {
@ -76,12 +78,15 @@ if (bg2csData.ld_json) {
              json_key = Object.keys(json).find(key => key.match(/^articlebody$/i)) || Object.keys(json).find(key => key.match(/^text$/i));
              json_text = parseHtmlEntities(json[json_key]);
            }
-            let content = document.querySelector(article_sel);
-            if (json_text && content) {
+            if (json_text && article.parentNode) {
              let parser = new DOMParser();
-              let doc = parser.parseFromString('<div>' + json_text + '</div>', 'text/html');
-              let content_new = doc.querySelector('div');
-              content.parentNode.replaceChild(content_new, content);
+              let doc = parser.parseFromString('<div style="margin: 25px 0px">' + json_text + '</div>', 'text/html');
+              let article_new = doc.querySelector('div');
+              if (article_append || !article.parentNode) {
+                article.innerHTML = '';
+                article.appendChild(article_new);
+              } else
+                article.parentNode.replaceChild(article_new, article);
            }
          } catch (err) {
            console.log(err);
@ -92,6 +97,40 @@ if (bg2csData.ld_json) {
  }
 }

+if (bg2csData.ld_json_next) {
+  if (bg2csData.ld_json_next.includes('|')) {
+    window.setTimeout(function () {
+      let ld_json_next_split = bg2csData.ld_json_next.split('|');
+      let paywall_sel = ld_json_next_split[0];
+      let article_sel = ld_json_next_split[1];
+      let paywall = document.querySelectorAll(paywall_sel);
+      let article = document.querySelector(article_sel);
+      let article_append = ld_json_next_split[2];// optional
+      if (paywall.length && article) {
+        removeDOMElement(...paywall);
+        let json_script = document.querySelector('script#__NEXT_DATA__');
+        let json = JSON.parse(json_script.text);
+        if (json) {
+          let url_next = findKeyJson(json, ['slug']);
+          if (url_next && !window.location.pathname.endsWith(url_next))
+            refreshCurrentTab();
+          let json_text = parseHtmlEntities(findKeyJson(json, ['body', 'content', 'description'], 500));
+          if (json_text && article.parentNode) {
+            let parser = new DOMParser();
+            let doc = parser.parseFromString('<div>' + json_text + '</div>', 'text/html');
+              let article_new = doc.querySelector('div');
+              if (article_append || !article.parentNode) {
+                article.innerHTML = '';
+                article.appendChild(article_new);
+              } else
+                article.parentNode.replaceChild(article_new, article);
+          }
+        }
+      }
+    }, 1000);
+  }
+}
+
 // custom/updated sites: load text from Google webcache
 if (bg2csData.ld_google_webcache) {
  if (bg2csData.ld_google_webcache.includes('|')) {
@ -5154,6 +5193,24 @@ function getArticleJsonScript() {
  return json_script;
 }

+function findKeyJson(json, keys, min_val_len = 0) {
+  let source;
+  if (Array.isArray(json)) {
+    for (let elem of json)
+      source = source || findKeyJson(json[elem], keys, min_val_len);
+  } else if (typeof json === 'object') {
+    for (let elem in json) {
+      let json_elem = json[elem];
+      if (typeof json_elem === 'string' && keys.includes(elem)) {
+        if (json_elem.length > min_val_len)
+          return json_elem;
+      } else
+        source = source || findKeyJson(json_elem, keys, min_val_len);
+    }
+  }
+  return source;
+}
+
 function genHexString(len) {
  let output = '';
  for (let i = 0; i < len; i++)
--- a/custom/manifest.json
+++ b/custom/manifest.json
@ -51,5 +51,5 @@
    "webRequestBlocking",
    "*://*/*"
  ],
-  "version": "3.1.6.7"
+  "version": "3.1.6.8"
 }
--- a/custom/sites_custom.json
+++ b/custom/sites_custom.json
@ -29,6 +29,11 @@
    "allow_cookies": 1,
    "domain": "bild.de"
  },
+  "Bisnisindonesia.id": {
+    "allow_cookies": 1,
+    "domain": "bisnisindonesia.id",
+    "ld_json_next": "div.bottom-10|article"
+  },
  "Bizwest.com": {
    "allow_cookies": 1,
    "cs_code": "[{\"cond\":\"div.fp-paywall\", \"rm_elem\":1, \"elems\":[{\"cond\":\"div.fp-content\", \"rm_attrib\":\"class\"}]}]",
--- a/manifest.json
+++ b/manifest.json
@ -756,5 +756,5 @@
    "*://html.onlineviewer.net/*",
    "*://webcache.googleusercontent.com/*"
  ],
-  "version": "3.1.6.7"
+  "version": "3.1.6.8"
 }
--- a/options/options_custom.js
+++ b/options/options_custom.js
@ -224,6 +224,7 @@ function edit_options() {
    document.querySelector('input[data-key="amp_unhide"]').checked = (edit_site.amp_unhide > 0);
    document.querySelector('input[data-key="amp_redirect"]').value = edit_site.amp_redirect ? edit_site.amp_redirect : '';
    document.querySelector('input[data-key="ld_json"]').value = edit_site.ld_json ? edit_site.ld_json : '';
+    document.querySelector('input[data-key="ld_json_next"]').value = edit_site.ld_json_next ? edit_site.ld_json_next : '';
    document.querySelector('input[data-key="ld_google_webcache"]').value = edit_site.ld_google_webcache ? edit_site.ld_google_webcache : '';
    document.querySelector('input[data-key="add_ext_link"]').value = edit_site.add_ext_link ? edit_site.add_ext_link : '';
    document.querySelector('select[data-key="add_ext_link_type"]').selectedIndex = add_ext_link_type_options.indexOf(edit_site.add_ext_link_type);
@ -296,6 +297,7 @@ function renderOptions() {
      'amp_unhide': 1,
      'amp_redirect': 0,
      'ld_json': 0,
+      'ld_json_next': 0,
      'ld_google_webcache': 0,
      'add_ext_link': 0,
      'add_ext_link_type': 0,
@ -337,6 +339,7 @@ function renderOptions() {
            block_regex: '\\.example\\.com\\/js\\/',
            amp_redirect: 'div.paywall',
            ld_json: 'div.paywall|div.article',
+            ld_json_next: 'div.paywall|div.article',
            ld_google_webcache: 'div.paywall|div.article',
            add_ext_link: 'div.paywall|div.article',
            cs_code: 'for dev: check GitLab examples',
@ -382,6 +385,7 @@ function renderOptions() {
      (sites_custom[key]['amp_unhide'] > 0 ? ' | amp_unhide' : '') +
      (sites_custom[key]['amp_redirect'] ? ' | amp_redirect' : '') +
      (sites_custom[key]['ld_json'] ? ' | ld_json' : '') +
+      (sites_custom[key]['ld_json_next'] ? ' | ld_json_next' : '') +
      (sites_custom[key]['ld_google_webcache'] ? ' | ld_google_webcache' : '') +
      (sites_custom[key]['add_ext_link'] && sites_custom[key]['add_ext_link_type'] ? ' | add_ext_link' : '') +
      (sites_custom[key]['cs_code'] ? ' | cs_code' : '');