feat: Add a custom extractor for www.engadget.com. (#552)

* feat:Add a custom extractor for ma.ttias.be.

When parsing content for cron.weekly issues, such as the one at https://ma.ttias.be/cronweekly/issue-130/, Mercury Parser would remove headings and ordered lists that were part of the content. This resolves that as follows:

* Remove "id" attributes from "h1" and "h2" elements. Those attributes would result in the elements having a low weight.
* Since Mercury Parser demotes "h1" elements to "h2", demote "h2" elements to "h3".
* Add class="entry-content-asset" to "ul" elements to avoid them being removed.

* removed redundant comment.

* feat: Add a custom extractor for engadget.com.

Co-authored-by: John Holdun <john@johnholdun.com>

cli.js

@@ -26,7 +26,7 @@ const {
   extendedListTypes,
   headers,
   addExtractor,
-  version,
+  version
 ) => {
   if (version) {
     console.log(package_info.version);

src/extractors/custom/index.js

@@ -138,3 +138,4 @@ export * from './ma.ttias.be';
 export * from './pastebin.com';
 export * from './www.abendblatt.de';
 export * from './www.gruene.de';
+export * from './www.engadget.com';

src/extractors/custom/www.engadget.com/index.js

@@ -0,0 +1,53 @@
+export const WwwEngadgetComExtractor = {
+  domain: 'www.engadget.com',
+
+  title: {
+    selectors: [['meta[name="og:title"]', 'value']],
+  },
+
+  author: {
+    selectors: ['a.th-meta[data-ylk*="subsec:author"]'],
+  },
+
+  // Engadget stories have publish dates, but the only representation of them on the page
+  // is in a format like "2h ago". There are also these tags with blank values:
+  // <meta class="swiftype" name="published_at" data-type="date" value="">
+  date_published: {
+    selectors: [
+      // enter selectors
+    ],
+  },
+
+  dek: {
+    selectors: ['div[class*="o-title_mark"] div'],
+  },
+
+  // Engadget stories do have lead images specified by an og:image meta tag, but selecting
+  // the value attribute of that tag fails. I believe the "&#x2111;" sequence of characters
+  // is triggering this inability to select the attribute value.
+  lead_image_url: {
+    selectors: [
+      // enter selectors
+    ],
+  },
+
+  content: {
+    selectors: [
+      [
+        // Some figures will be inside div.article-text, but some header figures/images
+        // will not.
+        '#page_body figure:not(div.article-text figure)',
+        'div.article-text',
+      ],
+    ],
+
+    // Is there anything in the content you selected that needs transformed
+    // before it's consumable content? E.g., unusual lazy loaded images
+    transforms: {},
+
+    // Is there anything that is in the result that shouldn't be?
+    // The clean selectors will remove anything that matches from
+    // the result
+    clean: [],
+  },
+};

src/extractors/custom/www.engadget.com/index.test.js

@@ -0,0 +1,120 @@
+import assert from 'assert';
+import URL from 'url';
+import cheerio from 'cheerio';
+
+import Mercury from 'mercury';
+import getExtractor from 'extractors/get-extractor';
+import { excerptContent } from 'utils/text';
+
+const fs = require('fs');
+
+describe('WwwEngadgetComExtractor', () => {
+  describe('initial test case', () => {
+    let result;
+    let url;
+    beforeAll(() => {
+      url =
+        'https://www.engadget.com/lyft-extends-free-scooter-rides-for-critical-workers-184144961.html';
+      const html = fs.readFileSync(
+        './fixtures/www.engadget.com/1587759947079.html'
+      );
+      result = Mercury.parse(url, { html, fallback: false });
+    });
+
+    it('is selected properly', () => {
+      // This test should be passing by default.
+      // It sanity checks that the correct parser
+      // is being selected for URLs from this domain
+      const extractor = getExtractor(url);
+      assert.equal(extractor.domain, URL.parse(url).hostname);
+    });
+
+    it('returns the title', async () => {
+      // To pass this test, fill out the title selector
+      // in ./src/extractors/custom/www.engadget.com/index.js.
+      const { title } = await result;
+
+      // Update these values with the expected values from
+      // the article.
+      assert.equal(
+        title,
+        `Lyft extends free scooter rides for critical workers through May`
+      );
+    });
+
+    it('returns the author', async () => {
+      // To pass this test, fill out the author selector
+      // in ./src/extractors/custom/www.engadget.com/index.js.
+      const { author } = await result;
+
+      // Update these values with the expected values from
+      // the article.
+      assert.equal(author, 'Jon Fingas');
+    });
+
+    it('returns the date_published', async () => {
+      // To pass this test, fill out the date_published selector
+      // in ./src/extractors/custom/www.engadget.com/index.js.
+      const { date_published } = await result;
+
+      // Update these values with the expected values from
+      // the article.
+      assert.equal(date_published, null);
+    });
+
+    it('returns the dek', async () => {
+      // To pass this test, fill out the dek selector
+      // in ./src/extractors/custom/www.engadget.com/index.js.
+      const { dek } = await result;
+
+      // Update these values with the expected values from
+      // the article.
+      assert.equal(
+        dek,
+        "Emergency, healthcare and transit workers won't need to pay."
+      );
+    });
+
+    it('returns the lead_image_url', async () => {
+      // To pass this test, fill out the lead_image_url selector
+      // in ./src/extractors/custom/www.engadget.com/index.js.
+      const { lead_image_url } = await result;
+
+      // Update these values with the expected values from
+      // the article.
+      assert.equal(lead_image_url, null);
+    });
+
+    it('returns the content', async () => {
+      // To pass this test, fill out the content selector
+      // in ./src/extractors/custom/www.engadget.com/index.js.
+      // You may also want to make use of the clean and transform
+      // options.
+      const { content } = await result;
+
+      const $ = cheerio.load(content || '');
+
+      const first13 = excerptContent(
+        $('*')
+          .first()
+          .text(),
+        13
+      );
+
+      // Update these values with the expected values from
+      // the article.
+      assert.equal(
+        first13,
+        'REUTERS/Brendan McDermid Lyft isn’t waiting until the end of April to decide on'
+      );
+
+      // Ensure that the figure is present.
+      const figure = $('figure');
+      assert.equal(figure.length, 1);
+
+      // Ensure that there are 3 paragraphs
+      const p = $('p');
+      assert.equal(p.length, 3);
+    });
+  });
+});