feat: custom medium extractor

dist/mercury.js

@@ -10,10 +10,11 @@ var cheerio = _interopDefault(require('cheerio'));
 var _Promise = _interopDefault(require('babel-runtime/core-js/promise'));
 var request = _interopDefault(require('request'));
 var _Reflect$ownKeys = _interopDefault(require('babel-runtime/core-js/reflect/own-keys'));
+var _slicedToArray = _interopDefault(require('babel-runtime/helpers/slicedToArray'));
 var stringDirection = _interopDefault(require('string-direction'));
 var _getIterator = _interopDefault(require('babel-runtime/core-js/get-iterator'));
+var _toConsumableArray = _interopDefault(require('babel-runtime/helpers/toConsumableArray'));
 var _defineProperty = _interopDefault(require('babel-runtime/helpers/defineProperty'));
-var _slicedToArray = _interopDefault(require('babel-runtime/helpers/slicedToArray'));
 var _typeof = _interopDefault(require('babel-runtime/helpers/typeof'));
 var validUrl = _interopDefault(require('valid-url'));
 var moment = _interopDefault(require('moment'));
@@ -1098,6 +1099,76 @@ var ApartmentTherapyExtractor = {
   }
 };
 
+var MediumExtractor = {
+  domain: 'medium.com',
+  title: {
+    selectors: ['h1']
+  },
+
+  author: {
+    selectors: [['meta[name="author"]', 'value']]
+  },
+
+  content: {
+    selectors: ['.section-content'],
+
+    // Is there anything in the content you selected that needs transformed
+    // before it's consumable content? E.g., unusual lazy loaded images
+    transforms: {
+      // Re-write lazy-loaded youtube videos
+      iframe: function iframe($node) {
+        var ytRe = /https:\/\/i.embed.ly\/.+url=https:\/\/i\.ytimg\.com\/vi\/(\w+)\//;
+        var thumb = decodeURIComponent($node.attr('data-thumbnail'));
+
+        if (ytRe.test(thumb)) {
+          var _thumb$match = thumb.match(ytRe);
+
+          var _thumb$match2 = _slicedToArray(_thumb$match, 2);
+
+          var _ = _thumb$match2[0];
+          var youtubeId = _thumb$match2[1]; // eslint-disable-line
+
+          $node.attr('src', 'https://www.youtube.com/embed/' + youtubeId);
+          var $parent = $node.parents('figure');
+          $parent.prepend($node.clone());
+          $node.remove();
+        }
+      }
+    },
+
+    // Is there anything that is in the result that shouldn't be?
+    // The clean selectors will remove anything that matches from
+    // the result
+    clean: []
+  },
+
+  date_published: {
+    selectors: [['time[datetime]', 'datetime']]
+  },
+
+  lead_image_url: {
+    selectors: [['meta[name="og:image"]', 'value']]
+  },
+
+  dek: {
+    selectors: [
+      // enter selectors
+    ]
+  },
+
+  next_page_url: {
+    selectors: [
+      // enter selectors
+    ]
+  },
+
+  excerpt: {
+    selectors: [
+      // enter selectors
+    ]
+  }
+};
+
 var Extractors = {
   'nymag.com': NYMagExtractor,
   'blogspot.com': BloggerExtractor,
@@ -1120,7 +1191,8 @@ var Extractors = {
   'gizmodo.com': DeadspinExtractor,
   'jalopnik.com': DeadspinExtractor,
   'www.broadwayworld.com': BroadwayWorldExtractor,
-  'www.apartmenttherapy.com': ApartmentTherapyExtractor
+  'www.apartmenttherapy.com': ApartmentTherapyExtractor,
+  'medium.com': MediumExtractor
 };
 
 // Spacer images to be removed
@@ -1443,13 +1515,22 @@ function cleanImages($article, $) {
   return $;
 }
 
-function markToKeep(article, $) {
-  var tags = arguments.length > 2 && arguments[2] !== undefined ? arguments[2] : [];
+function markToKeep(article, $, url) {
+  var tags = arguments.length > 3 && arguments[3] !== undefined ? arguments[3] : [];
 
   if (tags.length === 0) {
     tags = KEEP_SELECTORS;
   }
 
+  if (url) {
+    var _URL$parse = URL.parse(url);
+
+    var protocol = _URL$parse.protocol;
+    var hostname = _URL$parse.hostname;
+
+    tags = [].concat(_toConsumableArray(tags), ['iframe[src^="' + protocol + '//' + hostname + '"]']);
+  }
+
   $(tags.join(','), article).addClass(KEEP_CLASS);
 
   return $;
@@ -2593,7 +2674,7 @@ function extractCleanNode(article, _ref) {
   // Mark elements to keep that would normally be removed.
   // E.g., stripJunkTags will remove iframes, so we're going to mark
   // YouTube/Vimeo videos as elements we want to keep.
-  markToKeep(article, $);
+  markToKeep(article, $, url);
 
   // Drop certain tags like <title>, etc
   // This is -mostly- for cleanliness, not security.

src/extractors/all.js

@@ -15,6 +15,7 @@ import { PoliticoExtractor } from './custom/www.politico.com';
 import { DeadspinExtractor } from './custom/deadspin.com';
 import { BroadwayWorldExtractor } from './custom/www.broadwayworld.com';
 import { ApartmentTherapyExtractor } from './custom/www.apartmenttherapy.com';
+import { MediumExtractor } from './custom/medium.com';
 
 const Extractors = {
   'nymag.com': NYMagExtractor,
@@ -39,6 +40,7 @@ const Extractors = {
   'jalopnik.com': DeadspinExtractor,
   'www.broadwayworld.com': BroadwayWorldExtractor,
   'www.apartmenttherapy.com': ApartmentTherapyExtractor,
+  'medium.com': MediumExtractor,
 };
 
 export default Extractors;

src/extractors/custom/medium.com/index.js

@@ -0,0 +1,76 @@
+export const MediumExtractor = {
+  domain: 'medium.com',
+  title: {
+    selectors: [
+      'h1',
+    ],
+  },
+
+  author: {
+    selectors: [
+      ['meta[name="author"]', 'value'],
+    ],
+  },
+
+  content: {
+    selectors: [
+      '.section-content',
+    ],
+
+    // Is there anything in the content you selected that needs transformed
+    // before it's consumable content? E.g., unusual lazy loaded images
+    transforms: {
+      // Re-write lazy-loaded youtube videos
+      iframe: ($node) => {
+        const ytRe =
+          /https:\/\/i.embed.ly\/.+url=https:\/\/i\.ytimg\.com\/vi\/(\w+)\//;
+        const thumb = decodeURIComponent($node.attr('data-thumbnail'));
+
+        if (ytRe.test(thumb)) {
+          const [_, youtubeId] = thumb.match(ytRe) // eslint-disable-line
+          $node.attr('src', `https://www.youtube.com/embed/${youtubeId}`);
+          const $parent = $node.parents('figure');
+          $parent.prepend($node.clone());
+          $node.remove();
+        }
+      },
+    },
+
+    // Is there anything that is in the result that shouldn't be?
+    // The clean selectors will remove anything that matches from
+    // the result
+    clean: [
+
+    ],
+  },
+
+  date_published: {
+    selectors: [
+      ['time[datetime]', 'datetime'],
+    ],
+  },
+
+  lead_image_url: {
+    selectors: [
+      ['meta[name="og:image"]', 'value'],
+    ],
+  },
+
+  dek: {
+    selectors: [
+      // enter selectors
+    ],
+  },
+
+  next_page_url: {
+    selectors: [
+      // enter selectors
+    ],
+  },
+
+  excerpt: {
+    selectors: [
+      // enter selectors
+    ],
+  },
+};

src/extractors/custom/medium.com/index.test.js

@@ -0,0 +1,110 @@
+import assert from 'assert';
+import fs from 'fs';
+import URL from 'url';
+import cheerio from 'cheerio';
+
+import Mercury from 'mercury';
+import getExtractor from 'extractors/get-extractor';
+import { excerptContent } from 'utils/text';
+
+describe('MediumExtractor', () => {
+  it('is selected properly', () => {
+    // To pass this test, rename your extractor in
+    // ./src/extractors/custom/medium.com/index.js
+    // (e.g., CustomExtractor => NYTimesExtractor)
+    // then add your new extractor to
+    // src/extractors/all.js
+    const url =
+      'https://medium.com/the-wtf-economy/wtf-whats-the-future-e52ab9515573#.ilwrgwsks';
+    const extractor = getExtractor(url);
+    assert.equal(extractor.domain, URL.parse(url).hostname);
+  });
+
+  it('returns the title', async () => {
+    const html =
+      fs.readFileSync('./fixtures/medium.com/1477523363921.html');
+    const articleUrl =
+      'https://medium.com/the-wtf-economy/wtf-whats-the-future-e52ab9515573#.ilwrgwsks';
+
+    const { title } =
+      await Mercury.parse(articleUrl, html, { fallback: false });
+
+    assert.equal(title, 'WTF? What’s The Future?');
+  });
+
+
+  it('returns the author', async () => {
+    const html =
+      fs.readFileSync('./fixtures/medium.com/1477523363921.html');
+    const articleUrl =
+      'https://medium.com/the-wtf-economy/wtf-whats-the-future-e52ab9515573#.ilwrgwsks';
+
+    const { author } =
+      await Mercury.parse(articleUrl, html, { fallback: false });
+
+    assert.equal(author, 'Tim O\'Reilly');
+  });
+
+
+  it('returns the date_published', async () => {
+    const html =
+      fs.readFileSync('./fixtures/medium.com/1477523363921.html');
+    const articleUrl =
+      'https://medium.com/the-wtf-economy/wtf-whats-the-future-e52ab9515573#.ilwrgwsks';
+
+    const { date_published } =
+      await Mercury.parse(articleUrl, html, { fallback: false });
+
+    assert.equal(date_published, '2016-10-19T14:24:20.323Z');
+  });
+
+
+  it('returns the dek', async () => {
+    const html =
+      fs.readFileSync('./fixtures/medium.com/1477523363921.html');
+    const articleUrl =
+      'https://medium.com/the-wtf-economy/wtf-whats-the-future-e52ab9515573#.ilwrgwsks';
+
+    const { dek } =
+      await Mercury.parse(articleUrl, html, { fallback: false });
+
+    assert.equal(dek, null);
+  });
+
+
+  it('returns the lead_image_url', async () => {
+    // To pass this test, fill out the lead_image_url selector
+    // in ./src/extractors/custom/medium.com/index.js.
+    const html =
+      fs.readFileSync('./fixtures/medium.com/1477523363921.html');
+    const articleUrl =
+      'https://medium.com/the-wtf-economy/wtf-whats-the-future-e52ab9515573#.ilwrgwsks';
+
+    const { lead_image_url } =
+      await Mercury.parse(articleUrl, html, { fallback: false });
+
+    // Update these values with the expected values from
+    // the article.
+    assert.equal(lead_image_url, 'https://cdn-images-1.medium.com/max/1200/1*3Gzaug9mRc8vvx1cuQWkog.png');
+  });
+
+
+  it('returns the content', async () => {
+    const html =
+      fs.readFileSync('./fixtures/medium.com/1477523363921.html');
+    const url =
+      'https://medium.com/the-wtf-economy/wtf-whats-the-future-e52ab9515573#.ilwrgwsks';
+
+    const { content } =
+      await Mercury.parse(url, html, { fallback: false });
+
+    const $ = cheerio.load(content || '');
+
+    const first13 = excerptContent($('*').first().text(), 13);
+
+    // testing that youtube video transform is working
+    assert.equal(/IAoy3ia2ivI/.test(content), true);
+
+    assert.equal(first13, 'Video of WTF? My talk at the White House Frontiers ConferenceLast Thursday, I');
+  });
+});