feat: add money.cnn custom parser (#26)

* feat: add money.cnn custom parser

* added timezone to cnn custom parser

.gitignore

@@ -6,6 +6,6 @@ read
 preview.html
 preview.json
 coverage
-dist/mercury.test.js
-dist/mercury.test.js.map
-dist/mercury.test.web.js
+dist/mercury_test.js
+dist/mercury_test.js.map
+dist/mercury_test.web.js

dist/mercury.js

@@ -2787,6 +2787,43 @@ var WwwWashingtonpostComExtractor = {
   }
 };
 
+var MoneyCnnComExtractor = {
+  domain: 'money.cnn.com',
+
+  title: {
+    selectors: ['.article-title']
+  },
+
+  author: {
+    selectors: ['.byline a']
+  },
+
+  date_published: {
+    selectors: [['meta[name="date"]', 'value']]
+  },
+
+  dek: {
+    selectors: ['#storytext h2']
+  },
+
+  lead_image_url: {
+    selectors: [['meta[name="og:image"]', 'value']]
+  },
+
+  content: {
+    selectors: ['#storytext'],
+
+    // Is there anything in the content you selected that needs transformed
+    // before it's consumable content? E.g., unusual lazy loaded images
+    transforms: {},
+
+    // Is there anything that is in the result that shouldn't be?
+    // The clean selectors will remove anything that matches from
+    // the result
+    clean: ['.inStoryHeading']
+  }
+};
+
 
 
 var CustomExtractors = Object.freeze({
@@ -2809,7 +2846,8 @@ var CustomExtractors = Object.freeze({
 	ApartmentTherapyExtractor: ApartmentTherapyExtractor,
 	MediumExtractor: MediumExtractor,
 	WwwTmzComExtractor: WwwTmzComExtractor,
-	WwwWashingtonpostComExtractor: WwwWashingtonpostComExtractor
+	WwwWashingtonpostComExtractor: WwwWashingtonpostComExtractor,
+	MoneyCnnComExtractor: MoneyCnnComExtractor
 });
 
 var Extractors = _Object$keys(CustomExtractors).reduce(function (acc, key) {

package.json

@@ -17,7 +17,7 @@
     "test:web": "./node_modules/karma/bin/karma start karma.conf.js",
     "test:build": "cd ./scripts && jest check-build.test.js",
     "test:build:web": "node ./scripts/proxy-browser-test.js",
-    "watch:test": "jest ./src --watch",
+    "watch:test": "jest --watch",
     "generate-parser": "node ./dist/generate-custom-parser.js",
     "add-contributor": "all-contributors add",
     "generate-contributors": "all-contributors generate"

preview

@@ -5,7 +5,7 @@ var execSync = require('child_process').execSync
 console.log('Rebuilding Mercury')
 execSync('MERCURY_TEST_BUILD=true npm run build')
 
-var Mercury = require('./dist/mercury.test')
+var Mercury = require('./dist/mercury_test')
 
 var url = process.argv[2]
 

rollup.config.js

@@ -13,6 +13,6 @@ export default {
     babel(babelOpts),
   ],
   format: 'cjs',
-  dest: process.env.MERCURY_TEST_BUILD ? 'dist/mercury.test.js' : 'dist/mercury.js',
+  dest: process.env.MERCURY_TEST_BUILD ? 'dist/mercury_test.js' : 'dist/mercury.js',
   sourceMap: true,
 };

rollup.config.web.js

@@ -26,6 +26,6 @@ export default {
   ],
   format: 'iife',
   moduleName: 'Mercury',
-  dest: process.env.MERCURY_TEST_BUILD ? 'dist/mercury.test.web.js' : 'dist/mercury.web.js',
+  dest: process.env.MERCURY_TEST_BUILD ? 'dist/mercury_test.web.js' : 'dist/mercury.web.js',
   sourceMap: false,
 };

src/extractors/custom/index.js

@@ -18,3 +18,4 @@ export * from './www.apartmenttherapy.com';
 export * from './medium.com';
 export * from './www.tmz.com';
 export * from './www.washingtonpost.com';
+export * from './money.cnn.com';

src/extractors/custom/money.cnn.com/index.js

@@ -0,0 +1,53 @@
+export const MoneyCnnComExtractor = {
+  domain: 'money.cnn.com',
+
+  title: {
+    selectors: [
+      '.article-title',
+    ],
+  },
+
+  author: {
+    selectors: [
+      '.byline a',
+    ],
+  },
+
+  date_published: {
+    selectors: [
+      ['meta[name="date"]', 'value'],
+    ],
+
+    timezone: 'GMT',
+  },
+
+  dek: {
+    selectors: [
+      '#storytext h2',
+    ],
+  },
+
+  lead_image_url: {
+    selectors: [
+      ['meta[name="og:image"]', 'value'],
+    ],
+  },
+
+  content: {
+    selectors: [
+      '#storytext',
+    ],
+
+    // Is there anything in the content you selected that needs transformed
+    // before it's consumable content? E.g., unusual lazy loaded images
+    transforms: {
+    },
+
+    // Is there anything that is in the result that shouldn't be?
+    // The clean selectors will remove anything that matches from
+    // the result
+    clean: [
+      '.inStoryHeading',
+    ],
+  },
+};

src/extractors/custom/money.cnn.com/index.test.js

@@ -0,0 +1,122 @@
+import assert from 'assert';
+import fs from 'fs';
+import URL from 'url';
+import cheerio from 'cheerio';
+
+import Mercury from 'mercury';
+import getExtractor from 'extractors/get-extractor';
+import { excerptContent } from 'utils/text';
+
+describe('MoneyCnnComExtractor', () => {
+  it('is selected properly', () => {
+    // This test should be passing by default.
+    // It sanity checks that the correct parser
+    // is being selected for URLs from this domain
+    const url =
+      'http://money.cnn.com/2016/11/29/news/ohare-workers-strike/index.html';
+    const extractor = getExtractor(url);
+    assert.equal(extractor.domain, URL.parse(url).hostname);
+  });
+
+  it('returns the title', async () => {
+    // To pass this test, fill out the title selector
+    // in ./src/extractors/custom/money.cnn.com/index.js.
+    const html =
+      fs.readFileSync('./fixtures/money.cnn.com/1480437611330.html');
+    const articleUrl =
+      'http://money.cnn.com/2016/11/29/news/ohare-workers-strike/index.html';
+
+    const { title } =
+      await Mercury.parse(articleUrl, html, { fallback: false });
+
+    // Update these values with the expected values from
+    // the article.
+    assert.equal(title, 'Hundreds of Chicago O\'Hare airport workers go on strike');
+  });
+
+  it('returns the author', async () => {
+    // To pass this test, fill out the author selector
+    // in ./src/extractors/custom/money.cnn.com/index.js.
+    const html =
+      fs.readFileSync('./fixtures/money.cnn.com/1480437611330.html');
+    const articleUrl =
+      'http://money.cnn.com/2016/11/29/news/ohare-workers-strike/index.html';
+
+    const { author } =
+      await Mercury.parse(articleUrl, html, { fallback: false });
+
+    // Update these values with the expected values from
+    // the article.
+    assert.equal(author, 'Julia Horowitz');
+  });
+
+  it('returns the date_published', async () => {
+    // To pass this test, fill out the date_published selector
+    // in ./src/extractors/custom/money.cnn.com/index.js.
+    const html =
+      fs.readFileSync('./fixtures/money.cnn.com/1480437611330.html');
+    const articleUrl =
+      'http://money.cnn.com/2016/11/29/news/ohare-workers-strike/index.html';
+
+    const { date_published } =
+      await Mercury.parse(articleUrl, html, { fallback: false });
+
+    // Update these values with the expected values from
+    // the article.
+    assert.equal(date_published, '2016-11-29T03:33:08.000Z');
+  });
+
+  it('returns the dek', async () => {
+    // To pass this test, fill out the dek selector
+    // in ./src/extractors/custom/money.cnn.com/index.js.
+    const html =
+      fs.readFileSync('./fixtures/money.cnn.com/1480437611330.html');
+    const articleUrl =
+      'http://money.cnn.com/2016/11/29/news/ohare-workers-strike/index.html';
+
+    const { dek } =
+      await Mercury.parse(articleUrl, html, { fallback: false });
+
+    // Update these values with the expected values from
+    // the article.
+    assert.equal(dek, 'Heads up, travelers: Hundreds of workers are striking at Chicago O\'Hare International Airport on Tuesday.');
+  });
+
+  it('returns the lead_image_url', async () => {
+    // To pass this test, fill out the lead_image_url selector
+    // in ./src/extractors/custom/money.cnn.com/index.js.
+    const html =
+      fs.readFileSync('./fixtures/money.cnn.com/1480437611330.html');
+    const articleUrl =
+      'http://money.cnn.com/2016/11/29/news/ohare-workers-strike/index.html';
+
+    const { lead_image_url } =
+      await Mercury.parse(articleUrl, html, { fallback: false });
+
+    // Update these values with the expected values from
+    // the article.
+    assert.equal(lead_image_url, 'http://i2.cdn.turner.com/money/dam/assets/161118102423-ohare-airport-strike-780x439.jpg');
+  });
+
+  it('returns the content', async () => {
+    // To pass this test, fill out the content selector
+    // in ./src/extractors/custom/money.cnn.com/index.js.
+    // You may also want to make use of the clean and transform
+    // options.
+    const html =
+      fs.readFileSync('./fixtures/money.cnn.com/1480437611330.html');
+    const url =
+      'http://money.cnn.com/2016/11/29/news/ohare-workers-strike/index.html';
+
+    const { content } =
+      await Mercury.parse(url, html, { fallback: false });
+
+    const $ = cheerio.load(content || '');
+
+    const first13 = excerptContent($('*').first().text(), 13);
+
+    // Update these values with the expected values from
+    // the article.
+    assert.equal(first13, 'Janitors, baggage handlers, cabin cleaners and wheelchair attendants are asking for a $15');
+  });
+});