making all.js export a generic function to decrease possiblity of error

pull/17/head
Adam Pash 8 years ago
parent de5b120b79
commit 7fa90f59b7

@ -2,6 +2,7 @@
"presets": ["es2015"],
"plugins": [
[
"transform-export-extensions",
"transform-runtime",
{
"polyfill": true,

@ -25,6 +25,7 @@
"babel-plugin-module-resolver": "^2.2.0",
"babel-plugin-transform-async-to-generator": "^6.8.0",
"babel-plugin-transform-es2015-destructuring": "^6.9.0",
"babel-plugin-transform-export-extensions": "^6.8.0",
"babel-plugin-transform-object-rest-spread": "^6.8.0",
"babel-plugin-transform-runtime": "6.12.0",
"babel-preset-es2015": "^6.13.2",

@ -1,42 +1,10 @@
import mergeSupportedDomains from 'utils/merge-supported-domains';
import { NYMagExtractor } from './custom/nymag.com';
import { BloggerExtractor } from './custom/blogspot.com';
import { WikipediaExtractor } from './custom/wikipedia.org';
import { TwitterExtractor } from './custom/twitter.com';
import { NYTimesExtractor } from './custom/www.nytimes.com';
import { TheAtlanticExtractor } from './custom/www.theatlantic.com';
import { NewYorkerExtractor } from './custom/www.newyorker.com';
import { WiredExtractor } from './custom/www.wired.com';
import { MSNExtractor } from './custom/www.msn.com';
import { YahooExtractor } from './custom/www.yahoo.com';
import { BuzzfeedExtractor } from './custom/www.buzzfeed.com';
import { WikiaExtractor } from './custom/fandom.wikia.com';
import { LittleThingsExtractor } from './custom/www.littlethings.com';
import { PoliticoExtractor } from './custom/www.politico.com';
import { DeadspinExtractor } from './custom/deadspin.com';
import { BroadwayWorldExtractor } from './custom/www.broadwayworld.com';
import { ApartmentTherapyExtractor } from './custom/www.apartmenttherapy.com';
import { MediumExtractor } from './custom/medium.com';
import * as CustomExtractors from './custom/index';
const Extractors = {
'nymag.com': NYMagExtractor,
'blogspot.com': BloggerExtractor,
'wikipedia.org': WikipediaExtractor,
'twitter.com': TwitterExtractor,
'www.nytimes.com': NYTimesExtractor,
'www.theatlantic.com': TheAtlanticExtractor,
'www.newyorker.com': NewYorkerExtractor,
'www.wired.com': WiredExtractor,
'www.msn.com': MSNExtractor,
'www.yahoo.com': YahooExtractor,
'www.buzzfeed.com': BuzzfeedExtractor,
'fandom.wikia.com': WikiaExtractor,
'www.littlethings.com': LittleThingsExtractor,
'www.politico.com': PoliticoExtractor,
...mergeSupportedDomains(DeadspinExtractor),
'www.broadwayworld.com': BroadwayWorldExtractor,
'www.apartmenttherapy.com': ApartmentTherapyExtractor,
...mergeSupportedDomains(MediumExtractor),
};
export default Extractors;
export default Object.keys(CustomExtractors).reduce((acc, key) => {
const extractor = CustomExtractors[key]
return {
...acc,
...mergeSupportedDomains(extractor),
}
}, {})

@ -7,8 +7,7 @@ import Mercury from 'mercury';
import getExtractor from 'extractors/get-extractor';
import { excerptContent } from 'utils/text';
// Rename CustomExtractor
describe('CustomExtractor', () => {
describe('DeadspinExtractor', () => {
it('is selected properly', () => {
// To pass this test, rename your extractor in
// ./src/extractors/custom/deadspin.com/index.js

@ -0,0 +1,18 @@
export * from './blogspot.com';
export * from './nymag.com';
export * from './wikipedia.org';
export * from './twitter.com';
export * from './www.nytimes.com';
export * from './www.theatlantic.com';
export * from './www.newyorker.com';
export * from './www.wired.com';
export * from './www.msn.com';
export * from './www.yahoo.com';
export * from './www.buzzfeed.com';
export * from './fandom.wikia.com';
export * from './www.littlethings.com';
export * from './www.politico.com';
export * from './deadspin.com';
export * from './www.broadwayworld.com';
export * from './www.apartmenttherapy.com';
export * from './medium.com';

@ -1,4 +1,6 @@
export const NYTimesExtractor = {
domain: 'www.nytimes.com',
title: {
selectors: [
'.g-headline',

@ -1,10 +1,20 @@
import assert from 'assert';
import fs from 'fs';
import cheerio from 'cheerio';
import URL from 'url';
import getExtractor from 'extractors/get-extractor';
import Mercury from 'mercury';
describe('NYTimesExtractor', () => {
it('is selected properly', () => {
const url =
'http://www.nytimes.com/interactive/2016/09/15/arts/design/national-museum-of-african-american-history-and-culture.html';
const extractor = getExtractor(url);
console.log(extractor.domain, URL.parse(url).hostname)
assert.equal(extractor.domain, URL.parse(url).hostname);
});
it('works with a feature story', async () => {
const html = fs.readFileSync('./fixtures/www.nytimes.com/1474061823854.html');
const uri = 'http://www.nytimes.com/interactive/2016/09/15/arts/design/national-museum-of-african-american-history-and-culture.html';

Loading…
Cancel
Save