varmercury=createCommonjsModule(function(module){'use strict';function_interopDefault(ex){returnex&&(typeofex==='undefined'?'undefined':_typeof2(ex))==='object'&&'default'inex?ex['default']:ex;}var_regeneratorRuntime=_interopDefault(regenerator);var_extends$$=_interopDefault(_extends);var_asyncToGenerator=_interopDefault(asyncToGenerator);varURL$$=_interopDefault(URL);varbabelPolyfill$$=babelPolyfill;varcheerio$$=_interopDefault(cheerio);var_Promise=_interopDefault(promise);varrequest$$=_interopDefault(request);var_Reflect$ownKeys=_interopDefault(ownKeys);varstringDirection$$=_interopDefault(stringDirection);var_getIterator=_interopDefault(getIterator);var_defineProperty=_interopDefault(defineProperty);var_slicedToArray$$=_interopDefault(_slicedToArray);var_typeof=_interopDefault(_typeof2);varvalidUrl$$=_interopDefault(validUrl);varmoment$$=_interopDefault(moment);varwuzzy$$=_interopDefault(wuzzy);vardifflib$$=_interopDefault(difflib);var_Array$from=_interopDefault(from);varellipsize$$=_interopDefault(ellipsize);var_marked=[range].map(_regeneratorRuntime.mark);functionrange(){varstart=arguments.length<=0||arguments[0]===undefined?1:arguments[0];varend=arguments.length<=1||arguments[1]===undefined?1:arguments[1];return_regeneratorRuntime.wrap(functionrange$(_context){while(1){switch(_context.prev=_context.next){case0:if(!(start<=end)){_context.next=5;break;}_context.next=3;returnstart+=1;case3:_context.next=0;break;case5:case"end":return_context.stop();}}},_marked[0],this);}// extremely simple url validation as a first step
varmercury=createCommonjsModule(function(module){'use strict';function_interopDefault(ex){returnex&&(typeofex==='undefined'?'undefined':_typeof(ex))==='object'&&'default'inex?ex['default']:ex;}var_regeneratorRuntime=_interopDefault(regenerator);var_extends$$=_interopDefault(_extends);var_asyncToGenerator=_interopDefault(asyncToGenerator);varURL$$=_interopDefault(URL);varbabelPolyfill$$=babelPolyfill;varcheerio$$=_interopDefault(cheerio);var_Promise=_interopDefault(promise);varrequest$$=_interopDefault(request);var_Reflect$ownKeys$$=_interopDefault(_Reflect$ownKeys);varstringDirection$$=_interopDefault(stringDirection);var_getIterator$$=_interopDefault(_getIterator);var_defineProperty=_interopDefault(defineProperty);var_slicedToArray$$=_interopDefault(_slicedToArray);var_typeof$$=_interopDefault(_typeof);varvalidUrl$$=_interopDefault(validUrl);varmoment$$=_interopDefault(moment);varwuzzy$$=_interopDefault(wuzzy);vardifflib$$=_interopDefault(difflib);var_Array$from=_interopDefault(from);varellipsize$$=_interopDefault(ellipsize);var_marked=[range].map(_regeneratorRuntime.mark);functionrange(){varstart=arguments.length<=0||arguments[0]===undefined?1:arguments[0];varend=arguments.length<=1||arguments[1]===undefined?1:arguments[1];return_regeneratorRuntime.wrap(functionrange$(_context){while(1){switch(_context.prev=_context.next){case0:if(!(start<=end)){_context.next=5;break;}_context.next=3;returnstart+=1;case3:_context.next=0;break;case5:case"end":return_context.stop();}}},_marked[0],this);}// extremely simple url validation as a first step
functionvalidateUrl(_ref){varhostname=_ref.hostname;// If this isn't a valid url, return an error message
return!!hostname;}varErrors={badUrl:{error:true,messages:'The url parameter passed does not look like a valid URL. Please check your data and try again.'}};varREQUEST_HEADERS={'User-Agent':'Readability - http://readability.com/about/'};// The number of milliseconds to attempt to fetch a resource before timing out.
varFETCH_TIMEOUT=10000;// Content types that we do not extract content from
@ -65,7 +65,7 @@ function normalizeMetaTags($){$=convertMetaProp($,'content','value');$=convertMe
// Many sites will have img tags with no source, or an image tag with a src
// attribute that a is a placeholer. We need to be able to properly fill in
// the src attribute so the images are no longer lazy loaded.
functionconvertLazyLoadedImages($){$('img').each(function(_,img){_Reflect$ownKeys(img.attribs).forEach(function(attr){varvalue=img.attribs[attr];if(attr!=='src'&&IS_LINK.test(value)&&IS_IMAGE.test(value)){$(img).attr('src',value);}});});return$;}functionisComment(index,node){returnnode.type==='comment';}functioncleanComments($){$.root().find('*').contents().filter(isComment).remove();return$;}functionclean($){$(TAGS_TO_REMOVE).remove();$=cleanComments($);return$;}varResource={// Create a Resource.
functionconvertLazyLoadedImages($){$('img').each(function(_,img){_Reflect$ownKeys$$(img.attribs).forEach(function(attr){varvalue=img.attribs[attr];if(attr!=='src'&&IS_LINK.test(value)&&IS_IMAGE.test(value)){$(img).attr('src',value);}});});return$;}functionisComment(index,node){returnnode.type==='comment';}functioncleanComments($){$.root().find('*').contents().filter(isComment).remove();return$;}functionclean($){$(TAGS_TO_REMOVE).remove();$=cleanComments($);return$;}varResource={// Create a Resource.
//
// :param url: The URL for the document we should retrieve.
// :param response: If set, use as the response rather than
@ -83,7 +83,7 @@ clean:['.ad','.single-related-story'],// Object of tranformations to make on mat
// the transformation.
transforms:{// Convert h1s to h2s
h1:'h2',// Convert lazy-loaded noscript images to figures
noscript:functionnoscript($node){var$children=$node.children();if($children.length===1&&$children.get(0).tagName==='img'){return'figure';}returnnull;}}},title:{selectors:['h1.lede-feature-title','h1.headline-primary','h1']},author:{selectors:['.by-authors','.lede-feature-author']},dek:{selectors:['.lede-feature-teaser']},date_published:{selectors:['time.article-timestamp[datetime]','time.article-timestamp']}};varBloggerExtractor={domain:'blogspot.com',content:{// Blogger is insane and does not load its content
noscript:functionnoscript($node){var$children=$node.children();if($children.length===1&&$children.get(0).tagName==='img'){return'figure';}returnnull;}}},title:{selectors:['h1.lede-feature-title','h1.headline-primary','h1']},author:{selectors:['.by-authors','.lede-feature-author']},dek:{selectors:['.lede-feature-teaser']},date_published:{selectors:[['time.article-timestamp[datetime]','datetime'],'time.article-timestamp']}};varBloggerExtractor={domain:'blogspot.com',content:{// Blogger is insane and does not load its content
// initially in the page, but it's all there
// in noscript
selectors:['.post-content noscript'],// Selectors to remove from the extracted content
'.permalink[role=main]':functionpermalinkRoleMain($node,$){vartweets=$node.find('.tweet');var$tweetContainer=$('<div id="TWEETS_GO_HERE"></div>');$tweetContainer.append(tweets);$node.replaceWith($tweetContainer);},// Twitter wraps @ with s, which
@ -113,10 +113,18 @@ var TheAtlanticExtractor={domain:'www.theatlantic.com',title:{selectors:['h1.hed
transforms:[],// Is there anything that is in the result that shouldn't be?
// The clean selectors will remove anything that matches from
// the result
clean:[]},date_published:null,lead_image_url:null,dek:null,next_page_url:null,excerpt:null};varExtractors={'nymag.com':NYMagExtractor,'blogspot.com':BloggerExtractor,'wikipedia.org':WikipediaExtractor,'twitter.com':TwitterExtractor,'www.nytimes.com':NYTimesExtractor,'www.theatlantic.com':TheAtlanticExtractor};// Spacer images to be removed
varNewYorkerExtractor={domain:'www.newyorker.com',title:{selectors:['h1.title']},author:{selectors:['.contributors']},content:{selectors:['div#articleBody','div.articleBody'],// Is there anything in the content you selected that needs transformed
// before it's consumable content? E.g., unusual lazy loaded images
transforms:[],// Is there anything that is in the result that shouldn't be?
// The clean selectors will remove anything that matches from
// the result
clean:[]},date_published:{selectors:[['meta[name="article:published_time"]','value']]},lead_image_url:{selectors:[['meta[name="og:image"]','value']]},dek:{selectors:[['meta[name="og:description"]','value']]},next_page_url:null,excerpt:null};varExtractors={'nymag.com':NYMagExtractor,'blogspot.com':BloggerExtractor,'wikipedia.org':WikipediaExtractor,'twitter.com':TwitterExtractor,'www.nytimes.com':NYTimesExtractor,'www.theatlantic.com':TheAtlanticExtractor,'www.newyorker.com':NewYorkerExtractor};// Spacer images to be removed
varSPACER_RE=newRegExp('trans|transparent|spacer|blank','i');// A list of tags to strip from the output if we encounter them.
// (By-reference mutation, though. Returned just for convenience.)
functionconvertToParagraphs($){$=brsToPs($);$=convertDivs($);$=convertSpans($);return$;}functionconvertNodeTo($node,$){vartag=arguments.length<=2||arguments[2]===undefined?'p':arguments[2];varnode=$node.get(0);if(!node){return$;}var_$node$get=$node.get(0);varattribs=_$node$get.attribs;varattribString=_Reflect$ownKeys(attribs).map(function(key){returnkey+'='+attribs[key];}).join(' ');$node.replaceWith('<'+tag+' '+attribString+'>'+$node.contents()+'</'+tag+'>');return$;}functioncleanForHeight($img,$){varheight=parseInt($img.attr('height'),10);varwidth=parseInt($img.attr('width'),10)||20;// Remove images that explicitly have very small heights or
functionconvertToParagraphs($){$=brsToPs($);$=convertDivs($);$=convertSpans($);return$;}functionconvertNodeTo($node,$){vartag=arguments.length<=2||arguments[2]===undefined?'p':arguments[2];varnode=$node.get(0);if(!node){return$;}var_$node$get=$node.get(0);varattribs=_$node$get.attribs;varattribString=_Reflect$ownKeys$$(attribs).map(function(key){returnkey+'='+attribs[key];}).join(' ');$node.replaceWith('<'+tag+' '+attribString+'>'+$node.contents()+'</'+tag+'>');return$;}functioncleanForHeight($img,$){varheight=parseInt($img.attr('height'),10);varwidth=parseInt($img.attr('width'),10)||20;// Remove images that explicitly have very small heights or
// widths, because they are most likely shims or icons,
// which aren't very useful for reading.
if((height||20)<10||width<10){$img.remove();}elseif(height){// Don't ever specify a height on images, so that we can
@ -219,11 +227,11 @@ if((height||20)<10||width<10){$img.remove();}else if(height){// Don't ever speci
// aspect ratio.
$img.removeAttr('height');}return$;}// Cleans out images where the source string matches transparent/spacer/etc
// TODO This seems very aggressive - AP
functionremoveSpacers($img,$){if(SPACER_RE.test($img.attr('src'))){$img.remove();}return$;}functioncleanImages($article,$){$article.find('img').each(function(index,img){var$img=$(img);cleanForHeight($img,$);removeSpacers($img,$);});return$;}functionstripJunkTags(article,$){$(STRIP_OUTPUT_TAGS.join(','),article).remove();return$;}// H1 tags are typically the article title, which should be extracted
functionremoveSpacers($img,$){if(SPACER_RE.test($img.attr('src'))){$img.remove();}return$;}functioncleanImages($article,$){$article.find('img').each(function(index,img){var$img=$(img);cleanForHeight($img,$);removeSpacers($img,$);});return$;}functionstripJunkTags(article,$){vartags=arguments.length<=2||arguments[2]===undefined?[]:arguments[2];if(tags.length===0){tags=STRIP_OUTPUT_TAGS;}$(tags.join(','),article).remove();return$;}// H1 tags are typically the article title, which should be extracted
// by the title extractor instead. If there's less than 3 of them (<3),
$article.find('*').each(function(index,node){node.attribs=_Reflect$ownKeys(node.attribs).reduce(function(acc,attr){if(WHITELIST_ATTRS_RE.test(attr)){return_extends$$({},acc,_defineProperty({},attr,node.attribs[attr]));}returnacc;},{});});}// function removeAttrs(article, $) {
$article.find('*').each(function(index,node){node.attribs=_Reflect$ownKeys$$(node.attribs).reduce(function(acc,attr){if(WHITELIST_ATTRS_RE.test(attr)){return_extends$$({},acc,_defineProperty({},attr,node.attribs[attr]));}returnacc;},{});});}// function removeAttrs(article, $) {
// REMOVE_ATTRS.forEach((attr) => {
// $(`[${attr}]`, article).removeAttr(attr);
// });
@ -444,14 +452,14 @@ var values=nodes.map(function(index,node){return $(node).attr(value);}).toArray(
// also.
if(values.length===1){varmetaValue=void0;// Meta values that contain HTML should be stripped, as they
// weren't subject to cleaning previously.
if(cleanTags){metaValue=stripTags(values[0],$);}else{metaValue=values[0];}return{v:metaValue};}};for(var_iterator=_getIterator(foundNames),_step;!(_iteratorNormalCompletion=(_step=_iterator.next()).done);_iteratorNormalCompletion=true){var_ret=_loop();if((typeof_ret==='undefined'?'undefined':_typeof(_ret))==="object")return_ret.v;}// If nothing is found, return null
if(cleanTags){metaValue=stripTags(values[0],$);}else{metaValue=values[0];}return{v:metaValue};}};for(var_iterator=_getIterator$$(foundNames),_step;!(_iteratorNormalCompletion=(_step=_iterator.next()).done);_iteratorNormalCompletion=true){var_ret=_loop();if((typeof_ret==='undefined'?'undefined':_typeof$$(_ret))==="object")return_ret.v;}// If nothing is found, return null
}catch(err){_didIteratorError=true;_iteratorError=err;}finally{try{if(!_iteratorNormalCompletion&&_iterator.return){_iterator.return();}}finally{if(_didIteratorError){throw_iteratorError;}}}returnnull;}functionisGoodNode($node,maxChildren){// If it has a number of children, it's more likely a container
// element. Skip it.
if($node.children().length>maxChildren){returnfalse;}// If it looks to be within a comment, skip it.
if(withinComment($node)){returnfalse;}returntrue;}// Given a a list of selectors find content that may
// be extractable from the document. This is for flat
// meta-information, like author, title, date published, etc.
functionextractFromSelectors($,selectors){varmaxChildren=arguments.length<=2||arguments[2]===undefined?1:arguments[2];vartextOnly=arguments.length<=3||arguments[3]===undefined?true:arguments[3];var_iteratorNormalCompletion=true;var_didIteratorError=false;var_iteratorError=undefined;try{for(var_iterator=_getIterator(selectors),_step;!(_iteratorNormalCompletion=(_step=_iterator.next()).done);_iteratorNormalCompletion=true){varselector=_step.value;varnodes=$(selector);// If we didn't get exactly one of this selector, this may be
functionextractFromSelectors($,selectors){varmaxChildren=arguments.length<=2||arguments[2]===undefined?1:arguments[2];vartextOnly=arguments.length<=3||arguments[3]===undefined?true:arguments[3];var_iteratorNormalCompletion=true;var_didIteratorError=false;var_iteratorError=undefined;try{for(var_iterator=_getIterator$$(selectors),_step;!(_iteratorNormalCompletion=(_step=_iterator.next()).done);_iteratorNormalCompletion=true){varselector=_step.value;varnodes=$(selector);// If we didn't get exactly one of this selector, this may be
// a list of articles or comments. Skip it.
if(nodes.length===1){var$node=$(nodes[0]);if(isGoodNode($node,maxChildren)){varcontent=void0;if(textOnly){content=$node.text();}else{content=$node.html();}if(content){returncontent;}}}}}catch(err){_didIteratorError=true;_iteratorError=err;}finally{try{if(!_iteratorNormalCompletion&&_iterator.return){_iterator.return();}}finally{if(_didIteratorError){throw_iteratorError;}}}returnnull;}// strips all tags from a string of text
functionstripTags(text,$){// Wrapping text in html element prevents errors when text
@ -480,10 +488,12 @@ function cleanDateString(dateString){return(dateString.match(SPLIT_DATE_STRING)|
// it. Return none if we fail.
functioncleanDatePublished(dateString){// If string is in milliseconds or seconds, convert to int
if(MS_DATE_STRING.test(dateString)||SEC_DATE_STRING.test(dateString)){dateString=parseInt(dateString,10);}vardate=moment$$(newDate(dateString));if(!date.isValid()){dateString=cleanDateString(dateString);date=moment$$(newDate(dateString));}returndate.isValid()?date.toISOString():null;}// Clean our article content, returning a new, cleaned node.
functionextractCleanNode(article,_ref){var$=_ref.$;var_ref$cleanConditional=_ref.cleanConditionally;varcleanConditionally=_ref$cleanConditional===undefined?true:_ref$cleanConditional;var_ref$title=_ref.title;vartitle=_ref$title===undefined?'':_ref$title;var_ref$url=_ref.url;varurl=_ref$url===undefined?'':_ref$url;// Rewrite the tag name to div if it's a top level node like body or
functionextractCleanNode(article,_ref){var$=_ref.$;var_ref$cleanConditional=_ref.cleanConditionally;varcleanConditionally=_ref$cleanConditional===undefined?true:_ref$cleanConditional;var_ref$title=_ref.title;vartitle=_ref$title===undefined?'':_ref$title;var_ref$url=_ref.url;varurl=_ref$url===undefined?'':_ref$url;var_ref$defaultCleaner=_ref.defaultCleaner;vardefaultCleaner=_ref$defaultCleaner===undefined?true:_ref$defaultCleaner;// Rewrite the tag name to div if it's a top level node like body or
// html to avoid later complications with multiple body tags.
rewriteTopLevel(article,$);// Drop small images and spacer images
cleanImages(article,$);// Drop certain tags like <title>, etc
// Only do this is defaultCleaner is set to true;
// this can sometimes be too aggressive.
if(defaultCleaner)cleanImages(article,$);// Drop certain tags like <title>, etc
// This is -mostly- for cleanliness, not security.
stripJunkTags(article,$);// H1 tags are typically the article title, which should be extracted
// by the title extractor instead. If there's less than 3 of them (<3),
if(splitTitle.length>=6){var_ret=function(){// Look to see if we can find a breadcrumb splitter that happens
// more than once. If we can, we'll be able to better pull out
// the title.
vartermCounts=splitTitle.reduce(function(acc,titleText){acc[titleText]=acc[titleText]?acc[titleText]+1:1;returnacc;},{});var_Reflect$ownKeys$redu=_Reflect$ownKeys(termCounts).reduce(function(acc,key){if(acc[1]<termCounts[key]){return[key,termCounts[key]];}returnacc;},[0,0]);var_Reflect$ownKeys$redu2=_slicedToArray$$(_Reflect$ownKeys$redu,2);varmaxTerm=_Reflect$ownKeys$redu2[0];vartermCount=_Reflect$ownKeys$redu2[1];// We found a splitter that was used more than once, so it
vartermCounts=splitTitle.reduce(function(acc,titleText){acc[titleText]=acc[titleText]?acc[titleText]+1:1;returnacc;},{});var_Reflect$ownKeys$redu=_Reflect$ownKeys$$(termCounts).reduce(function(acc,key){if(acc[1]<termCounts[key]){return[key,termCounts[key]];}returnacc;},[0,0]);var_Reflect$ownKeys$redu2=_slicedToArray$$(_Reflect$ownKeys$redu,2);varmaxTerm=_Reflect$ownKeys$redu2[0];vartermCount=_Reflect$ownKeys$redu2[1];// We found a splitter that was used more than once, so it
// is probably the breadcrumber. Split our title on that instead.
// Note: max_term should be <= 4 characters, so that " >> "
// will match, but nothing longer than that.
if(termCount>=2&&maxTerm.length<=4){splitTitle=text.split(maxTerm);}varsplitEnds=[splitTitle[0],splitTitle.slice(-1)];varlongestEnd=splitEnds.reduce(function(acc,end){returnacc.length>end.length?acc:end;},'');if(longestEnd.length>10){return{v:longestEnd};}return{v:text};}();if((typeof_ret==='undefined'?'undefined':_typeof(_ret))==="object")return_ret.v;}returnnull;}functioncleanDomainFromTitle(splitTitle,url){// Search the ends of the title, looking for bits that fuzzy match
if(termCount>=2&&maxTerm.length<=4){splitTitle=text.split(maxTerm);}varsplitEnds=[splitTitle[0],splitTitle.slice(-1)];varlongestEnd=splitEnds.reduce(function(acc,end){returnacc.length>end.length?acc:end;},'');if(longestEnd.length>10){return{v:longestEnd};}return{v:text};}();if((typeof_ret==='undefined'?'undefined':_typeof$$(_ret))==="object")return_ret.v;}returnnull;}functioncleanDomainFromTitle(splitTitle,url){// Search the ends of the title, looking for bits that fuzzy match
// the URL too closely. If one is found, discard it and return the
varnode=this.getContentNode($,title,url,opts);if(nodeIsSufficient(node)){returnthis.cleanAndReturnNode(node,$);}// We didn't succeed on first pass, one by one disable our
// extraction opts and try again.
var_iteratorNormalCompletion=true;var_didIteratorError=false;var_iteratorError=undefined;try{for(var_iterator=_getIterator(_Reflect$ownKeys(opts).filter(function(k){returnopts[k]===true;})),_step;!(_iteratorNormalCompletion=(_step=_iterator.next()).done);_iteratorNormalCompletion=true){varkey=_step.value;opts[key]=false;$=cheerio$$.load(html);node=this.getContentNode($,title,url,opts);if(nodeIsSufficient(node)){break;}}}catch(err){_didIteratorError=true;_iteratorError=err;}finally{try{if(!_iteratorNormalCompletion&&_iterator.return){_iterator.return();}}finally{if(_didIteratorError){throw_iteratorError;}}}returnthis.cleanAndReturnNode(node,$);},// Get node given current options
var_iteratorNormalCompletion=true;var_didIteratorError=false;var_iteratorError=undefined;try{for(var_iterator=_getIterator$$(_Reflect$ownKeys$$(opts).filter(function(k){returnopts[k]===true;})),_step;!(_iteratorNormalCompletion=(_step=_iterator.next()).done);_iteratorNormalCompletion=true){varkey=_step.value;opts[key]=false;$=cheerio$$.load(html);node=this.getContentNode($,title,url,opts);if(nodeIsSufficient(node)){break;}}}catch(err){_didIteratorError=true;_iteratorError=err;}finally{try{if(!_iteratorNormalCompletion&&_iterator.return){_iterator.return();}}finally{if(_didIteratorError){throw_iteratorError;}}}returnthis.cleanAndReturnNode(node,$);},// Get node given current options
getContentNode:functiongetContentNode($,title,url,opts){returnextractCleanNode(extractBestNode($,opts),{$:$,cleanConditionally:opts.cleanConditionally,title:title,url:url});},// Once we got here, either we're at our last-resort node, or
// we broke early. Make sure we at least have -something- before we
// move forward.
@ -611,7 +622,7 @@ var bylineRe=/^[\n\s]*By/i;var BYLINE_SELECTORS_RE=[['#byline',bylineRe],['.byli
author=extractFromMeta($,AUTHOR_META_TAGS,metaCache);if(author&&author.length<AUTHOR_MAX_LENGTH){returncleanAuthor(author);}// Second, look through our selectors looking for potential authors.
author=extractFromSelectors($,AUTHOR_SELECTORS,2);if(author&&author.length<AUTHOR_MAX_LENGTH){returncleanAuthor(author);}// Last, use our looser regular-expression based selectors for
// potential authors.
var_iteratorNormalCompletion=true;var_didIteratorError=false;var_iteratorError=undefined;try{for(var_iterator=_getIterator(BYLINE_SELECTORS_RE),_step;!(_iteratorNormalCompletion=(_step=_iterator.next()).done);_iteratorNormalCompletion=true){var_ref4=_step.value;var_ref3=_slicedToArray$$(_ref4,2);varselector=_ref3[0];varregex=_ref3[1];varnode=$(selector);if(node.length===1){vartext=node.text();if(regex.test(text)){returncleanAuthor(text);}}}}catch(err){_didIteratorError=true;_iteratorError=err;}finally{try{if(!_iteratorNormalCompletion&&_iterator.return){_iterator.return();}}finally{if(_didIteratorError){throw_iteratorError;}}}returnnull;}};// An ordered list of meta tag names that denote
var_iteratorNormalCompletion=true;var_didIteratorError=false;var_iteratorError=undefined;try{for(var_iterator=_getIterator$$(BYLINE_SELECTORS_RE),_step;!(_iteratorNormalCompletion=(_step=_iterator.next()).done);_iteratorNormalCompletion=true){var_ref4=_step.value;var_ref3=_slicedToArray$$(_ref4,2);varselector=_ref3[0];varregex=_ref3[1];varnode=$(selector);if(node.length===1){vartext=node.text();if(regex.test(text)){returncleanAuthor(text);}}}}catch(err){_didIteratorError=true;_iteratorError=err;}finally{try{if(!_iteratorNormalCompletion&&_iterator.return){_iterator.return();}}finally{if(_didIteratorError){throw_iteratorError;}}}returnnull;}};// An ordered list of meta tag names that denote
// likely date published dates. All attributes
// should be lowercase for faster case-insensitive matching.
// From most distinct to least distinct.
@ -677,9 +688,9 @@ var GenericLeadImageUrlExtractor={extract:function extract(_ref){var $=_ref.$;va
varimageUrl=extractFromMeta($,LEAD_IMAGE_URL_META_TAGS,metaCache,false);if(imageUrl){cleanUrl=clean$1(imageUrl);if(cleanUrl)returncleanUrl;}// Next, try to find the "best" image via the content.
// We'd rather not have to fetch each image and check dimensions,
// so try to do some analysis and determine them instead.
varimgs=$('img',content).toArray();varimgScores={};imgs.forEach(function(img,index){var$img=$(img);varsrc=$img.attr('src');if(!src)return;varscore=scoreImageUrl(src);score+=scoreAttr($img);score+=scoreByParents($img);score+=scoreBySibling($img);score+=scoreByDimensions($img);score+=scoreByPosition(imgs,index);imgScores[src]=score;});var_Reflect$ownKeys$redu=_Reflect$ownKeys(imgScores).reduce(function(acc,key){returnimgScores[key]>acc[1]?[key,imgScores[key]]:acc;},[null,0]);var_Reflect$ownKeys$redu2=_slicedToArray$$(_Reflect$ownKeys$redu,2);vartopUrl=_Reflect$ownKeys$redu2[0];vartopScore=_Reflect$ownKeys$redu2[1];if(topScore>0){cleanUrl=clean$1(topUrl);if(cleanUrl)returncleanUrl;}// If nothing else worked, check to see if there are any really
varimgs=$('img',content).toArray();varimgScores={};imgs.forEach(function(img,index){var$img=$(img);varsrc=$img.attr('src');if(!src)return;varscore=scoreImageUrl(src);score+=scoreAttr($img);score+=scoreByParents($img);score+=scoreBySibling($img);score+=scoreByDimensions($img);score+=scoreByPosition(imgs,index);imgScores[src]=score;});var_Reflect$ownKeys$redu=_Reflect$ownKeys$$(imgScores).reduce(function(acc,key){returnimgScores[key]>acc[1]?[key,imgScores[key]]:acc;},[null,0]);var_Reflect$ownKeys$redu2=_slicedToArray$$(_Reflect$ownKeys$redu,2);vartopUrl=_Reflect$ownKeys$redu2[0];vartopScore=_Reflect$ownKeys$redu2[1];if(topScore>0){cleanUrl=clean$1(topUrl);if(cleanUrl)returncleanUrl;}// If nothing else worked, check to see if there are any really
// probable nodes in the doc, like <link rel="image_src" />.
var_iteratorNormalCompletion=true;var_didIteratorError=false;var_iteratorError=undefined;try{for(var_iterator=_getIterator(LEAD_IMAGE_URL_SELECTORS),_step;!(_iteratorNormalCompletion=(_step=_iterator.next()).done);_iteratorNormalCompletion=true){varselector=_step.value;var$node=$(selector).first();varsrc=$node.attr('src');if(src){cleanUrl=clean$1(src);if(cleanUrl)returncleanUrl;}varhref=$node.attr('href');if(href){cleanUrl=clean$1(href);if(cleanUrl)returncleanUrl;}varvalue=$node.attr('value');if(value){cleanUrl=clean$1(value);if(cleanUrl)returncleanUrl;}}}catch(err){_didIteratorError=true;_iteratorError=err;}finally{try{if(!_iteratorNormalCompletion&&_iterator.return){_iterator.return();}}finally{if(_didIteratorError){throw_iteratorError;}}}returnnull;}};functionscoreSimilarity(score,articleUrl,href){// Do this last and only if we have a real candidate, because it's
var_iteratorNormalCompletion=true;var_didIteratorError=false;var_iteratorError=undefined;try{for(var_iterator=_getIterator$$(LEAD_IMAGE_URL_SELECTORS),_step;!(_iteratorNormalCompletion=(_step=_iterator.next()).done);_iteratorNormalCompletion=true){varselector=_step.value;var$node=$(selector).first();varsrc=$node.attr('src');if(src){cleanUrl=clean$1(src);if(cleanUrl)returncleanUrl;}varhref=$node.attr('href');if(href){cleanUrl=clean$1(href);if(cleanUrl)returncleanUrl;}varvalue=$node.attr('value');if(value){cleanUrl=clean$1(value);if(cleanUrl)returncleanUrl;}}}catch(err){_didIteratorError=true;_iteratorError=err;}finally{try{if(!_iteratorNormalCompletion&&_iterator.return){_iterator.return();}}finally{if(_didIteratorError){throw_iteratorError;}}}returnnull;}};functionscoreSimilarity(score,articleUrl,href){// Do this last and only if we have a real candidate, because it's
// potentially expensive computationally. Compare the link to this
// URL using difflib to get the % similarity of these URLs. On a
// sliding scale, subtract points from this link based on
@ -753,48 +764,257 @@ var scoredPages=links.reduce(function(possiblePages,link){// Remove any anchor d
// standardizing URLs (it's hard), we're going to do
// some checking with and without a trailing slash
varhref=removeAnchor(link.attribs.href);var$link=$(link);varlinkText=$link.text();if(!shouldScore(href,articleUrl,baseUrl,parsedUrl,linkText,previousUrls)){returnpossiblePages;}// ## PASSED THE FIRST-PASS TESTS. Start scoring. ##
if(!possiblePages[href]){possiblePages[href]={score:0,linkText:linkText,href:href};}else{possiblePages[href].linkText=possiblePages[href].linkText+'|'+linkText;}varpossiblePage=possiblePages[href];varlinkData=makeSig($link,linkText);varpageNum=pageNumFromUrl(href);varscore=scoreBaseUrl(href,baseRegex);score+=scoreNextLinkText(linkData);score+=scoreCapLinks(linkData);score+=scorePrevLink(linkData);score+=scoreByParents$1($link);score+=scoreExtraneousLinks(href);score+=scorePageInLink(pageNum,isWp);score+=scoreLinkText(linkText,pageNum);score+=scoreSimilarity(score,articleUrl,href);possiblePage.score=score;returnpossiblePages;},{});return_Reflect$ownKeys(scoredPages).length===0?null:scoredPages;}// Looks for and returns next page url
if(!possiblePages[href]){possiblePages[href]={score:0,linkText:linkText,href:href};}else{possiblePages[href].linkText=possiblePages[href].linkText+'|'+linkText;}varpossiblePage=possiblePages[href];varlinkData=makeSig($link,linkText);varpageNum=pageNumFromUrl(href);varscore=scoreBaseUrl(href,baseRegex);score+=scoreNextLinkText(linkData);score+=scoreCapLinks(linkData);score+=scorePrevLink(linkData);score+=scoreByParents$1($link);score+=scoreExtraneousLinks(href);score+=scorePageInLink(pageNum,isWp);score+=scoreLinkText(linkText,pageNum);score+=scoreSimilarity(score,articleUrl,href);possiblePage.score=score;returnpossiblePages;},{});return_Reflect$ownKeys$$(scoredPages).length===0?null:scoredPages;}// Looks for and returns next page url
// for multi-page articles
varGenericNextPageUrlExtractor={extract:functionextract(_ref){var$=_ref.$;varurl=_ref.url;varparsedUrl=_ref.parsedUrl;var_ref$previousUrls=_ref.previousUrls;varpreviousUrls=_ref$previousUrls===undefined?[]:_ref$previousUrls;parsedUrl=parsedUrl||URL$$.parse(url);vararticleUrl=removeAnchor(url);varbaseUrl=articleBaseUrl(url,parsedUrl);varlinks=$('a[href]').toArray();varscoredLinks=scoreLinks({links:links,articleUrl:articleUrl,baseUrl:baseUrl,parsedUrl:parsedUrl,$:$,previousUrls:previousUrls});// If no links were scored, return null
if(!scoredLinks)returnnull;// now that we've scored all possible pages,
// find the biggest one.
vartopPage=_Reflect$ownKeys(scoredLinks).reduce(function(acc,link){varscoredLink=scoredLinks[link];returnscoredLink.score>acc.score?scoredLink:acc;},{score:-100});// If the score is less than 50, we're not confident enough to use it,
vartopPage=_Reflect$ownKeys$$(scoredLinks).reduce(function(acc,link){varscoredLink=scoredLinks[link];returnscoredLink.score>acc.score?scoredLink:acc;},{score:-100});// If the score is less than 50, we're not confident enough to use it,
// so we fail.
if(topPage.score>=50){returntopPage.href;}returnnull;}};varCANONICAL_META_SELECTORS=['og:url'];functionparseDomain(url){varparsedUrl=URL$$.parse(url);varhostname=parsedUrl.hostname;returnhostname;}functionresult(url){return{url:url,domain:parseDomain(url)};}varGenericUrlExtractor={extract:functionextract(_ref){var$=_ref.$;varurl=_ref.url;varmetaCache=_ref.metaCache;var$canonical=$('link[rel=canonical]');if($canonical.length!==0){varhref=$canonical.attr('href');if(href){returnresult(href);}}varmetaUrl=extractFromMeta($,CANONICAL_META_SELECTORS,metaCache);if(metaUrl){returnresult(metaUrl);}returnresult(url);}};varEXCERPT_META_SELECTORS=['og:description','twitter:description'];functionclean$2(content,$){varmaxLength=arguments.length<=2||arguments[2]===undefined?200:arguments[2];content=content.replace(/[\s\n]+/g,' ').trim();returnellipsize$$(content,maxLength,{ellipse:'…'});}varGenericExcerptExtractor={extract:functionextract(_ref){var$=_ref.$;varcontent=_ref.content;varmetaCache=_ref.metaCache;varexcerpt=extractFromMeta($,EXCERPT_META_SELECTORS,metaCache);if(excerpt){returnclean$2(stripTags(excerpt,$));}// Fall back to excerpting from the extracted content
varmaxLength=200;varshortContent=content.slice(0,maxLength*5);returnclean$2($(shortContent).text(),$,maxLength);}};varGenericWordCountExtractor={extract:functionextract(_ref){varcontent=_ref.content;var$=cheerio$$.load(content);vartext=normalizeSpaces($('div').first().text());returntext.split(/\s/).length;}};varGenericExtractor={// This extractor is the default for all domains
domain:'*',title:GenericTitleExtractor.extract,date_published:GenericDatePublishedExtractor.extract,author:GenericAuthorExtractor.extract,content:GenericContentExtractor.extract.bind(GenericContentExtractor),lead_image_url:GenericLeadImageUrlExtractor.extract,dek:GenericDekExtractor.extract,next_page_url:GenericNextPageUrlExtractor.extract,url_and_domain:GenericUrlExtractor.extract,excerpt:GenericExcerptExtractor.extract,word_count:GenericWordCountExtractor.extract,direction:functiondirection(_ref){vartitle=_ref.title;returnstringDirection$$.getDirection(title);},extract:functionextract(options){varhtml=options.html;if(html){var$=cheerio$$.load(html);options.$=$;}vartitle=this.title(options);vardate_published=this.date_published(options);varauthor=this.author(options);varcontent=this.content(_extends$$({},options,{title:title}));varlead_image_url=this.lead_image_url(_extends$$({},options,{content:content}));vardek=this.dek(_extends$$({},options,{content:content}));varnext_page_url=this.next_page_url(options);varexcerpt=this.excerpt(_extends$$({},options,{content:content}));varword_count=this.word_count(_extends$$({},options,{content:content}));vardirection=this.direction({title:title});var_url_and_domain=this.url_and_domain(options);varurl=_url_and_domain.url;vardomain=_url_and_domain.domain;return{title:title,author:author,date_published:date_published||null,dek:dek,lead_image_url:lead_image_url,content:content,next_page_url:next_page_url,url:url,domain:domain,excerpt:excerpt,word_count:word_count,direction:direction};}};functiongetExtractor(url,parsedUrl){parsedUrl=parsedUrl||URL$$.parse(url);var_parsedUrl=parsedUrl;varhostname=_parsedUrl.hostname;varbaseDomain=hostname.split('.').slice(-2).join('.');returnExtractors[hostname]||Extractors[baseDomain]||GenericExtractor;}varATTR_RE=/\[([\w-]+)\]/;// Remove elements by an array of selectors
domain:'*',title:GenericTitleExtractor.extract,date_published:GenericDatePublishedExtractor.extract,author:GenericAuthorExtractor.extract,content:GenericContentExtractor.extract.bind(GenericContentExtractor),lead_image_url:GenericLeadImageUrlExtractor.extract,dek:GenericDekExtractor.extract,next_page_url:GenericNextPageUrlExtractor.extract,url_and_domain:GenericUrlExtractor.extract,excerpt:GenericExcerptExtractor.extract,word_count:GenericWordCountExtractor.extract,direction:functiondirection(_ref){vartitle=_ref.title;returnstringDirection$$.getDirection(title);},extract:functionextract(options){varhtml=options.html;if(html){var$=cheerio$$.load(html);options.$=$;}vartitle=this.title(options);vardate_published=this.date_published(options);varauthor=this.author(options);varcontent=this.content(_extends$$({},options,{title:title}));varlead_image_url=this.lead_image_url(_extends$$({},options,{content:content}));vardek=this.dek(_extends$$({},options,{content:content}));varnext_page_url=this.next_page_url(options);varexcerpt=this.excerpt(_extends$$({},options,{content:content}));varword_count=this.word_count(_extends$$({},options,{content:content}));vardirection=this.direction({title:title});var_url_and_domain=this.url_and_domain(options);varurl=_url_and_domain.url;vardomain=_url_and_domain.domain;return{title:title,author:author,date_published:date_published||null,dek:dek,lead_image_url:lead_image_url,content:content,next_page_url:next_page_url,url:url,domain:domain,excerpt:excerpt,word_count:word_count,direction:direction};}};functiongetExtractor(url,parsedUrl){parsedUrl=parsedUrl||URL$$.parse(url);var_parsedUrl=parsedUrl;varhostname=_parsedUrl.hostname;varbaseDomain=hostname.split('.').slice(-2).join('.');returnExtractors[hostname]||Extractors[baseDomain]||GenericExtractor;}// Remove elements by an array of selectors
functioncleanBySelectors($content,$,_ref){varclean=_ref.clean;if(!clean)return$content;$(clean.join(','),$content).remove();return$content;}// Transform matching elements
functiontransformElements($content,$,_ref2){vartransforms=_ref2.transforms;if(!transforms)return$content;_Reflect$ownKeys(transforms).forEach(function(key){var$matches=$(key,$content);varvalue=transforms[key];// If value is a string, convert directly
functiontransformElements($content,$,_ref2){vartransforms=_ref2.transforms;if(!transforms)return$content;_Reflect$ownKeys$$(transforms).forEach(function(key){var$matches=$(key,$content);varvalue=transforms[key];// If value is a string, convert directly
if(typeofvalue==='string'){$matches.each(function(index,node){convertNodeTo($(node),$,transforms[key]);});}elseif(typeofvalue==='function'){// If value is function, apply function to node
$matches.each(function(index,node){varresult=value($(node),$);// If function returns a string, convert node to that value
if(typeofresult==='string'){convertNodeTo($(node),$,result);}});}});return$content;}function select(opts){var$=opts.$;vartype=opts.type;varextractionOpts=opts.extractionOpts;var_opts$extractHtml=opts.extractHtml;varextractHtml=_opts$extractHtml===undefined?false:_opts$extractHtml;// Skip if there's not extraction for this type
if(typeofresult==='string'){convertNodeTo($(node),$,result);}});}});return$content;}functionfindMatchingSelector($,selectors){returnselectors.find(function(selector){if(Array.isArray(selector)){var_selector=_slicedToArray$$(selector,2);vars=_selector[0];varattr=_selector[1];return$(s).length===1&&$(s).attr(attr)&&$(s).attr(attr).trim()!=='';}return$(selector).length===1&&$(selector).text().trim()!=='';});}functionselect(opts){var$=opts.$;vartype=opts.type;varextractionOpts=opts.extractionOpts;var_opts$extractHtml=opts.extractHtml;varextractHtml=_opts$extractHtml===undefined?false:_opts$extractHtml;// Skip if there's not extraction for this type
if(!extractionOpts)returnnull;// If a string is hardcoded for a type (e.g., Wikipedia
// contributors), return the string
if(typeofextractionOpts==='string')returnextractionOpts;varselectors=extractionOpts.selectors;var_extractionOpts$defau=extractionOpts.defaultCleaner;vardefaultCleaner=_extractionOpts$defau===undefined?true:_extractionOpts$defau;varmatchingSelector=selectors.find(function(selector){return$(selector).length===1&&$(selector).text().trim()!=='';});if(!matchingSelector)returnnull;// Declaring result; will contain either
if(typeofextractionOpts==='string')returnextractionOpts;varselectors=extractionOpts.selectors;var_extractionOpts$defau=extractionOpts.defaultCleaner;vardefaultCleaner=_extractionOpts$defau===undefined?true:_extractionOpts$defau;varmatchingSelector=findMatchingSelector($,selectors);if(!matchingSelector)returnnull;// Declaring result; will contain either
// text or html, which will be cleaned
// by the appropriate cleaner type
// If the selector type requests html as its return type
// transform and clean the element with provided selectors
if(extractHtml){var$content=$(matchingSelector);// Wrap in div so transformation can take place on root element
$content.wrap($('<div></div>'));$content=$content.parent();$content=transformElements($content,$,extractionOpts);$content=cleanBySelectors($content,$,extractionOpts);if(defaultCleaner){$content=Cleaners[type]($content,opts);}return$.html($content);}// if selector includes an attr (e.g., img[src]),
$content.wrap($('<div></div>'));$content=$content.parent();$content=transformElements($content,$,extractionOpts);$content=cleanBySelectors($content,$,extractionOpts);$content=Cleaners[type]($content,_extends$$({},opts,{defaultCleaner:defaultCleaner}));return$.html($content);}varresult=void0;// if selector is an array (e.g., ['img', 'src']),
// extract the attr
varattr=matchingSelector.match(ATTR_RE);varresult=void0;if(attr){result=$(matchingSelector).attr(attr[1]);}else{// otherwise use the text of the node
result=$(matchingSelector).text();}// Allow custom extractor to skip default cleaner
if(Array.isArray(matchingSelector)){var_matchingSelector=_slicedToArray$$(matchingSelector,2);varselector=_matchingSelector[0];varattr=_matchingSelector[1];result=$(selector).attr(attr).trim();}else{result=$(matchingSelector).text().trim();}// Allow custom extractor to skip default cleaner
// for this type; defaults to true
if(defaultCleaner){returnCleaners[type](result,opts);}returnresult;}functionextractResult(opts){vartype=opts.type;varextractor=opts.extractor;// If nothing matches the selector,
if(defaultCleaner){returnCleaners[type](result,opts);}returnresult;}functionextractResult(opts){vartype=opts.type;varextractor=opts.extractor;var_opts$fallback=opts.fallback;varfallback=_opts$fallback===undefined?true:_opts$fallback;varresult=select(_extends$$({},opts,{extractionOpts:extractor[type]}));// If custom parser succeeds, return the result
if(result){returnresult;}// If nothing matches the selector, and fallback is enabled,
// run the Generic extraction
return select(_extends$$({},opts,{extractionOpts:extractor[type]}))||GenericExtractor[type](opts);}varRootExtractor={extract:functionextract(){varextractor=arguments.length<=0||arguments[0]===undefined?GenericExtractor:arguments[0];varopts=arguments[1];var_opts=opts;varcontentOnly=_opts.contentOnly;varextractedTitle=_opts.extractedTitle;// This is the generic extractor. Run its extract method
if(extractor.domain==='*')returnextractor.extract(opts);opts=_extends$$({},opts,{extractor:extractor});if(contentOnly){var_content=extractResult(_extends$$({},opts,{type:'content',extractHtml:true,title:extractedTitle}));return{content:_content};}vartitle=extractResult(_extends$$({},opts,{type:'title'}));vardate_published=extractResult(_extends$$({},opts,{type:'date_published'}));varauthor=extractResult(_extends$$({},opts,{type:'author'}));varnext_page_url=extractResult(_extends$$({},opts,{type:'next_page_url'}));varcontent=extractResult(_extends$$({},opts,{type:'content',extractHtml:true,title:title}));varlead_image_url=extractResult(_extends$$({},opts,{type:'lead_image_url',content:content}));vardek=extractResult(_extends$$({},opts,{type:'dek',content:content}));varexcerpt=extractResult(_extends$$({},opts,{type:'excerpt',content:content}));varword_count=extractResult(_extends$$({},opts,{type:'word_count',content:content}));vardirection=extractResult(_extends$$({},opts,{type:'direction',title:title}));var_extractResult=extractResult(_extends$$({},opts,{type:'url_and_domain'}));varurl=_extractResult.url;vardomain=_extractResult.domain;return{title:title,content:content,author:author,date_published:date_published,lead_image_url:lead_image_url,dek:dek,next_page_url:next_page_url,url:url,domain:domain,excerpt:excerpt,word_count:word_count,direction:direction};}};varcollectAllPages=function(){var_ref=_asyncToGenerator(_regeneratorRuntime.mark(function_callee(_ref2){varnext_page_url=_ref2.next_page_url;varhtml=_ref2.html;var$=_ref2.$;varmetaCache=_ref2.metaCache;varresult=_ref2.result;varExtractor=_ref2.Extractor;vartitle=_ref2.title;varurl=_ref2.url;varpages,previousUrls,extractorOpts,nextPageResult;return_regeneratorRuntime.wrap(function_callee$(_context){while(1){switch(_context.prev=_context.next){case0:// At this point, we've fetched just the first page
if(fallback)returnGenericExtractor[type](opts);returnnull;}varRootExtractor={extract:functionextract(){varextractor=arguments.length<=0||arguments[0]===undefined?GenericExtractor:arguments[0];varopts=arguments[1];var_opts=opts;varcontentOnly=_opts.contentOnly;varextractedTitle=_opts.extractedTitle;// This is the generic extractor. Run its extract method
if(extractor.domain==='*')returnextractor.extract(opts);opts=_extends$$({},opts,{extractor:extractor});if(contentOnly){var_content=extractResult(_extends$$({},opts,{type:'content',extractHtml:true,title:extractedTitle}));return{content:_content};}vartitle=extractResult(_extends$$({},opts,{type:'title'}));vardate_published=extractResult(_extends$$({},opts,{type:'date_published'}));varauthor=extractResult(_extends$$({},opts,{type:'author'}));varnext_page_url=extractResult(_extends$$({},opts,{type:'next_page_url'}));varcontent=extractResult(_extends$$({},opts,{type:'content',extractHtml:true,title:title}));varlead_image_url=extractResult(_extends$$({},opts,{type:'lead_image_url',content:content}));vardek=extractResult(_extends$$({},opts,{type:'dek',content:content}));varexcerpt=extractResult(_extends$$({},opts,{type:'excerpt',content:content}));varword_count=extractResult(_extends$$({},opts,{type:'word_count',content:content}));vardirection=extractResult(_extends$$({},opts,{type:'direction',title:title}));var_ref3=extractResult(_extends$$({},opts,{type:'url_and_domain'}))||{url:null,domain:null};varurl=_ref3.url;vardomain=_ref3.domain;return{title:title,content:content,author:author,date_published:date_published,lead_image_url:lead_image_url,dek:dek,next_page_url:next_page_url,url:url,domain:domain,excerpt:excerpt,word_count:word_count,direction:direction};}};varcollectAllPages=function(){var_ref=_asyncToGenerator(_regeneratorRuntime.mark(function_callee(_ref2){varnext_page_url=_ref2.next_page_url;varhtml=_ref2.html;var$=_ref2.$;varmetaCache=_ref2.metaCache;varresult=_ref2.result;varExtractor=_ref2.Extractor;vartitle=_ref2.title;varurl=_ref2.url;varpages,previousUrls,extractorOpts,nextPageResult,word_count;return_regeneratorRuntime.wrap(function_callee$(_context){while(1){switch(_context.prev=_context.next){case0:// At this point, we've fetched just the first page
pages=1;previousUrls=[removeAnchor(url)];// If we've gone over 26 pages, something has
_context.next=9;returnResource.create(url,html,parsedUrl);case9:$=_context.sent;if(!$.error){_context.next=12;break;}return_context.abrupt('return',$);case12:html=$.html();// Cached value of every meta name in our document.
_context.next=10;returnResource.create(url,html,parsedUrl);case10:$=_context.sent;if(!$.error){_context.next=13;break;}return_context.abrupt('return',$);case13:html=$.html();// Cached value of every meta name in our document.
// Used when extracting title/author/date_published/dek
metaCache=$('meta').map(function(_,node){return$(node).attr('name');}).toArray();result=RootExtractor.extract(Extractor,{url:url,html:html,$:$,metaCache:metaCache,parsedUrl:parsedUrl});_result=result;title=_result.title;next_page_url=_result.next_page_url;// Fetch more pages if next_page_url found
if(!(fetchAllPages&&next_page_url)){_context.next=24;break;}_context.next=21;returncollectAllPages({Extractor:Extractor,next_page_url:next_page_url,html:html,$:$,metaCache:metaCache,result:result,title:title,url:url});case21:result=_context.sent;_context.next=25;break;case24:result=_extends$$({},result,{total_pages:1,rendered_pages:1});case25:return_context.abrupt('return',result);case26:case'end':return_context.stop();}}},_callee,_this);}))();},// A convenience method for getting a resource
metaCache=$('meta').map(function(_,node){return$(node).attr('name');}).toArray();result=RootExtractor.extract(Extractor,{url:url,html:html,$:$,metaCache:metaCache,parsedUrl:parsedUrl,fallback:fallback});_result=result;title=_result.title;next_page_url=_result.next_page_url;// Fetch more pages if next_page_url found
if(!(fetchAllPages&&next_page_url)){_context.next=25;break;}_context.next=22;returncollectAllPages({Extractor:Extractor,next_page_url:next_page_url,html:html,$:$,metaCache:metaCache,result:result,title:title,url:url});case22:result=_context.sent;_context.next=26;break;case25:result=_extends$$({},result,{total_pages:1,rendered_pages:1});case26:return_context.abrupt('return',result);case27:case'end':return_context.stop();}}},_callee,_this);}))();},// A convenience method for getting a resource
// to work with, e.g., for custom extractor generator
// A list of strings that denote a negative scoring for this content as being
// an article container. Checked against className and id.
//
// TODO: Perhaps have these scale based on their odds of being quality?
varNEGATIVE_SCORE_HINTS=['adbox','advert','author','bio','bookmark','bottom','byline','clear','com-','combx','comment','comment\\B','contact','copy','credit','crumb','date','deck','excerpt','featured',// tnr.com has a featured_content which throws us off
// A list of strings that can be considered unlikely candidates when
// extracting content from a resource. These strings are joined together
// and then tested for existence using re:test, so may contain simple,
// non-pipe style regular expression queries if necessary.
varUNLIKELY_CANDIDATES_BLACKLIST$1=['ad-break','adbox','advert','addthis','agegate','aux','blogger-labels','combx','comment','conversation','disqus','entry-unrelated','extra','foot','form','header','hidden','loader','login',// Note: This can hit 'blogindex'.
'menu','meta','nav','pager','pagination','predicta',// readwriteweb inline ad box
'presence_control_external',// lifehacker.com container full of false positives
// A list of strings that denote a negative scoring for this content as being
// an article container. Checked against className and id.
//
// TODO: Perhaps have these scale based on their odds of being quality?
varNEGATIVE_SCORE_HINTS$1=['adbox','advert','author','bio','bookmark','bottom','byline','clear','com-','combx','comment','comment\\B','contact','copy','credit','crumb','date','deck','excerpt','featured',// tnr.com has a featured_content which throws us off
@ -851,16 +1071,28 @@ function template(strings) {
}).join('\n');
}
var_templateObject=_taggedTemplateLiteral(['\n // Rename CustomExtractor\n // to fit your publication\n export const CustomExtractor = {\n domain: \'','\',\n title: {\n selectors: [\n // enter title selectors\n ],\n },\n\n content: {\n selectors: [\n // enter content selectors\n ],\n\n // Is there anything in the content you selected that needs transformed\n // before it\'s consumable content? E.g., unusual lazy loaded images\n transforms: [\n ],\n\n // Is there anything that is in the result that shouldn\'t be?\n // The clean selectors will remove anything that matches from\n // the result\n clean: [\n\n ]\n },\n\n date_published: null,\n\n lead_image_url: null,\n\n dek: null,\n\n next_page_url: null,\n\n excerpt: null,\n }\n '],['\n // Rename CustomExtractor\n // to fit your publication\n export const CustomExtractor = {\n domain: \'','\',\n title: {\n selectors: [\n // enter title selectors\n ],\n },\n\n content: {\n selectors: [\n // enter content selectors\n ],\n\n // Is there anything in the content you selected that needs transformed\n // before it\'s consumable content? E.g., unusual lazy loaded images\n transforms: [\n ],\n\n // Is there anything that is in the result that shouldn\'t be?\n // The clean selectors will remove anything that matches from\n // the result\n clean: [\n\n ]\n },\n\n date_published: null,\n\n lead_image_url: null,\n\n dek: null,\n\n next_page_url: null,\n\n excerpt: null,\n }\n ']);
var_templateObject=_taggedTemplateLiteral(['\n // Rename CustomExtractor\n // to fit your publication\n // (e.g., NYTimesExtractor)\n export const CustomExtractor = {\n domain: \'','\',\n title: {\n selectors: [\n // enter title selectors\n ],\n },\n\n author: {\n selectors: [\n // enter author selectors\n ],\n },\n\n content: {\n selectors: [\n // enter content selectors\n ],\n\n // Is there anything in the content you selected that needs transformed\n // before it\'s consumable content? E.g., unusual lazy loaded images\n transforms: [\n ],\n\n // Is there anything that is in the result that shouldn\'t be?\n // The clean selectors will remove anything that matches from\n // the result\n clean: [\n\n ]\n },\n\n date_published: null,\n\n lead_image_url: null,\n\n dek: null,\n\n next_page_url: null,\n\n excerpt: null,\n }\n '],['\n // Rename CustomExtractor\n // to fit your publication\n // (e.g., NYTimesExtractor)\n export const CustomExtractor = {\n domain: \'','\',\n title: {\n selectors: [\n // enter title selectors\n ],\n },\n\n author: {\n selectors: [\n // enter author selectors\n ],\n },\n\n content: {\n selectors: [\n // enter content selectors\n ],\n\n // Is there anything in the content you selected that needs transformed\n // before it\'s consumable content? E.g., unusual lazy loaded images\n transforms: [\n ],\n\n // Is there anything that is in the result that shouldn\'t be?\n // The clean selectors will remove anything that matches from\n // the result\n clean: [\n\n ]\n },\n\n date_published: null,\n\n lead_image_url: null,\n\n dek: null,\n\n next_page_url: null,\n\n excerpt: null,\n }\n ']);
functionextractorTemplate(hostname){
returntemplate(_templateObject,hostname);
}
var_templateObject$1=_taggedTemplateLiteral(['\n import assert from \'assert\';\n import fs from \'fs\';\n import URL from \'url\';\n import cheerio from \'cheerio\';\n\n import Mercury from \'mercury\';\n import getExtractor from \'extractors/get-extractor\';\n\n // Rename CustomExtractor\n describe(\'CustomExtractor\', () => {\n it(\'is selected properly\', () => {\n // To pass this test, rename your extractor in\n // ','/index.js\n // then add your new extractor to\n // src/extractors/all.js\n const url = \'','\';\n const extractor = getExtractor(url);\n assert.equal(extractor.domain, URL.parse(url).hostname)\n })\n\n it(\'works with a starter story\', (async) () => {\n // To pass this test, begin filling out your\n // selectors in ','/index.js. This test is just\n // a stub; you can add more fields to test as much of\n // your parser as possible.\n const html = fs.readFileSync(\'','\');\n const uri = \'','\';\n\n const { content, title, author } = await Mercury.parse(uri, html);\n const $ = cheerio.load(content);\n const text = $(\'*\').first()\n .text()\n .trim()\n .slice(0, 20);\n\n // Update these values with the expected values from\n // the article.\n assert.equal(title, \'\');\n assert.equal(author, \'\');\n assert.equal(text, \'\');\n });\n });\n '],['\n import assert from \'assert\';\n import fs from \'fs\';\n import URL from \'url\';\n import cheerio from \'cheerio\';\n\n import Mercury from \'mercury\';\n import getExtractor from \'extractors/get-extractor\';\n\n // Rename CustomExtractor\n describe(\'CustomExtractor\', () => {\n it(\'is selected properly\', () => {\n // To pass this test, rename your extractor in\n // ','/index.js\n // then add your new extractor to\n // src/extractors/all.js\n const url = \'','\';\n const extractor = getExtractor(url);\n assert.equal(extractor.domain, URL.parse(url).hostname)\n })\n\n it(\'works with a starter story\', (async) () => {\n // To pass this test, begin filling out your\n // selectors in ','/index.js. This test is just\n // a stub; you can add more fields to test as much of\n // your parser as possible.\n const html = fs.readFileSync(\'','\');\n const uri = \'','\';\n\n const { content, title, author } = await Mercury.parse(uri, html);\n const $ = cheerio.load(content);\n const text = $(\'*\').first()\n .text()\n .trim()\n .slice(0, 20);\n\n // Update these values with the expected values from\n // the article.\n assert.equal(title, \'\');\n assert.equal(author, \'\');\n assert.equal(text, \'\');\n });\n });\n ']);
var_templateObject$1=_taggedTemplateLiteral(['\n it(\'returns the ','\', (async) () => {\n // To pass this test, fill out the ',' selector\n // in ','/index.js.\n const html =\n fs.readFileSync(\'','\');\n const articleUrl =\n \'','\';\n\n const { ',' } =\n await Mercury.parse(articleUrl, html, { fallback: false });\n\n // Update these values with the expected values from\n // the article.\n assert.equal(',', ',')\n });\n '],['\n it(\'returns the ','\', (async) () => {\n // To pass this test, fill out the ',' selector\n // in ','/index.js.\n const html =\n fs.readFileSync(\'','\');\n const articleUrl =\n \'','\';\n\n const { ',' } =\n await Mercury.parse(articleUrl, html, { fallback: false });\n\n // Update these values with the expected values from\n // the article.\n assert.equal(',', ',')\n });\n ']);
var_templateObject2=_taggedTemplateLiteral(['\n import assert from \'assert\';\n import fs from \'fs\';\n import URL from \'url\';\n import cheerio from \'cheerio\';\n\n import Mercury from \'mercury\';\n import getExtractor from \'extractors/get-extractor\';\n\n // Rename CustomExtractor\n describe(\'CustomExtractor\', () => {\n it(\'is selected properly\', () => {\n // To pass this test, rename your extractor in\n // ','/index.js\n // (e.g., CustomExtractor => NYTimesExtractor)\n // then add your new extractor to\n // src/extractors/all.js\n const url =\n \'','\';\n const extractor = getExtractor(url);\n assert.equal(extractor.domain, URL.parse(url).hostname)\n })\n\n ','\n\n it(\'returns the content\', (async) () => {\n // To pass this test, fill out the content selector\n // in ','/index.js.\n // You may also want to make use of the clean and transform\n // options.\n const html =\n fs.readFileSync(\'','\');\n const url =\n \'','\';\n\n const { content } =\n await Mercury.parse(url, html, { fallback: false });\n\n const $ = cheerio.load(content || \'\');\n\n const first13 = $(\'*\').first()\n .text()\n .trim()\n .split(/\\s+/)\n .slice(0, 13)\n .join(\' \')\n\n // Update these values with the expected values from\n // the article.\n assert.equal(first13, null);\n });\n });\n '],['\n import assert from \'assert\';\n import fs from \'fs\';\n import URL from \'url\';\n import cheerio from \'cheerio\';\n\n import Mercury from \'mercury\';\n import getExtractor from \'extractors/get-extractor\';\n\n // Rename CustomExtractor\n describe(\'CustomExtractor\', () => {\n it(\'is selected properly\', () => {\n // To pass this test, rename your extractor in\n // ','/index.js\n // (e.g., CustomExtractor => NYTimesExtractor)\n // then add your new extractor to\n // src/extractors/all.js\n const url =\n \'','\';\n const extractor = getExtractor(url);\n assert.equal(extractor.domain, URL.parse(url).hostname)\n })\n\n ','\n\n it(\'returns the content\', (async) () => {\n // To pass this test, fill out the content selector\n // in ','/index.js.\n // You may also want to make use of the clean and transform\n // options.\n const html =\n fs.readFileSync(\'','\');\n const url =\n \'','\';\n\n const { content } =\n await Mercury.parse(url, html, { fallback: false });\n\n const $ = cheerio.load(content || \'\');\n\n const first13 = $(\'*\').first()\n .text()\n .trim()\n .split(/\\\\s+/)\n .slice(0, 13)\n .join(\' \')\n\n // Update these values with the expected values from\n // the article.\n assert.equal(first13, null);\n });\n });\n ']);
confirm(generateScaffold,[url,file],'Generating parser and tests');
console.log('Your custom site extractor has been set up. To get started building it, run\n npm test -- '+getDir(url)+'/index.test.js');
}else{
console.log('It looks like you already have a custom parser for this url.\nThe page you linked to has been added to '+file+'. Copy and paste\nthe following code to use that page in your tests:\nconst html = fs.readFileSync(\''+file+'\');');
confirm(generateScaffold,[url,file,result],'Generating parser and tests');
console.log('Your custom site extractor has been set up. To get started building it, run\n npm run watch:test -- '+hostname);
}else{
console.log('\n It looks like you already have a custom parser for this url.\n The page you linked to has been added to '+file+'. Copy and paste\n the following code to use that page in your tests:\n const html = fs.readFileSync(\''+file+'\');');
Mercury can extract meaningful content from almost any web site, but custom parsers allow the Mercury parser to find the content more quickly and more accurately than it might otherwise do. Our goal is to include custom parsers as many sites as we can, and we'd love your help!
## How to generate a custom parser
First, you'll need to clone the Mercury Parser repository and install dependencies.
Now that you're ready to go, take a look at the live custom parsers in [`src/extractors/custom`](/src/extractors/custom) for examples and to check if the site you want to write a parser for already exists.
To generate a new custom parser, run:
```bash
npm run generate-custom-parser
```
This script will prompt you to paste a link to an article you want to parse. The URL you choose will serve as the example your parser tests against. The script will also generate your custom parser and a barebones (and failing) test for your parser.
At that point, you'll be prompted to run:
```bash
npm test
```
This will run the test for your parser, which will fail (which makes sense — you haven't written it yet!). Your goal now is to follow the instructions in the generated `<example.com>/index.test.js` and `<example.com>/index.js` files until they pass!
## How to write a custom parser
## The basics of parsing a site with a Mercury custom parser
Custom parsers allow you to write CSS selectors that will find the content you're looking for on the page you're testing against. If you're familiar with jQuery, the selectors work exactly the same way.
Custom parsers allow you to write CSS selectors that will find the content you're looking for on the page you're testing against. If you've written any CSS or jQuery, CSS selectors should be very familiar to you.
You can query for every field returned by the Mercury Parser:
@ -49,6 +19,7 @@ You can query for every field returned by the Mercury Parser:
### Using selectors
#### Basic selectors
To demonstrate, let's start with something simple: Your selector for the page's title might look something like this:
```javascript
@ -69,6 +40,32 @@ As you might guess, the selectors key provides an array of selectors that Mercur
The selector you choose should return one element. If more than one element is returned by your selector, it will fail (and Mercury will fall back to its generic extractor).
#### Selecting an attribute
Sometimes the information you want to return lives in an element's attribute rather than its text — e.g., sometimes a more exact ISO-formatted date/time will be stored in an attribute of an element.
The text you want isn't the text inside a matching element, but rather, inside the datetime attribute. To write a selector that returns an attribute, you provide your custom parser with a two-element array. The first element is your selector; the second element is the attribute you'd like to return.
```javascript
export const ExampleExtractor = {
...
// This example returns the datetime attribute if it exists; if not, it falls back to the text of time.article-timestamp
date_published: {
selectors: [
['time.article-timestamp[datetime]', 'datetime'],
'time.article-timestamp',
],
},
...
```
This is all you'll need to know to handle most of the fields Mercury parses (titles, authors, date published, etc.). Article content is the exception.
For much more complex tranforms, you can perform dom manipulation within the tranform function, but this is discouraged unless absolutely necessary. See, for example, the lazy-loaded image transform in [the NYTimesExtractor](www.nytimes.com/index.js#L25), which transforms the src attribute on the lazy-loaded image.
## How to generate a custom parser
Now that you know the basics of how custom extractors work, let's walk through the workflow for how to write and submit one. For our example, we're going to use [The New Yorker](http://www.newyorker.com/). (You can find the results of this tutorial [in the NewYorkerExtractor source](www.newyorker.com).)
### Step 0: Installation
First, you'll need to clone the Mercury Parser repository and install dependencies.
You should also create a new git branch for your custom extractor:
```bash
git checkout -b feat-new-yorker-extractor
```
Now that you're ready to go, take a look at the live custom parsers in [`src/extractors/custom`](/src/extractors/custom) for examples and to check if the site you want to write a parser for already exists.
### Step 1: Generate your custom parser
If we don't already have a parser for the site you want to contribute, you're ready to generate a new custom parser. To do so, run:
```bash
npm run generate-parser
```
This script will prompt you to paste a link to an article you want to parse. The URL you choose will serve as the example your parser will test against. The script will also generate your custom parser and some barebones (and failing) tests for your parser.
For our New Yorker example, we're going to use [this story](http://www.newyorker.com/tech/elements/hacking-cryptography-and-the-countdown-to-quantum-computing).
When the generator script completes, you'll be prompted to run:
```bash
npm watch:test -- www.newyorker.com
```
This will run the tests for the parser you just generated, which should fail (which makes sense — you haven't written it yet!). Your goal now is to follow the instructions in the generated `www.newyorker.com/index.test.js` and `www.newyorker.com/index.js` files until they pass!
### Step 2: Passing your first test
If you look at your parser's test file, you'll see a few instructions to guide you in making your tests pass. The first comment at the top of our test file is simple: It instructs you to rename CustomExtractor. In our case, we're going to rename it to NewYorkerExtractor.
So, from this:
```javascript
// Rename CustomExtractor
describe('CustomExtractor', () => {
...
```
...to this:
```javascript
describe('NewYorkerExtractor', () => {
...
```
That was easy, but when you save the file you may notice that this didn't make your test pass. So let's look more closely at the first test:
This test checks to see whether or not Mercury can find your custom parser. In our example, it ensures that Mercury correctly selects our custom parser when it's parsing a link from `http://www.newyorker.com`. At the very top of the test, you'll see the following instructions:
So let's follow those instructions. In `[www.newyorker.com/index.js](www.newyorker.com/index.js)`, we're coing to rename `CustomExtractor` to `NewYorkerExtractor`, just like we did in the test. The top of our custom parser should now look like this:
```javascript
export const NewYorkerExtractor = {
domain: 'www.newyorker.com',
...
}
```
Save the file, and you'll notice your test still isn't passing. If we refer back to the instructions above, we'll see that we need to add our new extractor to `[src/extractors/all.js](../extractors/all.js)`. So let's do that. First, we need to add the following import to the rest of the imports at top of the file:
```javascript
import { NewYorkerExtractor } from './custom/www.newyorker.com';
```
Next, we need to add our new custom extractor to the Extractors object. The key should be your site's domain (in our case, `www.newyorker.com`), and the value should be the extractor we imported above. So, it should look like this:
```javascript
const Extractors = {
...
'www.newyorker.com': NewYorkerExtractor,
};
```
When you save your changes, you'll notice that your first test is now passing — congrats! Now it's time to move onto your next test.
### Step 3: Performing a simple title extraction
The next test checks to see whether your extractor returns the correct title:
```javascript
it('returns the title', (async) () => {
// To pass this test, fill out the title selector
// in ./src/extractors/custom/www.newyorker.com/index.js.
// Update these values with the expected values from
// the article.
assert.equal(title, 'Schrödinger’s Hack');
});
```
As you can see, to pass this test, we need to fill out our title selector. In order to do this, you need to know what your selector is. To do this, open the html fixture the generator downloaded for you in the `[fixtures](/fixtures)` directory. In our example, that file is `fixtures/www.newyorker.com/1475248565793.html`. Now open that file in your web browser.
The page should look more or less exactly like the site you pointed it to, but this version is downloaded locally for test purposes. (You should always look for selectors using this local fixture rather than the actual web site; some sites re-write elements after the page loads, and we want to make sure we're looking at the page the same way Mercury will be.)
(For the purpose of this guide, we're going to assume you're using Chrome as your default browser; any browser should do, but we're going to refer specifically to Chrome's developer tools in this guide.)
So, back to the title: We want to make sure our test finds the same title we see on the page. In our example, the title of the article is "Hacking, Cryptography, and the Countdown to Quantum Computing." To find the selector (again, in Chrome), press Cmd+Shift+C on OS X, or Ctrl+Shift+C on Windows, then click on the title.
The selector for this title appears to be `h1.title`. To verify that we're right, click on the Console tab in Chrome's Developer Tools and run the following check:
```javascript
$$('h1.title')
```
If that returns only one match (i.e., an array with just one element), and the text of that element looks like the title we want, you're good to go!
Now, in your custom parser file, enter `h1.title` in your list of selectors for the title extractor:
```javascript
export const NewYorkerExtractor = {
domain: 'www.newyorker.com',
title: {
selectors: [
'h1.title',
],
},
...
}
```
Save the file, and... uh oh, our example still fails.
```javascript
AssertionError: 'Hacking, Cryptography, and the Countdown to Quantum Computing' == 'Schrödinger’s Hack'
```
When Mercury generated our test, it took a guess at the page's title, and in this case, it got it wrong. So update the test with thte title we expect, save it, and your test should pass!
### Step 4: Speed it up
We've been moving at a slow pace, but as you can see, once you understand the basics, extracting most items on the page is actually very easy. For example, if you follow the same instructions to find the author selector, you'll find that the `.contributors` selector will return the correct author (Alex Hutchinson).
For a slightly more complex example, you'll find after a bit of looking that the best place to get the most accurate datetime on the page is in the head of the document, in the value attribute of a meta tag:
As [explained above](#selecting-an-attribute), to return an attribute rather than the text inside an element, your selector should be an array where the first element is the element selector and the second element is the attribute you want to return. So, in this example, the date_published selector should look like this:
```javascript
...
date_published: {
selectors: [
['meta[name="article:published_time"]', 'value'],
]
},
...
```
You can refer to the [NewYorkerExtractor](www.newyorker.com/index.js) to see more the rest of the basic selectors.
### Step 5: Content extraction
I've left content extraction for last, since it's often the trickiest, sometimes requiring special passes to [clean](#cleaning-content) and [transform](#using-tranforms) the content. For the New Yorker, the first part is easy: The selector for this page is clearly `div#articleBody`. But that's just our frist step, because unlike the other tests, where we want to make sure we're matching a simple string, we need to sanity check that the page looks good when it's rendered, and that there aren't any elements returned by our selector that we don't want.
To aid you in previewing the results, you can run the `./preview` script to see what the title and content output look like. So, after you've chosen your selector, run the preview script on the URL you're testing:
This script will open both an `html` and `json` file allowing you to preview your results. Luckily for us, the New Yorker content is simple, and doesn't require any unusual cleaning or transformations — at least not in this example. Remember that if you do see content that needs cleaned or transformed in the selected content, you can follow the instructions in the [clean](#cleaning-content) and [transform](#using-tranforms) sections above.
## Submitting a custom extractor
If you've written a custom extractor, please send us a pull request! Passing tests that demonstrate your parser in action will help us evaluate the parser. If you need more guidance for your custom parser or your tests, peruse any of the [custom extractors](./) and their accompanying tests.
If you've written a custom extractor, please send us a pull request! Passing tests that demonstrate your parser in action will help us evaluate the parser.
Sometimes you may find that the site you're parsing doesn't provide certain information. For example, some sites don't have deks, and in those instances, you don't need to write a selector for that field. If there's a test for a selector you don't need, you can just remove that test and make note of it in your pull request.
Adam Pash
8 years ago