varUNLIKELY_CANDIDATES_BLACKLIST$1=['ad-break','adbox','advert','addthis','agegate','aux','blogger-labels','combx','comment','conversation','disqus','entry-unrelated','extra','foot','form','header','hidden','loader','login',// Note: This can hit 'blogindex'.
'menu','meta','nav','pager','pagination','predicta',// readwriteweb inline ad box
'presence_control_external',// lifehacker.com container full of false positives
varNEGATIVE_SCORE_HINTS$1=['adbox','advert','author','bio','bookmark','bottom','byline','clear','com-','combx','comment','comment\\B','contact','copy','credit','crumb','date','deck','excerpt','featured',// tnr.com has a featured_content which throws us off
function_interopDefault$1(ex){returnex&&(typeofex==='undefined'?'undefined':_typeof(ex))==='object'&&'default'inex?ex['default']:ex;}var_regeneratorRuntime=_interopDefault$1(regenerator);var_extends$1=_interopDefault$1(_extends);var_asyncToGenerator=_interopDefault$1(asyncToGenerator);varURL$1=_interopDefault$1(URL);varcheerio$1=_interopDefault$1(cheerio);var_Promise=_interopDefault$1(promise);varrequest$1=_interopDefault$1(request);var_Reflect$ownKeys$1=_interopDefault$1(_Reflect$ownKeys);var_toConsumableArray$1=_interopDefault$1(_toConsumableArray);var_defineProperty$1=_interopDefault$1(_defineProperty);var_slicedToArray$1=_interopDefault$1(_slicedToArray);var_typeof$1=_interopDefault$1(_typeof);var_getIterator$1=_interopDefault$1(_getIterator);var_Object$keys=_interopDefault$1(keys);varstringDirection$1=_interopDefault$1(stringDirection);varvalidUrl$1=_interopDefault$1(validUrl);varmoment$1=_interopDefault$1(moment);varwuzzy$1=_interopDefault$1(wuzzy);vardifflib$1=_interopDefault$1(difflib);var_Array$from=_interopDefault$1(from);varellipsize$1=_interopDefault$1(ellipsize);var_marked=[range].map(_regeneratorRuntime.mark);functionrange(){varstart=arguments.length>0&&arguments[0]!==undefined?arguments[0]:1;varend=arguments.length>1&&arguments[1]!==undefined?arguments[1]:1;return_regeneratorRuntime.wrap(functionrange$(_context){while(1){switch(_context.prev=_context.next){case0:if(!(start<=end)){_context.next=5;break;}_context.next=3;returnstart+=1;case3:_context.next=0;break;case5:case"end":return_context.stop();}}},_marked[0],this);}// extremely simple url validation as a first step
functionvalidateUrl(_ref){varhostname=_ref.hostname;// If this isn't a valid url, return an error message
return!!hostname;}varErrors={badUrl:{error:true,messages:'The url parameter passed does not look like a valid URL. Please check your data and try again.'}};varREQUEST_HEADERS={'User-Agent':'Readability - http://readability.com/about/'};// The number of milliseconds to attempt to fetch a resource before timing out.
varFETCH_TIMEOUT=10000;// Content types that we do not extract content from
varBAD_CONTENT_TYPES=['audio/mpeg','image/gif','image/jpeg','image/jpg'];varBAD_CONTENT_TYPES_RE=newRegExp('^('+BAD_CONTENT_TYPES.join('|')+')$','i');// Use this setting as the maximum size an article can be
// for us to attempt parsing. Defaults to 5 MB.
varMAX_CONTENT_LENGTH=5242880;// Turn the global proxy on or off
// Proxying is not currently enabled in Python source
// so not implementing logic in port.
functionget(options){// eslint-disable-line
returnnew_Promise(function(resolve,reject){request$1(options,function(err,response,body){if(err){reject(err);}else{resolve({body:body,response:response});}});});}// Evaluate a response to ensure it's something we should be keeping.
// This does not validate in the sense of a response being 200 level or
// not. Validation here means that we haven't found reason to bail from
// further processing of this url.
functionvalidateResponse(response){varparseNon2xx=arguments.length>1&&arguments[1]!==undefined?arguments[1]:false;// Check if we got a valid status code
// This isn't great, but I'm requiring a statusMessage to be set
// before short circuiting b/c nock doesn't set it in tests
// statusMessage only not set in nock response, in which case
// I check statusCode, which is currently only 200 for OK responses
// in tests
if(response.statusMessage&&response.statusMessage!=='OK'||response.statusCode!==200){if(!response.statusCode){thrownewError('Unable to fetch content. Original exception was '+response.error);}elseif(!parseNon2xx){thrownewError('Resource returned a response status code of '+response.statusCode+' and resource was instructed to reject non-2xx level status codes.');}}var_response$headers=response.headers,contentType=_response$headers['content-type'],contentLength=_response$headers['content-length'];// Check that the content is not in BAD_CONTENT_TYPES
if(BAD_CONTENT_TYPES_RE.test(contentType)){thrownewError('Content-type for this resource was '+contentType+' and is not allowed.');}// Check that the content length is below maximum
if(contentLength>MAX_CONTENT_LENGTH){thrownewError('Content for this resource was too large. Maximum content length is '+MAX_CONTENT_LENGTH+'.');}returntrue;}// Grabs the last two pieces of the URL and joins them back together
// This is to get the 'livejournal.com' from 'erotictrains.livejournal.com'
// Set our response attribute to the result of fetching our URL.
// TODO: This should gracefully handle timeouts and raise the
// proper exceptions on the many failure cases of HTTP.
// TODO: Ensure we are not fetching something enormous. Always return
// unicode content for HTML, with charset conversion.
varfetchResource$1=function(){var_ref2=_asyncToGenerator(_regeneratorRuntime.mark(function_callee(url,parsedUrl){varoptions,_ref3,response,body;return_regeneratorRuntime.wrap(function_callee$(_context){while(1){switch(_context.prev=_context.next){case0:parsedUrl=parsedUrl||URL$1.parse(encodeURI(url));options={url:parsedUrl,headers:_extends$1({},REQUEST_HEADERS),timeout:FETCH_TIMEOUT,// Don't set encoding; fixes issues
// w/gzipped responses
encoding:null,// Accept cookies
jar:true,// Accept and decode gzip
gzip:true,// Follow any redirect
followAllRedirects:true};_context.next=4;returnget(options);case4:_ref3=_context.sent;response=_ref3.response;body=_ref3.body;_context.prev=7;validateResponse(response);return_context.abrupt('return',{body:body,response:response});case12:_context.prev=12;_context.t0=_context['catch'](7);return_context.abrupt('return',Errors.badUrl);case15:case'end':return_context.stop();}}},_callee,this,[[7,12]]);}));functionfetchResource(_x2,_x3){return_ref2.apply(this,arguments);}returnfetchResource;}();functionconvertMetaProp($,from$$1,to){$('meta['+from$$1+']').each(function(_,node){var$node=$(node);varvalue=$node.attr(from$$1);$node.attr(to,value);$node.removeAttr(from$$1);});return$;}// For ease of use in extracting from meta tags,
// replace the "content" attribute on meta tags with the
// In addition, normalize 'property' attributes to 'name' for ease of
// querying later. See, e.g., og or twitter meta tags.
functionnormalizeMetaTags($){$=convertMetaProp($,'content','value');$=convertMetaProp($,'property','name');return$;}// Spacer images to be removed
varSPACER_RE$1=newRegExp('trans|transparent|spacer|blank','i');// The class we will use to mark elements we want to keep
// but would normally remove
varKEEP_CLASS$1='mercury-parser-keep';varKEEP_SELECTORS$1=['iframe[src^="https://www.youtube.com"]','iframe[src^="http://www.youtube.com"]','iframe[src^="https://player.vimeo"]','iframe[src^="http://player.vimeo"]'];// A list of tags to strip from the output if we encounter them.
// A list of strings that can be considered unlikely candidates when
// extracting content from a resource. These strings are joined together
// and then tested for existence using re:test, so may contain simple,
// non-pipe style regular expression queries if necessary.
varUNLIKELY_CANDIDATES_BLACKLIST$2=['ad-break','adbox','advert','addthis','agegate','aux','blogger-labels','combx','comment','conversation','disqus','entry-unrelated','extra','foot',// 'form', // This is too generic, has too many false positives
'header','hidden','loader','login',// Note: This can hit 'blogindex'.
'menu','meta','nav','outbrain','pager','pagination','predicta',// readwriteweb inline ad box
'presence_control_external',// lifehacker.com container full of false positives
'popup','printfriendly','related','remove','remark','rss','share','shoutbox','sidebar','sociable','sponsor','taboola','tools'];// A list of strings that can be considered LIKELY candidates when
// extracting content from a resource. Essentially, the inverse of the
// blacklist above - if something matches both blacklist and whitelist,
// it is kept. This is useful, for example, if something has a className
// of "rss-content entry-content". It matched 'rss', so it would normally
// be removed, however, it's also the entry content, so it should be left
// These strings are joined together and then tested for existence using
// re:test, so may contain simple, non-pipe style regular expression queries
// if necessary.
varUNLIKELY_CANDIDATES_WHITELIST$2=['and','article','body','blogindex','column','content','entry-content-asset','format',// misuse of form
'hfeed','hentry','hatom','main','page','posts','shadow'];// A list of tags which, if found inside, should cause a <div /> to NOT
// be turned into a paragraph tag. Shallow div tags without these elements
// should be turned into <p /> tags.
varDIV_TO_P_BLOCK_TAGS$2=['a','blockquote','dl','div','img','p','pre','table'].join(',');// A list of tags that should be ignored when trying to find the top candidate
// for a document.
// A list of selectors that specify, very clearly, either hNews or other
// very content-specific style content, like Blogger templates.
// More examples here: http://microformats.org/wiki/blog-post-formats
// A list of strings that denote a positive scoring for this content as being
// an article container. Checked against className and id.
// TODO: Perhaps have these scale based on their odds of being quality?
varNEGATIVE_SCORE_HINTS$2=['adbox','advert','author','bio','bookmark','bottom','byline','clear','com-','combx','comment','comment\\B','contact','copy','credit','crumb','date','deck','excerpt','featured',// tnr.com has a featured_content which throws us off
'scroll','secondary','share','shopping','shoutbox','side','sidebar','sponsor','stamp','sub','summary','tags','tools','widget'];// The above list, joined into a matching regular expression
varNEGATIVE_SCORE_RE$2=newRegExp(NEGATIVE_SCORE_HINTS$2.join('|'),'i');// XPath to try to determine if a page is wordpress. Not always successful.
varIS_WP_SELECTOR$1='meta[name=generator][value^=WordPress]';// Match a digit. Pretty clear.
// A list of words that, if found in link text or URLs, likely mean that
// this link is not a next page link.
// Match any phrase that looks like it could be page, or paging, or pagination
varPAGE_RE$1=newRegExp('pag(e|ing|inat)','i');// Match any link text/classname/id that looks like it could mean the next
// page. Things like: next, continue, >, >>, » but not >|, »| as those can
// mean last page.
// export const NEXT_LINK_TEXT_RE = new RegExp('(next|weiter|continue|>([^\|]|$)|»([^\|]|$))', 'i');
// Match any link text/classname/id that looks like it is an end link: things
// like "first", "last", "end", etc.
// Match any link text/classname/id that looks like it means the previous
// page.
// Match 2 or more consecutive <br> tags
// Match 1 BR tag.
// A list of all of the block level tags known in HTML5 and below. Taken from
// http://bit.ly/qneNIT
varBLOCK_LEVEL_TAGS$2=['article','aside','blockquote','body','br','button','canvas','caption','col','colgroup','dd','div','dl','dt','embed','fieldset','figcaption','figure','footer','form','h1','h2','h3','h4','h5','h6','header','hgroup','hr','li','map','object','ol','output','p','pre','progress','section','table','tbody','textarea','tfoot','th','thead','tr','ul','video'];varBLOCK_LEVEL_TAGS_RE$2=newRegExp('^('+BLOCK_LEVEL_TAGS$2.join('|')+')$','i');// The removal is implemented as a blacklist and whitelist, this test finds
// blacklisted elements that aren't whitelisted. We do this all in one
// expression-both because it's only one pass, and because this skips the
// serialization for whitelisted nodes.
varcandidatesBlacklist$2=UNLIKELY_CANDIDATES_BLACKLIST$2.join('|');varCANDIDATES_BLACKLIST$2=newRegExp(candidatesBlacklist$2,'i');varcandidatesWhitelist$2=UNLIKELY_CANDIDATES_WHITELIST$2.join('|');varCANDIDATES_WHITELIST$2=newRegExp(candidatesWhitelist$2,'i');functionstripUnlikelyCandidates$1($){// Loop through the provided document and remove any non-link nodes
// that are unlikely candidates for article content.
// :param node: The node to paragraphize; this is a raw node
// :param $: The cheerio object to handle dom manipulation
// :param br: Whether or not the passed node is a br
functionparagraphize$1(node,$){varbr=arguments.length>2&&arguments[2]!==undefined?arguments[2]:false;var$node=$(node);if(br){varsibling=node.nextSibling;varp=$('<p></p>');// while the next node is text or not a block level element
// append it to a new p node
while(sibling&&!(sibling.tagName&&BLOCK_LEVEL_TAGS_RE$2.test(sibling.tagName))){varnextSibling=sibling.nextSibling;$(sibling).appendTo(p);sibling=nextSibling;}$node.replaceWith(p);$node.remove();return$;}return$;}functionconvertDivs$1($){$('div').each(function(index,div){var$div=$(div);varconvertable=$div.children(DIV_TO_P_BLOCK_TAGS$2).length===0;if(convertable){convertNodeTo$$1($div,$,'p');}});return$;}functionconvertSpans$2($){$('span').each(function(index,span){var$span=$(span);varconvertable=$span.parents('p, div').length===0;if(convertable){convertNodeTo$$1($span,$,'p');}});return$;}// Loop through the provided doc, and convert any p-like elements to
// (By-reference mutation, though. Returned just for convenience.)
functionconvertToParagraphs$$1($){$=brsToPs$$1($);$=convertDivs$1($);$=convertSpans$2($);return$;}functionconvertNodeTo$$1($node,$){vartag=arguments.length>2&&arguments[2]!==undefined?arguments[2]:'p';varnode=$node.get(0);if(!node){return$;}varattrs=getAttrs$1(node)||{};varattribString=_Reflect$ownKeys$1(attrs).map(function(key){returnkey+'='+attrs[key];}).join(' ');varhtml=void0;if($.browser){// In the browser, the contents of noscript tags aren't rendered, therefore
// transforms on the noscript tag (commonly used for lazy-loading) don't work
// as expected. This test case handles that
html=node.tagName.toLowerCase()==='noscript'?$node.text():$node.html();}else{html=$node.contents();}$node.replaceWith('<'+tag+' '+attribString+'>'+html+'</'+tag+'>');return$;}functioncleanForHeight$1($img,$){varheight=parseInt($img.attr('height'),10);varwidth=parseInt($img.attr('width'),10)||20;// Remove images that explicitly have very small heights or
// widths, because they are most likely shims or icons,
// which aren't very useful for reading.
if((height||20)<10||width<10){$img.remove();}elseif(height){// Don't ever specify a height on images, so that we can
// scale with respect to width without screwing up the
// aspect ratio.
$img.removeAttr('height');}return$;}// Cleans out images where the source string matches transparent/spacer/etc
// TODO This seems very aggressive - AP
functionremoveSpacers$1($img,$){if(SPACER_RE$1.test($img.attr('src'))){$img.remove();}return$;}functioncleanImages$1($article,$){$article.find('img').each(function(index,img){var$img=$(img);cleanForHeight$1($img,$);removeSpacers$1($img,$);});return$;}functionmarkToKeep$1(article,$,url){vartags=arguments.length>3&&arguments[3]!==undefined?arguments[3]:[];if(tags.length===0){tags=KEEP_SELECTORS$1;}if(url){var_URL$parse=URL$1.parse(url),protocol=_URL$parse.protocol,hostname=_URL$parse.hostname;tags=[].concat(_toConsumableArray$1(tags),['iframe[src^="'+protocol+'//'+hostname+'"]']);}$(tags.join(','),article).addClass(KEEP_CLASS$1);return$;}functionstripJunkTags$1(article,$){vartags=arguments.length>2&&arguments[2]!==undefined?arguments[2]:[];if(tags.length===0){tags=STRIP_OUTPUT_TAGS$1;}// Remove matching elements, but ignore
// any element with a class of mercury-parser-keep
$(tags.join(','),article).not('.'+KEEP_CLASS$1).remove();// Remove the mercury-parser-keep class from result
$('.'+KEEP_CLASS$1,article).removeClass(KEEP_CLASS$1);return$;}// H1 tags are typically the article title, which should be extracted
// by the title extractor instead. If there's less than 3 of them (<3),
// strip them. Otherwise, turn 'em into H2s.
functioncleanHOnes$$1(article,$){var$hOnes=$('h1',article);if($hOnes.length<3){$hOnes.each(function(index,node){return$(node).remove();});}else{$hOnes.each(function(index,node){convertNodeTo$$1($(node),$,'h2');});}return$;}functionremoveAllButWhitelist$1($article){$article.find('*').each(function(index,node){varattrs=getAttrs$1(node);setAttrs$1(node,_Reflect$ownKeys$1(attrs).reduce(function(acc,attr){if(WHITELIST_ATTRS_RE$1.test(attr)){return_extends$1({},acc,_defineProperty$1({},attr,attrs[attr]));}returnacc;},{}));});return$article;}// function removeAttrs(article, $) {
// REMOVE_ATTRS.forEach((attr) => {
// $(`[${attr}]`, article).removeAttr(attr);
// });
// }
// Remove attributes like style or align
functioncleanAttributes$$1($article){// Grabbing the parent because at this point
// A list of strings that can be considered unlikely candidates when
// extracting content from a resource. These strings are joined together
// and then tested for existence using re:test, so may contain simple,
// non-pipe style regular expression queries if necessary.
varUNLIKELY_CANDIDATES_BLACKLIST$1$1=['ad-break','adbox','advert','addthis','agegate','aux','blogger-labels','combx','comment','conversation','disqus','entry-unrelated','extra','foot','form','header','hidden','loader','login',// Note: This can hit 'blogindex'.
'menu','meta','nav','pager','pagination','predicta',// readwriteweb inline ad box
'presence_control_external',// lifehacker.com container full of false positives
'popup','printfriendly','related','remove','remark','rss','share','shoutbox','sidebar','sociable','sponsor','tools'];// A list of strings that can be considered LIKELY candidates when
// extracting content from a resource. Essentially, the inverse of the
// blacklist above - if something matches both blacklist and whitelist,
// it is kept. This is useful, for example, if something has a className
// of "rss-content entry-content". It matched 'rss', so it would normally
// be removed, however, it's also the entry content, so it should be left
// These strings are joined together and then tested for existence using
// re:test, so may contain simple, non-pipe style regular expression queries
// if necessary.
varUNLIKELY_CANDIDATES_WHITELIST$1$1=['and','article','body','blogindex','column','content','entry-content-asset','format',// misuse of form
'hfeed','hentry','hatom','main','page','posts','shadow'];// A list of tags which, if found inside, should cause a <div /> to NOT
// be turned into a paragraph tag. Shallow div tags without these elements
// should be turned into <p /> tags.
varDIV_TO_P_BLOCK_TAGS$1$1=['a','blockquote','dl','div','img','p','pre','table'].join(',');// A list of tags that should be ignored when trying to find the top candidate
// for a document.
varNON_TOP_CANDIDATE_TAGS$1$1=['br','b','i','label','hr','area','base','basefont','input','img','link','meta'];varNON_TOP_CANDIDATE_TAGS_RE$1$1=newRegExp('^('+NON_TOP_CANDIDATE_TAGS$1$1.join('|')+')$','i');// A list of selectors that specify, very clearly, either hNews or other
// very content-specific style content, like Blogger templates.
// More examples here: http://microformats.org/wiki/blog-post-formats
varHNEWS_CONTENT_SELECTORS$1$1=[['.hentry','.entry-content'],['entry','.entry-content'],['.entry','.entry_content'],['.post','.postbody'],['.post','.post_body'],['.post','.post-body']];varPHOTO_HINTS$1$1=['figure','photo','image','caption'];varPHOTO_HINTS_RE$1$1=newRegExp(PHOTO_HINTS$1$1.join('|'),'i');// A list of strings that denote a positive scoring for this content as being
// an article container. Checked against className and id.
// TODO: Perhaps have these scale based on their odds of being quality?
varNEGATIVE_SCORE_HINTS$1$1=['adbox','advert','author','bio','bookmark','bottom','byline','clear','com-','combx','comment','comment\\B','contact','copy','credit','crumb','date','deck','excerpt','featured',// tnr.com has a featured_content which throws us off
'scroll','secondary','share','shopping','shoutbox','side','sidebar','sponsor','stamp','sub','summary','tags','tools','widget'];// The above list, joined into a matching regular expression
varNEGATIVE_SCORE_RE$1$1=newRegExp(NEGATIVE_SCORE_HINTS$1$1.join('|'),'i');// Match a digit. Pretty clear.
// Match 2 or more consecutive <br> tags
// Match 1 BR tag.
// A list of all of the block level tags known in HTML5 and below. Taken from
// http://bit.ly/qneNIT
// The removal is implemented as a blacklist and whitelist, this test finds
// blacklisted elements that aren't whitelisted. We do this all in one
// expression-both because it's only one pass, and because this skips the
// serialization for whitelisted nodes.
varcandidatesBlacklist$1$1=UNLIKELY_CANDIDATES_BLACKLIST$1$1.join('|');varcandidatesWhitelist$1$1=UNLIKELY_CANDIDATES_WHITELIST$1$1.join('|');varPARAGRAPH_SCORE_TAGS$1$1=newRegExp('^(p|li|span|pre)$','i');varCHILD_CONTENT_TAGS$1$1=newRegExp('^(td|blockquote|ol|ul|dl)$','i');varBAD_TAGS$1$1=newRegExp('^(address|form)$','i');// Get the score of a node based on its className and id.
functiongetWeight$1(node){varclasses=node.attr('class');varid=node.attr('id');varscore=0;if(id){// if id exists, try to score on both positive and negative
if(POSITIVE_SCORE_RE$1$1.test(id)){score+=25;}if(NEGATIVE_SCORE_RE$1$1.test(id)){score-=25;}}if(classes){if(score===0){// if classes exist and id did not contribute to score
// try to score on both positive and negative
if(POSITIVE_SCORE_RE$1$1.test(classes)){score+=25;}if(NEGATIVE_SCORE_RE$1$1.test(classes)){score-=25;}}// even if score has been set by id, add score for
// possible photo matches
// "try to keep photos if we can"
if(PHOTO_HINTS_RE$1$1.test(classes)){score+=10;}// add 25 if class matches entry-content-asset,
returnparseFloat($node.attr('score'))||null;}// return 1 for every comma in text
functionscoreCommas$1(text){return(text.match(/,/g)||[]).length;}varidkRe$1=newRegExp('^(p|pre)$','i');functionscoreLength$1(textLength){vartagName=arguments.length>1&&arguments[1]!==undefined?arguments[1]:'p';varchunks=textLength/50;if(chunks>0){varlengthBonus=void0;// No idea why p or pre are being tamped down here
// but just following the source for now
// Not even sure why tagName is included here,
// since this is only being called from the context
// of scoreParagraph
if(idkRe$1.test(tagName)){lengthBonus=chunks-2;}else{lengthBonus=chunks-1.25;}returnMath.min(Math.max(lengthBonus,0),3);}return0;}// Score a paragraph using various methods. Things like number of
// commas, etc. Higher is better.
functionscoreParagraph$$1(node){varscore=1;vartext=node.text().trim();vartextLength=text.length;// If this paragraph is less than 25 characters, don't count it.
if(textLength<25){return0;}// Add points for any commas within this paragraph
score+=scoreCommas$1(text);// For every 50 characters in this paragraph, add another point. Up
// to 3 points.
score+=scoreLength$1(textLength);// Articles can end with short paragraphs when people are being clever
// but they can also end with short paragraphs setting up lists of junk
// that we strip. This negative tweaks junk setup paragraphs just below
// the cutoff threshold.
if(text.slice(-1)===':'){score-=1;}returnscore;}functionsetScore$1($node,$,score){$node.attr('score',score);return$node;}functionaddScore$$1($node,$,amount){try{varscore=getOrInitScore$$1($node,$)+amount;setScore$1($node,$,score);}catch(e){// Ignoring; error occurs in scoreNode
}return$node;}// Adds 1/4 of a child's score to its parent
functionaddToParent$$1(node,$,score){varparent=node.parent();if(parent){addScore$$1(parent,$,score*0.25);}returnnode;}// gets and returns the score if it exists
// if not, initializes a score based on
// the node's tag type
functiongetOrInitScore$$1($node,$){varweightNodes=arguments.length>2&&arguments[2]!==undefined?arguments[2]:true;varscore=getScore$1($node);if(score){returnscore;}score=scoreNode$$1($node);if(weightNodes){score+=getWeight$1($node);}addToParent$$1($node,$,score);returnscore;}// Score an individual node. Has some smarts for paragraphs, otherwise
// just scores based on tag.
functionscoreNode$$1($node){var_$node$get=$node.get(0),tagName=_$node$get.tagName;// TODO: Consider ordering by most likely.
// E.g., if divs are a more common tag on a page,
// Could save doing that regex test on every node – AP
if(PARAGRAPH_SCORE_TAGS$1$1.test(tagName)){returnscoreParagraph$$1($node);}elseif(tagName.toLowerCase()==='div'){return5;}elseif(CHILD_CONTENT_TAGS$1$1.test(tagName)){return3;}elseif(BAD_TAGS$1$1.test(tagName)){return-3;}elseif(tagName.toLowerCase()==='th'){return-5;}return0;}functionconvertSpans$1$1($node,$){if($node.get(0)){var_$node$get=$node.get(0),tagName=_$node$get.tagName;if(tagName==='span'){// convert spans to divs
convertNodeTo$$1($node,$,'div');}}}functionaddScoreTo$1($node,$,score){if($node){convertSpans$1$1($node,$);addScore$$1($node,$,score);}}functionscorePs$1($,weightNodes){$('p, pre').not('[score]').each(function(index,node){// The raw score for this paragraph, before we add any parent/child
// scores.
var$node=$(node);$node=setScore$1($node,$,getOrInitScore$$1($node,$,weightNodes));var$parent=$node.parent();varrawScore=scoreNode$$1($node);addScoreTo$1($parent,$,rawScore,weightNodes);if($parent){// Add half of the individual content score to the
// grandparent
addScoreTo$1($parent.parent(),$,rawScore/2,weightNodes);}});return$;}// score content. Parents get the full value of their children's
// content score, grandparents half
functionscoreContent$$1($){varweightNodes=arguments.length>1&&arguments[1]!==undefined?arguments[1]:true;// First, look for special hNews based selectors and give them a big
// boost, if they exist
HNEWS_CONTENT_SELECTORS$1$1.forEach(function(_ref){var_ref2=_slicedToArray$1(_ref,2),parentSelector=_ref2[0],childSelector=_ref2[1];$(parentSelector+' '+childSelector).each(function(index,node){addScore$$1($(node).parent(parentSelector),$,80);});});// Doubling this again
// Previous solution caused a bug
// in which parents weren't retaining
// scores. This is not ideal, and
// should be fixed.
scorePs$1($,weightNodes);scorePs$1($,weightNodes);return$;}varNORMALIZE_RE$1=/\s{2,}/g;functionnormalizeSpaces$1(text){returntext.replace(NORMALIZE_RE$1,' ').trim();}// Given a node type to search for, and a list of regular expressions,
// look to see if this extraction can be found in the URL. Expects
// that each expression in r_list will return group(1) as the proper
returnpageNum<100?pageNum:null;}functionremoveAnchor$1(url){returnurl.split('#')[0].replace(/\/$/,'');}functionisGoodSegment$1(segment,index,firstSegmentHasLetters){vargoodSegment=true;// If this is purely a number, and it's the first or second
// url_segment, it's probably a page number. Remove it.
if(index<2&&IS_DIGIT_RE$1.test(segment)&&segment.length<3){goodSegment=true;}// If this is the first url_segment and it's just "index",
// remove it
if(index===0&&segment.toLowerCase()==='index'){goodSegment=false;}// If our first or second url_segment is smaller than 3 characters,
// and the first url_segment had no alphas, remove it.
if(index<2&&segment.length<3&&!firstSegmentHasLetters){goodSegment=false;}returngoodSegment;}// Take a URL, and return the article base of said URL. That is, no
// pagination data exists in it. Useful for comparing to other links
// that might have pagination data within them.
functionarticleBaseUrl$1(url,parsed){varparsedUrl=parsed||URL$1.parse(url);varprotocol=parsedUrl.protocol,host=parsedUrl.host,path=parsedUrl.path;varfirstSegmentHasLetters=false;varcleanedSegments=path.split('/').reverse().reduce(function(acc,rawSegment,index){varsegment=rawSegment;// Split off and save anything that looks like a file type.
if(segment.includes('.')){var_segment$split=segment.split('.'),_segment$split2=_slicedToArray$1(_segment$split,2),possibleSegment=_segment$split2[0],fileExt=_segment$split2[1];if(IS_ALPHA_RE$1.test(fileExt)){segment=possibleSegment;}}// If our first or second segment has anything looking like a page
// number, remove it.
if(PAGE_IN_HREF_RE$1.test(segment)&&index<2){segment=segment.replace(PAGE_IN_HREF_RE$1,'');}// If we're on the first segment, check to see if we have any
// characters in it. The first segment is actually the last bit of
// the URL, and this will be helpful to determine if we're on a URL
// segment that looks like "/2/" for example.
if(index===0){firstSegmentHasLetters=HAS_ALPHA_RE$1.test(segment);}// If it's not marked for deletion, push it to cleaned_segments.
if(isGoodSegment$1(segment,index,firstSegmentHasLetters)){acc.push(segment);}returnacc;},[]);returnprotocol+'//'+host+cleanedSegments.reverse().join('/');}// Given a string, return True if it appears to have an ending sentence
// within it, false otherwise.
varSENTENCE_END_RE$1=newRegExp('.( |$)');functionhasSentenceEnd$1(text){returnSENTENCE_END_RE$1.test(text);}functionexcerptContent$1(content){varwords=arguments.length>1&&arguments[1]!==undefined?arguments[1]:10;returncontent.trim().split(/\s+/).slice(0,words).join(' ');}// Now that we have a top_candidate, look through the siblings of
// it to see if any of them are decently scored. If they are, they
// may be split parts of the content (Like two divs, a preamble and
functionmergeSiblings$1($candidate,topScore,$){if(!$candidate.parent().length){return$candidate;}varsiblingScoreThreshold=Math.max(10,topScore*0.25);varwrappingDiv=$('<div></div>');$candidate.parent().children().each(function(index,sibling){var$sibling=$(sibling);// Ignore tags like BR, HR, etc
if(NON_TOP_CANDIDATE_TAGS_RE$1$1.test(sibling.tagName)){returnnull;}varsiblingScore=getScore$1($sibling);if(siblingScore){if($sibling.get(0)===$candidate.get(0)){wrappingDiv.append($sibling);}else{varcontentBonus=0;vardensity=linkDensity$1($sibling);// If sibling has a very low link density,
// give it a small bonus
if(density<0.05){contentBonus+=20;}// If sibling has a high link density,
// give it a penalty
if(density>=0.5){contentBonus-=20;}// If sibling node has the same class as
// candidate, give it a bonus
if($sibling.attr('class')===$candidate.attr('class')){contentBonus+=topScore*0.2;}varnewScore=siblingScore+contentBonus;if(newScore>=siblingScoreThreshold){returnwrappingDiv.append($sibling);}elseif(sibling.tagName==='p'){varsiblingContent=$sibling.text();varsiblingContentLength=textLength$1(siblingContent);if(siblingContentLength>80&&density<0.25){returnwrappingDiv.append($sibling);}elseif(siblingContentLength<=80&&density===0&&hasSentenceEnd$1(siblingContent)){returnwrappingDiv.append($sibling);}}}}returnnull;});if(wrappingDiv.children().length===1&&wrappingDiv.children().first().get(0)===$candidate.get(0)){return$candidate;}returnwrappingDiv;}// After we've calculated scores, loop through all of the possible
// candidate nodes we found and find the one with the highest score.
functionfindTopCandidate$$1($){var$candidate=void0;vartopScore=0;$('[score]').each(function(index,node){// Ignore tags like BR, HR, etc
if(NON_TOP_CANDIDATE_TAGS_RE$1$1.test(node.tagName)){return;}var$node=$(node);varscore=getScore$1($node);if(score>topScore){topScore=score;$candidate=$node;}});// If we don't have a candidate, return the body
functionremoveUnlessContent$1($node,$,weight){// Explicitly save entry-content-asset tags, which are
// noted as valuable in the Publisher guidelines. For now
// this works everywhere. We may want to consider making
// this less of a sure-thing later.
if($node.hasClass('entry-content-asset')){return;}varcontent=normalizeSpaces$1($node.text());if(scoreCommas$1(content)<10){varpCount=$('p',$node).length;varinputCount=$('input',$node).length;// Looks like a form, too many inputs.
if(inputCount>pCount/3){$node.remove();return;}varcontentLength=content.length;varimgCount=$('img',$node).length;// Content is too short, and there are no images, so
// this is probably junk content.
if(contentLength<25&&imgCount===0){$node.remove();return;}vardensity=linkDensity$1($node);// Too high of link density, is probably a menu or
// something similar.
// console.log(weight, density, contentLength)
if(weight<25&&density>0.2&&contentLength>75){$node.remove();return;}// Too high of a link density, despite the score being
// high.
if(weight>=25&&density>0.5){// Don't remove the node if it's a list and the
// previous sibling starts with a colon though. That
// means it's probably content.
vartagName=$node.get(0).tagName.toLowerCase();varnodeIsList=tagName==='ol'||tagName==='ul';if(nodeIsList){varpreviousNode=$node.prev();if(previousNode&&normalizeSpaces$1(previousNode.text()).slice(-1)===':'){return;}}$node.remove();return;}varscriptCount=$('script',$node).length;// Too many script tags, not enough content.
if(scriptCount>0&&contentLength<150){$node.remove();return;}}}// Given an article, clean it of some superfluous content specified by
functioncleanTags$$1($article,$){$(CLEAN_CONDITIONALLY_TAGS$1,$article).each(function(index,node){var$node=$(node);varweight=getScore$1($node);if(!weight){weight=getOrInitScore$$1($node,$);setScore$1($node,$,weight);}// drop node if its weight is < 0
if(weight<0){$node.remove();}else{// deteremine if node seems like content
removeUnlessContent$1($node,$,weight);}});return$;}functioncleanHeaders$1($article,$){vartitle=arguments.length>2&&arguments[2]!==undefined?arguments[2]:'';$(HEADER_TAG_LIST$1,$article).each(function(index,header){var$header=$(header);// Remove any headers that appear before all other p tags in the
// document. This probably means that it was part of the title, a
// subtitle or something else extraneous like a datestamp or byline,
// all of which should be handled by other metadata handling.
if($($header,$article).prevAll('p').length===0){return$header.remove();}// Remove any headers that match the title exactly.
if(normalizeSpaces$1($(header).text())===title){return$header.remove();}// If this header has a negative weight, it's probably junk.
// Get rid of it.
if(getWeight$1($(header))<0){return$header.remove();}return$header;});return$;}// Rewrite the tag name to div if it's a top level node like body or
// html to avoid later complications with multiple body tags.
functionrewriteTopLevel$$1(article,$){// I'm not using context here because
// it's problematic when converting the
// top-level/root node - AP
$=convertNodeTo$$1($('html'),$,'div');$=convertNodeTo$$1($('body'),$,'div');return$;}/* eslint-disable */functionabsolutize$1($,rootUrl,attr,$content){$('['+attr+']',$content).each(function(_,node){varattrs=getAttrs$1(node);varurl=attrs[attr];if(url){varabsoluteUrl=URL$1.resolve(rootUrl,url);setAttr$1(node,attr,absoluteUrl);}});}functionmakeLinksAbsolute$$1($content,$,url){['href','src'].forEach(function(attr){returnabsolutize$1($,url,attr,$content);});return$content;}functiontextLength$1(text){returntext.trim().replace(/\s+/g,' ').length;}// Determines what percentage of the text
// in a node is link text
// Takes a node, returns a float
functionlinkDensity$1($node){vartotalTextLength=textLength$1($node.text());varlinkText=$node.find('a').text();varlinkLength=textLength$1(linkText);if(totalTextLength>0){returnlinkLength/totalTextLength;}elseif(totalTextLength===0&&linkLength>0){return1;}return0;}// Given a node type to search for, and a list of meta tag names to
// search for, find a meta tag associated.
functionextractFromMeta$$1($,metaNames,cachedNames){varcleanTags$$1=arguments.length>3&&arguments[3]!==undefined?arguments[3]:true;varfoundNames=metaNames.filter(function(name){returncachedNames.indexOf(name)!==-1;});var_iteratorNormalCompletion=true;var_didIteratorError=false;var_iteratorError=undefined;try{var_loop=function_loop(){varname=_step.value;vartype='name';varvalue='value';varnodes=$('meta['+type+'="'+name+'"]');// Get the unique value of every matching node, in case there
// are two meta tags with the same name and value.
// Remove empty values.
varvalues=nodes.map(function(index,node){return$(node).attr(value);}).toArray().filter(function(text){returntext!=='';});// If we have more than one value for the same name, we have a
// conflict and can't trust any of them. Skip this name. If we have
// zero, that means our meta tags had no values. Skip this name
// also.
if(values.length===1){varmetaValue=void0;// Meta values that contain HTML should be stripped, as they
// weren't subject to cleaning previously.
if(cleanTags$$1){metaValue=stripTags$1(values[0],$);}else{metaValue=values[0];}return{v:metaValue};}};for(var_iterator=_getIterator$1(foundNames),_step;!(_iteratorNormalCompletion=(_step=_iterator.next()).done);_iteratorNormalCompletion=true){var_ret=_loop();if((typeof_ret==='undefined'?'undefined':_typeof$1(_ret))==="object")return_ret.v;}// If nothing is found, return null
}catch(err){_didIteratorError=true;_iteratorError=err;}finally{try{if(!_iteratorNormalCompletion&&_iterator.return){_iterator.return();}}finally{if(_didIteratorError){throw_iteratorError;}}}returnnull;}functionisGoodNode$1($node,maxChildren){// If it has a number of children, it's more likely a container
// element. Skip it.
if($node.children().length>maxChildren){returnfalse;}// If it looks to be within a comment, skip it.
if(withinComment$$1($node)){returnfalse;}returntrue;}// Given a a list of selectors find content that may
// be extractable from the document. This is for flat
// meta-information, like author, title, date published, etc.
functionextractFromSelectors$$1($,selectors){varmaxChildren=arguments.length>2&&arguments[2]!==undefined?arguments[2]:1;vartextOnly=arguments.length>3&&arguments[3]!==undefined?arguments[3]:true;var_iteratorNormalCompletion=true;var_didIteratorError=false;var_iteratorError=undefined;try{for(var_iterator=_getIterator$1(selectors),_step;!(_iteratorNormalCompletion=(_step=_iterator.next()).done);_iteratorNormalCompletion=true){varselector=_step.value;varnodes=$(selector);// If we didn't get exactly one of this selector, this may be
// a list of articles or comments. Skip it.
if(nodes.length===1){var$node=$(nodes[0]);if(isGoodNode$1($node,maxChildren)){varcontent=void0;if(textOnly){content=$node.text();}else{content=$node.html();}if(content){returncontent;}}}}}catch(err){_didIteratorError=true;_iteratorError=err;}finally{try{if(!_iteratorNormalCompletion&&_iterator.return){_iterator.return();}}finally{if(_didIteratorError){throw_iteratorError;}}}returnnull;}// strips all tags from a string of text
functionstripTags$1(text,$){// Wrapping text in html element prevents errors when text
// has no html
varcleanText=$('<span>'+text+'</span>').text();returncleanText===''?text:cleanText;}functionwithinComment$$1($node){varparents=$node.parents().toArray();varcommentParent=parents.find(function(parent){varattrs=getAttrs$1(parent);varnodeClass=attrs.class,id=attrs.id;varclassAndId=nodeClass+' '+id;returnclassAndId.includes('comment');});returncommentParent!==undefined;}// Given a node, determine if it's article-like enough to return
// param: node (a cheerio node)
// return: boolean
functionnodeIsSufficient$1($node){return$node.text().trim().length>=100;}functionisWordpress$1($){return$(IS_WP_SELECTOR$1).length>0;}functiongetAttrs$1(node){varattribs=node.attribs,attributes=node.attributes;if(!attribs&&attributes){varattrs=_Reflect$ownKeys$1(attributes).reduce(function(acc,index){varattr=attributes[index];acc[attr.name]=attr.value;returnacc;},{});returnattrs;}returnattribs;}functionsetAttr$1(node,attr,val){if(node.attribs){node.attribs[attr]=val;}elseif(node.attributes){node.setAttribute(attr,val);}returnnode;}/* eslint-disable */functionsetAttrs$1(node,attrs){if(node.attribs){node.attribs=attrs;}elseif(node.attributes){while(node.attributes.length>0){node.removeAttribute(node.attributes[0].name);}_Reflect$ownKeys$1(attrs).forEach(function(key){node.setAttribute(key,attrs[key]);});}returnnode;}// DOM manipulation
varIS_LINK=newRegExp('https?://','i');varIS_IMAGE=newRegExp('.(png|gif|jpe?g)','i');varTAGS_TO_REMOVE=['script','style','form'].join(',');// Convert all instances of images with potentially
// lazy loaded images into normal images.
// Many sites will have img tags with no source, or an image tag with a src
// attribute that a is a placeholer. We need to be able to properly fill in
// the src attribute so the images are no longer lazy loaded.
functionconvertLazyLoadedImages($){$('img').each(function(_,img){varattrs=getAttrs$1(img);_Reflect$ownKeys$1(attrs).forEach(function(attr){varvalue=attrs[attr];if(attr!=='src'&&IS_LINK.test(value)&&IS_IMAGE.test(value)){$(img).attr('src',value);}});});return$;}functionisComment(index,node){returnnode.type==='comment';}functioncleanComments($){$('*').first().contents().filter(isComment).remove();return$;}functionclean($){$(TAGS_TO_REMOVE).remove();$=cleanComments($);return$;}varResource={// Create a Resource.
// :param url: The URL for the document we should retrieve.
// :param response: If set, use as the response rather than
// attempting to fetch it ourselves. Expects a
// string.
create:functioncreate(url,preparedResponse,parsedUrl){var_this=this;return_asyncToGenerator(_regeneratorRuntime.mark(function_callee(){varresult,validResponse;return_regeneratorRuntime.wrap(function_callee$(_context){while(1){switch(_context.prev=_context.next){case0:result=void0;if(!preparedResponse){_context.next=6;break;}validResponse={statusMessage:'OK',statusCode:200,headers:{'content-type':'text/html','content-length':500}};result={body:preparedResponse,response:validResponse};_context.next=9;break;case6:_context.next=8;returnfetchResource$1(url,parsedUrl);case8:result=_context.sent;case9:if(!result.error){_context.next=11;break;}return_context.abrupt('return',result);case11:return_context.abrupt('return',_this.generateDoc(result));case12:case'end':return_context.stop();}}},_callee,_this);}))();},generateDoc:functiongenerateDoc(_ref){varcontent=_ref.body,response=_ref.response;varcontentType=response.headers['content-type'];// TODO: Implement is_text function from
if(!contentType.includes('html')&&!contentType.includes('text')){thrownewError('Content does not appear to be text.');}var$=cheerio$1.load(content,{normalizeWhitespace:true});if($('*').first().children().length===0){thrownewError('No children, likely a bad parse.');}$=normalizeMetaTags($);$=convertLazyLoadedImages($);$=clean($);return$;}};varmerge=functionmerge(extractor,domains){returndomains.reduce(function(acc,domain){acc[domain]=extractor;returnacc;},{});};functionmergeSupportedDomains(extractor){returnextractor.supportedDomains?merge(extractor,[extractor.domain].concat(_toConsumableArray$1(extractor.supportedDomains))):merge(extractor,[extractor.domain]);}varBloggerExtractor={domain:'blogspot.com',content:{// Blogger is insane and does not load its content
// initially in the page, but it's all there
// in noscript
selectors:['.post-content noscript'],// Selectors to remove from the extracted content
clean:[],// Convert the noscript tag to a div
transforms:{noscript:'div'}},author:{selectors:['.post-author-name']},title:{selectors:['.post h2.title']},date_published:{selectors:['span.publishdate']}};varNYMagExtractor={domain:'nymag.com',content:{// Order by most likely. Extractor will stop on first occurrence
selectors:['div.article-content','section.body','article.article'],// Selectors to remove from the extracted content
clean:['.ad','.single-related-story'],// Object of tranformations to make on matched elements
// Each key is the selector, each value is the tag to
// transform to.
// If a function is given, it should return a string
// to convert to or nothing (in which case it will not perform
// the transformation.
transforms:{// Convert h1s to h2s
h1:'h2',// Convert lazy-loaded noscript images to figures
noscript:functionnoscript($node,$){if($.browser){var$children=$($node.text());if($children.length===1&&$children.get(0)!==undefined&&$children.get(0).tagName.toLowerCase()==='img'){return'figure';}}else{var_$children=$node.children();if(_$children.length===1&&_$children.get(0).tagName==='img'){return'figure';}}returnnull;}}},title:{selectors:['h1.lede-feature-title','h1.headline-primary','h1']},author:{selectors:['.by-authors','.lede-feature-author']},dek:{selectors:['.lede-feature-teaser']},date_published:{selectors:[['time.article-timestamp[datetime]','datetime'],'time.article-timestamp']}};varWikipediaExtractor={domain:'wikipedia.org',content:{selectors:['#mw-content-text'],defaultCleaner:false,// transform top infobox to an image with caption
transforms:{'.infobox img':functioninfoboxImg($node){var$parent=$node.parents('.infobox');// Only prepend the first image in .infobox
if($parent.children('img').length===0){$parent.prepend($node);}},'.infobox caption':'figcaption','.infobox':'figure'},// Selectors to remove from the extracted content
// Twitter doesn't have nice selectors, so our initial
// selector grabs the whole page, then we're re-writing
// it to fit our needs before we clean it up.
'.permalink[role=main]':functionpermalinkRoleMain($node,$){vartweets=$node.find('.tweet');var$tweetContainer=$('<div id="TWEETS_GO_HERE"></div>');$tweetContainer.append(tweets);$node.replaceWith($tweetContainer);},// Twitter wraps @ with s, which
varTheAtlanticExtractor={domain:'www.theatlantic.com',title:{selectors:['h1.hed']},author:{selectors:['article#article .article-cover-extra .metadata .byline a']},content:{selectors:['.article-body'],// Is there anything in the content you selected that needs transformed
// before it's consumable content? E.g., unusual lazy loaded images
transforms:[],// Is there anything that is in the result that shouldn't be?
// The clean selectors will remove anything that matches from
varNewYorkerExtractor={domain:'www.newyorker.com',title:{selectors:['h1.title']},author:{selectors:['.contributors']},content:{selectors:['div#articleBody','div.articleBody'],// Is there anything in the content you selected that needs transformed
// before it's consumable content? E.g., unusual lazy loaded images
transforms:[],// Is there anything that is in the result that shouldn't be?
// The clean selectors will remove anything that matches from
varWiredExtractor={domain:'www.wired.com',title:{selectors:['h1.post-title']},author:{selectors:['a[rel="author"]']},content:{selectors:['article.content'],// Is there anything in the content you selected that needs transformed
// before it's consumable content? E.g., unusual lazy loaded images
transforms:[],// Is there anything that is in the result that shouldn't be?
// The clean selectors will remove anything that matches from
varMSNExtractor={domain:'www.msn.com',title:{selectors:['h1']},author:{selectors:['span.authorname-txt']},content:{selectors:['div.richtext'],// Is there anything in the content you selected that needs transformed
// before it's consumable content? E.g., unusual lazy loaded images
transforms:[],// Is there anything that is in the result that shouldn't be?
// The clean selectors will remove anything that matches from
varYahooExtractor={domain:'www.yahoo.com',title:{selectors:['header.canvas-header']},author:{selectors:['span.provider-name']},content:{selectors:[// enter content selectors
'.content-canvas'],// Is there anything in the content you selected that needs transformed
// before it's consumable content? E.g., unusual lazy loaded images
transforms:[],// Is there anything that is in the result that shouldn't be?
// The clean selectors will remove anything that matches from
varBuzzfeedExtractor={domain:'www.buzzfeed.com',title:{selectors:['h1[id="post-title"]']},author:{selectors:['a[data-action="user/username"]','byline__author']},content:{selectors:['#buzz_sub_buzz'],defaultCleaner:false,// Is there anything in the content you selected that needs transformed
// before it's consumable content? E.g., unusual lazy loaded images
transforms:{h2:'b'},// Is there anything that is in the result that shouldn't be?
// The clean selectors will remove anything that matches from
varWikiaExtractor={domain:'fandom.wikia.com',title:{selectors:['h1.entry-title']},author:{selectors:['.author vcard','.fn']},content:{selectors:['.grid-content','.entry-content'],// Is there anything in the content you selected that needs transformed
// before it's consumable content? E.g., unusual lazy loaded images
transforms:[],// Is there anything that is in the result that shouldn't be?
// The clean selectors will remove anything that matches from
varLittleThingsExtractor={domain:'www.littlethings.com',title:{selectors:['h1.post-title']},author:{selectors:[['meta[name="author"]','value']]},content:{selectors:[// enter content selectors
'.mainContentIntro','.content-wrapper'],// Is there anything in the content you selected that needs transformed
// before it's consumable content? E.g., unusual lazy loaded images
transforms:[],// Is there anything that is in the result that shouldn't be?
// The clean selectors will remove anything that matches from
varPoliticoExtractor={domain:'www.politico.com',title:{selectors:[// enter title selectors
['meta[name="og:title"]','value']]},author:{selectors:['.story-main-content .byline .vcard']},content:{selectors:[// enter content selectors
'.story-main-content','.content-group','.story-core','.story-text'],// Is there anything in the content you selected that needs transformed
// before it's consumable content? E.g., unusual lazy loaded images
transforms:[],// Is there anything that is in the result that shouldn't be?
// The clean selectors will remove anything that matches from
// the result
clean:['figcaption']},date_published:{selectors:[['.story-main-content .timestamp time[datetime]','datetime']]},lead_image_url:{selectors:[// enter lead_image_url selectors
['meta[name="og:image"]','value']]},dek:{selectors:[['meta[name="description"]','value']]},next_page_url:null,excerpt:null};varDeadspinExtractor={domain:'deadspin.com',supportedDomains:['jezebel.com','lifehacker.com','kotaku.com','gizmodo.com','jalopnik.com','kinja.com'],title:{selectors:['h1.headline']},author:{selectors:['.author']},content:{selectors:['.post-content','.entry-content'],// Is there anything in the content you selected that needs transformed
// before it's consumable content? E.g., unusual lazy loaded images
transforms:{'iframe.lazyload[data-recommend-id^="youtube://"]':functioniframeLazyloadDataRecommendIdYoutube($node){varyoutubeId=$node.attr('id').split('youtube-')[1];$node.attr('src','https://www.youtube.com/embed/'+youtubeId);}},// Is there anything that is in the result that shouldn't be?
// The clean selectors will remove anything that matches from
// the result
clean:[]},date_published:{selectors:[['time.updated[datetime]','datetime']]},lead_image_url:{selectors:[['meta[name="og:image"]','value']]},dek:{selectors:[// enter selectors
]},next_page_url:{selectors:[// enter selectors
]},excerpt:{selectors:[// enter selectors
]}};// Rename CustomExtractor
// to fit your publication
// (e.g., NYTimesExtractor)
varBroadwayWorldExtractor={domain:'www.broadwayworld.com',title:{selectors:['h1.article-title']},author:{selectors:['span[itemprop=author]']},content:{selectors:['div[itemprop=articlebody]'],// Is there anything in the content you selected that needs transformed
// before it's consumable content? E.g., unusual lazy loaded images
transforms:{},// Is there anything that is in the result that shouldn't be?
// The clean selectors will remove anything that matches from
// the result
clean:[]},date_published:{selectors:[['meta[itemprop=datePublished]','value']]},lead_image_url:{selectors:[['meta[name="og:image"]','value']]},dek:{selectors:[['meta[name="og:description"]','value']]},next_page_url:{selectors:[// enter selectors
]},excerpt:{selectors:[// enter selectors
]}};// Rename CustomExtractor
// to fit your publication
// (e.g., NYTimesExtractor)
varApartmentTherapyExtractor={domain:'www.apartmenttherapy.com',title:{selectors:['h1.headline']},author:{selectors:['.PostByline__name']},content:{selectors:['div.post__content'],// Is there anything in the content you selected that needs transformed
// before it's consumable content? E.g., unusual lazy loaded images
transforms:{'div[data-render-react-id="images/LazyPicture"]':functiondivDataRenderReactIdImagesLazyPicture($node,$){vardata=JSON.parse($node.attr('data-props'));varsrc=data.sources[0].src;var$img=$('<img />').attr('src',src);$node.replaceWith($img);}},// Is there anything that is in the result that shouldn't be?
// The clean selectors will remove anything that matches from
// the result
clean:[]},date_published:{selectors:[['.PostByline__timestamp[datetime]','datetime']]},lead_image_url:{selectors:[['meta[name="og:image"]','value']]},dek:{selectors:[['meta[name=description]','value']]},next_page_url:{selectors:[// enter selectors
]},excerpt:{selectors:[// enter selectors
]}};varMediumExtractor={domain:'medium.com',supportedDomains:['trackchanges.postlight.com'],title:{selectors:['h1']},author:{selectors:[['meta[name="author"]','value']]},content:{selectors:['.section-content'],// Is there anything in the content you selected that needs transformed
// before it's consumable content? E.g., unusual lazy loaded images
$node.attr('src','https://www.youtube.com/embed/'+youtubeId);var$parent=$node.parents('figure');$parent.prepend($node.clone());$node.remove();}}},// Is there anything that is in the result that shouldn't be?
// The clean selectors will remove anything that matches from
// the result
clean:[]},date_published:{selectors:[['time[datetime]','datetime']]},lead_image_url:{selectors:[['meta[name="og:image"]','value']]},dek:{selectors:[// enter selectors
// Should be more restrictive than not, as a failed dek can be pretty
// detrimental to the aesthetics of an article.
// CLEAN DATE PUBLISHED CONSTANTS
varMS_DATE_STRING=/^\d{13}$/i;varSEC_DATE_STRING=/^\d{10}$/i;varCLEAN_DATE_STRING_RE=/^\s*published\s*:?\s*(.*)/i;varTIME_MERIDIAN_SPACE_RE=/(.*\d)(am|pm)(.*)/i;varTIME_MERIDIAN_DOTS_RE=/\.m\./i;varmonths=['jan','feb','mar','apr','may','jun','jul','aug','sep','oct','nov','dec'];varallMonths=months.join('|');vartimestamp1='[0-9]{1,2}:[0-9]{2,2}( ?[ap].?m.?)?';vartimestamp2='[0-9]{1,2}[/-][0-9]{1,2}[/-][0-9]{2,4}';varSPLIT_DATE_STRING=newRegExp('('+timestamp1+')|('+timestamp2+')|([0-9]{1,4})|('+allMonths+')','ig');// CLEAN TITLE CONSTANTS
// A regular expression that will match separating characters on a
// title, that usually denote breadcrumbs or something similar.
varTITLE_SPLITTERS_RE=/(: | - | \| )/g;varDOMAIN_ENDINGS_RE=newRegExp('.com$|.net$|.org$|.co.uk$','g');// Take an author string (like 'By David Smith ') and clean it to
// just the name(s): 'David Smith'.
functioncleanAuthor(author){returnauthor.replace(CLEAN_AUTHOR_RE,'$2').trim();}functionclean$1(leadImageUrl){leadImageUrl=leadImageUrl.trim();if(validUrl$1.isWebUri(leadImageUrl)){returnleadImageUrl;}returnnull;}// Take a dek HTML fragment, and return the cleaned version of it.
// Return None if the dek wasn't good enough.
functioncleanDek(dek,_ref){var$=_ref.$,excerpt=_ref.excerpt;// Sanity check that we didn't get too short or long of a dek.
if(dek.length>1000||dek.length<5)returnnull;// Check that dek isn't the same as excerpt
if(excerpt&&excerptContent$1(excerpt,10)===excerptContent$1(dek,10))returnnull;vardekText=stripTags$1(dek,$);// Plain text links shouldn't exist in the dek. If we have some, it's
// not a good dek - bail.
if(TEXT_LINK_RE.test(dekText))returnnull;returndekText.trim();}// Is there a compelling reason to use moment here?
// Mostly only being used for the isValid() method,
// but could just check for 'Invalid Date' string.
functioncleanDateString(dateString){return(dateString.match(SPLIT_DATE_STRING)||[]).join(' ').replace(TIME_MERIDIAN_DOTS_RE,'m').replace(TIME_MERIDIAN_SPACE_RE,'$1 $2 $3').replace(CLEAN_DATE_STRING_RE,'$1').trim();}// Take a date published string, and hopefully return a date out of
// it. Return none if we fail.
functioncleanDatePublished(dateString){// If string is in milliseconds or seconds, convert to int
if(MS_DATE_STRING.test(dateString)||SEC_DATE_STRING.test(dateString)){dateString=parseInt(dateString,10);}vardate=moment$1(newDate(dateString));if(!date.isValid()){dateString=cleanDateString(dateString);date=moment$1(newDate(dateString));}returndate.isValid()?date.toISOString():null;}// Clean our article content, returning a new, cleaned node.
functionextractCleanNode(article,_ref){var$=_ref.$,_ref$cleanConditional=_ref.cleanConditionally,cleanConditionally=_ref$cleanConditional===undefined?true:_ref$cleanConditional,_ref$title=_ref.title,title=_ref$title===undefined?'':_ref$title,_ref$url=_ref.url,url=_ref$url===undefined?'':_ref$url,_ref$defaultCleaner=_ref.defaultCleaner,defaultCleaner=_ref$defaultCleaner===undefined?true:_ref$defaultCleaner;// Rewrite the tag name to div if it's a top level node like body or
// html to avoid later complications with multiple body tags.
rewriteTopLevel$$1(article,$);// Drop small images and spacer images
// Only do this is defaultCleaner is set to true;
// this can sometimes be too aggressive.
if(defaultCleaner)cleanImages$1(article,$);// Mark elements to keep that would normally be removed.
// E.g., stripJunkTags will remove iframes, so we're going to mark
// YouTube/Vimeo videos as elements we want to keep.
markToKeep$1(article,$,url);// Drop certain tags like <title>, etc
// This is -mostly- for cleanliness, not security.
stripJunkTags$1(article,$);// H1 tags are typically the article title, which should be extracted
// by the title extractor instead. If there's less than 3 of them (<3),
// strip them. Otherwise, turn 'em into H2s.
cleanHOnes$$1(article,$);// Clean headers
cleanHeaders$1(article,$,title);// Make links absolute
makeLinksAbsolute$$1(article,$,url);// We used to clean UL's and OL's here, but it was leading to
// too many in-article lists being removed. Consider a better
// way to detect menus particularly and remove them.
// Also optionally running, since it can be overly aggressive.
cleanAttributes$$1(article,$);returnarticle;}functioncleanTitle$$1(title,_ref){varurl=_ref.url,$=_ref.$;// If title has |, :, or - in it, see if
// we can clean it up.
if(TITLE_SPLITTERS_RE.test(title)){title=resolveSplitTitle(title,url);}// Final sanity check that we didn't get a crazy title.
// if (title.length > 150 || title.length < 15) {
if(title.length>150){// If we did, return h1 from the document if it exists
varh1=$('h1');if(h1.length===1){title=h1.text();}}// strip any html tags in the title text
returnstripTags$1(title,$).trim();}functionextractBreadcrumbTitle(splitTitle,text){// This must be a very breadcrumbed title, like:
// The Best Gadgets on Earth : Bits : Blogs : NYTimes.com
// NYTimes - Blogs - Bits - The Best Gadgets on Earth
if(splitTitle.length>=6){var_ret=function(){// Look to see if we can find a breadcrumb splitter that happens
// more than once. If we can, we'll be able to better pull out
// the title.
vartermCounts=splitTitle.reduce(function(acc,titleText){acc[titleText]=acc[titleText]?acc[titleText]+1:1;returnacc;},{});var_Reflect$ownKeys$redu=_Reflect$ownKeys$1(termCounts).reduce(function(acc,key){if(acc[1]<termCounts[key]){return[key,termCounts[key]];}returnacc;},[0,0]),_Reflect$ownKeys$redu2=_slicedToArray$1(_Reflect$ownKeys$redu,2),maxTerm=_Reflect$ownKeys$redu2[0],termCount=_Reflect$ownKeys$redu2[1];// We found a splitter that was used more than once, so it
// is probably the breadcrumber. Split our title on that instead.
// Note: max_term should be <= 4 characters, so that " >> "
// will match, but nothing longer than that.
if(termCount>=2&&maxTerm.length<=4){splitTitle=text.split(maxTerm);}varsplitEnds=[splitTitle[0],splitTitle.slice(-1)];varlongestEnd=splitEnds.reduce(function(acc,end){returnacc.length>end.length?acc:end;},'');if(longestEnd.length>10){return{v:longestEnd};}return{v:text};}();if((typeof_ret==='undefined'?'undefined':_typeof$1(_ret))==="object")return_ret.v;}returnnull;}functioncleanDomainFromTitle(splitTitle,url){// Search the ends of the title, looking for bits that fuzzy match
// the URL too closely. If one is found, discard it and return the
// Strip out the big TLDs - it just makes the matching a bit more
// accurate. Not the end of the world if it doesn't strip right.
var_URL$parse=URL$1.parse(url),host=_URL$parse.host;varnakedDomain=host.replace(DOMAIN_ENDINGS_RE,'');varstartSlug=splitTitle[0].toLowerCase().replace(' ','');varstartSlugRatio=wuzzy$1.levenshtein(startSlug,nakedDomain);if(startSlugRatio>0.4&&startSlug.length>5){returnsplitTitle.slice(2).join('');}varendSlug=splitTitle.slice(-1)[0].toLowerCase().replace(' ','');varendSlugRatio=wuzzy$1.levenshtein(endSlug,nakedDomain);if(endSlugRatio>0.4&&endSlug.length>=5){returnsplitTitle.slice(0,-2).join('');}returnnull;}// Given a title with separators in it (colons, dashes, etc),
// resolve whether any of the segments should be removed.
functionresolveSplitTitle(title){varurl=arguments.length>1&&arguments[1]!==undefined?arguments[1]:'';// Splits while preserving splitters, like:
// ['The New New York', ' - ', 'The Washington Post']
varsplitTitle=title.split(TITLE_SPLITTERS_RE);if(splitTitle.length===1){returntitle;}varnewTitle=extractBreadcrumbTitle(splitTitle,title);if(newTitle)returnnewTitle;newTitle=cleanDomainFromTitle(splitTitle,url);if(newTitle)returnnewTitle;// Fuzzy ratio didn't find anything, so this title is probably legit.
// Just return it all.
returntitle;}varCleaners={author:cleanAuthor,lead_image_url:clean$1,dek:cleanDek,date_published:cleanDatePublished,content:extractCleanNode,title:cleanTitle$$1};// Using a variety of scoring techniques, extract the content most
functionextractBestNode($,opts){// clone the node so we can get back to our
// initial parsed state if needed
// TODO Do I need this? – AP
// let $root = $.root().clone()
if(opts.stripUnlikelyCandidates){$=stripUnlikelyCandidates$1($);}$=convertToParagraphs$$1($);$=scoreContent$$1($,opts.weightNodes);var$topCandidate=findTopCandidate$$1($);return$topCandidate;}varGenericContentExtractor={defaultOpts:{stripUnlikelyCandidates:true,weightNodes:true,cleanConditionally:true},// Extract the content for this resource - initially, pass in our
// most restrictive opts which will return the highest quality
// content. On each failure, retry with slightly more lax opts.
// cleanConditionally: Clean the node to return of some
// superfluous content. Things like forms, ads, etc.
extract:functionextract(_ref,opts){var$=_ref.$,html=_ref.html,title=_ref.title,url=_ref.url,cheerio$$1=_ref.cheerio;opts=_extends$1({},this.defaultOpts,opts);$=$||cheerio$$1.load(html);// Cascade through our extraction-specific opts in an ordered fashion,
// turning them off as we try to extract content.
varnode=this.getContentNode($,title,url,opts);if(nodeIsSufficient$1(node)){returnthis.cleanAndReturnNode(node,$);}// We didn't succeed on first pass, one by one disable our
// extraction opts and try again.
var_iteratorNormalCompletion=true;var_didIteratorError=false;var_iteratorError=undefined;try{for(var_iterator=_getIterator$1(_Reflect$ownKeys$1(opts).filter(function(k){returnopts[k]===true;})),_step;!(_iteratorNormalCompletion=(_step=_iterator.next()).done);_iteratorNormalCompletion=true){varkey=_step.value;opts[key]=false;$=cheerio$$1.load(html);node=this.getContentNode($,title,url,opts);if(nodeIsSufficient$1(node)){break;}}}catch(err){_didIteratorError=true;_iteratorError=err;}finally{try{if(!_iteratorNormalCompletion&&_iterator.return){_iterator.return();}}finally{if(_didIteratorError){throw_iteratorError;}}}returnthis.cleanAndReturnNode(node,$);},// Get node given current options
getContentNode:functiongetContentNode($,title,url,opts){returnextractCleanNode(extractBestNode($,opts),{$:$,cleanConditionally:opts.cleanConditionally,title:title,url:url});},// Once we got here, either we're at our last-resort node, or
// we broke early. Make sure we at least have -something- before we
// move forward.
cleanAndReturnNode:functioncleanAndReturnNode(node,$){if(!node){returnnull;}returnnormalizeSpaces$1($.html(node));// if return_type == "html":
// return normalize_spaces(node_to_html(node))
// else:
// return node
}};// TODO: It would be great if we could merge the meta and selector lists into
// a list of objects, because we could then rank them better. For example,
// .hentry .entry-title is far better suited than <meta title>.
// An ordered list of meta tag names that denote likely article titles. All
// attributes should be lowercase for faster case-insensitive matching. From
// most distinct to least distinct.
varSTRONG_TITLE_META_TAGS=['tweetmeme-title','dc.title','rbtitle','headline','title'];// og:title is weak because it typically contains context that we don't like,
// for example the source site's name. Gotta get that brand into facebook!
varWEAK_TITLE_META_TAGS=['og:title'];// An ordered list of XPath Selectors to find likely article titles. From
// Note - this does not use classes like CSS. This checks to see if the string
// exists in the className, which is not as accurate as .className (which
// splits on spaces/endlines), but for our purposes it's close enough. The
// speed tradeoff is worth the accuracy hit.
varSTRONG_TITLE_SELECTORS=['.hentry .entry-title','h1#articleHeader','h1.articleHeader','h1.article','.instapaper_title','#meebo-title'];varWEAK_TITLE_SELECTORS=['article h1','#entry-title','.entry-title','#entryTitle','#entrytitle','.entryTitle','.entrytitle','#articleTitle','.articleTitle','post post-title','h1.title','h2.article','h1','html head title','title'];varGenericTitleExtractor={extract:functionextract(_ref){var$=_ref.$,url=_ref.url,metaCache=_ref.metaCache;// First, check to see if we have a matching meta tag that we can make
// use of that is strongly associated with the headline.
vartitle=void0;title=extractFromMeta$$1($,STRONG_TITLE_META_TAGS,metaCache);if(title)returncleanTitle$$1(title,{url:url,$:$});// Second, look through our content selectors for the most likely
// article title that is strongly associated with the headline.
title=extractFromSelectors$$1($,STRONG_TITLE_SELECTORS);if(title)returncleanTitle$$1(title,{url:url,$:$});// Third, check for weaker meta tags that may match.
title=extractFromMeta$$1($,WEAK_TITLE_META_TAGS,metaCache);if(title)returncleanTitle$$1(title,{url:url,$:$});// Last, look for weaker selector tags that may match.
title=extractFromSelectors$$1($,WEAK_TITLE_SELECTORS);if(title)returncleanTitle$$1(title,{url:url,$:$});// If no matches, return an empty string
return'';}};// An ordered list of meta tag names that denote likely article authors. All
// attributes should be lowercase for faster case-insensitive matching. From
// Note: "author" is too often the -developer- of the page, so it is not
// added here.
varAUTHOR_META_TAGS=['byl','clmst','dc.author','dcsext.author','dc.creator','rbauthors','authors'];varAUTHOR_MAX_LENGTH=300;// An ordered list of XPath Selectors to find likely article authors. From
// Note - this does not use classes like CSS. This checks to see if the string
// exists in the className, which is not as accurate as .className (which
// splits on spaces/endlines), but for our purposes it's close enough. The
// speed tradeoff is worth the accuracy hit.
varAUTHOR_SELECTORS=['.entry .entry-author','.author.vcard .fn','.author .vcard .fn','.byline.vcard .fn','.byline .vcard .fn','.byline .by .author','.byline .by','.byline .author','.post-author.vcard','.post-author .vcard','a[rel=author]','#by_author','.by_author','#entryAuthor','.entryAuthor','.byline a[href*=author]','#author .authorname','.author .authorname','#author','.author','.articleauthor','.ArticleAuthor','.byline'];// An ordered list of Selectors to find likely article authors, with
// regular expression for content.
varbylineRe=/^[\n\s]*By/i;varBYLINE_SELECTORS_RE=[['#byline',bylineRe],['.byline',bylineRe]];varGenericAuthorExtractor={extract:functionextract(_ref){var$=_ref.$,metaCache=_ref.metaCache;varauthor=void0;// First, check to see if we have a matching
// meta tag that we can make use of.
author=extractFromMeta$$1($,AUTHOR_META_TAGS,metaCache);if(author&&author.length<AUTHOR_MAX_LENGTH){returncleanAuthor(author);}// Second, look through our selectors looking for potential authors.
author=extractFromSelectors$$1($,AUTHOR_SELECTORS,2);if(author&&author.length<AUTHOR_MAX_LENGTH){returncleanAuthor(author);}// Last, use our looser regular-expression based selectors for
// potential authors.
var_iteratorNormalCompletion=true;var_didIteratorError=false;var_iteratorError=undefined;try{for(var_iterator=_getIterator$1(BYLINE_SELECTORS_RE),_step;!(_iteratorNormalCompletion=(_step=_iterator.next()).done);_iteratorNormalCompletion=true){var_ref4=_step.value;var_ref3=_slicedToArray$1(_ref4,2);varselector=_ref3[0];varregex=_ref3[1];varnode=$(selector);if(node.length===1){vartext=node.text();if(regex.test(text)){returncleanAuthor(text);}}}}catch(err){_didIteratorError=true;_iteratorError=err;}finally{try{if(!_iteratorNormalCompletion&&_iterator.return){_iterator.return();}}finally{if(_didIteratorError){throw_iteratorError;}}}returnnull;}};// An ordered list of meta tag names that denote
// likely date published dates. All attributes
// should be lowercase for faster case-insensitive matching.
// From most distinct to least distinct.
varDATE_PUBLISHED_META_TAGS=['article:published_time','displaydate','dc.date','dc.date.issued','rbpubdate','publish_date','pub_date','pagedate','pubdate','revision_date','doc_date','date_created','content_create_date','lastmodified','created','date'];// An ordered list of XPath Selectors to find
// likely date published dates. From most explicit
// to least explicit.
varDATE_PUBLISHED_SELECTORS=['.hentry .dtstamp.published','.hentry .published','.hentry .dtstamp.updated','.hentry .updated','.single .published','.meta .published','.meta .postDate','.entry-date','.byline .date','.postmetadata .date','.article_datetime','.date-header','.story-date','.dateStamp','#story .datetime','.dateline','.pubdate'];// An ordered list of compiled regular expressions to find likely date
// published dates from the URL. These should always have the first
// reference be a date string that is parseable by dateutil.parser.parse
varabbrevMonthsStr='(jan|feb|mar|apr|may|jun|jul|aug|sep|oct|nov|dec)';varDATE_PUBLISHED_URL_RES=[// /2012/01/27/ but not /2012/01/293
newRegExp('/(20\\d{2}/\\d{2}/\\d{2})/','i'),// 20120127 or 20120127T but not 2012012733 or 8201201733
newRegExp('/(20\\d{2}/'+abbrevMonthsStr+'/[0-3]\\d)/','i')];varGenericDatePublishedExtractor={extract:functionextract(_ref){var$=_ref.$,url=_ref.url,metaCache=_ref.metaCache;vardatePublished=void0;// First, check to see if we have a matching meta tag
// that we can make use of.
// Don't try cleaning tags from this string
datePublished=extractFromMeta$$1($,DATE_PUBLISHED_META_TAGS,metaCache,false);if(datePublished)returncleanDatePublished(datePublished);// Second, look through our selectors looking for potential
// date_published's.
datePublished=extractFromSelectors$$1($,DATE_PUBLISHED_SELECTORS);if(datePublished)returncleanDatePublished(datePublished);// Lastly, look to see if a dately string exists in the URL
// An ordered list of meta tag names that denote likely article leading images.
// All attributes should be lowercase for faster case-insensitive matching.
// From most distinct to least distinct.
varLEAD_IMAGE_URL_META_TAGS=['og:image','twitter:image','image_src'];varLEAD_IMAGE_URL_SELECTORS=['link[rel=image_src]'];varPOSITIVE_LEAD_IMAGE_URL_HINTS=['upload','wp-content','large','photo','wp-image'];varPOSITIVE_LEAD_IMAGE_URL_HINTS_RE=newRegExp(POSITIVE_LEAD_IMAGE_URL_HINTS.join('|'),'i');varNEGATIVE_LEAD_IMAGE_URL_HINTS=['spacer','sprite','blank','throbber','gradient','tile','bg','background','icon','social','header','hdr','advert','spinner','loader','loading','default','rating','share','facebook','twitter','theme','promo','ads','wp-includes'];varNEGATIVE_LEAD_IMAGE_URL_HINTS_RE=newRegExp(NEGATIVE_LEAD_IMAGE_URL_HINTS.join('|'),'i');varGIF_RE=/\.gif(\?.*)?$/i;varJPG_RE=/\.jpe?g(\?.*)?$/i;functiongetSig($node){return($node.attr('class')||'')+' '+($node.attr('id')||'');}// Scores image urls based on a variety of heuristics.
functionscoreImageUrl(url){url=url.trim();varscore=0;if(POSITIVE_LEAD_IMAGE_URL_HINTS_RE.test(url)){score+=20;}if(NEGATIVE_LEAD_IMAGE_URL_HINTS_RE.test(url)){score-=20;}// TODO: We might want to consider removing this as
// gifs are much more common/popular than they once were
if(GIF_RE.test(url)){score-=10;}if(JPG_RE.test(url)){score+=10;}// PNGs are neutral.
returnscore;}// Alt attribute usually means non-presentational image.
functionscoreAttr($img){if($img.attr('alt')){return5;}return0;}// Look through our parent and grandparent for figure-like
// container elements, give a bonus if we find them
functionscoreByParents($img){varscore=0;var$figParent=$img.parents('figure').first();if($figParent.length===1){score+=25;}var$parent=$img.parent();var$gParent=void0;if($parent.length===1){$gParent=$parent.parent();}[$parent,$gParent].forEach(function($node){if(PHOTO_HINTS_RE$1$1.test(getSig($node))){score+=15;}});returnscore;}// Look at our immediate sibling and see if it looks like it's a
// caption. Bonus if so.
functionscoreBySibling($img){varscore=0;var$sibling=$img.next();varsibling=$sibling.get(0);if(sibling&&sibling.tagName==='figcaption'){score+=25;}if(PHOTO_HINTS_RE$1$1.test(getSig($sibling))){score+=15;}returnscore;}functionscoreByDimensions($img){varscore=0;varwidth=parseFloat($img.attr('width'));varheight=parseFloat($img.attr('height'));varsrc=$img.attr('src');// Penalty for skinny images
if(width&&width<=50){score-=50;}// Penalty for short images
if(height&&height<=50){score-=50;}if(width&&height&&!src.includes('sprite')){vararea=width*height;if(area<5000){// Smaller than 50 x 100
score-=100;}else{score+=Math.round(area/1000);}}returnscore;}functionscoreByPosition($imgs,index){return$imgs.length/2-index;}// Given a resource, try to find the lead image URL from within
// it. Like content and next page extraction, uses a scoring system
// to determine what the most likely image may be. Short circuits
// on really probable things like og:image meta tags.
varGenericLeadImageUrlExtractor={extract:functionextract(_ref){var$=_ref.$,content=_ref.content,metaCache=_ref.metaCache,html=_ref.html;varcleanUrl=void0;if(!$.browser&&$('head').length===0){$('*').first().prepend(html);}// Check to see if we have a matching meta tag that we can make use of.
// Moving this higher because common practice is now to use large
// images on things like Open Graph or Twitter cards.
// images usually have for things like Open Graph.
varimageUrl=extractFromMeta$$1($,LEAD_IMAGE_URL_META_TAGS,metaCache,false);if(imageUrl){cleanUrl=clean$1(imageUrl);if(cleanUrl)returncleanUrl;}// Next, try to find the "best" image via the content.
// We'd rather not have to fetch each image and check dimensions,
// so try to do some analysis and determine them instead.
var$content=$(content);varimgs=$('img',$content).toArray();varimgScores={};imgs.forEach(function(img,index){var$img=$(img);varsrc=$img.attr('src');if(!src)return;varscore=scoreImageUrl(src);score+=scoreAttr($img);score+=scoreByParents($img);score+=scoreBySibling($img);score+=scoreByDimensions($img);score+=scoreByPosition(imgs,index);imgScores[src]=score;});var_Reflect$ownKeys$redu=_Reflect$ownKeys$1(imgScores).reduce(function(acc,key){returnimgScores[key]>acc[1]?[key,imgScores[key]]:acc;},[null,0]),_Reflect$ownKeys$redu2=_slicedToArray$1(_Reflect$ownKeys$redu,2),topUrl=_Reflect$ownKeys$redu2[0],topScore=_Reflect$ownKeys$redu2[1];if(topScore>0){cleanUrl=clean$1(topUrl);if(cleanUrl)returncleanUrl;}// If nothing else worked, check to see if there are any really
// probable nodes in the doc, like <link rel="image_src" />.
// if not clean_value and node.attrib.get('href'):
// clean_value = self.clean(node.attrib['href'])
//
// if not clean_value and node.attrib.get('value'):
// clean_value = self.clean(node.attrib['value'])
//
// if clean_value:
// logger.debug('Found lead image in probable nodes.')
// logger.debug('Node was: %s', node)
// return clean_value
//
// return None
functionscoreSimilarity(score,articleUrl,href){// Do this last and only if we have a real candidate, because it's
// potentially expensive computationally. Compare the link to this
// URL using difflib to get the % similarity of these URLs. On a
// sliding scale, subtract points from this link based on
// similarity.
if(score>0){varsimilarity=newdifflib$1.SequenceMatcher(null,articleUrl,href).ratio();// Subtract .1 from diff_percent when calculating modifier,
// which means that if it's less than 10% different, we give a
// bonus instead. Ex:
// 3% different = +17.5 points
// 10% different = 0 points
// 20% different = -25 points
vardiffPercent=1.0-similarity;vardiffModifier=-(250*(diffPercent-0.2));returnscore+diffModifier;}return0;}functionscoreLinkText(linkText,pageNum){// If the link text can be parsed as a number, give it a minor
// bonus, with a slight bias towards lower numbered pages. This is
// so that pages that might not have 'next' in their text can still
// get scored, and sorted properly by score.
varscore=0;if(IS_DIGIT_RE$1.test(linkText.trim())){varlinkTextAsNum=parseInt(linkText,10);// If it's the first page, we already got it on the first call.
// Give it a negative score. Otherwise, up to page 10, give a
// small bonus.
if(linkTextAsNum<2){score=-30;}else{score=Math.max(0,10-linkTextAsNum);}// If it appears that the current page number is greater than
// this links page number, it's a very bad sign. Give it a big
// penalty.
if(pageNum&&pageNum>=linkTextAsNum){score-=50;}}returnscore;}functionscorePageInLink(pageNum,isWp){// page in the link = bonus. Intentionally ignore wordpress because
// their ?p=123 link style gets caught by this even though it means
// separate documents entirely.
if(pageNum&&!isWp){return50;}return0;}varDIGIT_RE$2=/\d/;// A list of words that, if found in link text or URLs, likely mean that
// this link is not a next page link.
varEXTRANEOUS_LINK_HINTS$1=['print','archive','comment','discuss','e-mail','email','share','reply','all','login','sign','single','adx','entry-unrelated'];varEXTRANEOUS_LINK_HINTS_RE$1=newRegExp(EXTRANEOUS_LINK_HINTS$1.join('|'),'i');// Match any link text/classname/id that looks like it could mean the next
// page. Things like: next, continue, >, >>, » but not >|, »| as those can
// mean last page.
varNEXT_LINK_TEXT_RE$1=newRegExp('(next|weiter|continue|>([^|]|$)|»([^|]|$))','i');// Match any link text/classname/id that looks like it is an end link: things
// like "first", "last", "end", etc.
varCAP_LINK_TEXT_RE$1=newRegExp('(first|last|end)','i');// Match any link text/classname/id that looks like it means the previous
// page.
varPREV_LINK_TEXT_RE$1=newRegExp('(prev|earl|old|new|<|«)','i');// Match any phrase that looks like it could be page, or paging, or pagination
functionscoreExtraneousLinks(href){// If the URL itself contains extraneous values, give a penalty.
if(EXTRANEOUS_LINK_HINTS_RE$1.test(href)){return-25;}return0;}functionmakeSig$1($link){return($link.attr('class')||'')+' '+($link.attr('id')||'');}functionscoreByParents$1($link){// If a parent node contains paging-like classname or id, give a
// bonus. Additionally, if a parent_node contains bad content
// (like 'sponsor'), give a penalty.
var$parent=$link.parent();varpositiveMatch=false;varnegativeMatch=false;varscore=0;_Array$from(range(0,4)).forEach(function(){if($parent.length===0){return;}varparentData=makeSig$1($parent,' ');// If we have 'page' or 'paging' in our data, that's a good
// sign. Add a bonus.
if(!positiveMatch&&PAGE_RE$1.test(parentData)){positiveMatch=true;score+=25;}// If we have 'comment' or something in our data, and
// we don't have something like 'content' as well, that's
// a bad sign. Give a penalty.
if(!negativeMatch&&NEGATIVE_SCORE_RE$2.test(parentData)&&EXTRANEOUS_LINK_HINTS_RE$1.test(parentData)){if(!POSITIVE_SCORE_RE$2.test(parentData)){negativeMatch=true;score-=25;}}$parent=$parent.parent();});returnscore;}functionscorePrevLink(linkData){// If the link has something like "previous", its definitely
// an old link, skip it.
if(PREV_LINK_TEXT_RE$1.test(linkData)){return-200;}return0;}functionshouldScore(href,articleUrl,baseUrl,parsedUrl,linkText,previousUrls){// skip if we've already fetched this url
if(previousUrls.find(function(url){returnhref===url;})!==undefined){returnfalse;}// If we've already parsed this URL, or the URL matches the base
if(!baseRegex.test(href)){return-25;}return0;}functionscoreNextLinkText(linkData){// Things like "next", ">>", etc.
if(NEXT_LINK_TEXT_RE$1.test(linkData)){return50;}return0;}functionscoreCapLinks(linkData){// Cap links are links like "last", etc.
if(CAP_LINK_TEXT_RE$1.test(linkData)){// If we found a link like "last", but we've already seen that
// this link is also "next", it's fine. If it's not been
// previously marked as "next", then it's probably bad.
// Penalize.
if(NEXT_LINK_TEXT_RE$1.test(linkData)){return-65;}}return0;}functionmakeBaseRegex(baseUrl){returnnewRegExp('^'+baseUrl,'i');}functionmakeSig($link,linkText){return(linkText||$link.text())+' '+($link.attr('class')||'')+' '+($link.attr('id')||'');}functionscoreLinks(_ref){varlinks=_ref.links,articleUrl=_ref.articleUrl,baseUrl=_ref.baseUrl,parsedUrl=_ref.parsedUrl,$=_ref.$,_ref$previousUrls=_ref.previousUrls,previousUrls=_ref$previousUrls===undefined?[]:_ref$previousUrls;parsedUrl=parsedUrl||URL$1.parse(articleUrl);varbaseRegex=makeBaseRegex(baseUrl);varisWp=isWordpress$1($);// Loop through all links, looking for hints that they may be next-page
// links. Things like having "page" in their textContent, className or
// id, or being a child of a node with a page-y className or id.
//
// After we do that, assign each page a score, and pick the one that
// looks most like the next page link, as long as its score is strong
// enough to have decent confidence.
varscoredPages=links.reduce(function(possiblePages,link){// Remove any anchor data since we don't do a good job
// standardizing URLs (it's hard), we're going to do
// some checking with and without a trailing slash
varattrs=getAttrs$1(link);varhref=removeAnchor$1(attrs.href);var$link=$(link);varlinkText=$link.text();if(!shouldScore(href,articleUrl,baseUrl,parsedUrl,linkText,previousUrls)){returnpossiblePages;}// ## PASSED THE FIRST-PASS TESTS. Start scoring. ##
if(!possiblePages[href]){possiblePages[href]={score:0,linkText:linkText,href:href};}else{possiblePages[href].linkText=possiblePages[href].linkText+'|'+linkText;}varpossiblePage=possiblePages[href];varlinkData=makeSig($link,linkText);varpageNum=pageNumFromUrl$1(href);varscore=scoreBaseUrl(href,baseRegex);score+=scoreNextLinkText(linkData);score+=scoreCapLinks(linkData);score+=scorePrevLink(linkData);score+=scoreByParents$1($link);score+=scoreExtraneousLinks(href);score+=scorePageInLink(pageNum,isWp);score+=scoreLinkText(linkText,pageNum);score+=scoreSimilarity(score,articleUrl,href);possiblePage.score=score;returnpossiblePages;},{});return_Reflect$ownKeys$1(scoredPages).length===0?null:scoredPages;}/* eslint-disable */// Looks for and returns next page url
// for multi-page articles
varGenericNextPageUrlExtractor={extract:functionextract(_ref){var$=_ref.$,url=_ref.url,parsedUrl=_ref.parsedUrl,_ref$previousUrls=_ref.previousUrls,previousUrls=_ref$previousUrls===undefined?[]:_ref$previousUrls;parsedUrl=parsedUrl||URL$1.parse(url);vararticleUrl=removeAnchor$1(url);varbaseUrl=articleBaseUrl$1(url,parsedUrl);varlinks=$('a[href]').toArray();varscoredLinks=scoreLinks({links:links,articleUrl:articleUrl,baseUrl:baseUrl,parsedUrl:parsedUrl,$:$,previousUrls:previousUrls});// If no links were scored, return null
if(!scoredLinks)returnnull;// now that we've scored all possible pages,
// find the biggest one.
vartopPage=_Reflect$ownKeys$1(scoredLinks).reduce(function(acc,link){varscoredLink=scoredLinks[link];returnscoredLink.score>acc.score?scoredLink:acc;},{score:-100});// If the score is less than 50, we're not confident enough to use it,
// so we fail.
if(topPage.score>=50){returntopPage.href;}returnnull;}};varCANONICAL_META_SELECTORS=['og:url'];functionparseDomain(url){varparsedUrl=URL$1.parse(url);varhostname=parsedUrl.hostname;returnhostname;}functionresult(url){return{url:url,domain:parseDomain(url)};}varGenericUrlExtractor={extract:functionextract(_ref){var$=_ref.$,url=_ref.url,metaCache=_ref.metaCache;var$canonical=$('link[rel=canonical]');if($canonical.length!==0){varhref=$canonical.attr('href');if(href){returnresult(href);}}varmetaUrl=extractFromMeta$$1($,CANONICAL_META_SELECTORS,metaCache);if(metaUrl){returnresult(metaUrl);}returnresult(url);}};varEXCERPT_META_SELECTORS=['og:description','twitter:description'];functionclean$2(content,$){varmaxLength=arguments.length>2&&arguments[2]!==undefined?arguments[2]:200;content=content.replace(/[\s\n]+/g,' ').trim();returnellipsize$1(content,maxLength,{ellipse:'…'});}varGenericExcerptExtractor={extract:functionextract(_ref){var$=_ref.$,content=_ref.content,metaCache=_ref.metaCache;varexcerpt=extractFromMeta$$1($,EXCERPT_META_SELECTORS,metaCache);if(excerpt){returnclean$2(stripTags$1(excerpt,$));}// Fall back to excerpting from the extracted content
varmaxLength=200;varshortContent=content.slice(0,maxLength*5);returnclean$2($(shortContent).text(),$,maxLength);}};varGenericWordCountExtractor={extract:functionextract(_ref){varcontent=_ref.content;var$=cheerio$1.load(content);var$content=$('div').first();vartext=normalizeSpaces$1($content.text());returntext.split(/\s/).length;}};varGenericExtractor={// This extractor is the default for all domains
domain:'*',title:GenericTitleExtractor.extract,date_published:GenericDatePublishedExtractor.extract,author:GenericAuthorExtractor.extract,content:GenericContentExtractor.extract.bind(GenericContentExtractor),lead_image_url:GenericLeadImageUrlExtractor.extract,dek:GenericDekExtractor.extract,next_page_url:GenericNextPageUrlExtractor.extract,url_and_domain:GenericUrlExtractor.extract,excerpt:GenericExcerptExtractor.extract,word_count:GenericWordCountExtractor.extract,direction:functiondirection(_ref){vartitle=_ref.title;returnstringDirection$1.getDirection(title);},extract:functionextract(options){varhtml=options.html,cheerio$$1=options.cheerio,$=options.$;if(html&&!$){varloaded=cheerio$$1.load(html);options.$=loaded;}vartitle=this.title(options);vardate_published=this.date_published(options);varauthor=this.author(options);varcontent=this.content(_extends$1({},options,{title:title}));varlead_image_url=this.lead_image_url(_extends$1({},options,{content:content}));vardek=this.dek(_extends$1({},options,{content:content}));varnext_page_url=this.next_page_url(options);varexcerpt=this.excerpt(_extends$1({},options,{content:content}));varword_count=this.word_count(_extends$1({},options,{content:content}));vardirection=this.direction({title:title});var_url_and_domain=this.url_and_domain(options),url=_url_and_domain.url,domain=_url_and_domain.domain;return{title:title,author:author,date_published:date_published||null,dek:dek,lead_image_url:lead_image_url,content:content,next_page_url:next_page_url,url:url,domain:domain,excerpt:excerpt,word_count:word_count,direction:direction};}};functiongetExtractor(url,parsedUrl){parsedUrl=parsedUrl||URL$1.parse(url);var_parsedUrl=parsedUrl,hostname=_parsedUrl.hostname;varbaseDomain=hostname.split('.').slice(-2).join('.');returnExtractors[hostname]||Extractors[baseDomain]||GenericExtractor;}/* eslint-disable */// Remove elements by an array of selectors
functioncleanBySelectors($content,$,_ref){varclean=_ref.clean;if(!clean)return$content;$(clean.join(','),$content).remove();return$content;}// Transform matching elements
functiontransformElements($content,$,_ref2){vartransforms=_ref2.transforms;if(!transforms)return$content;_Reflect$ownKeys$1(transforms).forEach(function(key){var$matches=$(key,$content);varvalue=transforms[key];// If value is a string, convert directly
if(typeofvalue==='string'){$matches.each(function(index,node){convertNodeTo$$1($(node),$,transforms[key]);});}elseif(typeofvalue==='function'){// If value is function, apply function to node
$matches.each(function(index,node){varresult=value($(node),$);// If function returns a string, convert node to that value
return$(selector).length===1&&$(selector).text().trim()!=='';});}functionselect(opts){var$=opts.$,type=opts.type,extractionOpts=opts.extractionOpts,_opts$extractHtml=opts.extractHtml,extractHtml=_opts$extractHtml===undefined?false:_opts$extractHtml;// Skip if there's not extraction for this type
if(!extractionOpts)returnnull;// If a string is hardcoded for a type (e.g., Wikipedia
// contributors), return the string
if(typeofextractionOpts==='string')returnextractionOpts;varselectors=extractionOpts.selectors,_extractionOpts$defau=extractionOpts.defaultCleaner,defaultCleaner=_extractionOpts$defau===undefined?true:_extractionOpts$defau;varmatchingSelector=findMatchingSelector($,selectors);if(!matchingSelector)returnnull;// Declaring result; will contain either
// text or html, which will be cleaned
// by the appropriate cleaner type
// If the selector type requests html as its return type
// transform and clean the element with provided selectors
if(extractHtml){var$content=$(matchingSelector);// Wrap in div so transformation can take place on root element
$content.wrap($('<div></div>'));$content=$content.parent();$content=transformElements($content,$,extractionOpts);$content=cleanBySelectors($content,$,extractionOpts);$content=Cleaners[type]($content,_extends$1({},opts,{defaultCleaner:defaultCleaner}));return$.html($content);}varresult=void0;// if selector is an array (e.g., ['img', 'src']),
// extract the attr
if(Array.isArray(matchingSelector)){var_matchingSelector=_slicedToArray$1(matchingSelector,2),selector=_matchingSelector[0],attr=_matchingSelector[1];result=$(selector).attr(attr).trim();}else{result=$(matchingSelector).text().trim();}// Allow custom extractor to skip default cleaner
// for this type; defaults to true
if(defaultCleaner){returnCleaners[type](result,opts);}returnresult;}functionextractResult(opts){vartype=opts.type,extractor=opts.extractor,_opts$fallback=opts.fallback,fallback=_opts$fallback===undefined?true:_opts$fallback;varresult=select(_extends$1({},opts,{extractionOpts:extractor[type]}));// If custom parser succeeds, return the result
if(result){returnresult;}// If nothing matches the selector, and fallback is enabled,
// run the Generic extraction
if(fallback)returnGenericExtractor[type](opts);returnnull;}varRootExtractor={extract:functionextract(){varextractor=arguments.length>0&&arguments[0]!==undefined?arguments[0]:GenericExtractor;varopts=arguments[1];var_opts=opts,contentOnly=_opts.contentOnly,extractedTitle=_opts.extractedTitle;// This is the generic extractor. Run its extract method
if(extractor.domain==='*')returnextractor.extract(opts);opts=_extends$1({},opts,{extractor:extractor});if(contentOnly){var_content=extractResult(_extends$1({},opts,{type:'content',extractHtml:true,title:extractedTitle}));return{content:_content};}vartitle=extractResult(_extends$1({},opts,{type:'title'}));vardate_published=extractResult(_extends$1({},opts,{type:'date_published'}));varauthor=extractResult(_extends$1({},opts,{type:'author'}));varnext_page_url=extractResult(_extends$1({},opts,{type:'next_page_url'}));varcontent=extractResult(_extends$1({},opts,{type:'content',extractHtml:true,title:title}));varlead_image_url=extractResult(_extends$1({},opts,{type:'lead_image_url',content:content}));varexcerpt=extractResult(_extends$1({},opts,{type:'excerpt',content:content}));vardek=extractResult(_extends$1({},opts,{type:'dek',content:content,excerpt:excerpt}));varword_count=extractResult(_extends$1({},opts,{type:'word_count',content:content}));vardirection=extractResult(_extends$1({},opts,{type:'direction',title:title}));var_ref3=extractResult(_extends$1({},opts,{type:'url_and_domain'}))||{url:null,domain:null},url=_ref3.url,domain=_ref3.domain;return{title:title,content:content,author:author,date_published:date_published,lead_image_url:lead_image_url,dek:dek,next_page_url:next_page_url,url:url,domain:domain,excerpt:excerpt,word_count:word_count,direction:direction};}};varcollectAllPages=function(){var_ref=_asyncToGenerator(_regeneratorRuntime.mark(function_callee(_ref2){varnext_page_url=_ref2.next_page_url,html=_ref2.html,$=_ref2.$,metaCache=_ref2.metaCache,result=_ref2.result,Extractor=_ref2.Extractor,title=_ref2.title,url=_ref2.url,cheerio$$1=_ref2.cheerio;varpages,previousUrls,extractorOpts,nextPageResult,word_count;return_regeneratorRuntime.wrap(function_callee$(_context){while(1){switch(_context.prev=_context.next){case0:// At this point, we've fetched just the first page
pages=1;previousUrls=[removeAnchor$1(url)];// If we've gone over 26 pages, something has
_context.next=7;returnResource.create(url,html,parsedUrl);case7:$=_context.sent;if(!$.error){_context.next=10;break;}return_context.abrupt('return',$);case10:html=$.html();// Cached value of every meta name in our document.
// Used when extracting title/author/date_published/dek
metaCache=$('meta').map(function(_,node){return$(node).attr('name');}).toArray();result=RootExtractor.extract(Extractor,{url:url,html:html,$:$,metaCache:metaCache,parsedUrl:parsedUrl,fallback:fallback,cheerio:cheerio$1});_result=result,title=_result.title,next_page_url=_result.next_page_url;// Fetch more pages if next_page_url found
if(!(fetchAllPages&&next_page_url)){_context.next=20;break;}_context.next=17;returncollectAllPages({Extractor:Extractor,next_page_url:next_page_url,html:html,$:$,metaCache:metaCache,result:result,title:title,url:url,cheerio:cheerio$1});case17:result=_context.sent;_context.next=21;break;case20:result=_extends$1({},result,{total_pages:1,rendered_pages:1});case21:return_context.abrupt('return',result);case22:case'end':return_context.stop();}}},_callee,_this);}))();},// A convenience method for getting a resource
// to work with, e.g., for custom extractor generator
var_templateObject=_taggedTemplateLiteral(['\n export const ',' = {\n domain: \'','\',\n\n title: {\n selectors: [\n // enter title selectors\n ],\n },\n\n author: {\n selectors: [\n // enter author selectors\n ],\n },\n\n date_published: {\n selectors: [\n // enter selectors\n ],\n },\n\n dek: {\n selectors: [\n // enter selectors\n ],\n },\n\n lead_image_url: {\n selectors: [\n // enter selectors\n ],\n },\n\n content: {\n selectors: [\n // enter content selectors\n ],\n\n // Is there anything in the content you selected that needs transformed\n // before it\'s consumable content? E.g., unusual lazy loaded images\n transforms: {\n },\n\n // Is there anything that is in the result that shouldn\'t be?\n // The clean selectors will remove anything that matches from\n // the result\n clean: [\n\n ]\n },\n }\n '],['\n export const ',' = {\n domain: \'','\',\n\n title: {\n selectors: [\n // enter title selectors\n ],\n },\n\n author: {\n selectors: [\n // enter author selectors\n ],\n },\n\n date_published: {\n selectors: [\n // enter selectors\n ],\n },\n\n dek: {\n selectors: [\n // enter selectors\n ],\n },\n\n lead_image_url: {\n selectors: [\n // enter selectors\n ],\n },\n\n content: {\n selectors: [\n // enter content selectors\n ],\n\n // Is there anything in the content you selected that needs transformed\n // before it\'s consumable content? E.g., unusual lazy loaded images\n transforms: {\n },\n\n // Is there anything that is in the result that shouldn\'t be?\n // The clean selectors will remove anything that matches from\n // the result\n clean: [\n\n ]\n },\n }\n ']);
var_templateObject$1=_taggedTemplateLiteral(['\n it(\'returns the ','\', async () => {\n // To pass this test, fill out the ',' selector\n // in ','/index.js.\n const html =\n fs.readFileSync(\'','\');\n const articleUrl =\n \'','\';\n\n const { ',' } =\n await Mercury.parse(articleUrl, html, { fallback: false });\n\n // Update these values with the expected values from\n // the article.\n assert.equal(',', ',')\n });\n '],['\n it(\'returns the ','\', async () => {\n // To pass this test, fill out the ',' selector\n // in ','/index.js.\n const html =\n fs.readFileSync(\'','\');\n const articleUrl =\n \'','\';\n\n const { ',' } =\n await Mercury.parse(articleUrl, html, { fallback: false });\n\n // Update these values with the expected values from\n // the article.\n assert.equal(',', ',')\n });\n ']);
var_templateObject2=_taggedTemplateLiteral(['\n import assert from \'assert\';\n import fs from \'fs\';\n import URL from \'url\';\n import cheerio from \'cheerio\';\n\n import Mercury from \'mercury\';\n import getExtractor from \'extractors/get-extractor\';\n import { excerptContent } from \'utils/text\';\n\n describe(\'','\', () => {\n it(\'is selected properly\', () => {\n // This test should be passing by default.\n // It sanity checks that the correct parser\n // is being selected for URLs from this domain\n const url =\n \'','\';\n const extractor = getExtractor(url);\n assert.equal(extractor.domain, URL.parse(url).hostname)\n })\n\n ','\n\n it(\'returns the content\', async () => {\n // To pass this test, fill out the content selector\n // in ','/index.js.\n // You may also want to make use of the clean and transform\n // options.\n const html =\n fs.readFileSync(\'','\');\n const url =\n \'','\';\n\n const { content } =\n await Mercury.parse(url, html, { fallback: false });\n\n const $ = cheerio.load(content || \'\');\n\n const first13 = excerptContent($(\'*\').first().text(), 13)\n\n // Update these values with the expected values from\n // the article.\n assert.equal(first13, \'Add the first 13 words of the article here\');\n });\n });\n '],['\n import assert from \'assert\';\n import fs from \'fs\';\n import URL from \'url\';\n import cheerio from \'cheerio\';\n\n import Mercury from \'mercury\';\n import getExtractor from \'extractors/get-extractor\';\n import { excerptContent } from \'utils/text\';\n\n describe(\'','\', () => {\n it(\'is selected properly\', () => {\n // This test should be passing by default.\n // It sanity checks that the correct parser\n // is being selected for URLs from this domain\n const url =\n \'','\';\n const extractor = getExtractor(url);\n assert.equal(extractor.domain, URL.parse(url).hostname)\n })\n\n ','\n\n it(\'returns the content\', async () => {\n // To pass this test, fill out the content selector\n // in ','/index.js.\n // You may also want to make use of the clean and transform\n // options.\n const html =\n fs.readFileSync(\'','\');\n const url =\n \'','\';\n\n const { content } =\n await Mercury.parse(url, html, { fallback: false });\n\n const $ = cheerio.load(content || \'\');\n\n const first13 = excerptContent($(\'*\').first().text(), 13)\n\n // Update these values with the expected values from\n // the article.\n assert.equal(first13, \'Add the first 13 words of the article here\');\n });\n });\n ']);
console.log('Your custom site extractor has been set up. To get started building it, run\n yarn watch:test -- '+hostname+'\n -- OR --\n npm run watch:test -- '+hostname);
console.log('\n It looks like you already have a custom parser for this url.\n The page you linked to has been added to '+file+'. Copy and paste\n the following code to use that page in your tests:\n const html = fs.readFileSync(\''+file+'\');');
