tags into\n// tags instead.\n//\n// :param $: A cheerio object\n\nexport default function brsToPs($) {\n let collapsing = false;\n $('br').each((index, element) => {\n const $element = $(element);\n const nextElement = $element.next().get(0);\n\n if (nextElement && nextElement.tagName.toLowerCase() === 'br') {\n collapsing = true;\n $element.remove();\n } else if (collapsing) {\n collapsing = false;\n // $(element).replaceWith('')\n paragraphize(element, $, true);\n }\n });\n\n return $;\n}\n","import { BLOCK_LEVEL_TAGS_RE } from './constants';\n\n// Given a node, turn it into a P if it is not already a P, and\n// make sure it conforms to the constraints of a P tag (I.E. does\n// not contain any other block tags.)\n//\n// If the node is a
, it treats the following inline siblings\n// as if they were its children.\n//\n// :param node: The node to paragraphize; this is a raw node\n// :param $: The cheerio object to handle dom manipulation\n// :param br: Whether or not the passed node is a br\n\nexport default function paragraphize(node, $, br = false) {\n const $node = $(node);\n\n if (br) {\n let sibling = node.nextSibling;\n const p = $('');\n\n // while the next node is text or not a block level element\n // append it to a new p node\n while (sibling && !(sibling.tagName && BLOCK_LEVEL_TAGS_RE.test(sibling.tagName))) {\n const nextSibling = sibling.nextSibling;\n $(sibling).appendTo(p);\n sibling = nextSibling;\n }\n\n $node.replaceWith(p);\n $node.remove();\n return $;\n }\n\n return $;\n}\n","import { brsToPs, convertNodeTo } from 'utils/dom';\n\nimport { DIV_TO_P_BLOCK_TAGS } from './constants';\n\nfunction convertDivs($) {\n $('div').each((index, div) => {\n const $div = $(div);\n const convertable = $div.children(DIV_TO_P_BLOCK_TAGS).length === 0;\n\n if (convertable) {\n convertNodeTo($div, $, 'p');\n }\n });\n\n return $;\n}\n\nfunction convertSpans($) {\n $('span').each((index, span) => {\n const $span = $(span);\n const convertable = $span.parents('p, div').length === 0;\n if (convertable) {\n convertNodeTo($span, $, 'p');\n }\n });\n\n return $;\n}\n\n// Loop through the provided doc, and convert any p-like elements to\n// actual paragraph tags.\n//\n// Things fitting this criteria:\n// * Multiple consecutive
tags.\n// * tags without block level elements inside of them\n// * tags who are not children of or tags.\n//\n// :param $: A cheerio object to search\n// :return cheerio object with new p elements\n// (By-reference mutation, though. Returned just for convenience.)\n\nexport default function convertToParagraphs($) {\n $ = brsToPs($);\n $ = convertDivs($);\n $ = convertSpans($);\n\n return $;\n}\n","import { getAttrs } from 'utils/dom';\n\nexport default function convertNodeTo($node, $, tag = 'p') {\n const node = $node.get(0);\n if (!node) {\n return $;\n }\n const attrs = getAttrs(node) || {};\n // console.log(attrs)\n\n const attribString = Reflect.ownKeys(attrs)\n .map(key => `${key}=${attrs[key]}`)\n .join(' ');\n let html;\n\n if ($.browser) {\n // In the browser, the contents of noscript tags aren't rendered, therefore\n // transforms on the noscript tag (commonly used for lazy-loading) don't work\n // as expected. This test case handles that\n html = node.tagName.toLowerCase() === 'noscript' ? $node.text() : $node.html();\n } else {\n html = $node.contents();\n }\n $node.replaceWith(\n `<${tag} ${attribString}>${html}`\n );\n return $;\n}\n","import { SPACER_RE } from './constants';\n\nfunction cleanForHeight($img, $) {\n const height = parseInt($img.attr('height'), 10);\n const width = parseInt($img.attr('width'), 10) || 20;\n\n // Remove images that explicitly have very small heights or\n // widths, because they are most likely shims or icons,\n // which aren't very useful for reading.\n if ((height || 20) < 10 || width < 10) {\n $img.remove();\n } else if (height) {\n // Don't ever specify a height on images, so that we can\n // scale with respect to width without screwing up the\n // aspect ratio.\n $img.removeAttr('height');\n }\n\n return $;\n}\n\n// Cleans out images where the source string matches transparent/spacer/etc\n// TODO This seems very aggressive - AP\nfunction removeSpacers($img, $) {\n if (SPACER_RE.test($img.attr('src'))) {\n $img.remove();\n }\n\n return $;\n}\n\nexport default function cleanImages($article, $) {\n $article.find('img').each((index, img) => {\n const $img = $(img);\n\n cleanForHeight($img, $);\n removeSpacers($img, $);\n });\n\n return $;\n}\n","import {\n STRIP_OUTPUT_TAGS,\n KEEP_CLASS,\n} from './constants';\n\nexport default function stripJunkTags(article, $, tags = []) {\n if (tags.length === 0) {\n tags = STRIP_OUTPUT_TAGS;\n }\n\n // Remove matching elements, but ignore\n // any element with a class of mercury-parser-keep\n $(tags.join(','), article).not(`.${KEEP_CLASS}`).remove();\n\n // Remove the mercury-parser-keep class from result\n $(`.${KEEP_CLASS}`, article).removeClass(KEEP_CLASS);\n\n return $;\n}\n","import {\n getAttrs,\n setAttrs,\n} from 'utils/dom';\n\nimport { WHITELIST_ATTRS_RE } from './constants';\n\nfunction removeAllButWhitelist($article) {\n $article.find('*').each((index, node) => {\n const attrs = getAttrs(node);\n\n setAttrs(node, Reflect.ownKeys(attrs).reduce((acc, attr) => {\n if (WHITELIST_ATTRS_RE.test(attr)) {\n return { ...acc, [attr]: attrs[attr] };\n }\n\n return acc;\n }, {}));\n });\n\n return $article;\n}\n\n// function removeAttrs(article, $) {\n// REMOVE_ATTRS.forEach((attr) => {\n// $(`[${attr}]`, article).removeAttr(attr);\n// });\n// }\n\n// Remove attributes like style or align\nexport default function cleanAttributes($article) {\n // Grabbing the parent because at this point\n // $article will be wrapped in a div which will\n // have a score set on it.\n return removeAllButWhitelist(\n $article.parent().length ?\n $article.parent() : $article\n );\n}\n","// // CONTENT FETCHING CONSTANTS ////\n\n// A list of strings that can be considered unlikely candidates when\n// extracting content from a resource. These strings are joined together\n// and then tested for existence using re:test, so may contain simple,\n// non-pipe style regular expression queries if necessary.\nexport const UNLIKELY_CANDIDATES_BLACKLIST = [\n 'ad-break',\n 'adbox',\n 'advert',\n 'addthis',\n 'agegate',\n 'aux',\n 'blogger-labels',\n 'combx',\n 'comment',\n 'conversation',\n 'disqus',\n 'entry-unrelated',\n 'extra',\n 'foot',\n 'form',\n 'header',\n 'hidden',\n 'loader',\n 'login', // Note: This can hit 'blogindex'.\n 'menu',\n 'meta',\n 'nav',\n 'pager',\n 'pagination',\n 'predicta', // readwriteweb inline ad box\n 'presence_control_external', // lifehacker.com container full of false positives\n 'popup',\n 'printfriendly',\n 'related',\n 'remove',\n 'remark',\n 'rss',\n 'share',\n 'shoutbox',\n 'sidebar',\n 'sociable',\n 'sponsor',\n 'tools',\n];\n\n// A list of strings that can be considered LIKELY candidates when\n// extracting content from a resource. Essentially, the inverse of the\n// blacklist above - if something matches both blacklist and whitelist,\n// it is kept. This is useful, for example, if something has a className\n// of \"rss-content entry-content\". It matched 'rss', so it would normally\n// be removed, however, it's also the entry content, so it should be left\n// alone.\n//\n// These strings are joined together and then tested for existence using\n// re:test, so may contain simple, non-pipe style regular expression queries\n// if necessary.\nexport const UNLIKELY_CANDIDATES_WHITELIST = [\n 'and',\n 'article',\n 'body',\n 'blogindex',\n 'column',\n 'content',\n 'entry-content-asset',\n 'format', // misuse of form\n 'hfeed',\n 'hentry',\n 'hatom',\n 'main',\n 'page',\n 'posts',\n 'shadow',\n];\n\n// A list of tags which, if found inside, should cause a to NOT\n// be turned into a paragraph tag. Shallow div tags without these elements\n// should be turned into tags.\nexport const DIV_TO_P_BLOCK_TAGS = [\n 'a',\n 'blockquote',\n 'dl',\n 'div',\n 'img',\n 'p',\n 'pre',\n 'table',\n].join(',');\n\n// A list of tags that should be ignored when trying to find the top candidate\n// for a document.\nexport const NON_TOP_CANDIDATE_TAGS = [\n 'br',\n 'b',\n 'i',\n 'label',\n 'hr',\n 'area',\n 'base',\n 'basefont',\n 'input',\n 'img',\n 'link',\n 'meta',\n];\n\nexport const NON_TOP_CANDIDATE_TAGS_RE =\n new RegExp(`^(${NON_TOP_CANDIDATE_TAGS.join('|')})$`, 'i');\n\n// A list of selectors that specify, very clearly, either hNews or other\n// very content-specific style content, like Blogger templates.\n// More examples here: http://microformats.org/wiki/blog-post-formats\nexport const HNEWS_CONTENT_SELECTORS = [\n ['.hentry', '.entry-content'],\n ['entry', '.entry-content'],\n ['.entry', '.entry_content'],\n ['.post', '.postbody'],\n ['.post', '.post_body'],\n ['.post', '.post-body'],\n];\n\nexport const PHOTO_HINTS = [\n 'figure',\n 'photo',\n 'image',\n 'caption',\n];\nexport const PHOTO_HINTS_RE = new RegExp(PHOTO_HINTS.join('|'), 'i');\n\n// A list of strings that denote a positive scoring for this content as being\n// an article container. Checked against className and id.\n//\n// TODO: Perhaps have these scale based on their odds of being quality?\nexport const POSITIVE_SCORE_HINTS = [\n 'article',\n 'articlecontent',\n 'instapaper_body',\n 'blog',\n 'body',\n 'content',\n 'entry-content-asset',\n 'entry',\n 'hentry',\n 'main',\n 'Normal',\n 'page',\n 'pagination',\n 'permalink',\n 'post',\n 'story',\n 'text',\n '[-_]copy', // usatoday\n '\\\\Bcopy',\n];\n\n// The above list, joined into a matching regular expression\nexport const POSITIVE_SCORE_RE = new RegExp(POSITIVE_SCORE_HINTS.join('|'), 'i');\n\n// Readability publisher-specific guidelines\nexport const READABILITY_ASSET = new RegExp('entry-content-asset', 'i');\n\n// A list of strings that denote a negative scoring for this content as being\n// an article container. Checked against className and id.\n//\n// TODO: Perhaps have these scale based on their odds of being quality?\nexport const NEGATIVE_SCORE_HINTS = [\n 'adbox',\n 'advert',\n 'author',\n 'bio',\n 'bookmark',\n 'bottom',\n 'byline',\n 'clear',\n 'com-',\n 'combx',\n 'comment',\n 'comment\\\\B',\n 'contact',\n 'copy',\n 'credit',\n 'crumb',\n 'date',\n 'deck',\n 'excerpt',\n 'featured', // tnr.com has a featured_content which throws us off\n 'foot',\n 'footer',\n 'footnote',\n 'graf',\n 'head',\n 'info',\n 'infotext', // newscientist.com copyright\n 'instapaper_ignore',\n 'jump',\n 'linebreak',\n 'link',\n 'masthead',\n 'media',\n 'meta',\n 'modal',\n 'outbrain', // slate.com junk\n 'promo',\n 'pr_', // autoblog - press release\n 'related',\n 'respond',\n 'roundcontent', // lifehacker restricted content warning\n 'scroll',\n 'secondary',\n 'share',\n 'shopping',\n 'shoutbox',\n 'side',\n 'sidebar',\n 'sponsor',\n 'stamp',\n 'sub',\n 'summary',\n 'tags',\n 'tools',\n 'widget',\n];\n// The above list, joined into a matching regular expression\nexport const NEGATIVE_SCORE_RE = new RegExp(NEGATIVE_SCORE_HINTS.join('|'), 'i');\n\n// Match a digit. Pretty clear.\nexport const DIGIT_RE = new RegExp('[0-9]');\n\n// Match 2 or more consecutive
tags\nexport const BR_TAGS_RE = new RegExp('(
tags\n\n\n// Match 1 BR tag.\n\n\n// A list of all of the block level tags known in HTML5 and below. Taken from\n// http://bit.ly/qneNIT\nvar BLOCK_LEVEL_TAGS = ['article', 'aside', 'blockquote', 'body', 'br', 'button', 'canvas', 'caption', 'col', 'colgroup', 'dd', 'div', 'dl', 'dt', 'embed', 'fieldset', 'figcaption', 'figure', 'footer', 'form', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'header', 'hgroup', 'hr', 'li', 'map', 'object', 'ol', 'output', 'p', 'pre', 'progress', 'section', 'table', 'tbody', 'textarea', 'tfoot', 'th', 'thead', 'tr', 'ul', 'video'];\nvar BLOCK_LEVEL_TAGS_RE = new RegExp('^(' + BLOCK_LEVEL_TAGS.join('|') + ')$', 'i');\n\n// The removal is implemented as a blacklist and whitelist, this test finds\n// blacklisted elements that aren't whitelisted. We do this all in one\n// expression-both because it's only one pass, and because this skips the\n// serialization for whitelisted nodes.\nvar candidatesBlacklist = UNLIKELY_CANDIDATES_BLACKLIST.join('|');\nvar CANDIDATES_BLACKLIST = new RegExp(candidatesBlacklist, 'i');\n\nvar candidatesWhitelist = UNLIKELY_CANDIDATES_WHITELIST.join('|');\nvar CANDIDATES_WHITELIST = new RegExp(candidatesWhitelist, 'i');\n\nfunction stripUnlikelyCandidates($) {\n // Loop through the provided document and remove any non-link nodes\n // that are unlikely candidates for article content.\n //\n // Links are ignored because there are very often links to content\n // that are identified as non-body-content, but may be inside\n // article-like content.\n //\n // :param $: a cheerio object to strip nodes from\n // :return $: the cleaned cheerio object\n $('*').not('a').each(function (index, node) {\n var $node = $(node);\n var classes = $node.attr('class');\n var id = $node.attr('id');\n if (!id && !classes) return;\n\n var classAndId = (classes || '') + ' ' + (id || '');\n if (CANDIDATES_WHITELIST.test(classAndId)) {\n return;\n } else if (CANDIDATES_BLACKLIST.test(classAndId)) {\n $node.remove();\n }\n });\n\n return $;\n}\n\n// ## NOTES:\n// Another good candidate for refactoring/optimizing.\n// Very imperative code, I don't love it. - AP\n\n// Given cheerio object, convert consecutive
tags into\n// tags instead.\n//\n// :param $: A cheerio object\n\nfunction brsToPs$$1($) {\n var collapsing = false;\n $('br').each(function (index, element) {\n var $element = $(element);\n var nextElement = $element.next().get(0);\n\n if (nextElement && nextElement.tagName.toLowerCase() === 'br') {\n collapsing = true;\n $element.remove();\n } else if (collapsing) {\n collapsing = false;\n // $(element).replaceWith('')\n paragraphize(element, $, true);\n }\n });\n\n return $;\n}\n\n// Given a node, turn it into a P if it is not already a P, and\n// make sure it conforms to the constraints of a P tag (I.E. does\n// not contain any other block tags.)\n//\n// If the node is a
, it treats the following inline siblings\n// as if they were its children.\n//\n// :param node: The node to paragraphize; this is a raw node\n// :param $: The cheerio object to handle dom manipulation\n// :param br: Whether or not the passed node is a br\n\nfunction paragraphize(node, $) {\n var br = arguments.length > 2 && arguments[2] !== undefined ? arguments[2] : false;\n\n var $node = $(node);\n\n if (br) {\n var sibling = node.nextSibling;\n var p = $('');\n\n // while the next node is text or not a block level element\n // append it to a new p node\n while (sibling && !(sibling.tagName && BLOCK_LEVEL_TAGS_RE.test(sibling.tagName))) {\n var nextSibling = sibling.nextSibling;\n $(sibling).appendTo(p);\n sibling = nextSibling;\n }\n\n $node.replaceWith(p);\n $node.remove();\n return $;\n }\n\n return $;\n}\n\nfunction convertDivs($) {\n $('div').each(function (index, div) {\n var $div = $(div);\n var convertable = $div.children(DIV_TO_P_BLOCK_TAGS).length === 0;\n\n if (convertable) {\n convertNodeTo$$1($div, $, 'p');\n }\n });\n\n return $;\n}\n\nfunction convertSpans($) {\n $('span').each(function (index, span) {\n var $span = $(span);\n var convertable = $span.parents('p, div').length === 0;\n if (convertable) {\n convertNodeTo$$1($span, $, 'p');\n }\n });\n\n return $;\n}\n\n// Loop through the provided doc, and convert any p-like elements to\n// actual paragraph tags.\n//\n// Things fitting this criteria:\n// * Multiple consecutive
tags.\n// * tags without block level elements inside of them\n// * tags who are not children of or tags.\n//\n// :param $: A cheerio object to search\n// :return cheerio object with new p elements\n// (By-reference mutation, though. Returned just for convenience.)\n\nfunction convertToParagraphs$$1($) {\n $ = brsToPs$$1($);\n $ = convertDivs($);\n $ = convertSpans($);\n\n return $;\n}\n\nfunction convertNodeTo$$1($node, $) {\n var tag = arguments.length > 2 && arguments[2] !== undefined ? arguments[2] : 'p';\n\n var node = $node.get(0);\n if (!node) {\n return $;\n }\n var attrs = getAttrs(node) || {};\n // console.log(attrs)\n\n var attribString = _Reflect$ownKeys(attrs).map(function (key) {\n return key + '=' + attrs[key];\n }).join(' ');\n var html = void 0;\n\n if ($.browser) {\n // In the browser, the contents of noscript tags aren't rendered, therefore\n // transforms on the noscript tag (commonly used for lazy-loading) don't work\n // as expected. This test case handles that\n html = node.tagName.toLowerCase() === 'noscript' ? $node.text() : $node.html();\n } else {\n html = $node.contents();\n }\n $node.replaceWith('<' + tag + ' ' + attribString + '>' + html + '');\n return $;\n}\n\nfunction cleanForHeight($img, $) {\n var height = parseInt($img.attr('height'), 10);\n var width = parseInt($img.attr('width'), 10) || 20;\n\n // Remove images that explicitly have very small heights or\n // widths, because they are most likely shims or icons,\n // which aren't very useful for reading.\n if ((height || 20) < 10 || width < 10) {\n $img.remove();\n } else if (height) {\n // Don't ever specify a height on images, so that we can\n // scale with respect to width without screwing up the\n // aspect ratio.\n $img.removeAttr('height');\n }\n\n return $;\n}\n\n// Cleans out images where the source string matches transparent/spacer/etc\n// TODO This seems very aggressive - AP\nfunction removeSpacers($img, $) {\n if (SPACER_RE.test($img.attr('src'))) {\n $img.remove();\n }\n\n return $;\n}\n\nfunction cleanImages($article, $) {\n $article.find('img').each(function (index, img) {\n var $img = $(img);\n\n cleanForHeight($img, $);\n removeSpacers($img, $);\n });\n\n return $;\n}\n\nfunction markToKeep(article, $, url) {\n var tags = arguments.length > 3 && arguments[3] !== undefined ? arguments[3] : [];\n\n if (tags.length === 0) {\n tags = KEEP_SELECTORS;\n }\n\n if (url) {\n var _URL$parse = URL.parse(url),\n protocol = _URL$parse.protocol,\n hostname = _URL$parse.hostname;\n\n tags = [].concat(_toConsumableArray(tags), ['iframe[src^=\"' + protocol + '//' + hostname + '\"]']);\n }\n\n $(tags.join(','), article).addClass(KEEP_CLASS);\n\n return $;\n}\n\nfunction stripJunkTags(article, $) {\n var tags = arguments.length > 2 && arguments[2] !== undefined ? arguments[2] : [];\n\n if (tags.length === 0) {\n tags = STRIP_OUTPUT_TAGS;\n }\n\n // Remove matching elements, but ignore\n // any element with a class of mercury-parser-keep\n $(tags.join(','), article).not('.' + KEEP_CLASS).remove();\n\n // Remove the mercury-parser-keep class from result\n $('.' + KEEP_CLASS, article).removeClass(KEEP_CLASS);\n\n return $;\n}\n\n// H1 tags are typically the article title, which should be extracted\n// by the title extractor instead. If there's less than 3 of them (<3),\n// strip them. Otherwise, turn 'em into H2s.\nfunction cleanHOnes$$1(article, $) {\n var $hOnes = $('h1', article);\n\n if ($hOnes.length < 3) {\n $hOnes.each(function (index, node) {\n return $(node).remove();\n });\n } else {\n $hOnes.each(function (index, node) {\n convertNodeTo$$1($(node), $, 'h2');\n });\n }\n\n return $;\n}\n\nfunction removeAllButWhitelist($article) {\n $article.find('*').each(function (index, node) {\n var attrs = getAttrs(node);\n\n setAttrs(node, _Reflect$ownKeys(attrs).reduce(function (acc, attr) {\n if (WHITELIST_ATTRS_RE.test(attr)) {\n return _extends({}, acc, _defineProperty({}, attr, attrs[attr]));\n }\n\n return acc;\n }, {}));\n });\n\n return $article;\n}\n\n// function removeAttrs(article, $) {\n// REMOVE_ATTRS.forEach((attr) => {\n// $(`[${attr}]`, article).removeAttr(attr);\n// });\n// }\n\n// Remove attributes like style or align\nfunction cleanAttributes$$1($article) {\n // Grabbing the parent because at this point\n // $article will be wrapped in a div which will\n // have a score set on it.\n return removeAllButWhitelist($article.parent().length ? $article.parent() : $article);\n}\n\nfunction removeEmpty($article, $) {\n $article.find('p').each(function (index, p) {\n var $p = $(p);\n if ($p.find('iframe, img').length === 0 && $p.text().trim() === '') $p.remove();\n });\n\n return $;\n}\n\n// // CONTENT FETCHING CONSTANTS ////\n\n// A list of strings that can be considered unlikely candidates when\n// extracting content from a resource. These strings are joined together\n// and then tested for existence using re:test, so may contain simple,\n// non-pipe style regular expression queries if necessary.\nvar UNLIKELY_CANDIDATES_BLACKLIST$1 = ['ad-break', 'adbox', 'advert', 'addthis', 'agegate', 'aux', 'blogger-labels', 'combx', 'comment', 'conversation', 'disqus', 'entry-unrelated', 'extra', 'foot', 'form', 'header', 'hidden', 'loader', 'login', // Note: This can hit 'blogindex'.\n'menu', 'meta', 'nav', 'pager', 'pagination', 'predicta', // readwriteweb inline ad box\n'presence_control_external', // lifehacker.com container full of false positives\n'popup', 'printfriendly', 'related', 'remove', 'remark', 'rss', 'share', 'shoutbox', 'sidebar', 'sociable', 'sponsor', 'tools'];\n\n// A list of strings that can be considered LIKELY candidates when\n// extracting content from a resource. Essentially, the inverse of the\n// blacklist above - if something matches both blacklist and whitelist,\n// it is kept. This is useful, for example, if something has a className\n// of \"rss-content entry-content\". It matched 'rss', so it would normally\n// be removed, however, it's also the entry content, so it should be left\n// alone.\n//\n// These strings are joined together and then tested for existence using\n// re:test, so may contain simple, non-pipe style regular expression queries\n// if necessary.\nvar UNLIKELY_CANDIDATES_WHITELIST$1 = ['and', 'article', 'body', 'blogindex', 'column', 'content', 'entry-content-asset', 'format', // misuse of form\n'hfeed', 'hentry', 'hatom', 'main', 'page', 'posts', 'shadow'];\n\n// A list of tags which, if found inside, should cause a to NOT\n// be turned into a paragraph tag. Shallow div tags without these elements\n// should be turned into tags.\nvar DIV_TO_P_BLOCK_TAGS$1 = ['a', 'blockquote', 'dl', 'div', 'img', 'p', 'pre', 'table'].join(',');\n\n// A list of tags that should be ignored when trying to find the top candidate\n// for a document.\nvar NON_TOP_CANDIDATE_TAGS$1 = ['br', 'b', 'i', 'label', 'hr', 'area', 'base', 'basefont', 'input', 'img', 'link', 'meta'];\n\nvar NON_TOP_CANDIDATE_TAGS_RE$1 = new RegExp('^(' + NON_TOP_CANDIDATE_TAGS$1.join('|') + ')$', 'i');\n\n// A list of selectors that specify, very clearly, either hNews or other\n// very content-specific style content, like Blogger templates.\n// More examples here: http://microformats.org/wiki/blog-post-formats\nvar HNEWS_CONTENT_SELECTORS$1 = [['.hentry', '.entry-content'], ['entry', '.entry-content'], ['.entry', '.entry_content'], ['.post', '.postbody'], ['.post', '.post_body'], ['.post', '.post-body']];\n\nvar PHOTO_HINTS$1 = ['figure', 'photo', 'image', 'caption'];\nvar PHOTO_HINTS_RE$1 = new RegExp(PHOTO_HINTS$1.join('|'), 'i');\n\n// A list of strings that denote a positive scoring for this content as being\n// an article container. Checked against className and id.\n//\n// TODO: Perhaps have these scale based on their odds of being quality?\nvar POSITIVE_SCORE_HINTS$1 = ['article', 'articlecontent', 'instapaper_body', 'blog', 'body', 'content', 'entry-content-asset', 'entry', 'hentry', 'main', 'Normal', 'page', 'pagination', 'permalink', 'post', 'story', 'text', '[-_]copy', // usatoday\n'\\\\Bcopy'];\n\n// The above list, joined into a matching regular expression\nvar POSITIVE_SCORE_RE$1 = new RegExp(POSITIVE_SCORE_HINTS$1.join('|'), 'i');\n\n// Readability publisher-specific guidelines\nvar READABILITY_ASSET$1 = new RegExp('entry-content-asset', 'i');\n\n// A list of strings that denote a negative scoring for this content as being\n// an article container. Checked against className and id.\n//\n// TODO: Perhaps have these scale based on their odds of being quality?\nvar NEGATIVE_SCORE_HINTS$1 = ['adbox', 'advert', 'author', 'bio', 'bookmark', 'bottom', 'byline', 'clear', 'com-', 'combx', 'comment', 'comment\\\\B', 'contact', 'copy', 'credit', 'crumb', 'date', 'deck', 'excerpt', 'featured', // tnr.com has a featured_content which throws us off\n'foot', 'footer', 'footnote', 'graf', 'head', 'info', 'infotext', // newscientist.com copyright\n'instapaper_ignore', 'jump', 'linebreak', 'link', 'masthead', 'media', 'meta', 'modal', 'outbrain', // slate.com junk\n'promo', 'pr_', // autoblog - press release\n'related', 'respond', 'roundcontent', // lifehacker restricted content warning\n'scroll', 'secondary', 'share', 'shopping', 'shoutbox', 'side', 'sidebar', 'sponsor', 'stamp', 'sub', 'summary', 'tags', 'tools', 'widget'];\n// The above list, joined into a matching regular expression\nvar NEGATIVE_SCORE_RE$1 = new RegExp(NEGATIVE_SCORE_HINTS$1.join('|'), 'i');\n\n// Match a digit. Pretty clear.\n\n\n// Match 2 or more consecutive
tags\n\n\n// Match 1 BR tag.\n\n\n// A list of all of the block level tags known in HTML5 and below. Taken from\n// http://bit.ly/qneNIT\n\n\n\n// The removal is implemented as a blacklist and whitelist, this test finds\n// blacklisted elements that aren't whitelisted. We do this all in one\n// expression-both because it's only one pass, and because this skips the\n// serialization for whitelisted nodes.\nvar candidatesBlacklist$1 = UNLIKELY_CANDIDATES_BLACKLIST$1.join('|');\n\n\nvar candidatesWhitelist$1 = UNLIKELY_CANDIDATES_WHITELIST$1.join('|');\n\n\n\n\nvar PARAGRAPH_SCORE_TAGS$1 = new RegExp('^(p|li|span|pre)$', 'i');\nvar CHILD_CONTENT_TAGS$1 = new RegExp('^(td|blockquote|ol|ul|dl)$', 'i');\nvar BAD_TAGS$1 = new RegExp('^(address|form)$', 'i');\n\n// Get the score of a node based on its className and id.\nfunction getWeight(node) {\n var classes = node.attr('class');\n var id = node.attr('id');\n var score = 0;\n\n if (id) {\n // if id exists, try to score on both positive and negative\n if (POSITIVE_SCORE_RE$1.test(id)) {\n score += 25;\n }\n if (NEGATIVE_SCORE_RE$1.test(id)) {\n score -= 25;\n }\n }\n\n if (classes) {\n if (score === 0) {\n // if classes exist and id did not contribute to score\n // try to score on both positive and negative\n if (POSITIVE_SCORE_RE$1.test(classes)) {\n score += 25;\n }\n if (NEGATIVE_SCORE_RE$1.test(classes)) {\n score -= 25;\n }\n }\n\n // even if score has been set by id, add score for\n // possible photo matches\n // \"try to keep photos if we can\"\n if (PHOTO_HINTS_RE$1.test(classes)) {\n score += 10;\n }\n\n // add 25 if class matches entry-content-asset,\n // a class apparently instructed for use in the\n // Readability publisher guidelines\n // https://www.readability.com/developers/guidelines\n if (READABILITY_ASSET$1.test(classes)) {\n score += 25;\n }\n }\n\n return score;\n}\n\n// returns the score of a node based on\n// the node's score attribute\n// returns null if no score set\nfunction getScore($node) {\n return parseFloat($node.attr('score')) || null;\n}\n\n// return 1 for every comma in text\nfunction scoreCommas(text) {\n return (text.match(/,/g) || []).length;\n}\n\nvar idkRe = new RegExp('^(p|pre)$', 'i');\n\nfunction scoreLength(textLength) {\n var tagName = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : 'p';\n\n var chunks = textLength / 50;\n\n if (chunks > 0) {\n var lengthBonus = void 0;\n\n // No idea why p or pre are being tamped down here\n // but just following the source for now\n // Not even sure why tagName is included here,\n // since this is only being called from the context\n // of scoreParagraph\n if (idkRe.test(tagName)) {\n lengthBonus = chunks - 2;\n } else {\n lengthBonus = chunks - 1.25;\n }\n\n return Math.min(Math.max(lengthBonus, 0), 3);\n }\n\n return 0;\n}\n\n// Score a paragraph using various methods. Things like number of\n// commas, etc. Higher is better.\nfunction scoreParagraph$$1(node) {\n var score = 1;\n var text = node.text().trim();\n var textLength = text.length;\n\n // If this paragraph is less than 25 characters, don't count it.\n if (textLength < 25) {\n return 0;\n }\n\n // Add points for any commas within this paragraph\n score += scoreCommas(text);\n\n // For every 50 characters in this paragraph, add another point. Up\n // to 3 points.\n score += scoreLength(textLength);\n\n // Articles can end with short paragraphs when people are being clever\n // but they can also end with short paragraphs setting up lists of junk\n // that we strip. This negative tweaks junk setup paragraphs just below\n // the cutoff threshold.\n if (text.slice(-1) === ':') {\n score -= 1;\n }\n\n return score;\n}\n\nfunction setScore($node, $, score) {\n $node.attr('score', score);\n return $node;\n}\n\nfunction addScore$$1($node, $, amount) {\n try {\n var score = getOrInitScore$$1($node, $) + amount;\n setScore($node, $, score);\n } catch (e) {\n // Ignoring; error occurs in scoreNode\n }\n\n return $node;\n}\n\n// Adds 1/4 of a child's score to its parent\nfunction addToParent$$1(node, $, score) {\n var parent = node.parent();\n if (parent) {\n addScore$$1(parent, $, score * 0.25);\n }\n\n return node;\n}\n\n// gets and returns the score if it exists\n// if not, initializes a score based on\n// the node's tag type\nfunction getOrInitScore$$1($node, $) {\n var weightNodes = arguments.length > 2 && arguments[2] !== undefined ? arguments[2] : true;\n\n var score = getScore($node);\n\n if (score) {\n return score;\n }\n\n score = scoreNode$$1($node);\n\n if (weightNodes) {\n score += getWeight($node);\n }\n\n addToParent$$1($node, $, score);\n\n return score;\n}\n\n// Score an individual node. Has some smarts for paragraphs, otherwise\n// just scores based on tag.\nfunction scoreNode$$1($node) {\n var _$node$get = $node.get(0),\n tagName = _$node$get.tagName;\n\n // TODO: Consider ordering by most likely.\n // E.g., if divs are a more common tag on a page,\n // Could save doing that regex test on every node – AP\n\n\n if (PARAGRAPH_SCORE_TAGS$1.test(tagName)) {\n return scoreParagraph$$1($node);\n } else if (tagName.toLowerCase() === 'div') {\n return 5;\n } else if (CHILD_CONTENT_TAGS$1.test(tagName)) {\n return 3;\n } else if (BAD_TAGS$1.test(tagName)) {\n return -3;\n } else if (tagName.toLowerCase() === 'th') {\n return -5;\n }\n\n return 0;\n}\n\nfunction convertSpans$1($node, $) {\n if ($node.get(0)) {\n var _$node$get = $node.get(0),\n tagName = _$node$get.tagName;\n\n if (tagName === 'span') {\n // convert spans to divs\n convertNodeTo$$1($node, $, 'div');\n }\n }\n}\n\nfunction addScoreTo($node, $, score) {\n if ($node) {\n convertSpans$1($node, $);\n addScore$$1($node, $, score);\n }\n}\n\nfunction scorePs($, weightNodes) {\n $('p, pre').not('[score]').each(function (index, node) {\n // The raw score for this paragraph, before we add any parent/child\n // scores.\n var $node = $(node);\n $node = setScore($node, $, getOrInitScore$$1($node, $, weightNodes));\n\n var $parent = $node.parent();\n var rawScore = scoreNode$$1($node);\n\n addScoreTo($parent, $, rawScore, weightNodes);\n if ($parent) {\n // Add half of the individual content score to the\n // grandparent\n addScoreTo($parent.parent(), $, rawScore / 2, weightNodes);\n }\n });\n\n return $;\n}\n\n// score content. Parents get the full value of their children's\n// content score, grandparents half\nfunction scoreContent$$1($) {\n var weightNodes = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : true;\n\n // First, look for special hNews based selectors and give them a big\n // boost, if they exist\n HNEWS_CONTENT_SELECTORS$1.forEach(function (_ref) {\n var _ref2 = _slicedToArray(_ref, 2),\n parentSelector = _ref2[0],\n childSelector = _ref2[1];\n\n $(parentSelector + ' ' + childSelector).each(function (index, node) {\n addScore$$1($(node).parent(parentSelector), $, 80);\n });\n });\n\n // Doubling this again\n // Previous solution caused a bug\n // in which parents weren't retaining\n // scores. This is not ideal, and\n // should be fixed.\n scorePs($, weightNodes);\n scorePs($, weightNodes);\n\n return $;\n}\n\n// Now that we have a top_candidate, look through the siblings of\n// it to see if any of them are decently scored. If they are, they\n// may be split parts of the content (Like two divs, a preamble and\n// a body.) Example:\n// http://articles.latimes.com/2009/oct/14/business/fi-bigtvs14\nfunction mergeSiblings($candidate, topScore, $) {\n if (!$candidate.parent().length) {\n return $candidate;\n }\n\n var siblingScoreThreshold = Math.max(10, topScore * 0.25);\n var wrappingDiv = $('');\n\n $candidate.parent().children().each(function (index, sibling) {\n var $sibling = $(sibling);\n // Ignore tags like BR, HR, etc\n if (NON_TOP_CANDIDATE_TAGS_RE$1.test(sibling.tagName)) {\n return null;\n }\n\n var siblingScore = getScore($sibling);\n if (siblingScore) {\n if ($sibling.get(0) === $candidate.get(0)) {\n wrappingDiv.append($sibling);\n } else {\n var contentBonus = 0;\n var density = linkDensity($sibling);\n\n // If sibling has a very low link density,\n // give it a small bonus\n if (density < 0.05) {\n contentBonus += 20;\n }\n\n // If sibling has a high link density,\n // give it a penalty\n if (density >= 0.5) {\n contentBonus -= 20;\n }\n\n // If sibling node has the same class as\n // candidate, give it a bonus\n if ($sibling.attr('class') === $candidate.attr('class')) {\n contentBonus += topScore * 0.2;\n }\n\n var newScore = siblingScore + contentBonus;\n\n if (newScore >= siblingScoreThreshold) {\n return wrappingDiv.append($sibling);\n } else if (sibling.tagName === 'p') {\n var siblingContent = $sibling.text();\n var siblingContentLength = textLength(siblingContent);\n\n if (siblingContentLength > 80 && density < 0.25) {\n return wrappingDiv.append($sibling);\n } else if (siblingContentLength <= 80 && density === 0 && hasSentenceEnd(siblingContent)) {\n return wrappingDiv.append($sibling);\n }\n }\n }\n }\n\n return null;\n });\n\n if (wrappingDiv.children().length === 1 && wrappingDiv.children().first().get(0) === $candidate.get(0)) {\n return $candidate;\n }\n\n return wrappingDiv;\n}\n\n// After we've calculated scores, loop through all of the possible\n// candidate nodes we found and find the one with the highest score.\nfunction findTopCandidate$$1($) {\n var $candidate = void 0;\n var topScore = 0;\n\n $('[score]').each(function (index, node) {\n // Ignore tags like BR, HR, etc\n if (NON_TOP_CANDIDATE_TAGS_RE$1.test(node.tagName)) {\n return;\n }\n\n var $node = $(node);\n var score = getScore($node);\n\n if (score > topScore) {\n topScore = score;\n $candidate = $node;\n }\n });\n\n // If we don't have a candidate, return the body\n // or whatever the first element is\n if (!$candidate) {\n return $('body') || $('*').first();\n }\n\n $candidate = mergeSiblings($candidate, topScore, $);\n\n return $candidate;\n}\n\n// Scoring\n\nfunction removeUnlessContent($node, $, weight) {\n // Explicitly save entry-content-asset tags, which are\n // noted as valuable in the Publisher guidelines. For now\n // this works everywhere. We may want to consider making\n // this less of a sure-thing later.\n if ($node.hasClass('entry-content-asset')) {\n return;\n }\n\n var content = normalizeSpaces($node.text());\n\n if (scoreCommas(content) < 10) {\n var pCount = $('p', $node).length;\n var inputCount = $('input', $node).length;\n\n // Looks like a form, too many inputs.\n if (inputCount > pCount / 3) {\n $node.remove();\n return;\n }\n\n var contentLength = content.length;\n var imgCount = $('img', $node).length;\n\n // Content is too short, and there are no images, so\n // this is probably junk content.\n if (contentLength < 25 && imgCount === 0) {\n $node.remove();\n return;\n }\n\n var density = linkDensity($node);\n\n // Too high of link density, is probably a menu or\n // something similar.\n // console.log(weight, density, contentLength)\n if (weight < 25 && density > 0.2 && contentLength > 75) {\n $node.remove();\n return;\n }\n\n // Too high of a link density, despite the score being\n // high.\n if (weight >= 25 && density > 0.5) {\n // Don't remove the node if it's a list and the\n // previous sibling starts with a colon though. That\n // means it's probably content.\n var tagName = $node.get(0).tagName.toLowerCase();\n var nodeIsList = tagName === 'ol' || tagName === 'ul';\n if (nodeIsList) {\n var previousNode = $node.prev();\n if (previousNode && normalizeSpaces(previousNode.text()).slice(-1) === ':') {\n return;\n }\n }\n\n $node.remove();\n return;\n }\n\n var scriptCount = $('script', $node).length;\n\n // Too many script tags, not enough content.\n if (scriptCount > 0 && contentLength < 150) {\n $node.remove();\n return;\n }\n }\n}\n\n// Given an article, clean it of some superfluous content specified by\n// tags. Things like forms, ads, etc.\n//\n// Tags is an array of tag name's to search through. (like div, form,\n// etc)\n//\n// Return this same doc.\nfunction cleanTags$$1($article, $) {\n $(CLEAN_CONDITIONALLY_TAGS, $article).each(function (index, node) {\n var $node = $(node);\n var weight = getScore($node);\n if (!weight) {\n weight = getOrInitScore$$1($node, $);\n setScore($node, $, weight);\n }\n\n // drop node if its weight is < 0\n if (weight < 0) {\n $node.remove();\n } else {\n // deteremine if node seems like content\n removeUnlessContent($node, $, weight);\n }\n });\n\n return $;\n}\n\nfunction cleanHeaders($article, $) {\n var title = arguments.length > 2 && arguments[2] !== undefined ? arguments[2] : '';\n\n $(HEADER_TAG_LIST, $article).each(function (index, header) {\n var $header = $(header);\n // Remove any headers that appear before all other p tags in the\n // document. This probably means that it was part of the title, a\n // subtitle or something else extraneous like a datestamp or byline,\n // all of which should be handled by other metadata handling.\n if ($($header, $article).prevAll('p').length === 0) {\n return $header.remove();\n }\n\n // Remove any headers that match the title exactly.\n if (normalizeSpaces($(header).text()) === title) {\n return $header.remove();\n }\n\n // If this header has a negative weight, it's probably junk.\n // Get rid of it.\n if (getWeight($(header)) < 0) {\n return $header.remove();\n }\n\n return $header;\n });\n\n return $;\n}\n\n// Rewrite the tag name to div if it's a top level node like body or\n// html to avoid later complications with multiple body tags.\nfunction rewriteTopLevel$$1(article, $) {\n // I'm not using context here because\n // it's problematic when converting the\n // top-level/root node - AP\n $ = convertNodeTo$$1($('html'), $, 'div');\n $ = convertNodeTo$$1($('body'), $, 'div');\n\n return $;\n}\n\nfunction absolutize($, rootUrl, attr, $content) {\n $('[' + attr + ']', $content).each(function (_, node) {\n var attrs = getAttrs(node);\n var url = attrs[attr];\n\n if (url) {\n var absoluteUrl = URL.resolve(rootUrl, url);\n setAttr(node, attr, absoluteUrl);\n }\n });\n}\n\nfunction makeLinksAbsolute$$1($content, $, url) {\n ['href', 'src'].forEach(function (attr) {\n return absolutize($, url, attr, $content);\n });\n\n return $content;\n}\n\nfunction textLength(text) {\n return text.trim().replace(/\\s+/g, ' ').length;\n}\n\n// Determines what percentage of the text\n// in a node is link text\n// Takes a node, returns a float\nfunction linkDensity($node) {\n var totalTextLength = textLength($node.text());\n\n var linkText = $node.find('a').text();\n var linkLength = textLength(linkText);\n\n if (totalTextLength > 0) {\n return linkLength / totalTextLength;\n } else if (totalTextLength === 0 && linkLength > 0) {\n return 1;\n }\n\n return 0;\n}\n\n// Given a node type to search for, and a list of meta tag names to\n// search for, find a meta tag associated.\nfunction extractFromMeta$$1($, metaNames, cachedNames) {\n var cleanTags$$1 = arguments.length > 3 && arguments[3] !== undefined ? arguments[3] : true;\n\n var foundNames = metaNames.filter(function (name) {\n return cachedNames.indexOf(name) !== -1;\n });\n\n var _iteratorNormalCompletion = true;\n var _didIteratorError = false;\n var _iteratorError = undefined;\n\n try {\n var _loop = function _loop() {\n var name = _step.value;\n\n var type = 'name';\n var value = 'value';\n\n var nodes = $('meta[' + type + '=\"' + name + '\"]');\n\n // Get the unique value of every matching node, in case there\n // are two meta tags with the same name and value.\n // Remove empty values.\n var values = nodes.map(function (index, node) {\n return $(node).attr(value);\n }).toArray().filter(function (text) {\n return text !== '';\n });\n\n // If we have more than one value for the same name, we have a\n // conflict and can't trust any of them. Skip this name. If we have\n // zero, that means our meta tags had no values. Skip this name\n // also.\n if (values.length === 1) {\n var metaValue = void 0;\n // Meta values that contain HTML should be stripped, as they\n // weren't subject to cleaning previously.\n if (cleanTags$$1) {\n metaValue = stripTags(values[0], $);\n } else {\n metaValue = values[0];\n }\n\n return {\n v: metaValue\n };\n }\n };\n\n for (var _iterator = _getIterator(foundNames), _step; !(_iteratorNormalCompletion = (_step = _iterator.next()).done); _iteratorNormalCompletion = true) {\n var _ret = _loop();\n\n if ((typeof _ret === 'undefined' ? 'undefined' : _typeof(_ret)) === \"object\") return _ret.v;\n }\n\n // If nothing is found, return null\n } catch (err) {\n _didIteratorError = true;\n _iteratorError = err;\n } finally {\n try {\n if (!_iteratorNormalCompletion && _iterator.return) {\n _iterator.return();\n }\n } finally {\n if (_didIteratorError) {\n throw _iteratorError;\n }\n }\n }\n\n return null;\n}\n\nfunction isGoodNode($node, maxChildren) {\n // If it has a number of children, it's more likely a container\n // element. Skip it.\n if ($node.children().length > maxChildren) {\n return false;\n }\n // If it looks to be within a comment, skip it.\n if (withinComment$$1($node)) {\n return false;\n }\n\n return true;\n}\n\n// Given a a list of selectors find content that may\n// be extractable from the document. This is for flat\n// meta-information, like author, title, date published, etc.\nfunction extractFromSelectors$$1($, selectors) {\n var maxChildren = arguments.length > 2 && arguments[2] !== undefined ? arguments[2] : 1;\n var textOnly = arguments.length > 3 && arguments[3] !== undefined ? arguments[3] : true;\n var _iteratorNormalCompletion = true;\n var _didIteratorError = false;\n var _iteratorError = undefined;\n\n try {\n for (var _iterator = _getIterator(selectors), _step; !(_iteratorNormalCompletion = (_step = _iterator.next()).done); _iteratorNormalCompletion = true) {\n var selector = _step.value;\n\n var nodes = $(selector);\n\n // If we didn't get exactly one of this selector, this may be\n // a list of articles or comments. Skip it.\n if (nodes.length === 1) {\n var $node = $(nodes[0]);\n\n if (isGoodNode($node, maxChildren)) {\n var content = void 0;\n if (textOnly) {\n content = $node.text();\n } else {\n content = $node.html();\n }\n\n if (content) {\n return content;\n }\n }\n }\n }\n } catch (err) {\n _didIteratorError = true;\n _iteratorError = err;\n } finally {\n try {\n if (!_iteratorNormalCompletion && _iterator.return) {\n _iterator.return();\n }\n } finally {\n if (_didIteratorError) {\n throw _iteratorError;\n }\n }\n }\n\n return null;\n}\n\n// strips all tags from a string of text\nfunction stripTags(text, $) {\n // Wrapping text in html element prevents errors when text\n // has no html\n var cleanText = $('' + text + '').text();\n return cleanText === '' ? text : cleanText;\n}\n\nfunction withinComment$$1($node) {\n var parents = $node.parents().toArray();\n var commentParent = parents.find(function (parent) {\n var attrs = getAttrs(parent);\n var nodeClass = attrs.class,\n id = attrs.id;\n\n var classAndId = nodeClass + ' ' + id;\n return classAndId.includes('comment');\n });\n\n return commentParent !== undefined;\n}\n\n// Given a node, determine if it's article-like enough to return\n// param: node (a cheerio node)\n// return: boolean\n\nfunction nodeIsSufficient($node) {\n return $node.text().trim().length >= 100;\n}\n\nfunction isWordpress($) {\n return $(IS_WP_SELECTOR).length > 0;\n}\n\nfunction getAttrs(node) {\n var attribs = node.attribs,\n attributes = node.attributes;\n\n\n if (!attribs && attributes) {\n var attrs = _Reflect$ownKeys(attributes).reduce(function (acc, index) {\n var attr = attributes[index];\n\n if (!attr.name || !attr.value) return acc;\n\n acc[attr.name] = attr.value;\n return acc;\n }, {});\n return attrs;\n }\n\n return attribs;\n}\n\nfunction setAttr(node, attr, val) {\n if (node.attribs) {\n node.attribs[attr] = val;\n } else if (node.attributes) {\n node.setAttribute(attr, val);\n }\n\n return node;\n}\n\nfunction setAttrs(node, attrs) {\n if (node.attribs) {\n node.attribs = attrs;\n } else if (node.attributes) {\n while (node.attributes.length > 0) {\n node.removeAttribute(node.attributes[0].name);\n }\n\n _Reflect$ownKeys(attrs).forEach(function (key) {\n node.setAttribute(key, attrs[key]);\n });\n }\n\n return node;\n}\n\n// DOM manipulation\n\nvar IS_LINK = new RegExp('https?://', 'i');\nvar IS_IMAGE = new RegExp('.(png|gif|jpe?g)', 'i');\n\nvar TAGS_TO_REMOVE = ['script', 'style', 'form'].join(',');\n\n// Convert all instances of images with potentially\n// lazy loaded images into normal images.\n// Many sites will have img tags with no source, or an image tag with a src\n// attribute that a is a placeholer. We need to be able to properly fill in\n// the src attribute so the images are no longer lazy loaded.\nfunction convertLazyLoadedImages($) {\n $('img').each(function (_, img) {\n var attrs = getAttrs(img);\n\n _Reflect$ownKeys(attrs).forEach(function (attr) {\n var value = attrs[attr];\n\n if (attr !== 'src' && IS_LINK.test(value) && IS_IMAGE.test(value)) {\n $(img).attr('src', value);\n }\n });\n });\n\n return $;\n}\n\nfunction isComment(index, node) {\n return node.type === 'comment';\n}\n\nfunction cleanComments($) {\n $.root().find('*').contents().filter(isComment).remove();\n\n return $;\n}\n\nfunction clean($) {\n $(TAGS_TO_REMOVE).remove();\n\n $ = cleanComments($);\n return $;\n}\n\nvar Resource = {\n\n // Create a Resource.\n //\n // :param url: The URL for the document we should retrieve.\n // :param response: If set, use as the response rather than\n // attempting to fetch it ourselves. Expects a\n // string.\n create: function create(url, preparedResponse, parsedUrl) {\n var _this = this;\n\n return _asyncToGenerator(_regeneratorRuntime.mark(function _callee() {\n var result, validResponse;\n return _regeneratorRuntime.wrap(function _callee$(_context) {\n while (1) {\n switch (_context.prev = _context.next) {\n case 0:\n result = void 0;\n\n if (!preparedResponse) {\n _context.next = 6;\n break;\n }\n\n validResponse = {\n statusMessage: 'OK',\n statusCode: 200,\n headers: {\n 'content-type': 'text/html',\n 'content-length': 500\n }\n };\n\n\n result = { body: preparedResponse, response: validResponse };\n _context.next = 9;\n break;\n\n case 6:\n _context.next = 8;\n return fetchResource$1(url, parsedUrl);\n\n case 8:\n result = _context.sent;\n\n case 9:\n if (!result.error) {\n _context.next = 12;\n break;\n }\n\n result.failed = true;\n return _context.abrupt('return', result);\n\n case 12:\n return _context.abrupt('return', _this.generateDoc(result));\n\n case 13:\n case 'end':\n return _context.stop();\n }\n }\n }, _callee, _this);\n }))();\n },\n generateDoc: function generateDoc(_ref) {\n var content = _ref.body,\n response = _ref.response;\n var contentType = response.headers['content-type'];\n\n // TODO: Implement is_text function from\n // https://github.com/ReadabilityHoldings/readability/blob/8dc89613241d04741ebd42fa9fa7df1b1d746303/readability/utils/text.py#L57\n\n if (!contentType.includes('html') && !contentType.includes('text')) {\n throw new Error('Content does not appear to be text.');\n }\n\n var $ = cheerio.load(content, { normalizeWhitespace: true });\n\n if ($.root().children().length === 0) {\n throw new Error('No children, likely a bad parse.');\n }\n\n $ = normalizeMetaTags($);\n $ = convertLazyLoadedImages($);\n $ = clean($);\n\n return $;\n }\n};\n\nvar merge = function merge(extractor, domains) {\n return domains.reduce(function (acc, domain) {\n acc[domain] = extractor;\n return acc;\n }, {});\n};\n\nfunction mergeSupportedDomains(extractor) {\n return extractor.supportedDomains ? merge(extractor, [extractor.domain].concat(_toConsumableArray(extractor.supportedDomains))) : merge(extractor, [extractor.domain]);\n}\n\nvar BloggerExtractor = {\n domain: 'blogspot.com',\n content: {\n // Blogger is insane and does not load its content\n // initially in the page, but it's all there\n // in noscript\n selectors: ['.post-content noscript'],\n\n // Selectors to remove from the extracted content\n clean: [],\n\n // Convert the noscript tag to a div\n transforms: {\n noscript: 'div'\n }\n },\n\n author: {\n selectors: ['.post-author-name']\n },\n\n title: {\n selectors: ['.post h2.title']\n },\n\n date_published: {\n selectors: ['span.publishdate']\n }\n};\n\nvar NYMagExtractor = {\n domain: 'nymag.com',\n content: {\n // Order by most likely. Extractor will stop on first occurrence\n selectors: ['div.article-content', 'section.body', 'article.article'],\n\n // Selectors to remove from the extracted content\n clean: ['.ad', '.single-related-story'],\n\n // Object of tranformations to make on matched elements\n // Each key is the selector, each value is the tag to\n // transform to.\n // If a function is given, it should return a string\n // to convert to or nothing (in which case it will not perform\n // the transformation.\n transforms: {\n // Convert h1s to h2s\n h1: 'h2',\n\n // Convert lazy-loaded noscript images to figures\n noscript: function noscript($node, $) {\n var $children = $.browser ? $($node.text()) : $node.children();\n if ($children.length === 1 && $children.get(0) !== undefined && $children.get(0).tagName.toLowerCase() === 'img') {\n return 'figure';\n }\n\n return null;\n }\n }\n },\n\n title: {\n selectors: ['h1.lede-feature-title', 'h1.headline-primary', 'h1']\n },\n\n author: {\n selectors: ['.by-authors', '.lede-feature-author']\n },\n\n dek: {\n selectors: ['.lede-feature-teaser']\n },\n\n date_published: {\n selectors: [['time.article-timestamp[datetime]', 'datetime'], 'time.article-timestamp']\n }\n};\n\nvar WikipediaExtractor = {\n domain: 'wikipedia.org',\n content: {\n selectors: ['#mw-content-text'],\n\n defaultCleaner: false,\n\n // transform top infobox to an image with caption\n transforms: {\n '.infobox img': function infoboxImg($node) {\n var $parent = $node.parents('.infobox');\n // Only prepend the first image in .infobox\n if ($parent.children('img').length === 0) {\n $parent.prepend($node);\n }\n },\n '.infobox caption': 'figcaption',\n '.infobox': 'figure'\n },\n\n // Selectors to remove from the extracted content\n clean: ['.mw-editsection', 'figure tr, figure td, figure tbody', '#toc', '.navbox']\n\n },\n\n author: 'Wikipedia Contributors',\n\n title: {\n selectors: ['h2.title']\n },\n\n date_published: {\n selectors: ['#footer-info-lastmod']\n }\n\n};\n\nvar TwitterExtractor = {\n domain: 'twitter.com',\n\n content: {\n transforms: {\n // We're transforming essentially the whole page here.\n // Twitter doesn't have nice selectors, so our initial\n // selector grabs the whole page, then we're re-writing\n // it to fit our needs before we clean it up.\n '.permalink[role=main]': function permalinkRoleMain($node, $) {\n var tweets = $node.find('.tweet');\n var $tweetContainer = $('');\n $tweetContainer.append(tweets);\n $node.replaceWith($tweetContainer);\n },\n\n // Twitter wraps @ with s, which\n // renders as a strikethrough\n s: 'span'\n },\n\n selectors: ['.permalink[role=main]'],\n\n defaultCleaner: false,\n\n clean: ['.stream-item-footer', 'button', '.tweet-details-fixer']\n },\n\n author: {\n selectors: ['.tweet.permalink-tweet .username']\n },\n\n date_published: {\n selectors: [['.permalink-tweet ._timestamp[data-time-ms]', 'data-time-ms']]\n }\n\n};\n\nvar NYTimesExtractor = {\n domain: 'www.nytimes.com',\n\n title: {\n selectors: ['.g-headline', 'h1.headline']\n },\n\n author: {\n selectors: [['meta[name=\"author\"]', 'value'], '.g-byline', '.byline']\n },\n\n content: {\n selectors: ['div.g-blocks', 'article#story'],\n\n transforms: {\n 'img.g-lazy': function imgGLazy($node) {\n var src = $node.attr('src');\n // const widths = $node.attr('data-widths')\n // .slice(1)\n // .slice(0, -1)\n // .split(',');\n // if (widths.length) {\n // width = widths.slice(-1);\n // } else {\n // width = '900';\n // }\n var width = 640;\n\n src = src.replace('{{size}}', width);\n $node.attr('src', src);\n }\n },\n\n clean: ['.ad', 'header#story-header', '.story-body-1 .lede.video', '.visually-hidden', '#newsletter-promo', '.promo', '.comments-button', '.hidden', '.comments', '.supplemental', '.nocontent', '.story-footer-links']\n },\n\n date_published: {\n selectors: [['meta[name=\"article:published\"]', 'value']]\n },\n\n lead_image_url: {\n selectors: [['meta[name=\"og:image\"]', 'value']]\n },\n\n dek: null,\n\n next_page_url: null,\n\n excerpt: null\n};\n\n// Rename CustomExtractor\n// to fit your publication\nvar TheAtlanticExtractor = {\n domain: 'www.theatlantic.com',\n title: {\n selectors: ['h1.hed']\n },\n\n author: {\n selectors: ['article#article .article-cover-extra .metadata .byline a']\n },\n\n content: {\n selectors: [['.article-cover figure.lead-img', '.article-body'], '.article-body'],\n\n // Is there anything in the content you selected that needs transformed\n // before it's consumable content? E.g., unusual lazy loaded images\n transforms: [],\n\n // Is there anything that is in the result that shouldn't be?\n // The clean selectors will remove anything that matches from\n // the result\n clean: ['.partner-box']\n },\n\n date_published: {\n selectors: [['time[itemProp=\"datePublished\"]', 'datetime']]\n },\n\n lead_image_url: null,\n\n next_page_url: null,\n\n excerpt: null\n};\n\n// Rename CustomExtractor\n// to fit your publication\n// (e.g., NYTimesExtractor)\nvar NewYorkerExtractor = {\n domain: 'www.newyorker.com',\n title: {\n selectors: ['h1.title']\n },\n\n author: {\n selectors: ['.contributors']\n },\n\n content: {\n selectors: ['div#articleBody', 'div.articleBody'],\n\n // Is there anything in the content you selected that needs transformed\n // before it's consumable content? E.g., unusual lazy loaded images\n transforms: [],\n\n // Is there anything that is in the result that shouldn't be?\n // The clean selectors will remove anything that matches from\n // the result\n clean: []\n },\n\n date_published: {\n selectors: [['meta[name=\"article:published_time\"]', 'value'], ['time[itemProp=\"datePublished\"]', 'content']],\n\n timezone: 'America/New_York'\n },\n\n lead_image_url: {\n selectors: [['meta[name=\"og:image\"]', 'value']]\n },\n\n dek: {\n selectors: ['.dek', 'h2.dek']\n },\n\n next_page_url: null,\n\n excerpt: null\n};\n\n// Rename CustomExtractor\n// to fit your publication\n// (e.g., NYTimesExtractor)\nvar WiredExtractor = {\n domain: 'www.wired.com',\n title: {\n selectors: ['h1.post-title']\n },\n\n author: {\n selectors: ['a[rel=\"author\"]']\n },\n\n content: {\n selectors: ['article.content'],\n\n // Is there anything in the content you selected that needs transformed\n // before it's consumable content? E.g., unusual lazy loaded images\n transforms: [],\n\n // Is there anything that is in the result that shouldn't be?\n // The clean selectors will remove anything that matches from\n // the result\n clean: ['.visually-hidden']\n },\n\n date_published: {\n selectors: [['meta[itemprop=\"datePublished\"]', 'value']]\n },\n\n lead_image_url: {\n selectors: [['meta[name=\"og:image\"]', 'value']]\n },\n\n dek: {\n selectors: []\n },\n\n next_page_url: null,\n\n excerpt: null\n};\n\n// Rename CustomExtractor\n// to fit your publication\n// (e.g., NYTimesExtractor)\nvar MSNExtractor = {\n domain: 'www.msn.com',\n title: {\n selectors: ['h1']\n },\n\n author: {\n selectors: ['span.authorname-txt']\n },\n\n content: {\n selectors: ['div.richtext'],\n\n // Is there anything in the content you selected that needs transformed\n // before it's consumable content? E.g., unusual lazy loaded images\n transforms: [],\n\n // Is there anything that is in the result that shouldn't be?\n // The clean selectors will remove anything that matches from\n // the result\n clean: ['span.caption']\n },\n\n date_published: {\n selectors: ['span.time']\n },\n\n lead_image_url: {\n selectors: []\n },\n\n dek: {\n selectors: []\n },\n\n next_page_url: null,\n\n excerpt: null\n};\n\n// Rename CustomExtractor\n// to fit your publication\n// (e.g., NYTimesExtractor)\nvar YahooExtractor = {\n domain: 'www.yahoo.com',\n title: {\n selectors: ['header.canvas-header']\n },\n\n author: {\n selectors: ['span.provider-name']\n },\n\n content: {\n selectors: [\n // enter content selectors\n '.content-canvas'],\n\n // Is there anything in the content you selected that needs transformed\n // before it's consumable content? E.g., unusual lazy loaded images\n transforms: [],\n\n // Is there anything that is in the result that shouldn't be?\n // The clean selectors will remove anything that matches from\n // the result\n clean: ['.figure-caption']\n },\n\n date_published: {\n selectors: [['time.date[datetime]', 'datetime']]\n },\n\n lead_image_url: {\n selectors: [['meta[name=\"og:image\"]', 'value']]\n },\n\n dek: {\n selectors: [\n // enter dek selectors\n ]\n },\n\n next_page_url: null,\n\n excerpt: null\n};\n\n// Rename CustomExtractor\n// to fit your publication\n// (e.g., NYTimesExtractor)\nvar BuzzfeedExtractor = {\n domain: 'www.buzzfeed.com',\n title: {\n selectors: ['h1[id=\"post-title\"]']\n },\n\n author: {\n selectors: ['a[data-action=\"user/username\"]', 'byline__author']\n },\n\n content: {\n selectors: [['.longform_custom_header_media', '#buzz_sub_buzz'], '#buzz_sub_buzz'],\n\n defaultCleaner: false,\n\n // Is there anything in the content you selected that needs transformed\n // before it's consumable content? E.g., unusual lazy loaded images\n transforms: {\n h2: 'b',\n\n 'div.longform_custom_header_media': function divLongform_custom_header_media($node) {\n if ($node.has('img') && $node.has('.longform_header_image_source')) {\n return 'figure';\n }\n\n return null;\n },\n\n 'figure.longform_custom_header_media .longform_header_image_source': 'figcaption'\n },\n\n // Is there anything that is in the result that shouldn't be?\n // The clean selectors will remove anything that matches from\n // the result\n clean: ['.instapaper_ignore', '.suplist_list_hide .buzz_superlist_item .buzz_superlist_number_inline', '.share-box', '.print']\n },\n\n date_published: {\n selectors: ['.buzz-datetime']\n },\n\n lead_image_url: {\n selectors: [['meta[name=\"og:image\"]', 'value']]\n },\n\n dek: {\n selectors: []\n },\n\n next_page_url: null,\n\n excerpt: null\n};\n\n// Rename CustomExtractor\n// to fit your publication\n// (e.g., NYTimesExtractor)\nvar WikiaExtractor = {\n domain: 'fandom.wikia.com',\n title: {\n selectors: ['h1.entry-title']\n },\n\n author: {\n selectors: ['.author vcard', '.fn']\n },\n\n content: {\n selectors: ['.grid-content', '.entry-content'],\n\n // Is there anything in the content you selected that needs transformed\n // before it's consumable content? E.g., unusual lazy loaded images\n transforms: [],\n\n // Is there anything that is in the result that shouldn't be?\n // The clean selectors will remove anything that matches from\n // the result\n clean: []\n },\n\n date_published: {\n selectors: [['meta[name=\"article:published_time\"]', 'value']]\n },\n\n lead_image_url: {\n selectors: [['meta[name=\"og:image\"]', 'value']]\n },\n\n dek: {\n selectors: []\n },\n\n next_page_url: null,\n\n excerpt: null\n};\n\n// Rename CustomExtractor\n// to fit your publication\n// (e.g., NYTimesExtractor)\nvar LittleThingsExtractor = {\n domain: 'www.littlethings.com',\n title: {\n selectors: ['h1.post-title']\n },\n\n author: {\n selectors: [['meta[name=\"author\"]', 'value']]\n },\n\n content: {\n selectors: [\n // enter content selectors\n '.mainContentIntro', '.content-wrapper'],\n\n // Is there anything in the content you selected that needs transformed\n // before it's consumable content? E.g., unusual lazy loaded images\n transforms: [],\n\n // Is there anything that is in the result that shouldn't be?\n // The clean selectors will remove anything that matches from\n // the result\n clean: []\n },\n\n lead_image_url: {\n selectors: [['meta[name=\"og:image\"]', 'value']]\n },\n\n next_page_url: null,\n\n excerpt: null\n};\n\n// Rename CustomExtractor\n// to fit your publication\n// (e.g., NYTimesExtractor)\nvar PoliticoExtractor = {\n domain: 'www.politico.com',\n title: {\n selectors: [\n // enter title selectors\n ['meta[name=\"og:title\"]', 'value']]\n },\n\n author: {\n selectors: ['.story-main-content .byline .vcard']\n },\n\n content: {\n selectors: [\n // enter content selectors\n '.story-main-content', '.content-group', '.story-core', '.story-text'],\n\n // Is there anything in the content you selected that needs transformed\n // before it's consumable content? E.g., unusual lazy loaded images\n transforms: [],\n\n // Is there anything that is in the result that shouldn't be?\n // The clean selectors will remove anything that matches from\n // the result\n clean: ['figcaption']\n },\n\n date_published: {\n selectors: [['.story-main-content .timestamp time[datetime]', 'datetime']]\n },\n\n lead_image_url: {\n selectors: [\n // enter lead_image_url selectors\n ['meta[name=\"og:image\"]', 'value']]\n },\n\n dek: {\n selectors: []\n },\n\n next_page_url: null,\n\n excerpt: null\n};\n\nvar DeadspinExtractor = {\n domain: 'deadspin.com',\n\n supportedDomains: ['jezebel.com', 'lifehacker.com', 'kotaku.com', 'gizmodo.com', 'jalopnik.com', 'kinja.com'],\n\n title: {\n selectors: ['h1.headline']\n },\n\n author: {\n selectors: ['.author']\n },\n\n content: {\n selectors: ['.post-content', '.entry-content'],\n\n // Is there anything in the content you selected that needs transformed\n // before it's consumable content? E.g., unusual lazy loaded images\n transforms: {\n 'iframe.lazyload[data-recommend-id^=\"youtube://\"]': function iframeLazyloadDataRecommendIdYoutube($node) {\n var youtubeId = $node.attr('id').split('youtube-')[1];\n $node.attr('src', 'https://www.youtube.com/embed/' + youtubeId);\n }\n },\n\n // Is there anything that is in the result that shouldn't be?\n // The clean selectors will remove anything that matches from\n // the result\n clean: ['.magnifier', '.lightbox']\n },\n\n date_published: {\n selectors: [['time.updated[datetime]', 'datetime']]\n },\n\n lead_image_url: {\n selectors: [['meta[name=\"og:image\"]', 'value']]\n },\n\n dek: {\n selectors: [\n // enter selectors\n ]\n },\n\n next_page_url: {\n selectors: [\n // enter selectors\n ]\n },\n\n excerpt: {\n selectors: [\n // enter selectors\n ]\n }\n};\n\n// Rename CustomExtractor\n// to fit your publication\n// (e.g., NYTimesExtractor)\nvar BroadwayWorldExtractor = {\n domain: 'www.broadwayworld.com',\n title: {\n selectors: ['h1.article-title']\n },\n\n author: {\n selectors: ['span[itemprop=author]']\n },\n\n content: {\n selectors: ['div[itemprop=articlebody]'],\n\n // Is there anything in the content you selected that needs transformed\n // before it's consumable content? E.g., unusual lazy loaded images\n transforms: {},\n\n // Is there anything that is in the result that shouldn't be?\n // The clean selectors will remove anything that matches from\n // the result\n clean: []\n },\n\n date_published: {\n selectors: [['meta[itemprop=datePublished]', 'value']]\n },\n\n lead_image_url: {\n selectors: [['meta[name=\"og:image\"]', 'value']]\n },\n\n dek: {\n selectors: []\n },\n\n next_page_url: {\n selectors: [\n // enter selectors\n ]\n },\n\n excerpt: {\n selectors: [\n // enter selectors\n ]\n }\n};\n\n// Rename CustomExtractor\n// to fit your publication\n// (e.g., NYTimesExtractor)\nvar ApartmentTherapyExtractor = {\n domain: 'www.apartmenttherapy.com',\n title: {\n selectors: ['h1.headline']\n },\n\n author: {\n selectors: ['.PostByline__name']\n },\n\n content: {\n selectors: ['div.post__content'],\n\n // Is there anything in the content you selected that needs transformed\n // before it's consumable content? E.g., unusual lazy loaded images\n transforms: {\n 'div[data-render-react-id=\"images/LazyPicture\"]': function divDataRenderReactIdImagesLazyPicture($node, $) {\n var data = JSON.parse($node.attr('data-props'));\n var src = data.sources[0].src;\n\n var $img = $('