import { removeAnchor } from 'utils/text'; import RootExtractor from 'extractors/root-extractor'; import GenericExtractor from 'extractors/generic'; import Resource from 'resource'; export default async function collectAllPages( { next_page_url, html, $, metaCache, result, Extractor, title, url, } ) { // At this point, we've fetched just the first page let pages = 1; const previousUrls = [removeAnchor(url)]; // If we've gone over 26 pages, something has // likely gone wrong. while (next_page_url && pages < 26) { pages += 1; $ = await Resource.create(next_page_url); html = $.html(); const extractorOpts = { url: next_page_url, html, $, metaCache, contentOnly: true, extractedTitle: title, previousUrls, }; const nextPageResult = RootExtractor.extract(Extractor, extractorOpts); previousUrls.push(next_page_url); result = { ...result, content: `${result.content}