fix: custom parser generator

- swap fs import
- fix rollup config
fix-custom-generator
Adam Pash 5 years ago
parent 0e27448866
commit a60d5a6332

File diff suppressed because it is too large Load Diff

28
dist/mercury.js vendored

@ -4,6 +4,7 @@ function _interopDefault (ex) { return (ex && (typeof ex === 'object') && 'defau
var _regeneratorRuntime = _interopDefault(require('@babel/runtime-corejs2/regenerator')); var _regeneratorRuntime = _interopDefault(require('@babel/runtime-corejs2/regenerator'));
var _objectSpread = _interopDefault(require('@babel/runtime-corejs2/helpers/objectSpread')); var _objectSpread = _interopDefault(require('@babel/runtime-corejs2/helpers/objectSpread'));
var _objectWithoutProperties = _interopDefault(require('@babel/runtime-corejs2/helpers/objectWithoutProperties'));
var _asyncToGenerator = _interopDefault(require('@babel/runtime-corejs2/helpers/asyncToGenerator')); var _asyncToGenerator = _interopDefault(require('@babel/runtime-corejs2/helpers/asyncToGenerator'));
var URL = _interopDefault(require('url')); var URL = _interopDefault(require('url'));
var cheerio = _interopDefault(require('cheerio')); var cheerio = _interopDefault(require('cheerio'));
@ -178,13 +179,16 @@ function excerptContent(content) {
function getEncoding(str) { function getEncoding(str) {
var encoding = DEFAULT_ENCODING; var encoding = DEFAULT_ENCODING;
var matches = ENCODING_RE.exec(str);
if (ENCODING_RE.test(str)) { if (matches !== null) {
var testEncode = ENCODING_RE.exec(str)[1]; var _matches = _slicedToArray(matches, 2);
if (iconv.encodingExists(testEncode)) { str = _matches[1];
encoding = testEncode; }
}
if (iconv.encodingExists(str)) {
encoding = str;
} }
return encoding; return encoding;
@ -1701,10 +1705,10 @@ var Resource = {
var decodedContent = iconv.decode(content, encoding); var decodedContent = iconv.decode(content, encoding);
var $ = cheerio.load(decodedContent); // after first cheerio.load, check to see if encoding matches var $ = cheerio.load(decodedContent); // after first cheerio.load, check to see if encoding matches
var metaContentType = $('meta[http-equiv=content-type]').attr('content'); var metaContentType = $('meta[http-equiv=content-type i]').attr('content') || $('meta[charset]').attr('charset');
var properEncoding = getEncoding(metaContentType); // if encodings in the header/body dont match, use the one in the body var properEncoding = getEncoding(metaContentType); // if encodings in the header/body dont match, use the one in the body
if (properEncoding !== encoding) { if (metaContentType && properEncoding !== encoding) {
decodedContent = iconv.decode(content, properEncoding); decodedContent = iconv.decode(content, properEncoding);
$ = cheerio.load(decodedContent); $ = cheerio.load(decodedContent);
} }
@ -6429,8 +6433,10 @@ var Mercury = {
parse: function () { parse: function () {
var _parse = _asyncToGenerator( var _parse = _asyncToGenerator(
/*#__PURE__*/ /*#__PURE__*/
_regeneratorRuntime.mark(function _callee(url, html) { _regeneratorRuntime.mark(function _callee(url) {
var opts, var _ref,
html,
opts,
_opts$fetchAllPages, _opts$fetchAllPages,
fetchAllPages, fetchAllPages,
_opts$fallback, _opts$fallback,
@ -6451,7 +6457,7 @@ var Mercury = {
while (1) { while (1) {
switch (_context.prev = _context.next) { switch (_context.prev = _context.next) {
case 0: case 0:
opts = _args.length > 2 && _args[2] !== undefined ? _args[2] : {}; _ref = _args.length > 1 && _args[1] !== undefined ? _args[1] : {}, html = _ref.html, opts = _objectWithoutProperties(_ref, ["html"]);
_opts$fetchAllPages = opts.fetchAllPages, fetchAllPages = _opts$fetchAllPages === void 0 ? true : _opts$fetchAllPages, _opts$fallback = opts.fallback, fallback = _opts$fallback === void 0 ? true : _opts$fallback, _opts$contentType = opts.contentType, contentType = _opts$contentType === void 0 ? 'html' : _opts$contentType; // if no url was passed and this is the browser version, _opts$fetchAllPages = opts.fetchAllPages, fetchAllPages = _opts$fetchAllPages === void 0 ? true : _opts$fetchAllPages, _opts$fallback = opts.fallback, fallback = _opts$fallback === void 0 ? true : _opts$fallback, _opts$contentType = opts.contentType, contentType = _opts$contentType === void 0 ? 'html' : _opts$contentType; // if no url was passed and this is the browser version,
// set url to window.location.href and load the html // set url to window.location.href and load the html
// from the current page // from the current page
@ -6549,7 +6555,7 @@ var Mercury = {
}, _callee, this); }, _callee, this);
})); }));
function parse(_x, _x2) { function parse(_x) {
return _parse.apply(this, arguments); return _parse.apply(this, arguments);
} }

@ -1,15 +1,20 @@
/* eslint-disable import/no-extraneous-dependencies */ /* eslint-disable import/no-extraneous-dependencies */
import babel from 'rollup-plugin-babel'; import babel from 'rollup-plugin-babel';
import babelrc from 'babelrc-rollup'; // eslint-disable-line import/extensions
import commonjs from 'rollup-plugin-commonjs'; import commonjs from 'rollup-plugin-commonjs';
const babelOpts = babelrc();
babelOpts.runtimeHelpers = true;
export default { export default {
entry: './scripts/generate-custom-parser.js', input: 'scripts/generate-custom-parser.js',
plugins: [commonjs(), babel(babelOpts)], plugins: [
format: 'cjs', commonjs(),
dest: 'dist/generate-custom-parser.js', // equivalent to --output babel({
sourceMap: true, externalHelpers: false,
runtimeHelpers: true,
}),
],
treeshake: true,
output: {
file: 'dist/generate-custom-parser.js',
format: 'cjs',
sourceMap: true,
},
}; };

Loading…
Cancel
Save