mercury-parser/scripts/update-fixtures.js

/* eslint-disable */

const { execFile, execFileSync } = require('child_process');
const fs = require('fs');
const path = require('path');
const URL = require('url');
const octokit = require('@octokit/rest')();

const Mercury = require('../dist/mercury');

// get all fixtures
execFile('find', ['fixtures', '-type', 'f'], (err, stdout) => {
  const fixtures = stdout.split('\n');

  const now = new Date();
  const twoWeeks = 2 * 7 * 24 * 60 * 60 * 1000;

  // iterate through fixtures for fixtures older than 2 weeks
  console.log('Finding fixtures to update...');
  const fixturesToUpdate = fixtures
    .filter(fixture => {
      const timestamp = path
        .basename(fixture)
        .split(/\.html$/)[0]
        .trim();
      try {
        const date = new Date(parseInt(timestamp, 10));
        return now - date > twoWeeks;
      } catch (e) {
        // if fixture isn't a timestamp, ignore it
        return false;
      }
    })
    .slice(0, 1);
  console.log(`${fixturesToUpdate.length} fixtures are out of date`);

  // iterate through fixtures and extract their URLs.
  console.log('Extracting urls...');
  const baseDomains = fixturesToUpdate.map(fixture => fixture.split('/')[1]);
  Promise.all(
    fixturesToUpdate.map((fixture, i) => {
      const html = fs.readFileSync(fixture);
      return Mercury.parse(`http://${baseDomains[i]}`, { html });
    })
  ).then(parsedFixture => {
    const fixturesAndUrls = fixturesToUpdate.reduce(
      (acc, fixture, i) =>
        acc.concat({
          fixture,
          url: parsedFixture[i].url,
          baseDomain: baseDomains[i],
        }),
      []
    );

    console.log('Updating all fixtures');
    const fns = fixturesAndUrls
      .map(fixtureAndUrl => {
        return () => {
          // console.log('Updating fixture for', fixtureAndUrl);
          return updateFixture(fixtureAndUrl);
        };
      })
      .concat(() => {
        return new Promise(res => {
          console.log('changed bases', changeBase);
          console.log(`otherMess`, otherMess);
          res();
        });
      });
    promiseSerial(fns);
  });
});

const changeBase = [];
const otherMess = [];
const updateFixture = ({ fixture, url, baseDomain }) => {
  return new Promise(res => {
    Mercury.parse(url)
      .then(({ url: updatedUrl }) => {
        if (!updatedUrl) {
          otherMess.push({ updatedUrl, url, fixture, baseDomain });
          return res();
        }
        console.log(`updatedUrl`, updatedUrl);
        const { hostname } = URL.parse(updatedUrl);
        if (hostname !== baseDomain) {
          console.log('Base URL has changed!!! Do something different');
          console.log(`url`, url);
          console.log(`updatedUrl`, updatedUrl);
          console.log(`hostname`, hostname);
          changeBase.push({
            fixture,
            url,
            baseDomain,
            newBaseDomain: hostname,
            updatedUrl,
          });
          return res();
        }
        execFile('yarn', ['generate-parser', url], (err, stdout) => {
          // console.log(`stdout`, stdout);
          const dirRe = new RegExp(`(${path.dirname(fixture)}\/\\d+\.html)`);
          const newFixture = stdout.match(dirRe)[0];

          console.log(`newFixture`, newFixture);
          // replace old fixture with new fixture in tests
          execFile(
            './scripts/find-and-replace.sh',
            [fixture, newFixture, 'src/extractors/custom/**/*.test.js'],
            (err, stdout) => {
              // remove old fixture
              fs.unlinkSync(fixture);
              const { branchName, commitMessage } = doTestsPass(baseDomain)
                ? {
                    branchName: `chore-update-${baseDomain}-fixture`,
                    commitMessage: `chore: update ${baseDomain} fixture`,
                  }
                : {
                    branchName: `fix-update-${baseDomain}-extractor`,
                    commitMessage: `fix: update ${baseDomain} extractor`,
                  };

              createAndPushBranch({ branchName, commitMessage });
              createPR({ branchName, title: commitMessage });
            }
          );
        });
      })
      .catch(e => {
        otherMess.push({ fixture, url, baseDomain, e });
      });
  });
};

const doTestsPass = site => {
  try {
    execFileSync('yarn', ['test:node', site]);
    return true;
  } catch (e) {
    return false;
  }
};

const promiseSerial = funcs =>
  funcs.reduce(
    (promise, func) =>
      promise.then(result => func().then(Array.prototype.concat.bind(result))),
    Promise.resolve([])
  );

const createAndPushBranch = ({ branchName, commitMessage }) => {
  execFileSync('git', [
    'config',
    'user.email',
    'adam.pash+postlight-bot@postlight.com',
  ]);
  execFileSync('git', ['config', 'user.name', 'Postlight Bot']);
  execFileSync('git', ['checkout', '-b', branchName]);
  execFileSync('git', ['add', '.']);
  execFileSync('git', ['commit', '-m', commitMessage]);
  execFileSync('git', [
    'push',
    '-q',
    `https://${
      process.env.GH_AUTH_TOKEN
    }@github.com/postlight/mercury-parser.git`,
  ]);
};

const createPR = ({ branchName, title, body = '' }) => {
  octokit.authenticate({
    type: 'token',
    token: process.env.GH_AUTH_TOKEN,
  });

  octokit.pulls.create({
    owner: 'postlight',
    repo: 'mercury-parser',
    title,
    head: branchName,
    base: 'master',
    body,
    maintainer_can_modify: true,
  });
};