From 8fe3bec6b6f612e132e36eedec984d93ba02aaea Mon Sep 17 00:00:00 2001 From: Adam Pash Date: Mon, 12 Sep 2016 10:08:49 -0400 Subject: [PATCH] fix: accepting cookies with request (required for sites like nytimes.com) --- src/iris.test.js | 7 +++++++ src/resource/utils/fetch-resource.js | 6 +++++- src/resource/utils/fetch-resource.test.js | 7 +++++++ 3 files changed, 19 insertions(+), 1 deletion(-) diff --git a/src/iris.test.js b/src/iris.test.js index 0fac38c4..7f1fff25 100644 --- a/src/iris.test.js +++ b/src/iris.test.js @@ -22,5 +22,12 @@ describe('Iris', function() { // console.log(result) }) + + it('does the nyt', async function() { + const result = await Iris.parse('http://www.nytimes.com/2016/08/16/upshot/the-state-of-the-clinton-trump-race-is-it-over.html?_r=0') + + // console.log(result) + }) + }) }) diff --git a/src/resource/utils/fetch-resource.js b/src/resource/utils/fetch-resource.js index 7263d9e6..fefc283e 100644 --- a/src/resource/utils/fetch-resource.js +++ b/src/resource/utils/fetch-resource.js @@ -21,9 +21,13 @@ export default async function fetchResource(url) { const options = { url: parsedUrl, - headers: REQUEST_HEADERS, + headers: { ...REQUEST_HEADERS }, timeout: FETCH_TIMEOUT, + // Don't set encoding; this fixes issues + // w/gzipped responses encoding: null, + // Accept cookies + jar: true, } const { response, body } = await get(options) diff --git a/src/resource/utils/fetch-resource.test.js b/src/resource/utils/fetch-resource.test.js index ff85bf72..94442d5d 100644 --- a/src/resource/utils/fetch-resource.test.js +++ b/src/resource/utils/fetch-resource.test.js @@ -16,6 +16,13 @@ describe('fetchResource(url)', function() { assert.equal(typeof body, 'object') }) + + it('fetches nyt', async () => { + const url = 'http://www.nytimes.com/2016/08/16/upshot/the-state-of-the-clinton-trump-race-is-it-over.html?_r=0' + const { body, response } = await fetchResource(url) + + assert.equal(typeof body, 'object') + }) }) describe('validateResponse(response)', () => {