mirror of https://github.com/searxng/searxng
Merge pull request #1075 from kvch/finish-jibe-b-engines
Finish PRs of @jibe-b: pubmed, oa_doi_rewrite, openaire, arxivpull/1/head
commit
3d50b0288d
@ -0,0 +1,76 @@
|
||||
#!/usr/bin/env python
|
||||
|
||||
"""
|
||||
ArXiV (Scientific preprints)
|
||||
@website https://arxiv.org
|
||||
@provide-api yes (export.arxiv.org/api/query)
|
||||
@using-api yes
|
||||
@results XML-RSS
|
||||
@stable yes
|
||||
@parse url, title, publishedDate, content
|
||||
More info on api: https://arxiv.org/help/api/user-manual
|
||||
"""
|
||||
|
||||
from lxml import html
|
||||
from datetime import datetime
|
||||
from searx.url_utils import urlencode
|
||||
|
||||
|
||||
categories = ['science']
|
||||
|
||||
base_url = 'http://export.arxiv.org/api/query?search_query=all:'\
|
||||
+ '{query}&start={offset}&max_results={number_of_results}'
|
||||
|
||||
# engine dependent config
|
||||
number_of_results = 10
|
||||
|
||||
|
||||
def request(query, params):
|
||||
# basic search
|
||||
offset = (params['pageno'] - 1) * number_of_results
|
||||
|
||||
string_args = dict(query=query,
|
||||
offset=offset,
|
||||
number_of_results=number_of_results)
|
||||
|
||||
params['url'] = base_url.format(**string_args)
|
||||
|
||||
return params
|
||||
|
||||
|
||||
def response(resp):
|
||||
results = []
|
||||
|
||||
dom = html.fromstring(resp.content)
|
||||
search_results = dom.xpath('//entry')
|
||||
|
||||
for entry in search_results:
|
||||
title = entry.xpath('.//title')[0].text
|
||||
|
||||
url = entry.xpath('.//id')[0].text
|
||||
|
||||
content_string = '{doi_content}{abstract_content}'
|
||||
|
||||
abstract = entry.xpath('.//summary')[0].text
|
||||
|
||||
# If a doi is available, add it to the snipppet
|
||||
try:
|
||||
doi_content = entry.xpath('.//link[@title="doi"]')[0].text
|
||||
content = content_string.format(doi_content=doi_content, abstract_content=abstract)
|
||||
except:
|
||||
content = content_string.format(doi_content="", abstract_content=abstract)
|
||||
|
||||
if len(content) > 300:
|
||||
content = content[0:300] + "..."
|
||||
# TODO: center snippet on query term
|
||||
|
||||
publishedDate = datetime.strptime(entry.xpath('.//published')[0].text, '%Y-%m-%dT%H:%M:%SZ')
|
||||
|
||||
res_dict = {'url': url,
|
||||
'title': title,
|
||||
'publishedDate': publishedDate,
|
||||
'content': content}
|
||||
|
||||
results.append(res_dict)
|
||||
|
||||
return results
|
@ -0,0 +1,98 @@
|
||||
#!/usr/bin/env python
|
||||
|
||||
"""
|
||||
PubMed (Scholar publications)
|
||||
@website https://www.ncbi.nlm.nih.gov/pubmed/
|
||||
@provide-api yes (https://www.ncbi.nlm.nih.gov/home/develop/api/)
|
||||
@using-api yes
|
||||
@results XML
|
||||
@stable yes
|
||||
@parse url, title, publishedDate, content
|
||||
More info on api: https://www.ncbi.nlm.nih.gov/books/NBK25501/
|
||||
"""
|
||||
|
||||
from flask_babel import gettext
|
||||
from lxml import etree
|
||||
from datetime import datetime
|
||||
from searx.url_utils import urlencode
|
||||
from searx.poolrequests import get
|
||||
|
||||
|
||||
categories = ['science']
|
||||
|
||||
base_url = 'https://eutils.ncbi.nlm.nih.gov/entrez/eutils/esearch.fcgi'\
|
||||
+ '?db=pubmed&{query}&retstart={offset}&retmax={hits}'
|
||||
|
||||
# engine dependent config
|
||||
number_of_results = 10
|
||||
pubmed_url = 'https://www.ncbi.nlm.nih.gov/pubmed/'
|
||||
|
||||
|
||||
def request(query, params):
|
||||
# basic search
|
||||
offset = (params['pageno'] - 1) * number_of_results
|
||||
|
||||
string_args = dict(query=urlencode({'term': query}),
|
||||
offset=offset,
|
||||
hits=number_of_results)
|
||||
|
||||
params['url'] = base_url.format(**string_args)
|
||||
|
||||
return params
|
||||
|
||||
|
||||
def response(resp):
|
||||
results = []
|
||||
|
||||
# First retrieve notice of each result
|
||||
pubmed_retrieve_api_url = 'https://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi?'\
|
||||
+ 'db=pubmed&retmode=xml&id={pmids_string}'
|
||||
|
||||
pmids_results = etree.XML(resp.content)
|
||||
pmids = pmids_results.xpath('//eSearchResult/IdList/Id')
|
||||
pmids_string = ''
|
||||
|
||||
for item in pmids:
|
||||
pmids_string += item.text + ','
|
||||
|
||||
retrieve_notice_args = dict(pmids_string=pmids_string)
|
||||
|
||||
retrieve_url_encoded = pubmed_retrieve_api_url.format(**retrieve_notice_args)
|
||||
|
||||
search_results_xml = get(retrieve_url_encoded).content
|
||||
search_results = etree.XML(search_results_xml).xpath('//PubmedArticleSet/PubmedArticle/MedlineCitation')
|
||||
|
||||
for entry in search_results:
|
||||
title = entry.xpath('.//Article/ArticleTitle')[0].text
|
||||
|
||||
pmid = entry.xpath('.//PMID')[0].text
|
||||
url = pubmed_url + pmid
|
||||
|
||||
try:
|
||||
content = entry.xpath('.//Abstract/AbstractText')[0].text
|
||||
except:
|
||||
content = gettext('No abstract is available for this publication.')
|
||||
|
||||
# If a doi is available, add it to the snipppet
|
||||
try:
|
||||
doi = entry.xpath('.//ELocationID[@EIdType="doi"]')[0].text
|
||||
content = 'DOI: {doi} Abstract: {content}'.format(doi=doi, content=content)
|
||||
except:
|
||||
pass
|
||||
|
||||
if len(content) > 300:
|
||||
content = content[0:300] + "..."
|
||||
# TODO: center snippet on query term
|
||||
|
||||
publishedDate = datetime.strptime(entry.xpath('.//DateCreated/Year')[0].text
|
||||
+ '-' + entry.xpath('.//DateCreated/Month')[0].text
|
||||
+ '-' + entry.xpath('.//DateCreated/Day')[0].text, '%Y-%m-%d')
|
||||
|
||||
res_dict = {'url': url,
|
||||
'title': title,
|
||||
'publishedDate': publishedDate,
|
||||
'content': content}
|
||||
|
||||
results.append(res_dict)
|
||||
|
||||
return results
|
@ -0,0 +1,37 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
from collections import defaultdict
|
||||
import mock
|
||||
from searx.engines import pubmed
|
||||
from searx.testing import SearxTestCase
|
||||
|
||||
|
||||
class TestPubmedEngine(SearxTestCase):
|
||||
|
||||
def test_request(self):
|
||||
query = 'test_query'
|
||||
dicto = defaultdict(dict)
|
||||
dicto['pageno'] = 1
|
||||
params = pubmed.request(query, dicto)
|
||||
self.assertIn('url', params)
|
||||
self.assertIn('eutils.ncbi.nlm.nih.gov/', params['url'])
|
||||
self.assertIn('term', params['url'])
|
||||
|
||||
def test_response(self):
|
||||
self.assertRaises(AttributeError, pubmed.response, None)
|
||||
self.assertRaises(AttributeError, pubmed.response, [])
|
||||
self.assertRaises(AttributeError, pubmed.response, '')
|
||||
self.assertRaises(AttributeError, pubmed.response, '[]')
|
||||
|
||||
response = mock.Mock(text='<PubmedArticleSet></PubmedArticleSet>')
|
||||
self.assertEqual(pubmed.response(response), [])
|
||||
|
||||
xml_mock = """<eSearchResult><Count>1</Count><RetMax>1</RetMax><RetStart>0</RetStart><IdList>
|
||||
<Id>1</Id>
|
||||
</IdList></eSearchResult>
|
||||
"""
|
||||
|
||||
response = mock.Mock(text=xml_mock.encode('utf-8'))
|
||||
results = pubmed.response(response)
|
||||
self.assertEqual(type(results), list)
|
||||
self.assertEqual(len(results), 1)
|
||||
self.assertEqual(results[0]['content'], 'No abstract is available for this publication.')
|
@ -0,0 +1,58 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
from collections import defaultdict
|
||||
import mock
|
||||
from searx.engines import arxiv
|
||||
from searx.testing import SearxTestCase
|
||||
|
||||
|
||||
class TestBaseEngine(SearxTestCase):
|
||||
|
||||
def test_request(self):
|
||||
query = 'test_query'
|
||||
dicto = defaultdict(dict)
|
||||
dicto['pageno'] = 1
|
||||
params = arxiv.request(query, dicto)
|
||||
self.assertIn('url', params)
|
||||
self.assertIn('export.arxiv.org/api/', params['url'])
|
||||
|
||||
def test_response(self):
|
||||
self.assertRaises(AttributeError, arxiv.response, None)
|
||||
self.assertRaises(AttributeError, arxiv.response, [])
|
||||
self.assertRaises(AttributeError, arxiv.response, '')
|
||||
self.assertRaises(AttributeError, arxiv.response, '[]')
|
||||
|
||||
response = mock.Mock(content=b'''<?xml version="1.0" encoding="UTF-8"?>
|
||||
<feed xmlns="http://www.w3.org/2005/Atom"></feed>''')
|
||||
self.assertEqual(arxiv.response(response), [])
|
||||
|
||||
xml_mock = b'''<?xml version="1.0" encoding="UTF-8"?>
|
||||
<feed xmlns="http://www.w3.org/2005/Atom">
|
||||
<title type="html">ArXiv Query: search_query=all:test_query&id_list=&start=0&max_results=1</title>
|
||||
<id>http://arxiv.org/api/1</id>
|
||||
<updated>2000-01-21T00:00:00-01:00</updated>
|
||||
<opensearch:totalResults xmlns:opensearch="http://a9.com/-/spec/opensearch/1.1/">1</opensearch:totalResults>
|
||||
<opensearch:startIndex xmlns:opensearch="http://a9.com/-/spec/opensearch/1.1/">0</opensearch:startIndex>
|
||||
<opensearch:itemsPerPage xmlns:opensearch="http://a9.com/-/spec/opensearch/1.1/">1</opensearch:itemsPerPage>
|
||||
<entry>
|
||||
<id>http://arxiv.org/1</id>
|
||||
<updated>2000-01-01T00:00:01Z</updated>
|
||||
<published>2000-01-01T00:00:01Z</published>
|
||||
<title>Mathematical proof.</title>
|
||||
<summary>Mathematical formula.</summary>
|
||||
<author>
|
||||
<name>A. B.</name>
|
||||
</author>
|
||||
<link href="http://arxiv.org/1" rel="alternate" type="text/html"/>
|
||||
<link title="pdf" href="http://arxiv.org/1" rel="related" type="application/pdf"/>
|
||||
<category term="math.QA" scheme="http://arxiv.org/schemas/atom"/>
|
||||
<category term="1" scheme="http://arxiv.org/schemas/atom"/>
|
||||
</entry>
|
||||
</feed>
|
||||
'''
|
||||
|
||||
response = mock.Mock(content=xml_mock)
|
||||
results = arxiv.response(response)
|
||||
self.assertEqual(type(results), list)
|
||||
self.assertEqual(len(results), 1)
|
||||
self.assertEqual(results[0]['title'], 'Mathematical proof.')
|
||||
self.assertEqual(results[0]['content'], 'Mathematical formula.')
|
Loading…
Reference in New Issue