From f1c10f4fe45f34c12994b9bbc4aca133202fd7ca Mon Sep 17 00:00:00 2001 From: Cqoicebordel Date: Fri, 6 Feb 2015 17:31:10 +0100 Subject: [PATCH] Startpage's unit test --- searx/engines/startpage.py | 13 +-- searx/tests/engines/test_startpage.py | 140 ++++++++++++++++++++++++++ searx/tests/test_engines.py | 1 + 3 files changed, 146 insertions(+), 8 deletions(-) create mode 100644 searx/tests/engines/test_startpage.py diff --git a/searx/engines/startpage.py b/searx/engines/startpage.py index d60ecd978..9d5b4befe 100644 --- a/searx/engines/startpage.py +++ b/searx/engines/startpage.py @@ -13,6 +13,7 @@ from lxml import html from cgi import escape import re +from searx.engines.xpath import extract_text # engine dependent config categories = ['general'] @@ -45,8 +46,7 @@ def request(query, params): # set language if specified if params['language'] != 'all': - params['data']['with_language'] = ('lang_' + - params['language'].split('_')[0]) + params['data']['with_language'] = ('lang_' + params['language'].split('_')[0]) return params @@ -64,18 +64,15 @@ def response(resp): continue link = links[0] url = link.attrib.get('href') - try: - title = escape(link.text_content()) - except UnicodeDecodeError: - continue # block google-ad url's if re.match("^http(s|)://www.google.[a-z]+/aclk.*$", url): continue + title = escape(extract_text(link)) + if result.xpath('./p[@class="desc"]'): - content = escape(result.xpath('./p[@class="desc"]')[0] - .text_content()) + content = escape(extract_text(result.xpath('./p[@class="desc"]'))) else: content = '' diff --git a/searx/tests/engines/test_startpage.py b/searx/tests/engines/test_startpage.py new file mode 100644 index 000000000..07f13ee27 --- /dev/null +++ b/searx/tests/engines/test_startpage.py @@ -0,0 +1,140 @@ +# -*- coding: utf-8 -*- +from collections import defaultdict +import mock +from searx.engines import startpage +from searx.testing import SearxTestCase + + +class TestStartpageEngine(SearxTestCase): + + def test_request(self): + query = 'test_query' + dicto = defaultdict(dict) + dicto['pageno'] = 1 + dicto['language'] = 'fr_FR' + params = startpage.request(query, dicto) + self.assertIn('url', params) + self.assertIn('startpage.com', params['url']) + self.assertIn('data', params) + self.assertIn('query', params['data']) + self.assertIn(query, params['data']['query']) + self.assertIn('with_language', params['data']) + self.assertIn('lang_fr', params['data']['with_language']) + + dicto['language'] = 'all' + params = startpage.request(query, dicto) + self.assertNotIn('with_language', params['data']) + + def test_response(self): + self.assertRaises(AttributeError, startpage.response, None) + self.assertRaises(AttributeError, startpage.response, []) + self.assertRaises(AttributeError, startpage.response, '') + self.assertRaises(AttributeError, startpage.response, '[]') + + response = mock.Mock(content='') + self.assertEqual(startpage.response(response), []) + + html = """ +
+

+ + This should be the title + + +

+

+ This should be the content. +

+

+ www.speedtest.net/fr/ + + - + + Navigation avec Ixquick Proxy + + - + + Mis en surbrillance + +

+
+ """ + response = mock.Mock(content=html) + results = startpage.response(response) + self.assertEqual(type(results), list) + self.assertEqual(len(results), 1) + self.assertEqual(results[0]['title'], 'This should be the title') + self.assertEqual(results[0]['url'], 'http://this.should.be.the.link/') + self.assertEqual(results[0]['content'], 'This should be the content.') + + html = """ +
+

+ + This should be the title + + +

+

+ This should be the content. +

+

+ www.speedtest.net/fr/ + + - + + Navigation avec Ixquick Proxy + + - + + Mis en surbrillance + +

+
+
+

+ +

+

+ This should be the content. +

+

+ www.speedtest.net/fr/ + +

+
+
+

+ + This should be the title + + +

+

+ www.speedtest.net/fr/ + + - + + Navigation avec Ixquick Proxy + + - + + Mis en surbrillance + +

+
+ """ + response = mock.Mock(content=html) + results = startpage.response(response) + self.assertEqual(type(results), list) + self.assertEqual(len(results), 1) + self.assertEqual(results[0]['content'], '') diff --git a/searx/tests/test_engines.py b/searx/tests/test_engines.py index 7fa1e2b8b..0a3559665 100644 --- a/searx/tests/test_engines.py +++ b/searx/tests/test_engines.py @@ -23,6 +23,7 @@ from searx.tests.engines.test_searchcode_code import * # noqa from searx.tests.engines.test_searchcode_doc import * # noqa from searx.tests.engines.test_soundcloud import * # noqa from searx.tests.engines.test_stackoverflow import * # noqa +from searx.tests.engines.test_startpage import * # noqa from searx.tests.engines.test_subtitleseeker import * # noqa from searx.tests.engines.test_twitter import * # noqa from searx.tests.engines.test_vimeo import * # noqa