From d07cfd9089d05a6a81109518f03ba82660a4aef7 Mon Sep 17 00:00:00 2001 From: dalf Date: Wed, 21 Jan 2015 11:33:16 +0100 Subject: [PATCH] [enh] use one single http connection pool : improve response time. close #100 --- searx/autocomplete.py | 2 +- searx/engines/wikidata.py | 2 +- searx/poolrequests.py | 61 +++++++++++++++++++++++++++++++++++++++ searx/search.py | 3 +- searx/webapp.py | 2 +- 5 files changed, 65 insertions(+), 5 deletions(-) create mode 100644 searx/poolrequests.py diff --git a/searx/autocomplete.py b/searx/autocomplete.py index 7ebebf1de..e37d34d71 100644 --- a/searx/autocomplete.py +++ b/searx/autocomplete.py @@ -17,13 +17,13 @@ along with searx. If not, see < http://www.gnu.org/licenses/ >. from lxml import etree -from requests import get from json import loads from urllib import urlencode from searx.languages import language_codes from searx.engines import ( categories, engines, engine_shortcuts ) +from searx.poolrequests import get def searx_bang(full_query): diff --git a/searx/engines/wikidata.py b/searx/engines/wikidata.py index da3152ac8..43f72761e 100644 --- a/searx/engines/wikidata.py +++ b/searx/engines/wikidata.py @@ -1,6 +1,6 @@ import json -from requests import get from urllib import urlencode +from searx.poolrequests import get from searx.utils import format_date_by_locale result_count = 1 diff --git a/searx/poolrequests.py b/searx/poolrequests.py new file mode 100644 index 000000000..ef7a195f5 --- /dev/null +++ b/searx/poolrequests.py @@ -0,0 +1,61 @@ +import requests + + +the_http_adapter = requests.adapters.HTTPAdapter(pool_connections=100) +the_https_adapter = requests.adapters.HTTPAdapter(pool_connections=100) + + +class SessionSinglePool(requests.Session): + + def __init__(self): + global the_https_adapter, the_http_adapter + super(SessionSinglePool, self).__init__() + + # reuse the same adapters + self.adapters.clear() + self.mount('https://', the_https_adapter) + self.mount('http://', the_http_adapter) + + def close(self): + """Call super, but clear adapters since there are managed globaly""" + self.adapters.clear() + super(SessionSinglePool, self).close() + + +def request(method, url, **kwargs): + """same as requests/requests/api.py request(...) except it use SessionSinglePool""" + session = SessionSinglePool() + response = session.request(method=method, url=url, **kwargs) + session.close() + return response + + +def get(url, **kwargs): + kwargs.setdefault('allow_redirects', True) + return request('get', url, **kwargs) + + +def options(url, **kwargs): + kwargs.setdefault('allow_redirects', True) + return request('options', url, **kwargs) + + +def head(url, **kwargs): + kwargs.setdefault('allow_redirects', False) + return request('head', url, **kwargs) + + +def post(url, data=None, json=None, **kwargs): + return request('post', url, data=data, json=json, **kwargs) + + +def put(url, data=None, **kwargs): + return request('put', url, data=data, **kwargs) + + +def patch(url, data=None, **kwargs): + return request('patch', url, data=data, **kwargs) + + +def delete(url, **kwargs): + return request('delete', url, **kwargs) diff --git a/searx/search.py b/searx/search.py index 0324d4aaf..b6cf84e94 100644 --- a/searx/search.py +++ b/searx/search.py @@ -15,9 +15,9 @@ along with searx. If not, see < http://www.gnu.org/licenses/ >. (C) 2013- by Adam Tauber, ''' -import requests as requests_lib import threading import re +import searx.poolrequests as requests_lib from itertools import izip_longest, chain from operator import itemgetter from Queue import Queue @@ -31,7 +31,6 @@ from searx.utils import gen_useragent from searx.query import Query from searx import logger - logger = logger.getChild('search') number_of_searches = 0 diff --git a/searx/webapp.py b/searx/webapp.py index 41bad5ea6..53576b4dd 100644 --- a/searx/webapp.py +++ b/searx/webapp.py @@ -27,7 +27,6 @@ import cStringIO import os from datetime import datetime, timedelta -from requests import get as http_get from itertools import chain from urllib import urlencode from flask import ( @@ -36,6 +35,7 @@ from flask import ( ) from flask.ext.babel import Babel, gettext, format_date from searx import settings, searx_dir +from searx.poolrequests import get as http_get from searx.engines import ( categories, engines, get_engines_stats, engine_shortcuts )