From 10a15e06e1bf6f1a22d1625950d8c221d99d6a07 Mon Sep 17 00:00:00 2001 From: Ben Busby Date: Mon, 6 Dec 2021 21:39:50 -0700 Subject: [PATCH] Fix incorrect request type for image searches Previously had hardcoded POST requests for all requests that didn't use the header template (which currently is only the image tab). Also refactored how the Filter class works. It now requires a valid Config model to be provided, which is then set up as a class var that the filtering functions can use as needed, rather than setting specific values from the config as individual values (which was confusing and sloppy). Fixes #561 --- app/filter.py | 43 ++++++++++--------------------------------- app/request.py | 6 +++--- app/utils/search.py | 5 ++--- test/test_results.py | 5 +++-- 4 files changed, 18 insertions(+), 41 deletions(-) diff --git a/app/filter.py b/app/filter.py index 452b71c..e03abf1 100644 --- a/app/filter.py +++ b/app/filter.py @@ -1,3 +1,4 @@ +from app.models.config import Config from app.models.endpoint import Endpoint from app.request import VALID_PARAMS, MAPS_URL from app.utils.misc import read_config_bool @@ -45,18 +46,8 @@ class Filter: # type result (such as "people also asked", "related searches", etc) RESULT_CHILD_LIMIT = 7 - def __init__(self, user_key: str, mobile=False, config=None) -> None: - if config is None: - config = {} - self.near = config['near'] if 'near' in config else '' - self.dark = config['dark'] if 'dark' in config else False - self.nojs = config['nojs'] if 'nojs' in config else False - self.new_tab = config['new_tab'] if 'new_tab' in config else False - self.alt_redirect = config['alts'] if 'alts' in config else False - self.block_title = ( - config['block_title'] if 'block_title' in config else '') - self.block_url = ( - config['block_url'] if 'block_url' in config else '') + def __init__(self, user_key: str, config: Config, mobile=False) -> None: + self.config = config self.mobile = mobile self.user_key = user_key self.main_divs = ResultSet('') @@ -69,16 +60,6 @@ class Filter: def elements(self): return self._elements - def reskin(self, page: str) -> str: - # Aesthetic only re-skinning - if self.dark: - page = page.replace( - 'fff', '000').replace( - '202124', 'ddd').replace( - '1967D2', '3b85ea') - - return page - def encrypt_path(self, path, is_element=False) -> str: # Encrypts path to avoid plaintext results in logs if is_element: @@ -109,7 +90,7 @@ class Filter: input_form = soup.find('form') if input_form is not None: - input_form['method'] = 'POST' + input_form['method'] = 'GET' if self.config.get_only else 'POST' # Ensure no extra scripts passed through for script in soup('script'): @@ -143,9 +124,7 @@ class Filter: _ = div.decompose() if len(div_ads) else None def remove_block_titles(self) -> None: - if not self.main_divs: - return - if self.block_title == '': + if not self.main_divs or not self.config.block_title: return block_title = re.compile(self.block_title) for div in [_ for _ in self.main_divs.find_all('div', recursive=True)]: @@ -154,9 +133,7 @@ class Filter: _ = div.decompose() if len(block_divs) else None def remove_block_url(self) -> None: - if not self.main_divs: - return - if self.block_url == '': + if not self.main_divs or not self.config.block_url: return block_url = re.compile(self.block_url) for div in [_ for _ in self.main_divs.find_all('div', recursive=True)]: @@ -244,7 +221,7 @@ class Filter: if src.startswith(LOGO_URL): # Re-brand with Whoogle logo element.replace_with(BeautifulSoup( - render_template('logo.html', dark=self.dark), + render_template('logo.html'), features='html.parser')) return elif src.startswith(GOOG_IMG) or GOOG_STATIC in src: @@ -323,10 +300,10 @@ class Filter: link['href'] = filter_link_args(q) # Add no-js option - if self.nojs: + if self.config.nojs: append_nojs(link) - if self.new_tab: + if self.config.new_tab: link['target'] = '_blank' else: if href.startswith(MAPS_URL): @@ -336,7 +313,7 @@ class Filter: link['href'] = href # Replace link location if "alts" config is enabled - if self.alt_redirect: + if self.config.alts: # Search and replace all link descriptions # with alternative location link['href'] = get_site_alt(link['href']) diff --git a/app/request.py b/app/request.py index ae36824..c1d3902 100644 --- a/app/request.py +++ b/app/request.py @@ -59,7 +59,7 @@ def gen_user_agent(is_mobile) -> str: return DESKTOP_UA.format("Mozilla", linux, firefox) -def gen_query(query, args, config, near_city=None) -> str: +def gen_query(query, args, config) -> str: param_dict = {key: '' for key in VALID_PARAMS} # Use :past(hour/day/week/month/year) if available @@ -96,8 +96,8 @@ def gen_query(query, args, config, near_city=None) -> str: param_dict['start'] = '&start=' + args.get('start') # Search for results near a particular city, if available - if near_city: - param_dict['near'] = '&near=' + urlparse.quote(near_city) + if config.near: + param_dict['near'] = '&near=' + urlparse.quote(config.near) # Set language for results (lr) if source isn't set, otherwise use the # result language param provided in the results diff --git a/app/utils/search.py b/app/utils/search.py index 0bc9335..546ec88 100644 --- a/app/utils/search.py +++ b/app/utils/search.py @@ -119,8 +119,7 @@ class Search: config=self.config) full_query = gen_query(self.query, self.request_params, - self.config, - content_filter.near) + self.config) # force mobile search when view image is true and # the request is not already made by a mobile @@ -132,7 +131,7 @@ class Search: force_mobile=view_image) # Produce cleanable html soup from response - html_soup = bsoup(content_filter.reskin(get_body.text), 'html.parser') + html_soup = bsoup(get_body.text, 'html.parser') # Replace current soup if view_image is active if view_image: diff --git a/test/test_results.py b/test/test_results.py index b462242..7d895e6 100644 --- a/test/test_results.py +++ b/test/test_results.py @@ -1,5 +1,6 @@ from bs4 import BeautifulSoup from app.filter import Filter +from app.models.config import Config from app.models.endpoint import Endpoint from app.utils.session import generate_user_key from datetime import datetime @@ -11,7 +12,7 @@ from test.conftest import demo_config def get_search_results(data): secret_key = generate_user_key() - soup = Filter(user_key=secret_key).clean( + soup = Filter(user_key=secret_key, config=Config(**demo_config)).clean( BeautifulSoup(data, 'html.parser')) main_divs = soup.find('div', {'id': 'main'}) @@ -74,7 +75,7 @@ def test_block_results(client): assert has_pinterest - demo_config['block'] = 'pinterest.com' + demo_config['block'] = 'pinterest.com,help.pinterest.com' rv = client.post(f'/{Endpoint.config}', data=demo_config) assert rv._status_code == 302