@ -3,6 +3,8 @@
Startpage ( Web )
Startpage ( Web )
"""
"""
from urllib . parse import urlencode
from lxml import html
from lxml import html
from dateutil import parser
from dateutil import parser
from datetime import datetime , timedelta
from datetime import datetime , timedelta
@ -33,7 +35,7 @@ supported_languages_url = 'https://www.startpage.com/do/settings'
# search-url
# search-url
base_url = ' https://startpage.com/ '
base_url = ' https://startpage.com/ '
search_url = base_url + ' do/search '
search_url = base_url + ' sp/search? '
# specific xpath variables
# specific xpath variables
# ads xpath //div[@id="results"]/div[@id="sponsored"]//div[@class="result"]
# ads xpath //div[@id="results"]/div[@id="sponsored"]//div[@class="result"]
@ -46,14 +48,12 @@ content_xpath = './/p[@class="w-gl__description"]'
# do search-request
# do search-request
def request ( query , params ) :
def request ( query , params ) :
params [ ' url ' ] = search_url
args = {
params [ ' method ' ] = ' POST '
params [ ' data ' ] = {
' query ' : query ,
' query ' : query ,
' page ' : params [ ' pageno ' ] ,
' page ' : params [ ' pageno ' ] ,
' cat ' : ' web ' ,
' cat ' : ' web ' ,
' cmd ' : ' process_search ' ,
# 'abp': "-1" ,
' engine0' : ' v1all ' ,
' sc' : ' Mj4jZy61QETj20 ' ,
}
}
# set language if specified
# set language if specified
@ -61,9 +61,10 @@ def request(query, params):
lang_code = match_language ( params [ ' language ' ] , supported_languages , fallback = None )
lang_code = match_language ( params [ ' language ' ] , supported_languages , fallback = None )
if lang_code :
if lang_code :
language_name = supported_languages [ lang_code ] [ ' alias ' ]
language_name = supported_languages [ lang_code ] [ ' alias ' ]
params[ ' data ' ] [ ' language ' ] = language_name
args [ ' language ' ] = language_name
params[ ' data ' ] [ ' lui ' ] = language_name
args [ ' lui ' ] = language_name
params [ ' url ' ] = search_url + urlencode ( args )
return params
return params