From 2f83750619faa9f0a6de8e4e26d42251a69e44a6 Mon Sep 17 00:00:00 2001 From: stepshal Date: Fri, 8 Jul 2016 23:43:28 +0700 Subject: [PATCH] Remove trailing whitespace --- examples/basic_engine.py | 1 - utils/fetch_currencies.py | 50 +++++++++++++++++++-------------------- 2 files changed, 25 insertions(+), 26 deletions(-) diff --git a/examples/basic_engine.py b/examples/basic_engine.py index d786564d3..3428f67fb 100644 --- a/examples/basic_engine.py +++ b/examples/basic_engine.py @@ -22,4 +22,3 @@ def response(resp): resp: requests response object ''' return [{'url': '', 'title': '', 'content': ''}] - diff --git a/utils/fetch_currencies.py b/utils/fetch_currencies.py index cb055dde7..201b32195 100644 --- a/utils/fetch_currencies.py +++ b/utils/fetch_currencies.py @@ -5,17 +5,17 @@ import unicodedata import string from urllib import urlencode from requests import get - + languages = {'de', 'en', 'es', 'fr', 'hu', 'it', 'nl', 'jp'} - + url_template = 'https://www.wikidata.org/w/api.php?action=wbgetentities&format=json&{query}&props=labels%7Cdatatype%7Cclaims%7Caliases&languages=' + '|'.join(languages) -url_wmflabs_template = 'http://wdq.wmflabs.org/api?q=' +url_wmflabs_template = 'http://wdq.wmflabs.org/api?q=' url_wikidata_search_template='http://www.wikidata.org/w/api.php?action=query&list=search&format=json&srnamespace=0&srprop=sectiontitle&{query}' - -wmflabs_queries = [ + +wmflabs_queries = [ 'CLAIM[31:8142]', # all devise ] - + db = { 'iso4217' : { }, @@ -26,7 +26,7 @@ db = { def remove_accents(data): return unicodedata.normalize('NFKD', data).lower() - + def normalize_name(name): return re.sub(' +',' ', remove_accents(name.lower()).replace('-', ' ')) @@ -66,22 +66,22 @@ def get_property_value(data, name): prop = data.get('claims', {}).get(name, {}) if len(prop) == 0: return None - - value = prop[0].get('mainsnak', {}).get('datavalue', {}).get('value', '') + + value = prop[0].get('mainsnak', {}).get('datavalue', {}).get('value', '') if value == '': return None return value - + def parse_currency(data): iso4217 = get_property_value(data, 'P498') - + if iso4217 is not None: unit = get_property_value(data, 'P558') if unit is not None: add_currency_name(unit, iso4217) - + labels = data.get('labels', {}) for language in languages: name = labels.get(language, {}).get('value', None) @@ -95,22 +95,22 @@ def parse_currency(data): alias = aliases[language][i].get('value', None) add_currency_name(alias, iso4217) - + def fetch_data(wikidata_ids): url = url_template.format(query=urlencode({'ids' : '|'.join(wikidata_ids)})) htmlresponse = get(url) jsonresponse = json.loads(htmlresponse.content) entities = jsonresponse.get('entities', {}) - + for pname in entities: pvalue = entities.get(pname) parse_currency(pvalue) - - + + def add_q(i): return "Q" + str(i) - - + + def fetch_data_batch(wikidata_ids): while len(wikidata_ids) > 0: if len(wikidata_ids) > 50: @@ -119,8 +119,8 @@ def fetch_data_batch(wikidata_ids): else: fetch_data(wikidata_ids) wikidata_ids = [] - - + + def wdq_query(query): url = url_wmflabs_template + query htmlresponse = get(url) @@ -131,23 +131,23 @@ def wdq_query(query): print "error for query '" + query + "' :" + error fetch_data_batch(qlist) - - + + def wd_query(query, offset=0): qlist = [] - + url = url_wikidata_search_template.format(query=urlencode({'srsearch': query, 'srlimit': 50, 'sroffset': offset})) htmlresponse = get(url) jsonresponse = json.loads(htmlresponse.content) for r in jsonresponse.get('query', {}).get('search', {}): qlist.append(r.get('title', '')) fetch_data_batch(qlist) - + ## fetch ## for q in wmflabs_queries: wdq_query(q) -# static +# static add_currency_name(u"euro", 'EUR') add_currency_name(u"euros", 'EUR') add_currency_name(u"dollar", 'USD')