Archives/reddit-grab
2
0
Fork 0
mirror of https://github.com/ArchiveTeam/reddit-grab synced 2024-11-12 01:10:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

Version 20200726.06. Fix project name for ZSTD dictionary request.

Browse Source
This commit is contained in:
arkiver 2020-07-26 17:44:05 -04:00
parent 496c018eef
commit 910687b053
1 changed files with 3 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
pipeline.py
View File

@ -54,7 +54,7 @@ if not WGET_AT:
# #
# Update this each time you make a non-cosmetic change. # Update this each time you make a non-cosmetic change.
# It will be added to the WARC files and reported to the tracker. # It will be added to the WARC files and reported to the tracker.
VERSION = '20200726.05' VERSION = '20200726.06'
USER_AGENT = 'Archive Team' USER_AGENT = 'Archive Team'
TRACKER_ID = 'reddittest' TRACKER_ID = 'reddittest'
TRACKER_HOST = 'trackerproxy.meo.ws' TRACKER_HOST = 'trackerproxy.meo.ws'
@ -163,7 +163,7 @@ class ZstdDict(object):
response = requests.get( response = requests.get(
'http://tracker.archiveteam.org:25654/dictionary', 'http://tracker.archiveteam.org:25654/dictionary',
params={ params={
'project': TRACKER_ID 'project': 'reddit'
} }
) )
response.raise_for_status() response.raise_for_status()
@ -232,7 +232,7 @@ class WgetArgs(object):
with open(os.path.join(item['item_dir'], 'zstdict'), 'wb') as f: with open(os.path.join(item['item_dir'], 'zstdict'), 'wb') as f:
f.write(dict_data['dict']) f.write(dict_data['dict'])
item['dict_id'] = dict_data['id'] item['dict_id'] = dict_data['id']
item['dict_project'] = TRACKER_ID item['dict_project'] = 'reddit'
wget_args.extend([ wget_args.extend([
'--warc-zstd-dict', ItemInterpolation('%(item_dir)s/zstdict'), '--warc-zstd-dict', ItemInterpolation('%(item_dir)s/zstdict'),
]) ])