Archives/tubearchivist
2
0
Fork 0
mirror of https://github.com/tubearchivist/tubearchivist synced 2024-11-19 15:25:51 +00:00
Code Issues Projects Releases Wiki Activity

immediate feedback when adding long lists to queue

Browse Source
This commit is contained in:
simon 2021-09-10 15:21:46 +07:00
parent e3afe3947c
commit adafa98568
1 changed files with 17 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
tubearchivist/home/src/download.py
View File

@ -32,6 +32,15 @@ class PendingList:
""" extract youtube ids from list """ """ extract youtube ids from list """
missing_videos = [] missing_videos = []
for entry in youtube_ids: for entry in youtube_ids:
# notify
mess_dict = {
"status": "pending",
"level": "info",
"title": "Adding to download queue.",
"message": 'Extracting lists'
}
set_message('progress:download', mess_dict)
# extract
url = entry['url'] url = entry['url']
url_type = entry['type'] url_type = entry['type']
if url_type == 'video': if url_type == 'video':
@ -78,20 +87,20 @@ class PendingList:
action = {"create": {"_id": youtube_id, "_index": "ta_download"}} action = {"create": {"_id": youtube_id, "_index": "ta_download"}}
bulk_list.append(json.dumps(action)) bulk_list.append(json.dumps(action))
bulk_list.append(json.dumps(video)) bulk_list.append(json.dumps(video))
# notify
mess_dict = {
"status": "pending",
"level": "info",
"title": "Adding to download queue.",
"message": 'Processing IDs...'
}
set_message('progress:download', mess_dict)
# add last newline # add last newline
bulk_list.append('\n') bulk_list.append('\n')
query_str = '\n'.join(bulk_list) query_str = '\n'.join(bulk_list)
headers = {'Content-type': 'application/x-ndjson'} headers = {'Content-type': 'application/x-ndjson'}
url = self.ES_URL + '/_bulk' url = self.ES_URL + '/_bulk'
request = requests.post(url, data=query_str, headers=headers) request = requests.post(url, data=query_str, headers=headers)
# notify
mess_dict = {
"status": "pending",
"level": "info",
"title": "Adding to download queue.",
"message": 'Processing IDs...'
}
set_message('progress:download', mess_dict)
if not request.ok: if not request.ok:
print(request) print(request)