Index Comments, #build
Changed: - added comment download and index - [API] added comment api endpointspull/367/head
commit
2fa907c478
@ -0,0 +1,158 @@
|
||||
"""
|
||||
Functionality:
|
||||
- Download comments
|
||||
- Index comments in ES
|
||||
- Retrieve comments from ES
|
||||
"""
|
||||
|
||||
from datetime import datetime
|
||||
|
||||
from home.src.download.yt_dlp_base import YtWrap
|
||||
from home.src.es.connect import ElasticWrap
|
||||
from home.src.ta.config import AppConfig
|
||||
|
||||
|
||||
class Comments:
|
||||
"""hold all comments functionality"""
|
||||
|
||||
def __init__(self, youtube_id, config=False):
|
||||
self.youtube_id = youtube_id
|
||||
self.es_path = f"ta_comment/_doc/{youtube_id}"
|
||||
self.json_data = False
|
||||
self.config = config
|
||||
self.is_activated = False
|
||||
self.comments_format = False
|
||||
|
||||
def build_json(self):
|
||||
"""build json document for es"""
|
||||
print(f"{self.youtube_id}: get comments")
|
||||
self.check_config()
|
||||
|
||||
if not self.is_activated:
|
||||
return
|
||||
|
||||
comments_raw, channel_id = self.get_yt_comments()
|
||||
self.format_comments(comments_raw)
|
||||
|
||||
self.json_data = {
|
||||
"youtube_id": self.youtube_id,
|
||||
"comment_last_refresh": int(datetime.now().strftime("%s")),
|
||||
"comment_channel_id": channel_id,
|
||||
"comment_comments": self.comments_format,
|
||||
}
|
||||
|
||||
def check_config(self):
|
||||
"""read config if not attached"""
|
||||
if not self.config:
|
||||
self.config = AppConfig().config
|
||||
|
||||
self.is_activated = bool(self.config["downloads"]["comment_max"])
|
||||
|
||||
def build_yt_obs(self):
|
||||
"""
|
||||
get extractor config
|
||||
max-comments,max-parents,max-replies,max-replies-per-thread
|
||||
"""
|
||||
max_comments = self.config["downloads"]["comment_max"]
|
||||
max_comments_list = [i.strip() for i in max_comments.split(",")]
|
||||
comment_sort = self.config["downloads"]["comment_sort"]
|
||||
|
||||
yt_obs = {
|
||||
"skip_download": True,
|
||||
"quiet": False,
|
||||
"getcomments": True,
|
||||
"extractor_args": {
|
||||
"youtube": {
|
||||
"max_comments": max_comments_list,
|
||||
"comment_sort": [comment_sort],
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
return yt_obs
|
||||
|
||||
def get_yt_comments(self):
|
||||
"""get comments from youtube"""
|
||||
yt_obs = self.build_yt_obs()
|
||||
info_json = YtWrap(yt_obs).extract(self.youtube_id)
|
||||
comments_raw = info_json.get("comments")
|
||||
channel_id = info_json.get("channel_id")
|
||||
return comments_raw, channel_id
|
||||
|
||||
def format_comments(self, comments_raw):
|
||||
"""process comments to match format"""
|
||||
comments = []
|
||||
|
||||
for comment in comments_raw:
|
||||
cleaned_comment = self.clean_comment(comment)
|
||||
comments.append(cleaned_comment)
|
||||
|
||||
self.comments_format = comments
|
||||
|
||||
def clean_comment(self, comment):
|
||||
"""parse metadata from comment for indexing"""
|
||||
time_text_datetime = datetime.utcfromtimestamp(comment["timestamp"])
|
||||
|
||||
if time_text_datetime.hour == 0 and time_text_datetime.minute == 0:
|
||||
format_string = "%Y-%m-%d"
|
||||
else:
|
||||
format_string = "%Y-%m-%d %H:%M"
|
||||
|
||||
time_text = time_text_datetime.strftime(format_string)
|
||||
|
||||
cleaned_comment = {
|
||||
"comment_id": comment["id"],
|
||||
"comment_text": comment["text"].replace("\xa0", ""),
|
||||
"comment_timestamp": comment["timestamp"],
|
||||
"comment_time_text": time_text,
|
||||
"comment_likecount": comment["like_count"],
|
||||
"comment_is_favorited": comment["is_favorited"],
|
||||
"comment_author": comment["author"],
|
||||
"comment_author_id": comment["author_id"],
|
||||
"comment_author_thumbnail": comment["author_thumbnail"],
|
||||
"comment_author_is_uploader": comment["author_is_uploader"],
|
||||
"comment_parent": comment["parent"],
|
||||
}
|
||||
|
||||
return cleaned_comment
|
||||
|
||||
def upload_comments(self):
|
||||
"""upload comments to es"""
|
||||
if not self.is_activated:
|
||||
return
|
||||
|
||||
_, _ = ElasticWrap(self.es_path).put(self.json_data)
|
||||
|
||||
vid_path = f"ta_video/_update/{self.youtube_id}"
|
||||
data = {"doc": {"comment_count": len(self.comments_format)}}
|
||||
_, _ = ElasticWrap(vid_path).post(data=data)
|
||||
|
||||
def delete_comments(self):
|
||||
"""delete comments from es"""
|
||||
print(f"{self.youtube_id}: delete comments")
|
||||
_, _ = ElasticWrap(self.es_path).delete(refresh=True)
|
||||
|
||||
def get_es_comments(self):
|
||||
"""get comments from ES"""
|
||||
response, statuscode = ElasticWrap(self.es_path).get()
|
||||
if statuscode == 404:
|
||||
print(f"comments: not found {self.youtube_id}")
|
||||
return False
|
||||
|
||||
return response.get("_source")
|
||||
|
||||
def reindex_comments(self):
|
||||
"""update comments from youtube"""
|
||||
self.check_config()
|
||||
if not self.is_activated:
|
||||
return
|
||||
|
||||
self.build_json()
|
||||
es_comments = self.get_es_comments()
|
||||
|
||||
if not self.comments_format and es_comments["comment_comments"]:
|
||||
# don't overwrite comments in es
|
||||
return
|
||||
|
||||
self.delete_comments()
|
||||
self.upload_comments()
|
@ -0,0 +1,8 @@
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<!-- Generator: Adobe Illustrator 26.5.0, SVG Export Plug-In . SVG Version: 6.00 Build 0) -->
|
||||
<svg version="1.1" id="Layer_1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0px" y="0px"
|
||||
viewBox="0 0 500 500" style="enable-background:new 0 0 500 500;" xml:space="preserve">
|
||||
<path d="M499.9,159L499.9,159c0.1-1.7,0.1-3.4,0.1-5.2c0-69.5-58.6-129.7-130.9-129.7c-52.9,0-98.4,34-119,77.4h0
|
||||
c-20.7-43.4-66.2-77.4-119-77.4C58.6,24.1,0,84.4,0,153.9c0,1.7,0.1,3.4,0.1,5.2h0c0,0-7.4,82.6,84.5,172.7
|
||||
c41.8,41.9,88.5,81.6,165.4,144.1c76.9-62.5,123.6-102.3,165.4-144.1C507.2,241.6,499.9,159,499.9,159z"/>
|
||||
</svg>
|
After Width: | Height: | Size: 684 B |
Loading…
Reference in New Issue