mirror of
https://github.com/tubearchivist/tubearchivist
synced 2024-11-02 09:41:07 +00:00
Downloads channel filter, #build
Changed: - Added downloads channel filter to channel pages - API: Filter download list view by channel - Fixed: is_live status check
This commit is contained in:
simon
commit
4f1daeb18c
@ -61,6 +61,7 @@ The list views return a paginate object with the following keys:
|
||||
- prev_pages: *array of ints* of previous pages, if available
|
||||
- current_page: *int* current page from query
|
||||
- max_hits: *bool* if max of 10k results is reached
|
||||
- params: *str* additional url encoded query parameters
|
||||
- last_page: *int* of last page link
|
||||
- next_pages: *array of ints* of next pages
|
||||
- total_hits: *int* total results
|
||||
@ -169,6 +170,7 @@ GET /api/download/
|
||||
|
||||
Parameter:
|
||||
- filter: pending, ignore
|
||||
- channel: channel-id
|
||||
|
||||
### Add list of videos to download queue
|
||||
POST /api/download/
|
||||
|
||||
@ -53,9 +53,7 @@ class ApiBaseView(APIView):
|
||||
|
||||
def initiate_pagination(self, request):
|
||||
"""set initial pagination values"""
|
||||
user_id = request.user.id
|
||||
page_get = int(request.GET.get("page", 0))
|
||||
self.pagination_handler = Pagination(page_get, user_id)
|
||||
self.pagination_handler = Pagination(request)
|
||||
self.data.update(
|
||||
{
|
||||
"size": self.pagination_handler.pagination["page_size"],
|
||||
@ -368,13 +366,23 @@ class DownloadApiListView(ApiBaseView):
|
||||
"""get request"""
|
||||
query_filter = request.GET.get("filter", False)
|
||||
self.data.update({"sort": [{"timestamp": {"order": "asc"}}]})
|
||||
|
||||
must_list = []
|
||||
if query_filter:
|
||||
if query_filter not in self.valid_filter:
|
||||
message = f"invalid url query filder: {query_filter}"
|
||||
print(message)
|
||||
return Response({"message": message}, status=400)
|
||||
|
||||
self.data["query"] = {"term": {"status": {"value": query_filter}}}
|
||||
must_list.append({"term": {"status": {"value": query_filter}}})
|
||||
|
||||
filter_channel = request.GET.get("channel", False)
|
||||
if filter_channel:
|
||||
must_list.append(
|
||||
{"term": {"channel_id": {"value": filter_channel}}}
|
||||
)
|
||||
|
||||
self.data["query"] = {"bool": {"must": must_list}}
|
||||
|
||||
self.get_document_list(request)
|
||||
return Response(self.response)
|
||||
|
||||
@ -246,7 +246,7 @@ class PendingList(PendingIndex):
|
||||
print(f"{youtube_id}: skipping premium video, id not matching")
|
||||
return False
|
||||
# stop if video is streaming live now
|
||||
if vid["is_live"]:
|
||||
if vid["live_status"] in ["is_upcoming", "is_live"]:
|
||||
return False
|
||||
|
||||
return self._parse_youtube_details(vid)
|
||||
|
||||
@ -63,6 +63,7 @@ class ChannelSubscription:
|
||||
|
||||
for idx, channel in enumerate(all_channels):
|
||||
channel_id = channel["channel_id"]
|
||||
print(f"{channel_id}: find missing videos.")
|
||||
last_videos = self.get_last_youtube_videos(channel_id)
|
||||
|
||||
if last_videos:
|
||||
|
||||
@ -73,16 +73,25 @@ class Pagination:
|
||||
figure out the pagination based on page size and total_hits
|
||||
"""
|
||||
|
||||
def __init__(self, page_get, user_id, search_get=False):
|
||||
self.user_id = user_id
|
||||
def __init__(self, request):
|
||||
self.request = request
|
||||
self.page_get = False
|
||||
self.params = False
|
||||
self.get_params()
|
||||
self.page_size = self.get_page_size()
|
||||
self.page_get = page_get
|
||||
self.search_get = search_get
|
||||
self.pagination = self.first_guess()
|
||||
|
||||
def get_params(self):
|
||||
"""process url query parameters"""
|
||||
query_dict = self.request.GET.copy()
|
||||
self.page_get = int(query_dict.get("page", 0))
|
||||
|
||||
_ = query_dict.pop("page", False)
|
||||
self.params = query_dict.urlencode()
|
||||
|
||||
def get_page_size(self):
|
||||
"""get default or user modified page_size"""
|
||||
key = f"{self.user_id}:page_size"
|
||||
key = f"{self.request.user.id}:page_size"
|
||||
page_size = RedisArchivist().get_message(key)["status"]
|
||||
if not page_size:
|
||||
config = AppConfig().config
|
||||
@ -108,9 +117,9 @@ class Pagination:
|
||||
"prev_pages": prev_pages,
|
||||
"current_page": page_get,
|
||||
"max_hits": False,
|
||||
"params": self.params,
|
||||
}
|
||||
if self.search_get:
|
||||
pagination.update({"search_get": self.search_get})
|
||||
|
||||
return pagination
|
||||
|
||||
def validate(self, total_hits):
|
||||
|
||||
@ -79,16 +79,16 @@
|
||||
<div class="pagination">
|
||||
{% if pagination %}
|
||||
{% if pagination.current_page > 1 %}
|
||||
{% if pagination.search_get %}
|
||||
<a class="pagination-item" href="{{ request.path }}?search={{ pagination.search_get }}">First</a>
|
||||
{% if pagination.params %}
|
||||
<a class="pagination-item" href="{{ request.path }}?{{ pagination.params }}">First</a>
|
||||
{% else %}
|
||||
<a class="pagination-item" href="{{ request.path }}">First</a>
|
||||
{% endif %}
|
||||
{% endif %}
|
||||
{% if pagination.prev_pages %}
|
||||
{% for page in pagination.prev_pages %}
|
||||
{% if pagination.search_get %}
|
||||
<a class="pagination-item" href="?page={{ page }}&search={{ pagination.search_get }}">{{ page }}</a>
|
||||
{% if pagination.params %}
|
||||
<a class="pagination-item" href="?page={{ page }}&{{ pagination.params }}">{{ page }}</a>
|
||||
{% else %}
|
||||
<a class="pagination-item" href="?page={{ page }}">{{ page }}</a>
|
||||
{% endif %}
|
||||
@ -100,16 +100,16 @@
|
||||
{% if pagination.next_pages %}
|
||||
<span> ></span>
|
||||
{% for page in pagination.next_pages %}
|
||||
{% if pagination.search_get %}
|
||||
<a class="pagination-item" href="?page={{ page }}&search={{ pagination.search_get }}">{{ page }}</a>
|
||||
{% if pagination.params %}
|
||||
<a class="pagination-item" href="?page={{ page }}&{{ pagination.params }}">{{ page }}</a>
|
||||
{% else %}
|
||||
<a class="pagination-item" href="?page={{ page }}">{{ page }}</a>
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
{% endif %}
|
||||
{% if pagination.last_page > 0 %}
|
||||
{% if pagination.search_get %}
|
||||
<a class="pagination-item" href="?page={{ pagination.last_page }}&search={{ pagination.search_get }}">
|
||||
{% if pagination.params %}
|
||||
<a class="pagination-item" href="?page={{ pagination.last_page }}&{{ pagination.params }}">
|
||||
{% if pagination.max_hits %}
|
||||
Max ({{ pagination.last_page }})
|
||||
{% else %}
|
||||
|
||||
@ -10,6 +10,9 @@
|
||||
<a href="{% url 'channel_id' channel_info.channel_id %}"><h3>Videos</h3></a>
|
||||
<a href="{% url 'channel_id_playlist' channel_info.channel_id %}"><h3>Playlists</h3></a>
|
||||
<a href="{% url 'channel_id_about' channel_info.channel_id %}"><h3>About</h3></a>
|
||||
{% if has_pending %}
|
||||
<a href="{% url 'downloads' %}?channel={{ channel_info.channel_id }}"><h3>Downloads</h3></a>
|
||||
{% endif %}
|
||||
</div>
|
||||
<div id="notifications" data="channel_id"></div>
|
||||
<div class="info-box info-box-2">
|
||||
|
||||
@ -10,6 +10,9 @@
|
||||
<a href="{% url 'channel_id' channel_info.channel_id %}"><h3>Videos</h3></a>
|
||||
<a href="{% url 'channel_id_playlist' channel_info.channel_id %}"><h3>Playlists</h3></a>
|
||||
<a href="{% url 'channel_id_about' channel_info.channel_id %}"><h3>About</h3></a>
|
||||
{% if has_pending %}
|
||||
<a href="{% url 'downloads' %}?channel={{ channel_info.channel_id }}"><h3>Downloads</h3></a>
|
||||
{% endif %}
|
||||
</div>
|
||||
<div class="info-box info-box-3">
|
||||
<div class="info-box-item">
|
||||
|
||||
@ -10,6 +10,9 @@
|
||||
<a href="{% url 'channel_id' channel_info.channel_id %}"><h3>Videos</h3></a>
|
||||
<a href="{% url 'channel_id_playlist' channel_info.channel_id %}"><h3>Playlists</h3></a>
|
||||
<a href="{% url 'channel_id_about' channel_info.channel_id %}"><h3>About</h3></a>
|
||||
{% if has_pending %}
|
||||
<a href="{% url 'downloads' %}?channel={{ channel_info.channel_id }}"><h3>Downloads</h3></a>
|
||||
{% endif %}
|
||||
</div>
|
||||
<div class="view-controls">
|
||||
<div class="toggle">
|
||||
|
||||
@ -3,7 +3,7 @@
|
||||
{% block content %}
|
||||
<div class="boxed-content">
|
||||
<div class="title-bar">
|
||||
<h1>Downloads</h1>
|
||||
<h1>Downloads {% if channel_filter_id %} for {{ channel_filter_name }}{% endif %}</h1>
|
||||
</div>
|
||||
<div id="notifications" data="download"></div>
|
||||
<div id="downloadControl"></div>
|
||||
@ -55,7 +55,7 @@
|
||||
<img src="{% static 'img/icon-listview.svg' %}" onclick="changeView(this)" data-origin="downloads" data-value="list" alt="list view">
|
||||
</div>
|
||||
</div>
|
||||
<h3>Total videos: {{ max_hits }}{% if max_hits == 10000 %}+{% endif %}</h3>
|
||||
<h3>Total videos: {{ max_hits }}{% if max_hits == 10000 %}+{% endif %} {% if channel_filter_id %} - from channel <i>{{ channel_filter_name }}</i>{% endif %}</h3>
|
||||
</div>
|
||||
<div class="boxed-content {% if view_style == "grid" %}boxed-{{ grid_items }}{% endif %}">
|
||||
<div class="video-list {{ view_style }} {% if view_style == "grid" %}grid-{{ grid_items }}{% endif %}">
|
||||
|
||||
@ -31,7 +31,7 @@ from home.src.frontend.forms import (
|
||||
UserSettingsForm,
|
||||
)
|
||||
from home.src.frontend.searching import SearchHandler
|
||||
from home.src.index.channel import channel_overwrites
|
||||
from home.src.index.channel import YoutubeChannel, channel_overwrites
|
||||
from home.src.index.generic import Pagination
|
||||
from home.src.index.playlist import YoutubePlaylist
|
||||
from home.src.ta.config import AppConfig, ScheduleBuilder
|
||||
@ -237,14 +237,10 @@ class ArchivistResultsView(ArchivistViewConfig):
|
||||
|
||||
def initiate_vars(self, request):
|
||||
"""search in es for vidoe hits"""
|
||||
page_get = int(request.GET.get("page", 0))
|
||||
self.user_id = request.user.id
|
||||
self.config_builder(self.user_id)
|
||||
self.search_get = request.GET.get("search", False)
|
||||
search_encoded = self._url_encode(self.search_get)
|
||||
self.pagination_handler = Pagination(
|
||||
page_get=page_get, user_id=self.user_id, search_get=search_encoded
|
||||
)
|
||||
self.pagination_handler = Pagination(request)
|
||||
self.sort_by = self._sort_by_overwrite()
|
||||
self._initial_data()
|
||||
|
||||
@ -362,7 +358,7 @@ class DownloadView(ArchivistResultsView):
|
||||
def get(self, request):
|
||||
"""handle get request"""
|
||||
self.initiate_vars(request)
|
||||
self._update_view_data()
|
||||
self._update_view_data(request)
|
||||
self.find_results()
|
||||
self.context.update(
|
||||
{
|
||||
@ -372,15 +368,33 @@ class DownloadView(ArchivistResultsView):
|
||||
)
|
||||
return render(request, "home/downloads.html", self.context)
|
||||
|
||||
def _update_view_data(self):
|
||||
def _update_view_data(self, request):
|
||||
"""update downloads view specific data dict"""
|
||||
if self.context["show_ignored_only"]:
|
||||
filter_view = "ignore"
|
||||
else:
|
||||
filter_view = "pending"
|
||||
|
||||
must_list = [{"term": {"status": {"value": filter_view}}}]
|
||||
|
||||
channel_filter = request.GET.get("channel", False)
|
||||
if channel_filter:
|
||||
must_list.append(
|
||||
{"term": {"channel_id": {"value": channel_filter}}}
|
||||
)
|
||||
|
||||
channel = YoutubeChannel(channel_filter)
|
||||
channel.get_from_es()
|
||||
self.context.update(
|
||||
{
|
||||
"channel_filter_id": channel_filter,
|
||||
"channel_filter_name": channel.json_data["channel_name"],
|
||||
}
|
||||
)
|
||||
|
||||
self.data.update(
|
||||
{
|
||||
"query": {"term": {"status": {"value": filter_view}}},
|
||||
"query": {"bool": {"must": must_list}},
|
||||
"sort": [{"timestamp": {"order": "asc"}}],
|
||||
}
|
||||
)
|
||||
@ -414,7 +428,37 @@ class DownloadView(ArchivistResultsView):
|
||||
return redirect("downloads", permanent=True)
|
||||
|
||||
|
||||
class ChannelIdView(ArchivistResultsView):
|
||||
class ChannelIdBaseView(ArchivistResultsView):
|
||||
"""base class for all channel-id views"""
|
||||
|
||||
def get_channel_meta(self, channel_id):
|
||||
"""get metadata for channel"""
|
||||
path = f"ta_channel/_doc/{channel_id}"
|
||||
response, _ = ElasticWrap(path).get()
|
||||
channel_info = SearchProcess(response).process()
|
||||
|
||||
return channel_info
|
||||
|
||||
def channel_has_pending(self, channel_id):
|
||||
"""check if channel has pending videos in queue"""
|
||||
path = "ta_download/_search"
|
||||
data = {
|
||||
"size": 1,
|
||||
"query": {
|
||||
"bool": {
|
||||
"must": [
|
||||
{"term": {"status": {"value": "pending"}}},
|
||||
{"term": {"channel_id": {"value": channel_id}}},
|
||||
]
|
||||
}
|
||||
},
|
||||
}
|
||||
response, _ = ElasticWrap(path).get(data=data)
|
||||
|
||||
self.context.update({"has_pending": bool(response["hits"]["hits"])})
|
||||
|
||||
|
||||
class ChannelIdView(ChannelIdBaseView):
|
||||
"""resolves to /channel/<channel-id>/
|
||||
display single channel page from channel_id
|
||||
"""
|
||||
@ -428,6 +472,7 @@ class ChannelIdView(ArchivistResultsView):
|
||||
self._update_view_data(channel_id)
|
||||
self.find_results()
|
||||
self.match_progress()
|
||||
self.channel_has_pending(channel_id)
|
||||
|
||||
if self.context["results"]:
|
||||
channel_info = self.context["results"][0]["source"]["channel"]
|
||||
@ -478,7 +523,7 @@ class ChannelIdView(ArchivistResultsView):
|
||||
return redirect("channel_id", channel_id, permanent=True)
|
||||
|
||||
|
||||
class ChannelIdAboutView(ArchivistResultsView):
|
||||
class ChannelIdAboutView(ChannelIdBaseView):
|
||||
"""resolves to /channel/<channel-id>/about/
|
||||
show metadata, handle per channel conf
|
||||
"""
|
||||
@ -488,6 +533,7 @@ class ChannelIdAboutView(ArchivistResultsView):
|
||||
def get(self, request, channel_id):
|
||||
"""handle get request"""
|
||||
self.initiate_vars(request)
|
||||
self.channel_has_pending(channel_id)
|
||||
|
||||
path = f"ta_channel/_doc/{channel_id}"
|
||||
response, _ = ElasticWrap(path).get()
|
||||
@ -521,7 +567,7 @@ class ChannelIdAboutView(ArchivistResultsView):
|
||||
return redirect("channel_id_about", channel_id, permanent=True)
|
||||
|
||||
|
||||
class ChannelIdPlaylistView(ArchivistResultsView):
|
||||
class ChannelIdPlaylistView(ChannelIdBaseView):
|
||||
"""resolves to /channel/<channel-id>/playlist/
|
||||
show all playlists of channel
|
||||
"""
|
||||
@ -534,8 +580,9 @@ class ChannelIdPlaylistView(ArchivistResultsView):
|
||||
self.initiate_vars(request)
|
||||
self._update_view_data(channel_id)
|
||||
self.find_results()
|
||||
self.channel_has_pending(channel_id)
|
||||
|
||||
channel_info = self._get_channel_meta(channel_id)
|
||||
channel_info = self.get_channel_meta(channel_id)
|
||||
channel_name = channel_info["channel_name"]
|
||||
self.context.update(
|
||||
{
|
||||
@ -556,14 +603,6 @@ class ChannelIdPlaylistView(ArchivistResultsView):
|
||||
|
||||
self.data["query"] = {"bool": {"must": must_list}}
|
||||
|
||||
def _get_channel_meta(self, channel_id):
|
||||
"""get metadata for channel"""
|
||||
path = f"ta_channel/_doc/{channel_id}"
|
||||
response, _ = ElasticWrap(path).get()
|
||||
channel_info = SearchProcess(response).process()
|
||||
|
||||
return channel_info
|
||||
|
||||
|
||||
class ChannelView(ArchivistResultsView):
|
||||
"""resolves to /channel/
|
||||
|
||||
@ -1,13 +1,13 @@
|
||||
beautifulsoup4==4.11.1
|
||||
celery==5.2.7
|
||||
Django==4.0.6
|
||||
Django==4.1.2
|
||||
django-auth-ldap==4.1.0
|
||||
django-cors-headers==3.13.0
|
||||
djangorestframework==3.13.1
|
||||
djangorestframework==3.14.0
|
||||
Pillow==9.2.0
|
||||
redis==4.3.4
|
||||
requests==2.28.1
|
||||
ryd-client==0.0.6
|
||||
uWSGI==2.0.20
|
||||
whitenoise==6.2.0
|
||||
yt_dlp==2022.9.1
|
||||
yt_dlp==2022.10.4
|
||||
|
||||
@ -140,7 +140,7 @@ function toggleCheckbox(checkbox) {
|
||||
var payload = JSON.stringify(payloadDict);
|
||||
sendPost(payload);
|
||||
setTimeout(function(){
|
||||
var currPage = window.location.pathname;
|
||||
var currPage = window.location.pathname + window.location.search;
|
||||
window.location.replace(currPage);
|
||||
return false;
|
||||
}, 500);
|
||||
|
||||
Loading…
Reference in New Issue
Block a user