From bb0b60e575ca4c7210a8ca23d860d72ebead4095 Mon Sep 17 00:00:00 2001 From: Omar Roth Date: Wed, 15 Aug 2018 10:22:36 -0500 Subject: [PATCH] Add playlist page and endpoint --- src/invidious.cr | 110 ++++++++++++---- src/invidious/comments.cr | 2 +- src/invidious/helpers/helpers.cr | 34 ----- src/invidious/helpers/utils.cr | 15 ++- src/invidious/playlists.cr | 160 +++++++++++++++++++++++ src/invidious/views/components/video.ecr | 7 +- src/invidious/views/playlist.ecr | 42 ++++++ 7 files changed, 311 insertions(+), 59 deletions(-) create mode 100644 src/invidious/playlists.cr create mode 100644 src/invidious/views/playlist.ecr diff --git a/src/invidious.cr b/src/invidious.cr index 616af019..18aa049a 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -355,6 +355,31 @@ get "/embed/:id" do |env| rendered "embed" end +# Playlists +get "/playlist" do |env| + plid = env.params.query["list"]? + if !plid + next env.redirect "/" + end + + page = env.params.query["page"]?.try &.to_i? + page ||= 1 + + if plid + begin + videos = extract_playlist(plid, page) + rescue ex + error_message = ex.message + next templated "error" + end + playlist = fetch_playlist(plid) + else + next env.redirect "/" + end + + templated "playlist" +end + # Search get "/results" do |env| @@ -1522,31 +1547,13 @@ get "/channel/:ucid" do |env| rss = XML.parse_html(rss.body) author = rss.xpath_node("//feed/author/name").not_nil!.content - url = produce_playlist_url(ucid, (page - 1) * 100) - response = client.get(url) - response = JSON.parse(response.body) - - if !response["content_html"]? - error_message = "This channel does not exist." + begin + videos = extract_playlist(ucid, page) + rescue ex + error_message = ex.message next templated "error" end - document = XML.parse_html(response["content_html"].as_s) - anchor = document.xpath_node(%q(//div[@class="pl-video-owner"]/a)) - if !anchor - videos = [] of ChannelVideo - next templated "channel" - end - - videos = [] of ChannelVideo - document.xpath_nodes(%q(//a[contains(@class,"pl-video-title-link")])).each do |node| - href = URI.parse(node["href"]) - id = HTTP::Params.parse(href.query.not_nil!)["v"] - title = node.content - - videos << ChannelVideo.new(id, title, Time.now, Time.now, "", "") - end - templated "channel" end @@ -2350,6 +2357,65 @@ get "/api/v1/search" do |env| response end +get "/api/v1/playlists/:plid" do |env| + plid = env.params.url["plid"] + + page = env.params.query["page"]?.try &.to_i? + page ||= 1 + + begin + videos = extract_playlist(plid, page) + rescue ex + env.response.content_type = "application/json" + response = {"error" => "Playlist is empty"}.to_json + halt env, status_code: 404, response: response + end + + playlist = fetch_playlist(plid) + + response = JSON.build do |json| + json.object do + json.field "title", playlist.title + json.field "id", playlist.id + + json.field "author", playlist.author + json.field "authorId", playlist.ucid + json.field "authorUrl", "/channel/#{playlist.ucid}" + + json.field "description", playlist.description + json.field "videoCount", playlist.video_count + + json.field "viewCount", playlist.views + json.field "updated", playlist.updated.epoch + + json.field "videos" do + json.array do + videos.each do |video| + json.object do + json.field "title", video.title + json.field "id", video.id + + json.field "author", video.author + json.field "authorId", video.ucid + json.field "authorUrl", "/channel/#{video.ucid}" + + json.field "videoThumbnails" do + generate_thumbnails(json, video.id) + end + + json.field "index", video.index + json.field "lengthSeconds", video.length_seconds + end + end + end + end + end + end + + env.response.content_type = "application/json" + response +end + get "/api/manifest/dash/id/videoplayback" do |env| env.response.headers["Access-Control-Allow-Origin"] = "*" env.redirect "/videoplayback?#{env.params.query}" diff --git a/src/invidious/comments.cr b/src/invidious/comments.cr index b7898d5e..d9f86f69 100644 --- a/src/invidious/comments.cr +++ b/src/invidious/comments.cr @@ -262,7 +262,7 @@ def fill_links(html, scheme, host) end if host == "www.youtube.com" - html = html.xpath_node(%q(//p[@id="eow-description"])).not_nil!.to_xml + html = html.xpath_node(%q(//body)).not_nil!.to_xml else html = html.to_xml(options: XML::SaveOptions::NO_DECL) end diff --git a/src/invidious/helpers/helpers.cr b/src/invidious/helpers/helpers.cr index 7104e1e9..5f29ee8d 100644 --- a/src/invidious/helpers/helpers.cr +++ b/src/invidious/helpers/helpers.cr @@ -116,40 +116,6 @@ def login_req(login_form, f_req) return HTTP::Params.encode(data) end -def produce_playlist_url(id, index) - if id.starts_with? "UC" - id = "UU" + id.lchop("UC") - end - ucid = "VL" + id - - continuation = [0x08_u8] + write_var_int(index) - slice = continuation.to_unsafe.to_slice(continuation.size) - slice = Base64.urlsafe_encode(slice, false) - - # Inner Base64 - continuation = "PT:" + slice - continuation = [0x7a_u8, continuation.bytes.size.to_u8] + continuation.bytes - slice = continuation.to_unsafe.to_slice(continuation.size) - slice = Base64.urlsafe_encode(slice) - slice = URI.escape(slice) - - # Outer Base64 - continuation = [0x1a.to_u8, slice.bytes.size.to_u8] + slice.bytes - continuation = ucid.bytes + continuation - continuation = [0x12_u8, ucid.size.to_u8] + continuation - continuation = [0xe2_u8, 0xa9_u8, 0x85_u8, 0xb2_u8, 2_u8, continuation.size.to_u8] + continuation - - # Wrap bytes - slice = continuation.to_unsafe.to_slice(continuation.size) - slice = Base64.urlsafe_encode(slice) - slice = URI.escape(slice) - continuation = slice - - url = "/browse_ajax?action_continuation=1&continuation=#{continuation}" - - return url -end - def produce_videos_url(ucid, page = 1) page = "#{page}" diff --git a/src/invidious/helpers/utils.cr b/src/invidious/helpers/utils.cr index 9ff411cd..d424fb9c 100644 --- a/src/invidious/helpers/utils.cr +++ b/src/invidious/helpers/utils.cr @@ -64,10 +64,23 @@ end def decode_date(string : String) # String matches 'YYYY' - if string.match(/\d{4}/) + if string.match(/^\d{4}/) return Time.new(string.to_i, 1, 1) end + # Try to parse as format Jul 10, 2000 + begin + return Time.parse(string, "%b %-d, %Y", Time::Location.local) + rescue ex + end + + case string + when "today" + return Time.now + when "yesterday" + return Time.now - 1.day + end + # String matches format "20 hours ago", "4 months ago"... date = string.split(" ")[-3, 3] delta = date[0].to_i diff --git a/src/invidious/playlists.cr b/src/invidious/playlists.cr new file mode 100644 index 00000000..9bd5724d --- /dev/null +++ b/src/invidious/playlists.cr @@ -0,0 +1,160 @@ +class Playlist + add_mapping({ + title: String, + id: String, + author: String, + ucid: String, + description: String, + video_count: Int32, + views: Int64, + updated: Time, + }) +end + +class PlaylistVideo + add_mapping({ + title: String, + id: String, + author: String, + ucid: String, + length_seconds: Int32, + published: Time, + playlists: Array(String), + index: Int32, + }) +end + +def extract_playlist(plid, page) + index = (page - 1) * 100 + url = produce_playlist_url(plid, index) + + client = make_client(YT_URL) + response = client.get(url) + response = JSON.parse(response.body) + if !response["content_html"]? || response["content_html"].as_s.empty? + raise "Playlist does not exist" + end + + videos = [] of PlaylistVideo + + document = XML.parse_html(response["content_html"].as_s) + anchor = document.xpath_node(%q(//div[@class="pl-video-owner"]/a)) + if anchor + document.xpath_nodes(%q(.//tr[contains(@class, "pl-video")])).each_with_index do |video, offset| + anchor = video.xpath_node(%q(.//td[@class="pl-video-title"])) + if !anchor + next + end + + title = anchor.xpath_node(%q(.//a)).not_nil!.content.strip(" \n") + id = anchor.xpath_node(%q(.//a)).not_nil!["href"].lchop("/watch?v=")[0, 11] + + anchor = anchor.xpath_node(%q(.//div[@class="pl-video-owner"]/a)) + if anchor + author = anchor.content + ucid = anchor["href"].split("/")[2] + else + author = "" + ucid = "" + end + + anchor = video.xpath_node(%q(.//td[@class="pl-video-time"]/div/div[1])) + if anchor && !anchor.content.empty? + length_seconds = decode_length_seconds(anchor.content) + else + length_seconds = 0 + end + + videos << PlaylistVideo.new( + title, + id, + author, + ucid, + length_seconds, + Time.now, + [plid], + index + offset, + ) + end + end + + return videos +end + +def produce_playlist_url(id, index) + if id.starts_with? "UC" + id = "UU" + id.lchop("UC") + end + ucid = "VL" + id + + continuation = [0x08_u8] + write_var_int(index) + slice = continuation.to_unsafe.to_slice(continuation.size) + slice = Base64.urlsafe_encode(slice, false) + + # Inner Base64 + continuation = "PT:" + slice + continuation = [0x7a_u8, continuation.bytes.size.to_u8] + continuation.bytes + slice = continuation.to_unsafe.to_slice(continuation.size) + slice = Base64.urlsafe_encode(slice) + slice = URI.escape(slice) + + # Outer Base64 + continuation = [0x1a.to_u8, slice.bytes.size.to_u8] + slice.bytes + continuation = ucid.bytes + continuation + continuation = [0x12_u8, ucid.size.to_u8] + continuation + continuation = [0xe2_u8, 0xa9_u8, 0x85_u8, 0xb2_u8, 2_u8, continuation.size.to_u8] + continuation + + # Wrap bytes + slice = continuation.to_unsafe.to_slice(continuation.size) + slice = Base64.urlsafe_encode(slice) + slice = URI.escape(slice) + continuation = slice + + url = "/browse_ajax?action_continuation=1&continuation=#{continuation}" + + return url +end + +def fetch_playlist(plid) + client = make_client(YT_URL) + response = client.get("/playlist?list=#{plid}&disable_polymer=1") + document = XML.parse_html(response.body) + + title = document.xpath_node(%q(//h1[@class="pl-header-title"])).not_nil!.content + title = title.strip(" \n") + + description = document.xpath_node(%q(//span[@class="pl-header-description-text"]/div/div[1])) + description ||= document.xpath_node(%q(//span[@class="pl-header-description-text"])) + + if description + description = description.to_xml.strip(" \n") + description = description.split("