From 7ace3fc989d5b24104af92537dc3a67cf9f608c3 Mon Sep 17 00:00:00 2001 From: Samantaz Fox Date: Fri, 4 Feb 2022 04:46:38 +0100 Subject: [PATCH 01/10] Move remaining user-related routes out of main file --- src/invidious.cr | 333 ++--------------------------- src/invidious/routes/account.cr | 358 ++++++++++++++++++++++++++++++++ 2 files changed, 372 insertions(+), 319 deletions(-) create mode 100644 src/invidious/routes/account.cr diff --git a/src/invidious.cr b/src/invidious.cr index f4cae7ea..6f4f575b 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -364,16 +364,30 @@ end Invidious::Routing.get "/results", Invidious::Routes::Search, :results Invidious::Routing.get "/search", Invidious::Routes::Search, :search + # User login/out Invidious::Routing.get "/login", Invidious::Routes::Login, :login_page Invidious::Routing.post "/login", Invidious::Routes::Login, :login Invidious::Routing.post "/signout", Invidious::Routes::Login, :signout + # User preferences Invidious::Routing.get "/preferences", Invidious::Routes::PreferencesRoute, :show Invidious::Routing.post "/preferences", Invidious::Routes::PreferencesRoute, :update Invidious::Routing.get "/toggle_theme", Invidious::Routes::PreferencesRoute, :toggle_theme Invidious::Routing.get "/data_control", Invidious::Routes::PreferencesRoute, :data_control Invidious::Routing.post "/data_control", Invidious::Routes::PreferencesRoute, :update_data_control + # User account management + Invidious::Routing.get "/change_password", Invidious::Routes::Account, :get_change_password + Invidious::Routing.post "/change_password", Invidious::Routes::Account, :post_change_password + Invidious::Routing.get "/delete_account", Invidious::Routes::Account, :get_delete + Invidious::Routing.post "/delete_account", Invidious::Routes::Account, :post_delete + Invidious::Routing.get "/clear_watch_history", Invidious::Routes::Account, :get_clear_history + Invidious::Routing.post "/clear_watch_history", Invidious::Routes::Account, :post_clear_history + Invidious::Routing.get "/authorize_token", Invidious::Routes::Account, :get_authorize_token + Invidious::Routing.post "/authorize_token", Invidious::Routes::Account, :post_authorize_token + Invidious::Routing.get "/token_manager", Invidious::Routes::Account, :token_manager + Invidious::Routing.post "/token_ajax", Invidious::Routes::Account, :token_ajax + # Feeds Invidious::Routing.get "/view_all_playlists", Invidious::Routes::Feeds, :view_all_playlists_redirect Invidious::Routing.get "/feed/playlists", Invidious::Routes::Feeds, :playlists @@ -412,325 +426,6 @@ define_v1_api_routes() define_api_manifest_routes() define_video_playback_routes() -get "/change_password" do |env| - locale = env.get("preferences").as(Preferences).locale - - user = env.get? "user" - sid = env.get? "sid" - referer = get_referer(env) - - if !user - next env.redirect referer - end - - user = user.as(User) - sid = sid.as(String) - csrf_token = generate_response(sid, {":change_password"}, HMAC_KEY) - - templated "change_password" -end - -post "/change_password" do |env| - locale = env.get("preferences").as(Preferences).locale - - user = env.get? "user" - sid = env.get? "sid" - referer = get_referer(env) - - if !user - next env.redirect referer - end - - user = user.as(User) - sid = sid.as(String) - token = env.params.body["csrf_token"]? - - # We don't store passwords for Google accounts - if !user.password - next error_template(400, "Cannot change password for Google accounts") - end - - begin - validate_request(token, sid, env.request, HMAC_KEY, locale) - rescue ex - next error_template(400, ex) - end - - password = env.params.body["password"]? - if !password - next error_template(401, "Password is a required field") - end - - new_passwords = env.params.body.select { |k, v| k.match(/^new_password\[\d+\]$/) }.map { |k, v| v } - - if new_passwords.size <= 1 || new_passwords.uniq.size != 1 - next error_template(400, "New passwords must match") - end - - new_password = new_passwords.uniq[0] - if new_password.empty? - next error_template(401, "Password cannot be empty") - end - - if new_password.bytesize > 55 - next error_template(400, "Password cannot be longer than 55 characters") - end - - if !Crypto::Bcrypt::Password.new(user.password.not_nil!).verify(password.byte_slice(0, 55)) - next error_template(401, "Incorrect password") - end - - new_password = Crypto::Bcrypt::Password.create(new_password, cost: 10) - Invidious::Database::Users.update_password(user, new_password.to_s) - - env.redirect referer -end - -get "/delete_account" do |env| - locale = env.get("preferences").as(Preferences).locale - - user = env.get? "user" - sid = env.get? "sid" - referer = get_referer(env) - - if !user - next env.redirect referer - end - - user = user.as(User) - sid = sid.as(String) - csrf_token = generate_response(sid, {":delete_account"}, HMAC_KEY) - - templated "delete_account" -end - -post "/delete_account" do |env| - locale = env.get("preferences").as(Preferences).locale - - user = env.get? "user" - sid = env.get? "sid" - referer = get_referer(env) - - if !user - next env.redirect referer - end - - user = user.as(User) - sid = sid.as(String) - token = env.params.body["csrf_token"]? - - begin - validate_request(token, sid, env.request, HMAC_KEY, locale) - rescue ex - next error_template(400, ex) - end - - view_name = "subscriptions_#{sha256(user.email)}" - Invidious::Database::Users.delete(user) - Invidious::Database::SessionIDs.delete(email: user.email) - PG_DB.exec("DROP MATERIALIZED VIEW #{view_name}") - - env.request.cookies.each do |cookie| - cookie.expires = Time.utc(1990, 1, 1) - env.response.cookies << cookie - end - - env.redirect referer -end - -get "/clear_watch_history" do |env| - locale = env.get("preferences").as(Preferences).locale - - user = env.get? "user" - sid = env.get? "sid" - referer = get_referer(env) - - if !user - next env.redirect referer - end - - user = user.as(User) - sid = sid.as(String) - csrf_token = generate_response(sid, {":clear_watch_history"}, HMAC_KEY) - - templated "clear_watch_history" -end - -post "/clear_watch_history" do |env| - locale = env.get("preferences").as(Preferences).locale - - user = env.get? "user" - sid = env.get? "sid" - referer = get_referer(env) - - if !user - next env.redirect referer - end - - user = user.as(User) - sid = sid.as(String) - token = env.params.body["csrf_token"]? - - begin - validate_request(token, sid, env.request, HMAC_KEY, locale) - rescue ex - next error_template(400, ex) - end - - Invidious::Database::Users.clear_watch_history(user) - env.redirect referer -end - -get "/authorize_token" do |env| - locale = env.get("preferences").as(Preferences).locale - - user = env.get? "user" - sid = env.get? "sid" - referer = get_referer(env) - - if !user - next env.redirect referer - end - - user = user.as(User) - sid = sid.as(String) - csrf_token = generate_response(sid, {":authorize_token"}, HMAC_KEY) - - scopes = env.params.query["scopes"]?.try &.split(",") - scopes ||= [] of String - - callback_url = env.params.query["callback_url"]? - if callback_url - callback_url = URI.parse(callback_url) - end - - expire = env.params.query["expire"]?.try &.to_i? - - templated "authorize_token" -end - -post "/authorize_token" do |env| - locale = env.get("preferences").as(Preferences).locale - - user = env.get? "user" - sid = env.get? "sid" - referer = get_referer(env) - - if !user - next env.redirect referer - end - - user = env.get("user").as(User) - sid = sid.as(String) - token = env.params.body["csrf_token"]? - - begin - validate_request(token, sid, env.request, HMAC_KEY, locale) - rescue ex - next error_template(400, ex) - end - - scopes = env.params.body.select { |k, v| k.match(/^scopes\[\d+\]$/) }.map { |k, v| v } - callback_url = env.params.body["callbackUrl"]? - expire = env.params.body["expire"]?.try &.to_i? - - access_token = generate_token(user.email, scopes, expire, HMAC_KEY) - - if callback_url - access_token = URI.encode_www_form(access_token) - url = URI.parse(callback_url) - - if url.query - query = HTTP::Params.parse(url.query.not_nil!) - else - query = HTTP::Params.new - end - - query["token"] = access_token - url.query = query.to_s - - env.redirect url.to_s - else - csrf_token = "" - env.set "access_token", access_token - templated "authorize_token" - end -end - -get "/token_manager" do |env| - locale = env.get("preferences").as(Preferences).locale - - user = env.get? "user" - sid = env.get? "sid" - referer = get_referer(env, "/subscription_manager") - - if !user - next env.redirect referer - end - - user = user.as(User) - tokens = Invidious::Database::SessionIDs.select_all(user.email) - - templated "token_manager" -end - -post "/token_ajax" do |env| - locale = env.get("preferences").as(Preferences).locale - - user = env.get? "user" - sid = env.get? "sid" - referer = get_referer(env) - - redirect = env.params.query["redirect"]? - redirect ||= "true" - redirect = redirect == "true" - - if !user - if redirect - next env.redirect referer - else - next error_json(403, "No such user") - end - end - - user = user.as(User) - sid = sid.as(String) - token = env.params.body["csrf_token"]? - - begin - validate_request(token, sid, env.request, HMAC_KEY, locale) - rescue ex - if redirect - next error_template(400, ex) - else - next error_json(400, ex) - end - end - - if env.params.query["action_revoke_token"]? - action = "action_revoke_token" - else - next env.redirect referer - end - - session = env.params.query["session"]? - session ||= "" - - case action - when .starts_with? "action_revoke_token" - Invidious::Database::SessionIDs.delete(sid: session, email: user.email) - else - next error_json(400, "Unsupported action #{action}") - end - - if redirect - env.redirect referer - else - env.response.content_type = "application/json" - "{}" - end -end - # Channels {"/channel/:ucid/live", "/user/:user/live", "/c/:user/live"}.each do |route| diff --git a/src/invidious/routes/account.cr b/src/invidious/routes/account.cr new file mode 100644 index 00000000..2be0de37 --- /dev/null +++ b/src/invidious/routes/account.cr @@ -0,0 +1,358 @@ +{% skip_file if flag?(:api_only) %} + +module Invidious::Routes::Account + extend self + + # ------------------- + # Password update + # ------------------- + + # Show the password change interface (GET request) + def get_change_password(env) + locale = env.get("preferences").as(Preferences).locale + + user = env.get? "user" + sid = env.get? "sid" + referer = get_referer(env) + + if !user + return env.redirect referer + end + + user = user.as(User) + sid = sid.as(String) + csrf_token = generate_response(sid, {":change_password"}, HMAC_KEY) + + templated "change_password" + end + + # Handle the password change (POST request) + def post_change_password(env) + locale = env.get("preferences").as(Preferences).locale + + user = env.get? "user" + sid = env.get? "sid" + referer = get_referer(env) + + if !user + return env.redirect referer + end + + user = user.as(User) + sid = sid.as(String) + token = env.params.body["csrf_token"]? + + # We don't store passwords for Google accounts + if !user.password + return error_template(400, "Cannot change password for Google accounts") + end + + begin + validate_request(token, sid, env.request, HMAC_KEY, locale) + rescue ex + return error_template(400, ex) + end + + password = env.params.body["password"]? + if !password + return error_template(401, "Password is a required field") + end + + new_passwords = env.params.body.select { |k, v| k.match(/^new_password\[\d+\]$/) }.map { |k, v| v } + + if new_passwords.size <= 1 || new_passwords.uniq.size != 1 + return error_template(400, "New passwords must match") + end + + new_password = new_passwords.uniq[0] + if new_password.empty? + return error_template(401, "Password cannot be empty") + end + + if new_password.bytesize > 55 + return error_template(400, "Password cannot be longer than 55 characters") + end + + if !Crypto::Bcrypt::Password.new(user.password.not_nil!).verify(password.byte_slice(0, 55)) + return error_template(401, "Incorrect password") + end + + new_password = Crypto::Bcrypt::Password.create(new_password, cost: 10) + Invidious::Database::Users.update_password(user, new_password.to_s) + + env.redirect referer + end + + # ------------------- + # Account deletion + # ------------------- + + # Show the account deletion confirmation prompt (GET request) + def get_delete(env) + locale = env.get("preferences").as(Preferences).locale + + user = env.get? "user" + sid = env.get? "sid" + referer = get_referer(env) + + if !user + return env.redirect referer + end + + user = user.as(User) + sid = sid.as(String) + csrf_token = generate_response(sid, {":delete_account"}, HMAC_KEY) + + templated "delete_account" + end + + # Handle the account deletion (POST request) + def post_delete(env) + locale = env.get("preferences").as(Preferences).locale + + user = env.get? "user" + sid = env.get? "sid" + referer = get_referer(env) + + if !user + return env.redirect referer + end + + user = user.as(User) + sid = sid.as(String) + token = env.params.body["csrf_token"]? + + begin + validate_request(token, sid, env.request, HMAC_KEY, locale) + rescue ex + return error_template(400, ex) + end + + view_name = "subscriptions_#{sha256(user.email)}" + Invidious::Database::Users.delete(user) + Invidious::Database::SessionIDs.delete(email: user.email) + PG_DB.exec("DROP MATERIALIZED VIEW #{view_name}") + + env.request.cookies.each do |cookie| + cookie.expires = Time.utc(1990, 1, 1) + env.response.cookies << cookie + end + + env.redirect referer + end + + # ------------------- + # Clear history + # ------------------- + + # Show the watch history deletion confirmation prompt (GET request) + def get_clear_history(env) + locale = env.get("preferences").as(Preferences).locale + + user = env.get? "user" + sid = env.get? "sid" + referer = get_referer(env) + + if !user + return env.redirect referer + end + + user = user.as(User) + sid = sid.as(String) + csrf_token = generate_response(sid, {":clear_watch_history"}, HMAC_KEY) + + templated "clear_watch_history" + end + + # Handle the watch history clearing (POST request) + def post_clear_history(env) + locale = env.get("preferences").as(Preferences).locale + + user = env.get? "user" + sid = env.get? "sid" + referer = get_referer(env) + + if !user + return env.redirect referer + end + + user = user.as(User) + sid = sid.as(String) + token = env.params.body["csrf_token"]? + + begin + validate_request(token, sid, env.request, HMAC_KEY, locale) + rescue ex + return error_template(400, ex) + end + + Invidious::Database::Users.clear_watch_history(user) + env.redirect referer + end + + # ------------------- + # Authorize tokens + # ------------------- + + # Show the "authorize token?" confirmation prompt (GET request) + def get_authorize_token(env) + locale = env.get("preferences").as(Preferences).locale + + user = env.get? "user" + sid = env.get? "sid" + referer = get_referer(env) + + if !user + return env.redirect referer + end + + user = user.as(User) + sid = sid.as(String) + csrf_token = generate_response(sid, {":authorize_token"}, HMAC_KEY) + + scopes = env.params.query["scopes"]?.try &.split(",") + scopes ||= [] of String + + callback_url = env.params.query["callback_url"]? + if callback_url + callback_url = URI.parse(callback_url) + end + + expire = env.params.query["expire"]?.try &.to_i? + + templated "authorize_token" + end + + # Handle token authorization (POST request) + def post_authorize_token(env) + locale = env.get("preferences").as(Preferences).locale + + user = env.get? "user" + sid = env.get? "sid" + referer = get_referer(env) + + if !user + return env.redirect referer + end + + user = env.get("user").as(User) + sid = sid.as(String) + token = env.params.body["csrf_token"]? + + begin + validate_request(token, sid, env.request, HMAC_KEY, locale) + rescue ex + return error_template(400, ex) + end + + scopes = env.params.body.select { |k, v| k.match(/^scopes\[\d+\]$/) }.map { |k, v| v } + callback_url = env.params.body["callbackUrl"]? + expire = env.params.body["expire"]?.try &.to_i? + + access_token = generate_token(user.email, scopes, expire, HMAC_KEY) + + if callback_url + access_token = URI.encode_www_form(access_token) + url = URI.parse(callback_url) + + if url.query + query = HTTP::Params.parse(url.query.not_nil!) + else + query = HTTP::Params.new + end + + query["token"] = access_token + url.query = query.to_s + + env.redirect url.to_s + else + csrf_token = "" + env.set "access_token", access_token + templated "authorize_token" + end + end + + # ------------------- + # Manage tokens + # ------------------- + + # Show the token manager page (GET request) + def token_manager(env) + locale = env.get("preferences").as(Preferences).locale + + user = env.get? "user" + sid = env.get? "sid" + referer = get_referer(env, "/subscription_manager") + + if !user + return env.redirect referer + end + + user = user.as(User) + tokens = Invidious::Database::SessionIDs.select_all(user.email) + + templated "token_manager" + end + + # ------------------- + # AJAX for tokens + # ------------------- + + # Handle internal (non-API) token actions (POST request) + def token_ajax(env) + locale = env.get("preferences").as(Preferences).locale + + user = env.get? "user" + sid = env.get? "sid" + referer = get_referer(env) + + redirect = env.params.query["redirect"]? + redirect ||= "true" + redirect = redirect == "true" + + if !user + if redirect + return env.redirect referer + else + return error_json(403, "No such user") + end + end + + user = user.as(User) + sid = sid.as(String) + token = env.params.body["csrf_token"]? + + begin + validate_request(token, sid, env.request, HMAC_KEY, locale) + rescue ex + if redirect + return error_template(400, ex) + else + return error_json(400, ex) + end + end + + if env.params.query["action_revoke_token"]? + action = "action_revoke_token" + else + return env.redirect referer + end + + session = env.params.query["session"]? + session ||= "" + + case action + when .starts_with? "action_revoke_token" + Invidious::Database::SessionIDs.delete(sid: session, email: user.email) + else + return error_json(400, "Unsupported action #{action}") + end + + if redirect + return env.redirect referer + else + env.response.content_type = "application/json" + return "{}" + end + end +end From fb3615502258c71249c6d77aebe684234756b416 Mon Sep 17 00:00:00 2001 From: Samantaz Fox Date: Fri, 4 Feb 2022 04:55:50 +0100 Subject: [PATCH 02/10] Move user routes definitions to a macro in routing.cr --- src/invidious.cr | 25 ++----------------------- src/invidious/routing.cr | 26 ++++++++++++++++++++++++++ 2 files changed, 28 insertions(+), 23 deletions(-) diff --git a/src/invidious.cr b/src/invidious.cr index 6f4f575b..fc498dbf 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -364,29 +364,8 @@ end Invidious::Routing.get "/results", Invidious::Routes::Search, :results Invidious::Routing.get "/search", Invidious::Routes::Search, :search - # User login/out - Invidious::Routing.get "/login", Invidious::Routes::Login, :login_page - Invidious::Routing.post "/login", Invidious::Routes::Login, :login - Invidious::Routing.post "/signout", Invidious::Routes::Login, :signout - - # User preferences - Invidious::Routing.get "/preferences", Invidious::Routes::PreferencesRoute, :show - Invidious::Routing.post "/preferences", Invidious::Routes::PreferencesRoute, :update - Invidious::Routing.get "/toggle_theme", Invidious::Routes::PreferencesRoute, :toggle_theme - Invidious::Routing.get "/data_control", Invidious::Routes::PreferencesRoute, :data_control - Invidious::Routing.post "/data_control", Invidious::Routes::PreferencesRoute, :update_data_control - - # User account management - Invidious::Routing.get "/change_password", Invidious::Routes::Account, :get_change_password - Invidious::Routing.post "/change_password", Invidious::Routes::Account, :post_change_password - Invidious::Routing.get "/delete_account", Invidious::Routes::Account, :get_delete - Invidious::Routing.post "/delete_account", Invidious::Routes::Account, :post_delete - Invidious::Routing.get "/clear_watch_history", Invidious::Routes::Account, :get_clear_history - Invidious::Routing.post "/clear_watch_history", Invidious::Routes::Account, :post_clear_history - Invidious::Routing.get "/authorize_token", Invidious::Routes::Account, :get_authorize_token - Invidious::Routing.post "/authorize_token", Invidious::Routes::Account, :post_authorize_token - Invidious::Routing.get "/token_manager", Invidious::Routes::Account, :token_manager - Invidious::Routing.post "/token_ajax", Invidious::Routes::Account, :token_ajax + # User routes + define_user_routes() # Feeds Invidious::Routing.get "/view_all_playlists", Invidious::Routes::Feeds, :view_all_playlists_redirect diff --git a/src/invidious/routing.cr b/src/invidious/routing.cr index 7551f22d..5efe1bd8 100644 --- a/src/invidious/routing.cr +++ b/src/invidious/routing.cr @@ -10,6 +10,32 @@ module Invidious::Routing {% end %} end +macro define_user_routes + # User login/out + Invidious::Routing.get "/login", Invidious::Routes::Login, :login_page + Invidious::Routing.post "/login", Invidious::Routes::Login, :login + Invidious::Routing.post "/signout", Invidious::Routes::Login, :signout + + # User preferences + Invidious::Routing.get "/preferences", Invidious::Routes::PreferencesRoute, :show + Invidious::Routing.post "/preferences", Invidious::Routes::PreferencesRoute, :update + Invidious::Routing.get "/toggle_theme", Invidious::Routes::PreferencesRoute, :toggle_theme + Invidious::Routing.get "/data_control", Invidious::Routes::PreferencesRoute, :data_control + Invidious::Routing.post "/data_control", Invidious::Routes::PreferencesRoute, :update_data_control + + # User account management + Invidious::Routing.get "/change_password", Invidious::Routes::Account, :get_change_password + Invidious::Routing.post "/change_password", Invidious::Routes::Account, :post_change_password + Invidious::Routing.get "/delete_account", Invidious::Routes::Account, :get_delete + Invidious::Routing.post "/delete_account", Invidious::Routes::Account, :post_delete + Invidious::Routing.get "/clear_watch_history", Invidious::Routes::Account, :get_clear_history + Invidious::Routing.post "/clear_watch_history", Invidious::Routes::Account, :post_clear_history + Invidious::Routing.get "/authorize_token", Invidious::Routes::Account, :get_authorize_token + Invidious::Routing.post "/authorize_token", Invidious::Routes::Account, :post_authorize_token + Invidious::Routing.get "/token_manager", Invidious::Routes::Account, :token_manager + Invidious::Routing.post "/token_ajax", Invidious::Routes::Account, :token_ajax +end + macro define_v1_api_routes {{namespace = Invidious::Routes::API::V1}} # Videos From c04f45d5e36499e6faefd163e92c58fa1abaa7ae Mon Sep 17 00:00:00 2001 From: Samantaz Fox Date: Fri, 4 Feb 2022 04:09:07 +0100 Subject: [PATCH 03/10] Move user struct to own file, under Invidious namespace --- src/invidious.cr | 2 +- src/invidious/search.cr | 2 +- src/invidious/user/user.cr | 27 ++++++++++++++++++++++++++ src/invidious/users.cr | 30 ++--------------------------- src/invidious/views/preferences.ecr | 2 +- src/invidious/views/template.ecr | 4 ++-- 6 files changed, 34 insertions(+), 33 deletions(-) create mode 100644 src/invidious/user/user.cr diff --git a/src/invidious.cr b/src/invidious.cr index fc498dbf..1e78ef5d 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -548,7 +548,7 @@ add_handler AuthHandler.new add_handler DenyFrame.new add_context_storage_type(Array(String)) add_context_storage_type(Preferences) -add_context_storage_type(User) +add_context_storage_type(Invidious::User) Kemal.config.logger = LOGGER Kemal.config.host_binding = Kemal.config.host_binding != "0.0.0.0" ? Kemal.config.host_binding : CONFIG.host_binding diff --git a/src/invidious/search.cr b/src/invidious/search.cr index d8971e79..ae106bf6 100644 --- a/src/invidious/search.cr +++ b/src/invidious/search.cr @@ -176,7 +176,7 @@ end def process_search_query(query, page, user, region) if user - user = user.as(User) + user = user.as(Invidious::User) view_name = "subscriptions_#{sha256(user.email)}" end diff --git a/src/invidious/user/user.cr b/src/invidious/user/user.cr new file mode 100644 index 00000000..a6d05fd1 --- /dev/null +++ b/src/invidious/user/user.cr @@ -0,0 +1,27 @@ +require "db" + +struct Invidious::User + include DB::Serializable + + property updated : Time + property notifications : Array(String) + property subscriptions : Array(String) + property email : String + + @[DB::Field(converter: Invidious::User::PreferencesConverter)] + property preferences : Preferences + property password : String? + property token : String + property watched : Array(String) + property feed_needs_update : Bool? + + module PreferencesConverter + def self.from_rs(rs) + begin + Preferences.from_json(rs.read(String)) + rescue ex + Preferences.from_json("{}") + end + end + end +end diff --git a/src/invidious/users.cr b/src/invidious/users.cr index 9810f8a2..b4995e95 100644 --- a/src/invidious/users.cr +++ b/src/invidious/users.cr @@ -3,32 +3,6 @@ require "crypto/bcrypt/password" # Materialized views may not be defined using bound parameters (`$1` as used elsewhere) MATERIALIZED_VIEW_SQL = ->(email : String) { "SELECT cv.* FROM channel_videos cv WHERE EXISTS (SELECT subscriptions FROM users u WHERE cv.ucid = ANY (u.subscriptions) AND u.email = E'#{email.gsub({'\'' => "\\'", '\\' => "\\\\"})}') ORDER BY published DESC" } -struct User - include DB::Serializable - - property updated : Time - property notifications : Array(String) - property subscriptions : Array(String) - property email : String - - @[DB::Field(converter: User::PreferencesConverter)] - property preferences : Preferences - property password : String? - property token : String - property watched : Array(String) - property feed_needs_update : Bool? - - module PreferencesConverter - def self.from_rs(rs) - begin - Preferences.from_json(rs.read(String)) - rescue ex - Preferences.from_json("{}") - end - end - end -end - def get_user(sid, headers, refresh = true) if email = Invidious::Database::SessionIDs.select_email(sid) user = Invidious::Database::Users.select!(email: email) @@ -84,7 +58,7 @@ def fetch_user(sid, headers) token = Base64.urlsafe_encode(Random::Secure.random_bytes(32)) - user = User.new({ + user = Invidious::User.new({ updated: Time.utc, notifications: [] of String, subscriptions: channels, @@ -102,7 +76,7 @@ def create_user(sid, email, password) password = Crypto::Bcrypt::Password.create(password, cost: 10) token = Base64.urlsafe_encode(Random::Secure.random_bytes(32)) - user = User.new({ + user = Invidious::User.new({ updated: Time.utc, notifications: [] of String, subscriptions: [] of String, diff --git a/src/invidious/views/preferences.ecr b/src/invidious/views/preferences.ecr index 96904259..3606d140 100644 --- a/src/invidious/views/preferences.ecr +++ b/src/invidious/views/preferences.ecr @@ -252,7 +252,7 @@ <% end %> <% end %> - <% if env.get?("user") && CONFIG.admins.includes? env.get?("user").as(User).email %> + <% if env.get?("user") && CONFIG.admins.includes? env.get?("user").as(Invidious::User).email %> <%= translate(locale, "preferences_category_admin") %>
diff --git a/src/invidious/views/template.ecr b/src/invidious/views/template.ecr index 92df1272..bd908dd6 100644 --- a/src/invidious/views/template.ecr +++ b/src/invidious/views/template.ecr @@ -52,7 +52,7 @@
" href="/feed/subscriptions" class="pure-menu-heading"> - <% notification_count = env.get("user").as(User).notifications.size %> + <% notification_count = env.get("user").as(Invidious::User).notifications.size %> <% if notification_count > 0 %> <%= notification_count %> <% else %> @@ -67,7 +67,7 @@
<% if env.get("preferences").as(Preferences).show_nick %>
- <%= env.get("user").as(User).email %> + <%= env.get("user").as(Invidious::User).email %>
<% end %>
From ad4a06fca5d11b57705540818d3eb4e86bb6ac14 Mon Sep 17 00:00:00 2001 From: Samantaz Fox Date: Fri, 4 Feb 2022 03:44:10 +0100 Subject: [PATCH 04/10] Move user captcha code to its own module --- src/invidious.cr | 15 ++++--- src/invidious/routes/login.cr | 4 +- src/invidious/user/captcha.cr | 78 +++++++++++++++++++++++++++++++++++ src/invidious/users.cr | 69 ------------------------------- 4 files changed, 87 insertions(+), 79 deletions(-) create mode 100644 src/invidious/user/captcha.cr diff --git a/src/invidious.cr b/src/invidious.cr index 1e78ef5d..06ce3ead 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -38,14 +38,13 @@ require "./invidious/jobs/**" CONFIG = Config.load HMAC_KEY = CONFIG.hmac_key || Random::Secure.hex(32) -PG_DB = DB.open CONFIG.database_url -ARCHIVE_URL = URI.parse("https://archive.org") -LOGIN_URL = URI.parse("https://accounts.google.com") -PUBSUB_URL = URI.parse("https://pubsubhubbub.appspot.com") -REDDIT_URL = URI.parse("https://www.reddit.com") -TEXTCAPTCHA_URL = URI.parse("https://textcaptcha.com") -YT_URL = URI.parse("https://www.youtube.com") -HOST_URL = make_host_url(Kemal.config) +PG_DB = DB.open CONFIG.database_url +ARCHIVE_URL = URI.parse("https://archive.org") +LOGIN_URL = URI.parse("https://accounts.google.com") +PUBSUB_URL = URI.parse("https://pubsubhubbub.appspot.com") +REDDIT_URL = URI.parse("https://www.reddit.com") +YT_URL = URI.parse("https://www.youtube.com") +HOST_URL = make_host_url(Kemal.config) CHARS_SAFE = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_" TEST_IDS = {"AgbeGFYluEA", "BaW_jenozKc", "a9LDPn-MO4I", "ddFvjfvPnqk", "iqKdEhx-dD4"} diff --git a/src/invidious/routes/login.cr b/src/invidious/routes/login.cr index f4859e6f..42ac0b1d 100644 --- a/src/invidious/routes/login.cr +++ b/src/invidious/routes/login.cr @@ -393,9 +393,9 @@ module Invidious::Routes::Login prompt = "" if captcha_type == "image" - captcha = generate_captcha(HMAC_KEY) + captcha = Invidious::User::Captcha.generate_image(HMAC_KEY) else - captcha = generate_text_captcha(HMAC_KEY) + captcha = Invidious::User::Captcha.generate_text(HMAC_KEY) end return templated "login" diff --git a/src/invidious/user/captcha.cr b/src/invidious/user/captcha.cr new file mode 100644 index 00000000..8a0f67e5 --- /dev/null +++ b/src/invidious/user/captcha.cr @@ -0,0 +1,78 @@ +require "openssl/hmac" + +struct Invidious::User + module Captcha + extend self + + private TEXTCAPTCHA_URL = URI.parse("https://textcaptcha.com") + + def generate_image(key) + second = Random::Secure.rand(12) + second_angle = second * 30 + second = second * 5 + + minute = Random::Secure.rand(12) + minute_angle = minute * 30 + minute = minute * 5 + + hour = Random::Secure.rand(12) + hour_angle = hour * 30 + minute_angle.to_f / 12 + if hour == 0 + hour = 12 + end + + clock_svg = <<-END_SVG + + + + 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8 + 9 + 10 + 11 + 12 + + + + + + + END_SVG + + image = "data:image/png;base64," + image += Process.run(%(rsvg-convert -w 400 -h 400 -b none -f png), shell: true, + input: IO::Memory.new(clock_svg), output: Process::Redirect::Pipe + ) do |proc| + Base64.strict_encode(proc.output.gets_to_end) + end + + answer = "#{hour}:#{minute.to_s.rjust(2, '0')}:#{second.to_s.rjust(2, '0')}" + answer = OpenSSL::HMAC.hexdigest(:sha256, key, answer) + + return { + question: image, + tokens: {generate_response(answer, {":login"}, key, use_nonce: true)}, + } + end + + def generate_text(key) + response = make_client(TEXTCAPTCHA_URL, &.get("/github.com/iv.org/invidious.json").body) + response = JSON.parse(response) + + tokens = response["a"].as_a.map do |answer| + generate_response(answer.as_s, {":login"}, key, use_nonce: true) + end + + return { + question: response["q"].as_s, + tokens: tokens, + } + end + end +end diff --git a/src/invidious/users.cr b/src/invidious/users.cr index b4995e95..b763596b 100644 --- a/src/invidious/users.cr +++ b/src/invidious/users.cr @@ -91,75 +91,6 @@ def create_user(sid, email, password) return user, sid end -def generate_captcha(key) - second = Random::Secure.rand(12) - second_angle = second * 30 - second = second * 5 - - minute = Random::Secure.rand(12) - minute_angle = minute * 30 - minute = minute * 5 - - hour = Random::Secure.rand(12) - hour_angle = hour * 30 + minute_angle.to_f / 12 - if hour == 0 - hour = 12 - end - - clock_svg = <<-END_SVG - - - - 1 - 2 - 3 - 4 - 5 - 6 - 7 - 8 - 9 - 10 - 11 - 12 - - - - - - - END_SVG - - image = "data:image/png;base64," - image += Process.run(%(rsvg-convert -w 400 -h 400 -b none -f png), shell: true, - input: IO::Memory.new(clock_svg), output: Process::Redirect::Pipe - ) do |proc| - Base64.strict_encode(proc.output.gets_to_end) - end - - answer = "#{hour}:#{minute.to_s.rjust(2, '0')}:#{second.to_s.rjust(2, '0')}" - answer = OpenSSL::HMAC.hexdigest(:sha256, key, answer) - - return { - question: image, - tokens: {generate_response(answer, {":login"}, key, use_nonce: true)}, - } -end - -def generate_text_captcha(key) - response = make_client(TEXTCAPTCHA_URL, &.get("/github.com/iv.org/invidious.json").body) - response = JSON.parse(response) - - tokens = response["a"].as_a.map do |answer| - generate_response(answer.as_s, {":login"}, key, use_nonce: true) - end - - return { - question: response["q"].as_s, - tokens: tokens, - } -end - def subscribe_ajax(channel_id, action, env_headers) headers = HTTP::Headers.new headers["Cookie"] = env_headers["Cookie"] From ef8dc7272beed31189df1568e59b14b805783a62 Mon Sep 17 00:00:00 2001 From: Samantaz Fox Date: Fri, 4 Feb 2022 05:19:22 +0100 Subject: [PATCH 05/10] Put CSV import function under its own module --- src/invidious/user/imports.cr | 42 ++++++++++++++++++++--------------- 1 file changed, 24 insertions(+), 18 deletions(-) diff --git a/src/invidious/user/imports.cr b/src/invidious/user/imports.cr index 2ae1dcb1..c8580038 100644 --- a/src/invidious/user/imports.cr +++ b/src/invidious/user/imports.cr @@ -1,27 +1,33 @@ require "csv" -def parse_subscription_export_csv(csv_content : String) - rows = CSV.new(csv_content, headers: true) - subscriptions = Array(String).new +struct Invidious::User + module Import + extend self - # Counter to limit the amount of imports. - # This is intended to prevent DoS. - row_counter = 0 + # Parse a youtube CSV subscription file + def parse_subscription_export_csv(csv_content : String) + rows = CSV.new(csv_content, headers: true) + subscriptions = Array(String).new - rows.each do |row| - # Limit to 1200 - row_counter += 1 - break if row_counter > 1_200 + # Counter to limit the amount of imports. + # This is intended to prevent DoS. + row_counter = 0 - # Channel ID is the first column in the csv export we can't use the header - # name, because the header name is localized depending on the - # language the user has set on their account - channel_id = row[0].strip + rows.each do |row| + # Limit to 1200 + row_counter += 1 + break if row_counter > 1_200 - next if channel_id.empty? + # Channel ID is the first column in the csv export we can't use the header + # name, because the header name is localized depending on the + # language the user has set on their account + channel_id = row[0].strip - subscriptions << channel_id + next if channel_id.empty? + subscriptions << channel_id + end + + return subscriptions + end end - - return subscriptions end From 2bbd424fce4ad1d19643b370250c9f8cee8f1e6f Mon Sep 17 00:00:00 2001 From: Samantaz Fox Date: Fri, 4 Feb 2022 06:01:07 +0100 Subject: [PATCH 06/10] Move import logic to its own module --- spec/invidious/user/imports_spec.cr | 4 +- src/invidious/routes/preferences.cr | 142 ++------------------- src/invidious/user/imports.cr | 191 +++++++++++++++++++++++++++- 3 files changed, 202 insertions(+), 135 deletions(-) diff --git a/spec/invidious/user/imports_spec.cr b/spec/invidious/user/imports_spec.cr index 5a682ec5..762ce0d8 100644 --- a/spec/invidious/user/imports_spec.cr +++ b/spec/invidious/user/imports_spec.cr @@ -25,9 +25,9 @@ def csv_sample CSV end -Spectator.describe "Invidious::User::Imports" do +Spectator.describe Invidious::User::Import do it "imports CSV" do - subscriptions = parse_subscription_export_csv(csv_sample) + subscriptions = Invidious::User::Import.parse_subscription_export_csv(csv_sample) expect(subscriptions).to be_an(Array(String)) expect(subscriptions.size).to eq(13) diff --git a/src/invidious/routes/preferences.cr b/src/invidious/routes/preferences.cr index 930c588b..b574c1c1 100644 --- a/src/invidious/routes/preferences.cr +++ b/src/invidious/routes/preferences.cr @@ -321,149 +321,27 @@ module Invidious::Routes::PreferencesRoute # TODO: Unify into single import based on content-type case part.name when "import_invidious" - body = JSON.parse(body) - - if body["subscriptions"]? - user.subscriptions += body["subscriptions"].as_a.map(&.as_s) - user.subscriptions.uniq! - - user.subscriptions = get_batch_channels(user.subscriptions) - - Invidious::Database::Users.update_subscriptions(user) - end - - if body["watch_history"]? - user.watched += body["watch_history"].as_a.map(&.as_s) - user.watched.uniq! - Invidious::Database::Users.update_watch_history(user) - end - - if body["preferences"]? - user.preferences = Preferences.from_json(body["preferences"].to_json) - Invidious::Database::Users.update_preferences(user) - end - - if playlists = body["playlists"]?.try &.as_a? - playlists.each do |item| - title = item["title"]?.try &.as_s?.try &.delete("<>") - description = item["description"]?.try &.as_s?.try &.delete("\r") - privacy = item["privacy"]?.try &.as_s?.try { |privacy| PlaylistPrivacy.parse? privacy } - - next if !title - next if !description - next if !privacy - - playlist = create_playlist(title, privacy, user) - Invidious::Database::Playlists.update_description(playlist.id, description) - - videos = item["videos"]?.try &.as_a?.try &.each_with_index do |video_id, idx| - raise InfoException.new("Playlist cannot have more than 500 videos") if idx > 500 - - video_id = video_id.try &.as_s? - next if !video_id - - begin - video = get_video(video_id) - rescue ex - next - end - - playlist_video = PlaylistVideo.new({ - title: video.title, - id: video.id, - author: video.author, - ucid: video.ucid, - length_seconds: video.length_seconds, - published: video.published, - plid: playlist.id, - live_now: video.live_now, - index: Random::Secure.rand(0_i64..Int64::MAX), - }) - - Invidious::Database::PlaylistVideos.insert(playlist_video) - Invidious::Database::Playlists.update_video_added(playlist.id, playlist_video.index) - end - end - end + Invidious::User::Import.from_invidious(user, body) when "import_youtube" filename = part.filename || "" - extension = filename.split(".").last + success = Invidious::User::Import.from_youtube(user, body, filename, type) - if extension == "xml" || type == "application/xml" || type == "text/xml" - subscriptions = XML.parse(body) - user.subscriptions += subscriptions.xpath_nodes(%q(//outline[@type="rss"])).map do |channel| - channel["xmlUrl"].match(/UC[a-zA-Z0-9_-]{22}/).not_nil![0] - end - elsif extension == "json" || type == "application/json" - subscriptions = JSON.parse(body) - user.subscriptions += subscriptions.as_a.compact_map do |entry| - entry["snippet"]["resourceId"]["channelId"].as_s - end - elsif extension == "csv" || type == "text/csv" - subscriptions = parse_subscription_export_csv(body) - user.subscriptions += subscriptions - else + if !success haltf(env, status_code: 415, response: error_template(415, "Invalid subscription file uploaded") ) end - - user.subscriptions.uniq! - user.subscriptions = get_batch_channels(user.subscriptions) - - Invidious::Database::Users.update_subscriptions(user) when "import_freetube" - user.subscriptions += body.scan(/"channelId":"(?[a-zA-Z0-9_-]{24})"/).map do |md| - md["channel_id"] - end - user.subscriptions.uniq! - - user.subscriptions = get_batch_channels(user.subscriptions) - - Invidious::Database::Users.update_subscriptions(user) + Invidious::User::Import.from_freetube(user, body) when "import_newpipe_subscriptions" - body = JSON.parse(body) - user.subscriptions += body["subscriptions"].as_a.compact_map do |channel| - if match = channel["url"].as_s.match(/\/channel\/(?UC[a-zA-Z0-9_-]{22})/) - next match["channel"] - elsif match = channel["url"].as_s.match(/\/user\/(?.+)/) - response = YT_POOL.client &.get("/user/#{match["user"]}?disable_polymer=1&hl=en&gl=US") - html = XML.parse_html(response.body) - ucid = html.xpath_node(%q(//link[@rel="canonical"])).try &.["href"].split("/")[-1] - next ucid if ucid - end - - nil - end - user.subscriptions.uniq! - - user.subscriptions = get_batch_channels(user.subscriptions) - - Invidious::Database::Users.update_subscriptions(user) + Invidious::User::Import.from_newpipe_subs(user, body) when "import_newpipe" - Compress::Zip::Reader.open(IO::Memory.new(body)) do |file| - file.each_entry do |entry| - if entry.filename == "newpipe.db" - tempfile = File.tempfile(".db") - File.write(tempfile.path, entry.io.gets_to_end) - db = DB.open("sqlite3://" + tempfile.path) + success = Invidious::User::Import.from_newpipe(user, body) - user.watched += db.query_all("SELECT url FROM streams", as: String).map(&.lchop("https://www.youtube.com/watch?v=")) - user.watched.uniq! - - Invidious::Database::Users.update_watch_history(user) - - user.subscriptions += db.query_all("SELECT url FROM subscriptions", as: String).map(&.lchop("https://www.youtube.com/channel/")) - user.subscriptions.uniq! - - user.subscriptions = get_batch_channels(user.subscriptions) - - Invidious::Database::Users.update_subscriptions(user) - - db.close - tempfile.delete - end - end + if !success + haltf(env, status_code: 415, + response: error_template(415, "Uploaded file is too large") + ) end else nil # Ignore end diff --git a/src/invidious/user/imports.cr b/src/invidious/user/imports.cr index c8580038..7404cd97 100644 --- a/src/invidious/user/imports.cr +++ b/src/invidious/user/imports.cr @@ -29,5 +29,194 @@ struct Invidious::User return subscriptions end - end + + # ------------------- + # Invidious + # ------------------- + + # Import from another invidious account + def from_invidious(user : User, body : String) + data = JSON.parse(body) + + if data["subscriptions"]? + user.subscriptions += data["subscriptions"].as_a.map(&.as_s) + user.subscriptions.uniq! + user.subscriptions = get_batch_channels(user.subscriptions) + + Invidious::Database::Users.update_subscriptions(user) + end + + if data["watch_history"]? + user.watched += data["watch_history"].as_a.map(&.as_s) + user.watched.uniq! + Invidious::Database::Users.update_watch_history(user) + end + + if data["preferences"]? + user.preferences = Preferences.from_json(data["preferences"].to_json) + Invidious::Database::Users.update_preferences(user) + end + + if playlists = data["playlists"]?.try &.as_a? + playlists.each do |item| + title = item["title"]?.try &.as_s?.try &.delete("<>") + description = item["description"]?.try &.as_s?.try &.delete("\r") + privacy = item["privacy"]?.try &.as_s?.try { |privacy| PlaylistPrivacy.parse? privacy } + + next if !title + next if !description + next if !privacy + + playlist = create_playlist(title, privacy, user) + Invidious::Database::Playlists.update_description(playlist.id, description) + + videos = item["videos"]?.try &.as_a?.try &.each_with_index do |video_id, idx| + raise InfoException.new("Playlist cannot have more than 500 videos") if idx > 500 + + video_id = video_id.try &.as_s? + next if !video_id + + begin + video = get_video(video_id) + rescue ex + next + end + + playlist_video = PlaylistVideo.new({ + title: video.title, + id: video.id, + author: video.author, + ucid: video.ucid, + length_seconds: video.length_seconds, + published: video.published, + plid: playlist.id, + live_now: video.live_now, + index: Random::Secure.rand(0_i64..Int64::MAX), + }) + + Invidious::Database::PlaylistVideos.insert(playlist_video) + Invidious::Database::Playlists.update_video_added(playlist.id, playlist_video.index) + end + end + end + end + + # ------------------- + # Youtube + # ------------------- + + # Import subscribed channels from Youtube + # Returns success status + def from_youtube(user : User, body : String, filename : String, type : String) : Bool + extension = filename.split(".").last + + if extension == "xml" || type == "application/xml" || type == "text/xml" + subscriptions = XML.parse(body) + user.subscriptions += subscriptions.xpath_nodes(%q(//outline[@type="rss"])).map do |channel| + channel["xmlUrl"].match(/UC[a-zA-Z0-9_-]{22}/).not_nil![0] + end + elsif extension == "json" || type == "application/json" + subscriptions = JSON.parse(body) + user.subscriptions += subscriptions.as_a.compact_map do |entry| + entry["snippet"]["resourceId"]["channelId"].as_s + end + elsif extension == "csv" || type == "text/csv" + subscriptions = parse_subscription_export_csv(body) + user.subscriptions += subscriptions + else + return false + end + + user.subscriptions.uniq! + user.subscriptions = get_batch_channels(user.subscriptions) + + Invidious::Database::Users.update_subscriptions(user) + return true + end + + # ------------------- + # Freetube + # ------------------- + + def from_freetube(user : User, body : String) + matches = body.scan(/"channelId":"(?[a-zA-Z0-9_-]{24})"/) + + user.subscriptions += matches.map(&.["channel_id"]) + user.subscriptions.uniq! + user.subscriptions = get_batch_channels(user.subscriptions) + + Invidious::Database::Users.update_subscriptions(user) + end + + # ------------------- + # Newpipe + # ------------------- + + def from_newpipe_subs(user : User, body : String) + data = JSON.parse(body) + + user.subscriptions += data["subscriptions"].as_a.compact_map do |channel| + if match = channel["url"].as_s.match(/\/channel\/(?UC[a-zA-Z0-9_-]{22})/) + next match["channel"] + elsif match = channel["url"].as_s.match(/\/user\/(?.+)/) + # Resolve URL using the API + resolved_url = YoutubeAPI.resolve_url("https://www.youtube.com/user/#{match["user"]}") + ucid = resolved_url.dig?("endpoint", "browseEndpoint", "browseId") + next ucid.as_s if ucid + end + + nil + end + + user.subscriptions.uniq! + user.subscriptions = get_batch_channels(user.subscriptions) + + Invidious::Database::Users.update_subscriptions(user) + end + + def from_newpipe(user : User, body : String) : Bool + io = IO::Memory.new(body) + + Compress::Zip::File.open(io) do |file| + file.entries.each do |entry| + entry.open do |file_io| + # Ensure max size of 4MB + io_sized = IO::Sized.new(file_io, 0x400000) + + next if entry.filename != "newpipe.db" + + tempfile = File.tempfile(".db") + + begin + File.write(tempfile.path, io_sized.gets_to_end) + rescue + return false + end + + db = DB.open("sqlite3://" + tempfile.path) + + user.watched += db.query_all("SELECT url FROM streams", as: String) + .map(&.lchop("https://www.youtube.com/watch?v=")) + + user.watched.uniq! + Invidious::Database::Users.update_watch_history(user) + + user.subscriptions += db.query_all("SELECT url FROM subscriptions", as: String) + .map(&.lchop("https://www.youtube.com/channel/")) + + user.subscriptions.uniq! + user.subscriptions = get_batch_channels(user.subscriptions) + + Invidious::Database::Users.update_subscriptions(user) + + db.close + tempfile.delete + end + end + end + + # Success! + return true + end + end # module end From 71a8867a4a719023230802f2927617d676bfa0b4 Mon Sep 17 00:00:00 2001 From: Samantaz Fox Date: Fri, 4 Feb 2022 06:43:43 +0100 Subject: [PATCH 07/10] Move user cookies to their own module --- src/invidious/routes/login.cr | 40 +++-------------------------- src/invidious/routes/preferences.cr | 30 ++-------------------- src/invidious/user/cookies.cr | 37 ++++++++++++++++++++++++++ 3 files changed, 42 insertions(+), 65 deletions(-) create mode 100644 src/invidious/user/cookies.cr diff --git a/src/invidious/routes/login.cr b/src/invidious/routes/login.cr index 42ac0b1d..8767ec22 100644 --- a/src/invidious/routes/login.cr +++ b/src/invidious/routes/login.cr @@ -282,18 +282,8 @@ module Invidious::Routes::Login host = URI.parse(env.request.headers["Host"]).host - if Kemal.config.ssl || CONFIG.https_only - secure = true - else - secure = false - end - cookies.each do |cookie| - if Kemal.config.ssl || CONFIG.https_only - cookie.secure = secure - else - cookie.secure = secure - end + cookie.secure = Invidious::User::Cookies::SECURE if cookie.extension cookie.extension = cookie.extension.not_nil!.gsub(".youtube.com", host) @@ -338,19 +328,7 @@ module Invidious::Routes::Login sid = Base64.urlsafe_encode(Random::Secure.random_bytes(32)) Invidious::Database::SessionIDs.insert(sid, email) - if Kemal.config.ssl || CONFIG.https_only - secure = true - else - secure = false - end - - if CONFIG.domain - env.response.cookies["SID"] = HTTP::Cookie.new(name: "SID", domain: "#{CONFIG.domain}", value: sid, expires: Time.utc + 2.years, - secure: secure, http_only: true) - else - env.response.cookies["SID"] = HTTP::Cookie.new(name: "SID", value: sid, expires: Time.utc + 2.years, - secure: secure, http_only: true) - end + env.response.cookies["SID"] = Invidious::User::Cookies.sid(CONFIG.domain, sid) else return error_template(401, "Wrong username or password") end @@ -455,19 +433,7 @@ module Invidious::Routes::Login view_name = "subscriptions_#{sha256(user.email)}" PG_DB.exec("CREATE MATERIALIZED VIEW #{view_name} AS #{MATERIALIZED_VIEW_SQL.call(user.email)}") - if Kemal.config.ssl || CONFIG.https_only - secure = true - else - secure = false - end - - if CONFIG.domain - env.response.cookies["SID"] = HTTP::Cookie.new(name: "SID", domain: "#{CONFIG.domain}", value: sid, expires: Time.utc + 2.years, - secure: secure, http_only: true) - else - env.response.cookies["SID"] = HTTP::Cookie.new(name: "SID", value: sid, expires: Time.utc + 2.years, - secure: secure, http_only: true) - end + env.response.cookies["SID"] = Invidious::User::Cookies.sid(CONFIG.domain, sid) if env.request.cookies["PREFS"]? user.preferences = env.get("preferences").as(Preferences) diff --git a/src/invidious/routes/preferences.cr b/src/invidious/routes/preferences.cr index b574c1c1..294932eb 100644 --- a/src/invidious/routes/preferences.cr +++ b/src/invidious/routes/preferences.cr @@ -214,19 +214,7 @@ module Invidious::Routes::PreferencesRoute File.write("config/config.yml", CONFIG.to_yaml) end else - if Kemal.config.ssl || CONFIG.https_only - secure = true - else - secure = false - end - - if CONFIG.domain - env.response.cookies["PREFS"] = HTTP::Cookie.new(name: "PREFS", domain: "#{CONFIG.domain}", value: URI.encode_www_form(preferences.to_json), expires: Time.utc + 2.years, - secure: secure, http_only: true) - else - env.response.cookies["PREFS"] = HTTP::Cookie.new(name: "PREFS", value: URI.encode_www_form(preferences.to_json), expires: Time.utc + 2.years, - secure: secure, http_only: true) - end + env.response.cookies["PREFS"] = Invidious::User::Cookies.prefs(CONFIG.domain, preferences) end env.redirect referer @@ -261,21 +249,7 @@ module Invidious::Routes::PreferencesRoute preferences.dark_mode = "dark" end - preferences = preferences.to_json - - if Kemal.config.ssl || CONFIG.https_only - secure = true - else - secure = false - end - - if CONFIG.domain - env.response.cookies["PREFS"] = HTTP::Cookie.new(name: "PREFS", domain: "#{CONFIG.domain}", value: URI.encode_www_form(preferences), expires: Time.utc + 2.years, - secure: secure, http_only: true) - else - env.response.cookies["PREFS"] = HTTP::Cookie.new(name: "PREFS", value: URI.encode_www_form(preferences), expires: Time.utc + 2.years, - secure: secure, http_only: true) - end + env.response.cookies["PREFS"] = Invidious::User::Cookies.prefs(CONFIG.domain, preferences) end if redirect diff --git a/src/invidious/user/cookies.cr b/src/invidious/user/cookies.cr new file mode 100644 index 00000000..99df1b07 --- /dev/null +++ b/src/invidious/user/cookies.cr @@ -0,0 +1,37 @@ +require "http/cookie" + +struct Invidious::User + module Cookies + extend self + + # Note: we use ternary operator because the two variables + # used in here are not booleans. + SECURE = (Kemal.config.ssl || CONFIG.https_only) ? true : false + + # Session ID (SID) cookie + # Parameter "domain" comes from the global config + def sid(domain : String?, sid) : HTTP::Cookie + return HTTP::Cookie.new( + name: "SID", + domain: domain, + value: sid, + expires: Time.utc + 2.years, + secure: SECURE, + http_only: true + ) + end + + # Preferences (PREFS) cookie + # Parameter "domain" comes from the global config + def prefs(domain : String?, preferences : Preferences) : HTTP::Cookie + return HTTP::Cookie.new( + name: "PREFS", + domain: domain, + value: URI.encode_www_form(preferences.to_json), + expires: Time.utc + 2.years, + secure: SECURE, + http_only: true + ) + end + end +end From 99d770be643d36fe40fbb8ac54a5fa5f692aebf0 Mon Sep 17 00:00:00 2001 From: Samantaz Fox Date: Sun, 6 Feb 2022 21:51:02 +0100 Subject: [PATCH 08/10] Move user pages (ECR files) to subfolder --- src/invidious/routes/account.cr | 12 ++++++------ src/invidious/routes/api/v1/authenticated.cr | 2 +- src/invidious/routes/login.cr | 8 ++++---- src/invidious/routes/preferences.cr | 4 ++-- src/invidious/routes/subscriptions.cr | 2 +- src/invidious/views/{ => user}/authorize_token.ecr | 0 src/invidious/views/{ => user}/change_password.ecr | 0 .../views/{ => user}/clear_watch_history.ecr | 0 src/invidious/views/{ => user}/data_control.ecr | 0 src/invidious/views/{ => user}/delete_account.ecr | 0 src/invidious/views/{ => user}/login.ecr | 0 src/invidious/views/{ => user}/preferences.ecr | 0 .../views/{ => user}/subscription_manager.ecr | 0 src/invidious/views/{ => user}/token_manager.ecr | 0 14 files changed, 14 insertions(+), 14 deletions(-) rename src/invidious/views/{ => user}/authorize_token.ecr (100%) rename src/invidious/views/{ => user}/change_password.ecr (100%) rename src/invidious/views/{ => user}/clear_watch_history.ecr (100%) rename src/invidious/views/{ => user}/data_control.ecr (100%) rename src/invidious/views/{ => user}/delete_account.ecr (100%) rename src/invidious/views/{ => user}/login.ecr (100%) rename src/invidious/views/{ => user}/preferences.ecr (100%) rename src/invidious/views/{ => user}/subscription_manager.ecr (100%) rename src/invidious/views/{ => user}/token_manager.ecr (100%) diff --git a/src/invidious/routes/account.cr b/src/invidious/routes/account.cr index 2be0de37..9bb73136 100644 --- a/src/invidious/routes/account.cr +++ b/src/invidious/routes/account.cr @@ -23,7 +23,7 @@ module Invidious::Routes::Account sid = sid.as(String) csrf_token = generate_response(sid, {":change_password"}, HMAC_KEY) - templated "change_password" + templated "user/change_password" end # Handle the password change (POST request) @@ -103,7 +103,7 @@ module Invidious::Routes::Account sid = sid.as(String) csrf_token = generate_response(sid, {":delete_account"}, HMAC_KEY) - templated "delete_account" + templated "user/delete_account" end # Handle the account deletion (POST request) @@ -161,7 +161,7 @@ module Invidious::Routes::Account sid = sid.as(String) csrf_token = generate_response(sid, {":clear_watch_history"}, HMAC_KEY) - templated "clear_watch_history" + templated "user/clear_watch_history" end # Handle the watch history clearing (POST request) @@ -220,7 +220,7 @@ module Invidious::Routes::Account expire = env.params.query["expire"]?.try &.to_i? - templated "authorize_token" + templated "user/authorize_token" end # Handle token authorization (POST request) @@ -268,7 +268,7 @@ module Invidious::Routes::Account else csrf_token = "" env.set "access_token", access_token - templated "authorize_token" + templated "user/authorize_token" end end @@ -291,7 +291,7 @@ module Invidious::Routes::Account user = user.as(User) tokens = Invidious::Database::SessionIDs.select_all(user.email) - templated "token_manager" + templated "user/token_manager" end # ------------------- diff --git a/src/invidious/routes/api/v1/authenticated.cr b/src/invidious/routes/api/v1/authenticated.cr index 4e9fc801..c27853ca 100644 --- a/src/invidious/routes/api/v1/authenticated.cr +++ b/src/invidious/routes/api/v1/authenticated.cr @@ -343,7 +343,7 @@ module Invidious::Routes::API::V1::Authenticated env.response.content_type = "text/html" csrf_token = generate_response(sid, {":authorize_token"}, HMAC_KEY, use_nonce: true) - return templated "authorize_token" + return templated "user/authorize_token" else env.response.content_type = "application/json" diff --git a/src/invidious/routes/login.cr b/src/invidious/routes/login.cr index 8767ec22..65b337d1 100644 --- a/src/invidious/routes/login.cr +++ b/src/invidious/routes/login.cr @@ -27,7 +27,7 @@ module Invidious::Routes::Login tfa = env.params.query["tfa"]? prompt = nil - templated "login" + templated "user/login" end def self.login(env) @@ -133,7 +133,7 @@ module Invidious::Routes::Login tfa = tfa_code captcha = {tokens: [token], question: ""} - return templated "login" + return templated "user/login" end if challenge_results[0][-1]?.try &.[5] == "INCORRECT_ANSWER_ENTERED" @@ -190,7 +190,7 @@ module Invidious::Routes::Login tfa = nil captcha = nil - return templated "login" + return templated "user/login" end tl = challenge_results[1][2] @@ -376,7 +376,7 @@ module Invidious::Routes::Login captcha = Invidious::User::Captcha.generate_text(HMAC_KEY) end - return templated "login" + return templated "user/login" end tokens = env.params.body.select { |k, _| k.match(/^token\[\d+\]$/) }.map { |_, v| v } diff --git a/src/invidious/routes/preferences.cr b/src/invidious/routes/preferences.cr index 294932eb..68d61fd1 100644 --- a/src/invidious/routes/preferences.cr +++ b/src/invidious/routes/preferences.cr @@ -8,7 +8,7 @@ module Invidious::Routes::PreferencesRoute preferences = env.get("preferences").as(Preferences) - templated "preferences" + templated "user/preferences" end def self.update(env) @@ -272,7 +272,7 @@ module Invidious::Routes::PreferencesRoute user = user.as(User) - templated "data_control" + templated "user/data_control" end def self.update_data_control(env) diff --git a/src/invidious/routes/subscriptions.cr b/src/invidious/routes/subscriptions.cr index ec8fe67b..7b1fa876 100644 --- a/src/invidious/routes/subscriptions.cr +++ b/src/invidious/routes/subscriptions.cr @@ -163,6 +163,6 @@ module Invidious::Routes::Subscriptions end end - templated "subscription_manager" + templated "user/subscription_manager" end end diff --git a/src/invidious/views/authorize_token.ecr b/src/invidious/views/user/authorize_token.ecr similarity index 100% rename from src/invidious/views/authorize_token.ecr rename to src/invidious/views/user/authorize_token.ecr diff --git a/src/invidious/views/change_password.ecr b/src/invidious/views/user/change_password.ecr similarity index 100% rename from src/invidious/views/change_password.ecr rename to src/invidious/views/user/change_password.ecr diff --git a/src/invidious/views/clear_watch_history.ecr b/src/invidious/views/user/clear_watch_history.ecr similarity index 100% rename from src/invidious/views/clear_watch_history.ecr rename to src/invidious/views/user/clear_watch_history.ecr diff --git a/src/invidious/views/data_control.ecr b/src/invidious/views/user/data_control.ecr similarity index 100% rename from src/invidious/views/data_control.ecr rename to src/invidious/views/user/data_control.ecr diff --git a/src/invidious/views/delete_account.ecr b/src/invidious/views/user/delete_account.ecr similarity index 100% rename from src/invidious/views/delete_account.ecr rename to src/invidious/views/user/delete_account.ecr diff --git a/src/invidious/views/login.ecr b/src/invidious/views/user/login.ecr similarity index 100% rename from src/invidious/views/login.ecr rename to src/invidious/views/user/login.ecr diff --git a/src/invidious/views/preferences.ecr b/src/invidious/views/user/preferences.ecr similarity index 100% rename from src/invidious/views/preferences.ecr rename to src/invidious/views/user/preferences.ecr diff --git a/src/invidious/views/subscription_manager.ecr b/src/invidious/views/user/subscription_manager.ecr similarity index 100% rename from src/invidious/views/subscription_manager.ecr rename to src/invidious/views/user/subscription_manager.ecr diff --git a/src/invidious/views/token_manager.ecr b/src/invidious/views/user/token_manager.ecr similarity index 100% rename from src/invidious/views/token_manager.ecr rename to src/invidious/views/user/token_manager.ecr From 60e870b27783bdcdb07d26489b50d18a85c49eeb Mon Sep 17 00:00:00 2001 From: Samantaz Fox Date: Sat, 12 Feb 2022 17:32:20 +0100 Subject: [PATCH 09/10] Fix OPML import --- src/invidious/user/imports.cr | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/src/invidious/user/imports.cr b/src/invidious/user/imports.cr index 7404cd97..61d10719 100644 --- a/src/invidious/user/imports.cr +++ b/src/invidious/user/imports.cr @@ -105,12 +105,25 @@ struct Invidious::User # Youtube # ------------------- + private def is_opml?(mimetype : String, extension : String) + opml_mimetypes = [ + "application/xml", + "text/xml", + "text/x-opml", + "text/x-opml+xml", + ] + + opml_extensions = ["xml", "opml"] + + return opml_mimetypes.any?(&.== mimetype) || opml_extensions.any?(&.== extension) + end + # Import subscribed channels from Youtube # Returns success status def from_youtube(user : User, body : String, filename : String, type : String) : Bool extension = filename.split(".").last - if extension == "xml" || type == "application/xml" || type == "text/xml" + if is_opml?(type, extension) subscriptions = XML.parse(body) user.subscriptions += subscriptions.xpath_nodes(%q(//outline[@type="rss"])).map do |channel| channel["xmlUrl"].match(/UC[a-zA-Z0-9_-]{22}/).not_nil![0] From 57353fe0c611f2065ccec8863d26ab586d72fdfb Mon Sep 17 00:00:00 2001 From: Samantaz Fox Date: Sun, 13 Feb 2022 22:35:22 +0100 Subject: [PATCH 10/10] Fix Freetube subscriptions import --- src/invidious/user/imports.cr | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/invidious/user/imports.cr b/src/invidious/user/imports.cr index 61d10719..f8b9e4e4 100644 --- a/src/invidious/user/imports.cr +++ b/src/invidious/user/imports.cr @@ -152,9 +152,16 @@ struct Invidious::User # ------------------- def from_freetube(user : User, body : String) + # Legacy import? matches = body.scan(/"channelId":"(?[a-zA-Z0-9_-]{24})"/) + subs = matches.map(&.["channel_id"]) - user.subscriptions += matches.map(&.["channel_id"]) + if subs.empty? + data = JSON.parse(body)["subscriptions"] + subs = data.as_a.map(&.["id"].as_s) + end + + user.subscriptions += subs user.subscriptions.uniq! user.subscriptions = get_batch_channels(user.subscriptions)