From 50bab26a3a6409b2a23ff5537f0469434fd4c0b9 Mon Sep 17 00:00:00 2001 From: Omar Roth Date: Sat, 9 Nov 2019 14:21:31 -0500 Subject: [PATCH] Add support for CONNECT proxy --- src/invidious.cr | 66 ++++++++++++++++++++++++- src/invidious/helpers/handlers.cr | 29 +++++++++++ src/invidious/helpers/helpers.cr | 4 ++ src/invidious/helpers/jobs.cr | 82 +++++++++++++++++++++++-------- src/invidious/helpers/proxy.cr | 8 ++- src/invidious/helpers/utils.cr | 5 +- src/invidious/mixes.cr | 2 +- src/invidious/trending.cr | 2 +- 8 files changed, 170 insertions(+), 28 deletions(-) diff --git a/src/invidious.cr b/src/invidious.cr index 6a197795..b0a99d21 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -5568,7 +5568,7 @@ get "/videoplayback" do |env| next env.redirect location end - IO.copy(response.body_io, env.response) + IO.copy response.body_io, env.response end rescue ex end @@ -5865,6 +5865,69 @@ get "/Captcha" do |env| response.body end +connect "*" do |env| + if CONFIG.proxy_address.empty? + env.response.status_code = 400 + next + end + + url = env.request.headers["Host"]?.try { |u| u.split(":") } + host = url.try &.[0]? + port = url.try &.[1]? + + host = "www.google.com" if !host || host.empty? + port = "443" if !port || port.empty? + + # if env.request.internal_uri + # env.request.internal_uri.not_nil!.path = "#{host}:#{port}" + # end + + user, pass = env.request.headers["Proxy-Authorization"]? + .try { |i| i.lchop("Basic ") } + .try { |i| Base64.decode_string(i) } + .try &.split(":", 2) || {nil, nil} + + if CONFIG.proxy_user != user || CONFIG.proxy_pass != pass + env.response.status_code = 403 + next + end + + begin + upstream = TCPSocket.new(host, port) + rescue ex + logger.puts("Exception: #{ex.message}") + env.response.status_code = 400 + next + end + + env.response.reset + env.response.upgrade do |downstream| + downstream = downstream.as(TCPSocket) + downstream.sync = true + + spawn do + begin + bytes = 1 + while bytes != 0 + bytes = IO.copy upstream, downstream + end + rescue ex + end + end + + begin + bytes = 1 + while bytes != 0 + bytes = IO.copy downstream, upstream + end + rescue ex + ensure + upstream.close + downstream.close + end + end +end + # Undocumented, creates anonymous playlist with specified 'video_ids', max 50 videos get "/watch_videos" do |env| response = YT_POOL.client &.get(env.request.resource) @@ -5939,6 +6002,7 @@ end public_folder "assets" Kemal.config.powered_by_header = false +add_handler ProxyHandler.new add_handler FilteredCompressHandler.new add_handler APIHandler.new add_handler AuthHandler.new diff --git a/src/invidious/helpers/handlers.cr b/src/invidious/helpers/handlers.cr index 87b10bc9..f3b99b7d 100644 --- a/src/invidious/helpers/handlers.cr +++ b/src/invidious/helpers/handlers.cr @@ -212,3 +212,32 @@ class DenyFrame < Kemal::Handler call_next env end end + +class ProxyHandler < Kemal::Handler + def call(env) + if env.request.headers["Proxy-Authorization"]? && env.request.method != "CONNECT" + user, pass = env.request.headers["Proxy-Authorization"]? + .try { |i| i.lchop("Basic ") } + .try { |i| Base64.decode_string(i) } + .try &.split(":", 2) || {nil, nil} + + if CONFIG.proxy_user != user || CONFIG.proxy_pass != pass + env.response.status_code = 403 + return + end + + HTTP::Client.exec(env.request.method, "#{env.request.headers["Host"]?}#{env.request.resource}", env.request.headers, env.request.body) do |response| + response.headers.each do |key, value| + if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase) && key.downcase != "transfer-encoding" + env.response.headers[key] = value + end + end + IO.copy response.body_io, env.response + end + env.response.close + return + else + call_next env + end + end +end diff --git a/src/invidious/helpers/helpers.cr b/src/invidious/helpers/helpers.cr index 96d14737..ba095bc3 100644 --- a/src/invidious/helpers/helpers.cr +++ b/src/invidious/helpers/helpers.cr @@ -263,6 +263,10 @@ struct Config admin_email: {type: String, default: "omarroth@protonmail.com"}, # Email for bug reports cookies: {type: HTTP::Cookies, default: HTTP::Cookies.new, converter: StringToCookies}, # Saved cookies in "name1=value1; name2=value2..." format captcha_key: {type: String?, default: nil}, # Key for Anti-Captcha + proxy_address: {type: String, default: ""}, + proxy_port: {type: Int32, default: 8080}, + proxy_user: {type: String, default: ""}, + proxy_pass: {type: String, default: ""}, }) end diff --git a/src/invidious/helpers/jobs.cr b/src/invidious/helpers/jobs.cr index c6e0ef42..02c3ab05 100644 --- a/src/invidious/helpers/jobs.cr +++ b/src/invidious/helpers/jobs.cr @@ -249,15 +249,34 @@ def bypass_captcha(captcha_key, logger) end headers = response.cookies.add_request_headers(HTTP::Headers.new) - - response = JSON.parse(HTTP::Client.post("https://api.anti-captcha.com/createTask", body: { - "clientKey" => CONFIG.captcha_key, - "task" => { - "type" => "NoCaptchaTaskProxyless", - "websiteURL" => "https://www.youtube.com/watch?v=CvFH_6DNRCY&gl=US&hl=en&disable_polymer=1&has_verified=1&bpctr=9999999999", - "websiteKey" => site_key, - }, - }.to_json).body) + captcha_client = HTTPClient.new(URI.parse("https://api.anti-captcha.com")) + captcha_client.family = CONFIG.force_resolve || Socket::Family::INET + if !CONFIG.proxy_address.empty? + response = JSON.parse(captcha_client.post("/createTask", body: { + "clientKey" => CONFIG.captcha_key, + "task" => { + "type" => "NoCaptchaTask", + "websiteURL" => "https://www.youtube.com#{path}", + "websiteKey" => site_key, + "proxyType" => "http", + "proxyAddress" => CONFIG.proxy_address, + "proxyPort" => CONFIG.proxy_port, + "proxyLogin" => CONFIG.proxy_user, + "proxyPassword" => CONFIG.proxy_pass, + "userAgent" => headers["user-agent"], + }, + }.to_json).body) + else + response = JSON.parse(captcha_client.post("/createTask", body: { + "clientKey" => CONFIG.captcha_key, + "task" => { + "type" => "NoCaptchaTaskProxyless", + "websiteURL" => "https://www.youtube.com#{path}", + "websiteKey" => site_key, + "userAgent" => headers["user-agent"], + }, + }.to_json).body) + end raise response["error"].as_s if response["error"]? task_id = response["taskId"].as_i @@ -265,7 +284,7 @@ def bypass_captcha(captcha_key, logger) loop do sleep 10.seconds - response = JSON.parse(HTTP::Client.post("https://api.anti-captcha.com/getTaskResult", body: { + response = JSON.parse(captcha_client.post("/getTaskResult", body: { "clientKey" => CONFIG.captcha_key, "taskId" => task_id, }.to_json).body) @@ -283,7 +302,11 @@ def bypass_captcha(captcha_key, logger) yield response.cookies.select { |cookie| cookie.name != "PREF" } elsif response.headers["Location"]?.try &.includes?("/sorry/index") location = response.headers["Location"].try { |u| URI.parse(u) } - headers = HTTP::Headers{":authority" => location.host.not_nil!} + headers = HTTP::Headers{ + ":authority" => location.host.not_nil!, + "origin" => "https://www.google.com", + "user-agent" => "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.61 Safari/537.36", + } response = YT_POOL.client &.get(location.full_path, headers) html = XML.parse_html(response.body) @@ -297,14 +320,32 @@ def bypass_captcha(captcha_key, logger) captcha_client = HTTPClient.new(URI.parse("https://api.anti-captcha.com")) captcha_client.family = CONFIG.force_resolve || Socket::Family::INET - response = JSON.parse(captcha_client.post("/createTask", body: { - "clientKey" => CONFIG.captcha_key, - "task" => { - "type" => "NoCaptchaTaskProxyless", - "websiteURL" => location.to_s, - "websiteKey" => site_key, - }, - }.to_json).body) + if !CONFIG.proxy_address.empty? + response = JSON.parse(captcha_client.post("/createTask", body: { + "clientKey" => CONFIG.captcha_key, + "task" => { + "type" => "NoCaptchaTask", + "websiteURL" => location.to_s, + "websiteKey" => site_key, + "proxyType" => "http", + "proxyAddress" => CONFIG.proxy_address, + "proxyPort" => CONFIG.proxy_port, + "proxyLogin" => CONFIG.proxy_user, + "proxyPassword" => CONFIG.proxy_pass, + "userAgent" => headers["user-agent"], + }, + }.to_json).body) + else + response = JSON.parse(captcha_client.post("/createTask", body: { + "clientKey" => CONFIG.captcha_key, + "task" => { + "type" => "NoCaptchaTaskProxyless", + "websiteURL" => location.to_s, + "websiteKey" => site_key, + "userAgent" => headers["user-agent"], + }, + }.to_json).body) + end raise response["error"].as_s if response["error"]? task_id = response["taskId"].as_i @@ -326,8 +367,7 @@ def bypass_captcha(captcha_key, logger) inputs["g-recaptcha-response"] = response["solution"]["gRecaptchaResponse"].as_s headers["content-type"] = "application/x-www-form-urlencoded" - headers["origin"] = "https://www.google.com" - headers["user-agent"] = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.61 Safari/537.36" + headers["referer"] = location.to_s response = YT_POOL.client &.post("/sorry/index", headers: headers, form: inputs) headers = HTTP::Headers{ diff --git a/src/invidious/helpers/proxy.cr b/src/invidious/helpers/proxy.cr index 4f415ba0..af114e29 100644 --- a/src/invidious/helpers/proxy.cr +++ b/src/invidious/helpers/proxy.cr @@ -1,3 +1,7 @@ +def connect(path : String, &block : HTTP::Server::Context -> _) + Kemal::RouteHandler::INSTANCE.add_route("CONNECT", path, &block) +end + # See https://github.com/crystal-lang/crystal/issues/2963 class HTTPProxy getter proxy_host : String @@ -124,7 +128,7 @@ def get_nova_proxies(country_code = "US") client.connect_timeout = 10.seconds headers = HTTP::Headers.new - headers["User-Agent"] = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36" + headers["User-Agent"] = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.61 Safari/537.36" headers["Accept"] = "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8" headers["Accept-Language"] = "Accept-Language: en-US,en;q=0.9" headers["Host"] = "www.proxynova.com" @@ -161,7 +165,7 @@ def get_spys_proxies(country_code = "US") client.connect_timeout = 10.seconds headers = HTTP::Headers.new - headers["User-Agent"] = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36" + headers["User-Agent"] = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.61 Safari/537.36" headers["Accept"] = "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8" headers["Accept-Language"] = "Accept-Language: en-US,en;q=0.9" headers["Host"] = "spys.one" diff --git a/src/invidious/helpers/utils.cr b/src/invidious/helpers/utils.cr index 1fff206d..52ca0f82 100644 --- a/src/invidious/helpers/utils.cr +++ b/src/invidious/helpers/utils.cr @@ -2,11 +2,12 @@ require "lsquic" require "pool/connection" def add_yt_headers(request) - request.headers["user-agent"] ||= "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.97 Safari/537.36" + return if request.resource.starts_with? "/sorry/index" + + request.headers["user-agent"] ||= "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.61 Safari/537.36" request.headers["accept-charset"] ||= "ISO-8859-1,utf-8;q=0.7,*;q=0.7" request.headers["accept"] ||= "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8" request.headers["accept-language"] ||= "en-us,en;q=0.5" - return if request.resource.starts_with? "/sorry/index" request.headers["x-youtube-client-name"] ||= "1" request.headers["x-youtube-client-version"] ||= "1.20180719" if !CONFIG.cookies.empty? diff --git a/src/invidious/mixes.cr b/src/invidious/mixes.cr index 04a37b87..f5ff40ef 100644 --- a/src/invidious/mixes.cr +++ b/src/invidious/mixes.cr @@ -20,7 +20,7 @@ end def fetch_mix(rdid, video_id, cookies = nil, locale = nil) headers = HTTP::Headers.new - headers["User-Agent"] = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36" + headers["User-Agent"] = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.61 Safari/537.36" if cookies headers = cookies.add_request_headers(headers) diff --git a/src/invidious/trending.cr b/src/invidious/trending.cr index 017c42f5..4f80be64 100644 --- a/src/invidious/trending.cr +++ b/src/invidious/trending.cr @@ -1,6 +1,6 @@ def fetch_trending(trending_type, region, locale) headers = HTTP::Headers.new - headers["User-Agent"] = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36" + headers["User-Agent"] = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.61 Safari/537.36" region ||= "US" region = region.upcase