From e84e1193fc6ab88b96b88735178dc84027a553cc Mon Sep 17 00:00:00 2001 From: Nutomic Date: Fri, 17 Sep 2021 15:44:20 +0000 Subject: [PATCH] Dont pass accept-encoding header to pictrs (ref #1734) (#1738) * Allow image upload in docker/dev/ (replaces #1765) * Dont pass accept-encoding header to pictrs (ref #1734) * Fix clippy errors --- Cargo.lock | 66 +-------------------- crates/routes/Cargo.toml | 5 +- crates/routes/src/images.rs | 5 ++ crates/utils/src/rate_limit/rate_limiter.rs | 10 +--- crates/utils/src/request.rs | 1 + docker/dev/docker-compose.yml | 16 +++-- docker/dev/nginx.conf | 51 ++++++++++++++++ docker/federation/docker-compose.yml | 2 +- 8 files changed, 73 insertions(+), 83 deletions(-) create mode 100644 docker/dev/nginx.conf diff --git a/Cargo.lock b/Cargo.lock index a1fff70a0..8cd935abb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -70,12 +70,10 @@ dependencies = [ "ahash", "base64 0.13.0", "bitflags", - "brotli2", "bytes", "bytestring", "derive_more", "encoding_rs", - "flate2", "futures-core", "futures-util", "h2", @@ -97,7 +95,6 @@ dependencies = [ "smallvec", "time 0.2.27", "tokio", - "zstd", ] [[package]] @@ -378,7 +375,6 @@ dependencies = [ "base64 0.13.0", "bytes", "cfg-if", - "cookie", "derive_more", "futures-core", "itoa", @@ -530,26 +526,6 @@ dependencies = [ "opaque-debug 0.3.0", ] -[[package]] -name = "brotli-sys" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4445dea95f4c2b41cde57cc9fee236ae4dbae88d8fcbdb4750fc1bb5d86aaecd" -dependencies = [ - "cc", - "libc", -] - -[[package]] -name = "brotli2" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0cb036c3eade309815c15ddbacec5b22c4d1f3983a774ab2eac2e3e9ea85568e" -dependencies = [ - "brotli-sys", - "libc", -] - [[package]] name = "bumpalo" version = "3.7.0" @@ -614,9 +590,6 @@ name = "cc" version = "1.0.69" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e70cc2f62c6ce1868963827bd677764c62d07c3d9a3e1fb1177ee1a9ab199eb2" -dependencies = [ - "jobserver", -] [[package]] name = "cfg-if" @@ -1559,15 +1532,6 @@ version = "0.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b71991ff56294aa922b450139ee08b3bfc70982c6b2c7562771375cf73542dd4" -[[package]] -name = "jobserver" -version = "0.1.24" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af25a77299a7f711a01975c35a6a424eb6862092cc2d6c72c4ed6cbc56dfc1fa" -dependencies = [ - "libc", -] - [[package]] name = "jpeg-decoder" version = "0.1.22" @@ -1867,6 +1831,7 @@ name = "lemmy_routes" version = "0.12.2-rc.1" dependencies = [ "actix", + "actix-http", "actix-web", "actix-web-actors", "anyhow", @@ -4001,32 +3966,3 @@ name = "xdg" version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d089681aa106a86fade1b0128fb5daf07d5867a509ab036d99988dec80429a57" - -[[package]] -name = "zstd" -version = "0.7.0+zstd.1.4.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9428752481d8372e15b1bf779ea518a179ad6c771cca2d2c60e4fbff3cc2cd52" -dependencies = [ - "zstd-safe", -] - -[[package]] -name = "zstd-safe" -version = "3.1.0+zstd.1.4.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5aa1926623ad7fe406e090555387daf73db555b948134b4d73eac5eb08fb666d" -dependencies = [ - "libc", - "zstd-sys", -] - -[[package]] -name = "zstd-sys" -version = "1.5.0+zstd.1.4.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e6c094340240369025fc6b731b054ee2a834328fa584310ac96aa4baebdc465" -dependencies = [ - "cc", - "libc", -] diff --git a/crates/routes/Cargo.toml b/crates/routes/Cargo.toml index 6ee6239e1..4167b681d 100644 --- a/crates/routes/Cargo.toml +++ b/crates/routes/Cargo.toml @@ -21,15 +21,14 @@ diesel = "1.4.7" actix = "0.12.0" actix-web = { version = "4.0.0-beta.8", default-features = false, features = ["rustls"] } actix-web-actors = { version = "4.0.0-beta.6", default-features = false } +actix-http = "3.0.0-beta.9" sha2 = "0.9.5" log = "0.4.14" anyhow = "1.0.43" chrono = { version = "0.4.19", features = ["serde"] } rss = "1.10.0" serde = { version = "1.0.129", features = ["derive"] } -# TODO awc should use default-features = false, because gzip is a heavy dependency. -# This must wait for pictrs to have a configurable disabled gzip -awc = "3.0.0-beta.7" +awc = { version = "3.0.0-beta.7", default-features = false } url = { version = "2.2.2", features = ["serde"] } strum = "0.21.0" lazy_static = "1.4.0" diff --git a/crates/routes/src/images.rs b/crates/routes/src/images.rs index a89bde0f1..7439e4a8e 100644 --- a/crates/routes/src/images.rs +++ b/crates/routes/src/images.rs @@ -1,3 +1,4 @@ +use actix_http::http::header::ACCEPT_ENCODING; use actix_web::{body::BodyStream, http::StatusCode, web::Data, *}; use anyhow::anyhow; use awc::Client; @@ -56,6 +57,8 @@ async fn upload( }; let mut client_req = client.request_from(format!("{}/image", pictrs_url()?), req.head()); + // remove content-encoding header so that pictrs doesnt send gzipped response + client_req.headers_mut().remove(ACCEPT_ENCODING); if let Some(addr) = req.head().peer_addr { client_req = client_req.insert_header(("X-Forwarded-For", addr.to_string())) @@ -103,6 +106,7 @@ async fn image( client: web::Data, ) -> Result { let mut client_req = client.request_from(url, req.head()); + client_req.headers_mut().remove(ACCEPT_ENCODING); if let Some(addr) = req.head().peer_addr { client_req = client_req.insert_header(("X-Forwarded-For", addr.to_string())) @@ -137,6 +141,7 @@ async fn delete( let url = format!("{}/image/delete/{}/{}", pictrs_url()?, &token, &file); let mut client_req = client.request_from(url, req.head()); + client_req.headers_mut().remove(ACCEPT_ENCODING); if let Some(addr) = req.head().peer_addr { client_req = client_req.insert_header(("X-Forwarded-For", addr.to_string())) diff --git a/crates/utils/src/rate_limit/rate_limiter.rs b/crates/utils/src/rate_limit/rate_limiter.rs index a965c79bb..46b6b0c72 100644 --- a/crates/utils/src/rate_limit/rate_limiter.rs +++ b/crates/utils/src/rate_limit/rate_limiter.rs @@ -18,19 +18,11 @@ pub(crate) enum RateLimitType { } /// Rate limiting based on rate type and IP addr -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Default)] pub struct RateLimiter { buckets: HashMap>, } -impl Default for RateLimiter { - fn default() -> Self { - Self { - buckets: HashMap::>::new(), - } - } -} - impl RateLimiter { fn insert_ip(&mut self, ip: &IpAddr) { for rate_limit_type in RateLimitType::iter() { diff --git a/crates/utils/src/request.rs b/crates/utils/src/request.rs index a881cf02c..f1655710e 100644 --- a/crates/utils/src/request.rs +++ b/crates/utils/src/request.rs @@ -114,6 +114,7 @@ pub(crate) struct PictrsResponse { #[derive(Deserialize, Debug, Clone)] pub(crate) struct PictrsFile { file: String, + #[allow(dead_code)] delete_token: String, } diff --git a/docker/dev/docker-compose.yml b/docker/dev/docker-compose.yml index bc42d4562..2474c74d6 100644 --- a/docker/dev/docker-compose.yml +++ b/docker/dev/docker-compose.yml @@ -1,6 +1,16 @@ version: '3.3' services: + nginx: + image: nginx:1-alpine + ports: + - "1236:1236" + volumes: + - ./nginx.conf:/etc/nginx/nginx.conf + restart: always + depends_on: + - pictrs + - lemmy-ui lemmy: image: lemmy-dev:latest @@ -17,12 +27,10 @@ services: lemmy-ui: image: dessalines/lemmy-ui:dev - ports: - - "1235:1234" restart: always environment: - LEMMY_INTERNAL_HOST=lemmy:8536 - - LEMMY_EXTERNAL_HOST=localhost:8536 + - LEMMY_EXTERNAL_HOST=localhost:1234 - LEMMY_HTTPS=false depends_on: - lemmy @@ -42,8 +50,6 @@ services: pictrs: image: asonix/pictrs:v0.2.6-r2 - ports: - - "8537:8080" user: 991:991 volumes: - ./volumes/pictrs:/mnt diff --git a/docker/dev/nginx.conf b/docker/dev/nginx.conf new file mode 100644 index 000000000..dfc4c07b6 --- /dev/null +++ b/docker/dev/nginx.conf @@ -0,0 +1,51 @@ +worker_processes 1; +events { + worker_connections 1024; +} +http { + upstream lemmy { + server "lemmy:8536"; + } + upstream lemmy-ui { + server "lemmy-ui:1234"; + } + server { + listen 1236; + server_name localhost; + + # frontend + location / { + set $proxpass "http://lemmy-ui"; + if ($http_accept = "application/activity+json") { + set $proxpass "http://lemmy"; + } + if ($http_accept = "application/ld+json; profile=\"https://www.w3.org/ns/activitystreams\"") { + set $proxpass "http://lemmy"; + } + if ($request_method = POST) { + set $proxpass "http://lemmy"; + } + proxy_pass $proxpass; + + rewrite ^(.+)/+$ $1 permanent; + + # Send actual client IP upstream + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + + # backend + location ~ ^/(api|pictrs|feeds|nodeinfo|.well-known) { + proxy_pass "http://lemmy"; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + + # Add IP forwarding headers + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + } +} diff --git a/docker/federation/docker-compose.yml b/docker/federation/docker-compose.yml index 3761ebadc..8b76a65a2 100644 --- a/docker/federation/docker-compose.yml +++ b/docker/federation/docker-compose.yml @@ -11,7 +11,7 @@ services: - "8580:8580" volumes: - ./nginx.conf:/etc/nginx/nginx.conf - restart: on-failure + restart: always depends_on: - pictrs - lemmy-alpha-ui