From 41b90bb162f172b289470368c169fffb69106658 Mon Sep 17 00:00:00 2001 From: Riley Date: Fri, 7 Jan 2022 08:53:45 -0600 Subject: [PATCH] Add console-subscriber (#2003) * Add console-subscriber * Add larg event buffer capacity * Add console port to lemmy prod container * Expose pict-rs console port * Update pictrs --- .cargo/config | 2 + Cargo.lock | 265 ++++++++++++++++++++++++-- Cargo.toml | 3 +- crates/api/Cargo.toml | 2 +- crates/api_crud/Cargo.toml | 2 +- crates/apub/Cargo.toml | 2 +- crates/apub_lib/src/activity_queue.rs | 3 +- crates/routes/Cargo.toml | 2 +- crates/utils/Cargo.toml | 2 +- docker/dev/Dockerfile | 2 + docker/dev/docker-compose.yml | 5 +- docker/prod/docker-compose.yml | 4 +- src/lib.rs | 27 ++- 13 files changed, 289 insertions(+), 32 deletions(-) create mode 100644 .cargo/config diff --git a/.cargo/config b/.cargo/config new file mode 100644 index 000000000..bff29e6e1 --- /dev/null +++ b/.cargo/config @@ -0,0 +1,2 @@ +[build] +rustflags = ["--cfg", "tokio_unstable"] diff --git a/Cargo.lock b/Cargo.lock index 923399f47..439e855be 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -690,6 +690,40 @@ dependencies = [ "xdg", ] +[[package]] +name = "console-api" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "033fddce299c93dd44ae21d5f5a6e749baa5d103784bcdde65701c07272a9fde" +dependencies = [ + "prost 0.9.0", + "prost-types 0.9.0", + "tonic 0.6.2", + "tonic-build 0.6.2", + "tracing-core", +] + +[[package]] +name = "console-subscriber" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2380cc150266375aeda8f9aeadc5527395c1a8807ecf9fa97a46d1bb760ec5b" +dependencies = [ + "console-api", + "futures", + "hdrhistogram", + "humantime", + "serde", + "serde_json", + "thread_local", + "tokio", + "tokio-stream", + "tonic 0.6.2", + "tracing", + "tracing-core", + "tracing-subscriber", +] + [[package]] name = "convert_case" version = "0.4.0" @@ -1058,6 +1092,70 @@ version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457" +[[package]] +name = "encoding" +version = "0.2.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b0d943856b990d12d3b55b359144ff341533e516d94098b1d3fc1ac666d36ec" +dependencies = [ + "encoding-index-japanese", + "encoding-index-korean", + "encoding-index-simpchinese", + "encoding-index-singlebyte", + "encoding-index-tradchinese", +] + +[[package]] +name = "encoding-index-japanese" +version = "1.20141219.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04e8b2ff42e9a05335dbf8b5c6f7567e5591d0d916ccef4e0b1710d32a0d0c91" +dependencies = [ + "encoding_index_tests", +] + +[[package]] +name = "encoding-index-korean" +version = "1.20141219.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4dc33fb8e6bcba213fe2f14275f0963fd16f0a02c878e3095ecfdf5bee529d81" +dependencies = [ + "encoding_index_tests", +] + +[[package]] +name = "encoding-index-simpchinese" +version = "1.20141219.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d87a7194909b9118fc707194baa434a4e3b0fb6a5a757c73c3adb07aa25031f7" +dependencies = [ + "encoding_index_tests", +] + +[[package]] +name = "encoding-index-singlebyte" +version = "1.20141219.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3351d5acffb224af9ca265f435b859c7c01537c0849754d3db3fdf2bfe2ae84a" +dependencies = [ + "encoding_index_tests", +] + +[[package]] +name = "encoding-index-tradchinese" +version = "1.20141219.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd0e20d5688ce3cab59eb3ef3a2083a5c77bf496cb798dc6fcdb75f323890c18" +dependencies = [ + "encoding_index_tests", +] + +[[package]] +name = "encoding_index_tests" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a246d82be1c9d791c5dfde9a2bd045fc3cbba3fa2b11ad558f27d01712f00569" + [[package]] name = "encoding_rs" version = "0.8.30" @@ -1115,6 +1213,12 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37ab347416e802de484e4d03c7316c48f1ecb56574dfd4a46a80f173ce1de04d" +[[package]] +name = "fixedbitset" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "279fb028e20b3c4c320317955b77c5e0c9701f05a1d309905d6fc702cdc5053e" + [[package]] name = "flate2" version = "1.0.22" @@ -1348,6 +1452,19 @@ dependencies = [ "ahash", ] +[[package]] +name = "hdrhistogram" +version = "7.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6490be71f07a5f62b564bc58e36953f675833df11c7e4a0647bee7a07ca1ec5e" +dependencies = [ + "base64 0.13.0", + "byteorder", + "flate2", + "nom 7.1.0", + "num-traits", +] + [[package]] name = "heck" version = "0.3.3" @@ -1494,6 +1611,12 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421" +[[package]] +name = "humantime" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" + [[package]] name = "hyper" version = "0.14.16" @@ -1971,6 +2094,7 @@ dependencies = [ "cargo-husky", "chrono", "clokwerk", + "console-subscriber", "diesel", "diesel_migrations", "doku", @@ -2018,6 +2142,7 @@ dependencies = [ "deser-hjson", "diesel", "doku", + "encoding", "futures", "http", "itertools", @@ -2542,11 +2667,11 @@ dependencies = [ "futures", "http", "opentelemetry", - "prost", + "prost 0.8.0", "thiserror", "tokio", - "tonic", - "tonic-build", + "tonic 0.5.2", + "tonic-build 0.5.2", ] [[package]] @@ -2646,7 +2771,17 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "467d164a6de56270bd7c4d070df81d07beace25012d5103ced4e9ff08d6afdb7" dependencies = [ - "fixedbitset", + "fixedbitset 0.2.0", + "indexmap", +] + +[[package]] +name = "petgraph" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a13a2fa9d0b63e5f22328828741e523766fff0ee9e779316902290dff3f824f" +dependencies = [ + "fixedbitset 0.4.1", "indexmap", ] @@ -2784,7 +2919,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "de5e2533f59d08fcf364fd374ebda0692a70bd6d7e66ef97f306f45c6c5d8020" dependencies = [ "bytes", - "prost-derive", + "prost-derive 0.8.0", +] + +[[package]] +name = "prost" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "444879275cb4fd84958b1a1d5420d15e6fcf7c235fe47f053c9c2a80aceb6001" +dependencies = [ + "bytes", + "prost-derive 0.9.0", ] [[package]] @@ -2798,9 +2943,29 @@ dependencies = [ "itertools", "log", "multimap", - "petgraph", - "prost", - "prost-types", + "petgraph 0.5.1", + "prost 0.8.0", + "prost-types 0.8.0", + "tempfile", + "which", +] + +[[package]] +name = "prost-build" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62941722fb675d463659e49c4f3fe1fe792ff24fe5bbaa9c08cd3b98a1c354f5" +dependencies = [ + "bytes", + "heck", + "itertools", + "lazy_static", + "log", + "multimap", + "petgraph 0.6.0", + "prost 0.9.0", + "prost-types 0.9.0", + "regex", "tempfile", "which", ] @@ -2818,6 +2983,19 @@ dependencies = [ "syn 1.0.82", ] +[[package]] +name = "prost-derive" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9cc1a3263e07e0bf68e96268f37665207b49560d98739662cdfaae215c720fe" +dependencies = [ + "anyhow", + "itertools", + "proc-macro2 1.0.33", + "quote 1.0.10", + "syn 1.0.82", +] + [[package]] name = "prost-types" version = "0.8.0" @@ -2825,7 +3003,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "603bbd6394701d13f3f25aada59c7de9d35a6a5887cfc156181234a44002771b" dependencies = [ "bytes", - "prost", + "prost 0.8.0", +] + +[[package]] +name = "prost-types" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "534b7a0e836e3c482d2693070f982e39e7611da9695d4d1f5a4b186b51faef0a" +dependencies = [ + "bytes", + "prost 0.9.0", ] [[package]] @@ -3693,11 +3881,10 @@ checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" [[package]] name = "tokio" -version = "1.14.0" +version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70e992e41e0d2fb9f755b37446f20900f64446ef54874f40a60c78f021ac6144" +checksum = "fbbf1c778ec206785635ce8ad57fe52b3009ae9e0c9f574a728f3049d3e55838" dependencies = [ - "autocfg", "bytes", "libc", "memchr", @@ -3708,6 +3895,7 @@ dependencies = [ "pin-project-lite", "signal-hook-registry", "tokio-macros", + "tracing", "winapi", ] @@ -3723,9 +3911,9 @@ dependencies = [ [[package]] name = "tokio-macros" -version = "1.6.0" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c9efc1aba077437943f7515666aa2b882dfabfbfdf89c819ea75a8d6e9eaba5e" +checksum = "b557f72f448c511a979e2564e55d74e6c4432fc96ff4f6241bc6bded342643b7" dependencies = [ "proc-macro2 1.0.33", "quote 1.0.10", @@ -3806,8 +3994,39 @@ dependencies = [ "hyper-timeout", "percent-encoding", "pin-project", - "prost", - "prost-derive", + "prost 0.8.0", + "prost-derive 0.8.0", + "tokio", + "tokio-stream", + "tokio-util", + "tower", + "tower-layer", + "tower-service", + "tracing", + "tracing-futures", +] + +[[package]] +name = "tonic" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff08f4649d10a70ffa3522ca559031285d8e421d727ac85c60825761818f5d0a" +dependencies = [ + "async-stream", + "async-trait", + "base64 0.13.0", + "bytes", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "hyper", + "hyper-timeout", + "percent-encoding", + "pin-project", + "prost 0.9.0", + "prost-derive 0.9.0", "tokio", "tokio-stream", "tokio-util", @@ -3825,7 +4044,19 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "12b52d07035516c2b74337d2ac7746075e7dcae7643816c1b12c5ff8a7484c08" dependencies = [ "proc-macro2 1.0.33", - "prost-build", + "prost-build 0.8.0", + "quote 1.0.10", + "syn 1.0.82", +] + +[[package]] +name = "tonic-build" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9403f1bafde247186684b230dc6f38b5cd514584e8bec1dd32514be4745fa757" +dependencies = [ + "proc-macro2 1.0.33", + "prost-build 0.9.0", "quote 1.0.10", "syn 1.0.82", ] diff --git a/Cargo.toml b/Cargo.toml index 5af893af9..524f03560 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -54,11 +54,12 @@ tracing-actix-web = { version = "0.5.0-beta.8", default-features = false } tracing-error = "0.2.0" tracing-log = "0.1.2" tracing-subscriber = { version = "0.3.3", features = ["env-filter"] } +console-subscriber = "0.1.0" strum = "0.23.0" url = { version = "2.2.2", features = ["serde"] } openssl = "0.10.38" http-signature-normalization-actix = { version = "0.5.0-beta.14", default-features = false, features = ["sha-2"] } -tokio = { version = "1.14.0", features = ["sync"] } +tokio = { version = "1.14.0", features = ["sync", "tracing"] } anyhow = "1.0.51" reqwest = { version = "0.11.7", features = ["json"] } reqwest-middleware = "0.1.3" diff --git a/crates/api/Cargo.toml b/crates/api/Cargo.toml index 07daf268c..21121f365 100644 --- a/crates/api/Cargo.toml +++ b/crates/api/Cargo.toml @@ -39,7 +39,7 @@ http = "0.2.5" http-signature-normalization-actix = { version = "0.5.0-beta.14", default-features = false, features = ["sha-2"] } base64 = "0.13.0" tokio = "1.14.0" -futures = "0.3.18" +futures = "0.3.17" itertools = "0.10.3" uuid = { version = "0.8.2", features = ["serde", "v4"] } sha2 = "0.10.0" diff --git a/crates/api_crud/Cargo.toml b/crates/api_crud/Cargo.toml index c2b97678d..d522e5dab 100644 --- a/crates/api_crud/Cargo.toml +++ b/crates/api_crud/Cargo.toml @@ -35,7 +35,7 @@ http = "0.2.5" http-signature-normalization-actix = { version = "0.5.0-beta.14", default-features = false, features = ["sha-2"] } base64 = "0.13.0" tokio = "1.14.0" -futures = "0.3.18" +futures = "0.3.17" itertools = "0.10.3" uuid = { version = "0.8.2", features = ["serde", "v4"] } sha2 = "0.10.0" diff --git a/crates/apub/Cargo.toml b/crates/apub/Cargo.toml index 881ca0d41..086d492c2 100644 --- a/crates/apub/Cargo.toml +++ b/crates/apub/Cargo.toml @@ -39,7 +39,7 @@ percent-encoding = "2.1.0" http = "0.2.5" http-signature-normalization-actix = { version = "0.5.0-beta.14", default-features = false, features = ["server", "sha-2"] } tokio = "1.14.0" -futures = "0.3.18" +futures = "0.3.17" itertools = "0.10.3" uuid = { version = "0.8.2", features = ["serde", "v4"] } sha2 = "0.10.0" diff --git a/crates/apub_lib/src/activity_queue.rs b/crates/apub_lib/src/activity_queue.rs index 7357b2d69..fac30251e 100644 --- a/crates/apub_lib/src/activity_queue.rs +++ b/crates/apub_lib/src/activity_queue.rs @@ -82,9 +82,10 @@ async fn do_send(task: SendActivityTask, client: &ClientWithMiddleware) -> Resul if !o.status().is_success() { let status = o.status(); let text = o.text().await?; + warn!( "Send {} to {} failed with status {}: {}", - task.activity_id, task.inbox, status, text + task.activity_id, task.inbox, status, text, ); } } diff --git a/crates/routes/Cargo.toml b/crates/routes/Cargo.toml index ff0b3d1c0..bd4cf3d1a 100644 --- a/crates/routes/Cargo.toml +++ b/crates/routes/Cargo.toml @@ -26,7 +26,7 @@ actix-http = "3.0.0-beta.15" sha2 = "0.10.0" anyhow = "1.0.51" chrono = { version = "0.4.19", features = ["serde"] } -futures = "0.3.18" +futures = "0.3.17" reqwest = { version = "0.11.7", features = ["stream"] } reqwest-middleware = "0.1.3" rss = "2.0.0" diff --git a/crates/utils/Cargo.toml b/crates/utils/Cargo.toml index 894ae0c13..66c049678 100644 --- a/crates/utils/Cargo.toml +++ b/crates/utils/Cargo.toml @@ -36,7 +36,7 @@ reqwest-middleware = "0.1.3" tokio = { version = "1.14.0", features = ["sync"] } strum = "0.23.0" strum_macros = "0.23.1" -futures = "0.3.18" +futures = "0.3.17" diesel = "1.4.8" http = "0.2.5" deser-hjson = "1.0.2" diff --git a/docker/dev/Dockerfile b/docker/dev/Dockerfile index 95329c015..f1dad5ce4 100644 --- a/docker/dev/Dockerfile +++ b/docker/dev/Dockerfile @@ -7,6 +7,7 @@ WORKDIR /app # Cargo chef plan FROM chef as planner +ENV RUSTFLAGS="--cfg tokio_unstable" # Copy dirs COPY . . @@ -16,6 +17,7 @@ RUN cargo chef prepare --recipe-path recipe.json FROM chef as builder ARG CARGO_BUILD_TARGET=x86_64-unknown-linux-musl ARG RUSTRELEASEDIR="debug" +ENV RUSTFLAGS="--cfg tokio_unstable" COPY --from=planner /app/recipe.json ./recipe.json RUN cargo chef cook --recipe-path recipe.json --target ${CARGO_BUILD_TARGET} diff --git a/docker/dev/docker-compose.yml b/docker/dev/docker-compose.yml index 9bfcbc449..9e05bca5a 100644 --- a/docker/dev/docker-compose.yml +++ b/docker/dev/docker-compose.yml @@ -16,6 +16,7 @@ services: image: lemmy-dev:latest ports: - "8536:8536" + - "6669:6669" restart: always environment: - RUST_LOG="warn,lemmy_server=debug,lemmy_api=debug,lemmy_api_common=debug,lemmy_api_crud=debug,lemmy_apub=debug,lemmy_db_schema=debug,lemmy_db_views=debug,lemmy_db_views_actor=debug,lemmy_db_views_moderator=debug,lemmy_routes=debug,lemmy_utils=debug,lemmy_websocket=debug" @@ -50,10 +51,12 @@ services: restart: always pictrs: - image: asonix/pictrs:0.3.0-beta.11 + image: asonix/pictrs:0.3.0-beta.12-r1 user: 991:991 environment: - PICTRS_OPENTELEMETRY_URL=http://otel:4137 + ports: + - "6670:6669" volumes: - ./volumes/pictrs:/mnt restart: always diff --git a/docker/prod/docker-compose.yml b/docker/prod/docker-compose.yml index 446afb036..88f5dbd51 100644 --- a/docker/prod/docker-compose.yml +++ b/docker/prod/docker-compose.yml @@ -15,6 +15,7 @@ services: image: dessalines/lemmy:0.14.3 ports: - "127.0.0.1:8536:8536" + - "127.0.0.1:6669:6669" restart: always environment: - RUST_LOG="warn,lemmy_server=info,lemmy_api=info,lemmy_api_common=info,lemmy_api_crud=info,lemmy_apub=info,lemmy_db_schema=info,lemmy_db_views=info,lemmy_db_views_actor=info,lemmy_db_views_moderator=info,lemmy_routes=info,lemmy_utils=info,lemmy_websocket=info" @@ -37,9 +38,10 @@ services: - lemmy pictrs: - image: asonix/pictrs:0.3.0-beta.11 + image: asonix/pictrs:0.3.0-beta.12-r1 ports: - "127.0.0.1:8537:8080" + - "127.0.0.1:6670:6669" user: 991:991 volumes: - ./volumes/pictrs:/mnt diff --git a/src/lib.rs b/src/lib.rs index 29b144690..fc68ba382 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -4,6 +4,7 @@ pub mod code_migrations; pub mod root_span_builder; pub mod scheduled_tasks; +use console_subscriber::ConsoleLayer; use lemmy_utils::LemmyError; use opentelemetry::{ sdk::{propagation::TraceContextPropagator, Resource}, @@ -13,20 +14,32 @@ use opentelemetry_otlp::WithExportConfig; use tracing::subscriber::set_global_default; use tracing_error::ErrorLayer; use tracing_log::LogTracer; -use tracing_subscriber::{layer::SubscriberExt, EnvFilter, Registry}; +use tracing_subscriber::{filter::Targets, layer::SubscriberExt, Layer, Registry}; pub fn init_tracing(opentelemetry_url: Option<&str>) -> Result<(), LemmyError> { LogTracer::init()?; opentelemetry::global::set_text_map_propagator(TraceContextPropagator::new()); - let env_filter = EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info")); - let format_layer = tracing_subscriber::fmt::layer(); + let log_description = std::env::var("RUST_LOG").unwrap_or_else(|_| "info".into()); + + let targets = log_description + .trim() + .trim_matches('"') + .parse::()?; + + let format_layer = tracing_subscriber::fmt::layer().with_filter(targets.clone()); + + let console_layer = ConsoleLayer::builder() + .with_default_env() + .server_addr(([0, 0, 0, 0], 6669)) + .event_buffer_capacity(1024 * 1024) + .spawn(); let subscriber = Registry::default() - .with(env_filter) .with(format_layer) - .with(ErrorLayer::default()); + .with(ErrorLayer::default()) + .with(console_layer); if let Some(url) = opentelemetry_url { let tracer = opentelemetry_otlp::new_pipeline() @@ -42,7 +55,9 @@ pub fn init_tracing(opentelemetry_url: Option<&str>) -> Result<(), LemmyError> { ) .install_batch(opentelemetry::runtime::Tokio)?; - let otel_layer = tracing_opentelemetry::layer().with_tracer(tracer); + let otel_layer = tracing_opentelemetry::layer() + .with_tracer(tracer) + .with_filter(targets); let subscriber = subscriber.with(otel_layer);