From 29355c749e07976a5ad59dc49581ac42440bcf54 Mon Sep 17 00:00:00 2001 From: Felix Ableitner Date: Mon, 14 Sep 2020 12:23:05 +0200 Subject: [PATCH 1/2] Remove brotli dependency --- server/Cargo.lock | 53 +++--------------------------- server/Cargo.toml | 10 +++--- server/lemmy_rate_limit/Cargo.toml | 2 +- server/lemmy_utils/Cargo.toml | 2 +- 4 files changed, 12 insertions(+), 55 deletions(-) diff --git a/server/Cargo.lock b/server/Cargo.lock index 4ed9e059b..0c50f464d 100644 --- a/server/Cargo.lock +++ b/server/Cargo.lock @@ -139,14 +139,12 @@ dependencies = [ "actix-utils 2.0.0", "base64 0.12.3", "bitflags 1.2.1", - "brotli2", "bytes", "cookie", "copyless", "derive_more", "either", "encoding_rs", - "flate2", "futures-channel", "futures-core", "futures-util", @@ -325,9 +323,9 @@ dependencies = [ [[package]] name = "actix-web" -version = "3.0.0" +version = "3.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e665de333edabd0421799822dac3e7d8a25a63bb995ae1f60cd99619d8ddda8" +checksum = "cd7fc56022da91a4dc00ccae7d7bb82e539749ca36df181695f4efdf5d413b2e" dependencies = [ "actix-codec 0.3.0", "actix-http", @@ -700,26 +698,6 @@ dependencies = [ "opaque-debug 0.3.0", ] -[[package]] -name = "brotli-sys" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4445dea95f4c2b41cde57cc9fee236ae4dbae88d8fcbdb4750fc1bb5d86aaecd" -dependencies = [ - "cc", - "libc", -] - -[[package]] -name = "brotli2" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0cb036c3eade309815c15ddbacec5b22c4d1f3983a774ab2eac2e3e9ea85568e" -dependencies = [ - "brotli-sys", - "libc", -] - [[package]] name = "buf-min" version = "0.1.1" @@ -931,15 +909,6 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8aebca1129a03dc6dc2b127edd729435bbc4a37e1d5f4d7513165089ceb02634" -[[package]] -name = "crc32fast" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba125de2af0df55319f41944744ad91c71113bf74a4646efff39afe1f6842db1" -dependencies = [ - "cfg-if", -] - [[package]] name = "crossbeam-channel" version = "0.4.4" @@ -1289,18 +1258,6 @@ dependencies = [ "ascii_utils", ] -[[package]] -name = "flate2" -version = "1.0.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "766d0e77a2c1502169d4a93ff3b8c15a71fd946cd0126309752104e5f3c46d94" -dependencies = [ - "cfg-if", - "crc32fast", - "libc", - "miniz_oxide", -] - [[package]] name = "fnv" version = "1.0.7" @@ -1591,12 +1548,12 @@ dependencies = [ [[package]] name = "http-signature-normalization-actix" -version = "0.4.0-alpha.3" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b44149de8286e9a07aeb72f4dee198530c0fb95df77f36b11138a748788f5603" +checksum = "88c2cc504dd6a2af53b5f2c3dba63aa5d797359df5b64a21e2230bf78146a373" dependencies = [ - "actix-http", "actix-web", + "awc", "base64 0.12.3", "bytes", "chrono", diff --git a/server/Cargo.toml b/server/Cargo.toml index 37fac2808..ec8a29d00 100644 --- a/server/Cargo.toml +++ b/server/Cargo.toml @@ -30,10 +30,10 @@ serde_json = { version = "1.0.52", features = ["preserve_order"]} serde = { version = "1.0.105", features = ["derive"] } actix = "0.10.0" actix-web = { version = "3.0.0", default-features = false, features = ["rustls"] } -actix-files = "0.3.0" -actix-web-actors = "3.0.0" -actix-rt = "1.1.1" -awc = "2.0.0-alpha.2" +actix-files = { version = "0.3.0", default-features = false } +actix-web-actors = { version = "3.0.0", default-features = false } +actix-rt = { version = "1.1.1", default-features = false } +awc = { version = "2.0.0", default-features = false } log = "0.4.0" env_logger = "0.7.1" rand = "0.7.3" @@ -46,7 +46,7 @@ url = { version = "2.1.1", features = ["serde"] } percent-encoding = "2.1.0" openssl = "0.10" http = "0.2.1" -http-signature-normalization-actix = { version = "0.4.0-alpha.2", default-features = false, features = ["sha-2"] } +http-signature-normalization-actix = { version = "0.4.0", default-features = false, features = ["sha-2"] } base64 = "0.12.1" tokio = "0.2.21" futures = "0.3.5" diff --git a/server/lemmy_rate_limit/Cargo.toml b/server/lemmy_rate_limit/Cargo.toml index 949f72f99..dc86523ff 100644 --- a/server/lemmy_rate_limit/Cargo.toml +++ b/server/lemmy_rate_limit/Cargo.toml @@ -14,5 +14,5 @@ tokio = "0.2.21" strum = "0.18.0" strum_macros = "0.18.0" futures = "0.3.5" -actix-web = { version = "3.0.0-alpha.3", features = ["rustls"] } +actix-web = { version = "3.0.0", default-features = false, features = ["rustls"] } log = "0.4.0" diff --git a/server/lemmy_utils/Cargo.toml b/server/lemmy_utils/Cargo.toml index a852cb852..be6cb6535 100644 --- a/server/lemmy_utils/Cargo.toml +++ b/server/lemmy_utils/Cargo.toml @@ -25,5 +25,5 @@ comrak = "0.7" lazy_static = "1.3.0" openssl = "0.10" url = { version = "2.1.1", features = ["serde"] } -actix-web = "3.0.0-alpha.3" +actix-web = {version = "3.0.0", default-features = false } anyhow = "1.0.32" From 1870dc8cd9ba182ab8753c09af93abc6dd4e219e Mon Sep 17 00:00:00 2001 From: nutomic Date: Mon, 14 Sep 2020 15:29:50 +0000 Subject: [PATCH 2/2] Split lemmy_utils into multiple files (#96) Update dependencies Move send_local_notifs into lemmy_api_structs (ref #1115) Split lemmy_utils into multiple files Co-authored-by: Felix Ableitner Reviewed-on: https://yerbamate.dev/LemmyNet/lemmy/pulls/96 --- server/Cargo.lock | 73 ++++- server/Cargo.toml | 8 +- server/lemmy_api_structs/Cargo.toml | 4 + server/lemmy_api_structs/src/lib.rs | 152 +++++++++ server/lemmy_db/Cargo.toml | 6 +- server/lemmy_db/src/activity.rs | 2 +- server/lemmy_db/src/category.rs | 2 +- server/lemmy_db/src/comment_view.rs | 4 +- server/lemmy_db/src/community_view.rs | 4 +- server/lemmy_db/src/lib.rs | 2 + server/lemmy_db/src/moderator_views.rs | 38 +-- server/lemmy_db/src/post_view.rs | 6 +- server/lemmy_db/src/private_message_view.rs | 6 +- server/lemmy_db/src/schema.rs | 64 ++-- server/lemmy_db/src/site_view.rs | 6 +- server/lemmy_db/src/user.rs | 2 +- server/lemmy_db/src/user_mention_view.rs | 6 +- server/lemmy_db/src/user_view.rs | 6 +- server/lemmy_rate_limit/Cargo.toml | 4 +- server/lemmy_rate_limit/src/lib.rs | 2 +- server/lemmy_utils/Cargo.toml | 6 +- server/lemmy_utils/src/apub.rs | 64 ++++ server/lemmy_utils/src/email.rs | 55 ++++ server/lemmy_utils/src/lib.rs | 324 +------------------- server/lemmy_utils/src/test.rs | 86 ++++++ server/lemmy_utils/src/utils.rs | 130 ++++++++ server/src/api/comment.rs | 136 +------- server/src/api/community.rs | 20 +- server/src/api/mod.rs | 18 +- server/src/api/post.rs | 18 +- server/src/api/site.rs | 21 +- server/src/api/user.rs | 24 +- server/src/apub/activities.rs | 2 +- server/src/apub/comment.rs | 7 +- server/src/apub/community.rs | 10 +- server/src/apub/fetcher.rs | 5 +- server/src/apub/inbox/activities/create.rs | 11 +- server/src/apub/inbox/activities/delete.rs | 2 +- server/src/apub/inbox/activities/dislike.rs | 3 +- server/src/apub/inbox/activities/like.rs | 3 +- server/src/apub/inbox/activities/remove.rs | 2 +- server/src/apub/inbox/activities/undo.rs | 2 +- server/src/apub/inbox/activities/update.rs | 11 +- server/src/apub/inbox/community_inbox.rs | 2 +- server/src/apub/inbox/user_inbox.rs | 3 +- server/src/apub/mod.rs | 7 +- server/src/apub/post.rs | 9 +- server/src/apub/private_message.rs | 4 +- server/src/apub/user.rs | 9 +- server/src/code_migrations.rs | 5 +- server/src/lib.rs | 21 +- server/src/main.rs | 2 +- server/src/routes/feeds.rs | 5 +- server/src/routes/nodeinfo.rs | 5 +- server/src/routes/webfinger.rs | 3 +- server/src/routes/websocket.rs | 2 +- server/test.sh | 2 +- 57 files changed, 736 insertions(+), 700 deletions(-) create mode 100644 server/lemmy_utils/src/apub.rs create mode 100644 server/lemmy_utils/src/email.rs create mode 100644 server/lemmy_utils/src/test.rs create mode 100644 server/lemmy_utils/src/utils.rs diff --git a/server/Cargo.lock b/server/Cargo.lock index 0c50f464d..388f8d0b4 100644 --- a/server/Cargo.lock +++ b/server/Cargo.lock @@ -139,12 +139,14 @@ dependencies = [ "actix-utils 2.0.0", "base64 0.12.3", "bitflags 1.2.1", + "brotli2", "bytes", "cookie", "copyless", "derive_more", "either", "encoding_rs", + "flate2", "futures-channel", "futures-core", "futures-util", @@ -698,6 +700,26 @@ dependencies = [ "opaque-debug 0.3.0", ] +[[package]] +name = "brotli-sys" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4445dea95f4c2b41cde57cc9fee236ae4dbae88d8fcbdb4750fc1bb5d86aaecd" +dependencies = [ + "cc", + "libc", +] + +[[package]] +name = "brotli2" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0cb036c3eade309815c15ddbacec5b22c4d1f3983a774ab2eac2e3e9ea85568e" +dependencies = [ + "brotli-sys", + "libc", +] + [[package]] name = "buf-min" version = "0.1.1" @@ -837,9 +859,9 @@ checksum = "0dbbb57365263e881e805dc77d94697c9118fd94d8da011240555aa7b23445bd" [[package]] name = "comrak" -version = "0.7.0" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e17058cc536cf290563e88787d7b9e6030ce4742943017cc2ffb71f88034021c" +checksum = "0d325e4f2ffff52ca77d995bb675494d5364aa332499d5f7c7fbb28c25e671f6" dependencies = [ "clap", "entities", @@ -847,9 +869,11 @@ dependencies = [ "pest", "pest_derive", "regex", + "shell-words", "twoway", "typed-arena", "unicode_categories", + "xdg", ] [[package]] @@ -909,6 +933,15 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8aebca1129a03dc6dc2b127edd729435bbc4a37e1d5f4d7513165089ceb02634" +[[package]] +name = "crc32fast" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba125de2af0df55319f41944744ad91c71113bf74a4646efff39afe1f6842db1" +dependencies = [ + "cfg-if", +] + [[package]] name = "crossbeam-channel" version = "0.4.4" @@ -1258,6 +1291,18 @@ dependencies = [ "ascii_utils", ] +[[package]] +name = "flate2" +version = "1.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "766d0e77a2c1502169d4a93ff3b8c15a71fd946cd0126309752104e5f3c46d94" +dependencies = [ + "cfg-if", + "crc32fast", + "libc", + "miniz_oxide", +] + [[package]] name = "fnv" version = "1.0.7" @@ -1773,7 +1818,11 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" name = "lemmy_api_structs" version = "0.1.0" dependencies = [ + "actix-web", + "diesel", "lemmy_db", + "lemmy_utils", + "log", "serde 1.0.115", ] @@ -3057,6 +3106,12 @@ dependencies = [ "opaque-debug 0.3.0", ] +[[package]] +name = "shell-words" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6fa3938c99da4914afedd13bf3d79bcb6c277d1b2c398d23257a304d9e1b074" + [[package]] name = "signal-hook-registry" version = "1.2.1" @@ -3186,15 +3241,15 @@ checksum = "6446ced80d6c486436db5c078dde11a9f73d42b57fb273121e160b84f63d894c" [[package]] name = "strum" -version = "0.18.0" +version = "0.19.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57bd81eb48f4c437cadc685403cad539345bf703d78e63707418431cecd4522b" +checksum = "3924a58d165da3b7b2922c667ab0673c7b5fd52b5c19ea3442747bcb3cd15abe" [[package]] name = "strum_macros" -version = "0.18.0" +version = "0.19.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87c85aa3f8ea653bfd3ddf25f7ee357ee4d204731f6aa9ad04002306f6e2774c" +checksum = "2d2ab682ecdcae7f5f45ae85cd7c1e6c8e68ea42c8a612d47fedf831c037146a" dependencies = [ "heck", "proc-macro2", @@ -3864,3 +3919,9 @@ dependencies = [ "winapi 0.2.8", "winapi-build", ] + +[[package]] +name = "xdg" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d089681aa106a86fade1b0128fb5daf07d5867a509ab036d99988dec80429a57" diff --git a/server/Cargo.toml b/server/Cargo.toml index ec8a29d00..721ce51e7 100644 --- a/server/Cargo.toml +++ b/server/Cargo.toml @@ -22,7 +22,7 @@ lemmy_rate_limit = { path = "./lemmy_rate_limit" } diesel = "1.4.4" diesel_migrations = "1.4.0" dotenv = "0.15.0" -activitystreams = "0.7.0-alpha.3" +activitystreams = "0.7.0-alpha.4" activitystreams-ext = "0.1.0-alpha.2" bcrypt = "0.8.0" chrono = { version = "0.4.7", features = ["serde"] } @@ -37,8 +37,8 @@ awc = { version = "2.0.0", default-features = false } log = "0.4.0" env_logger = "0.7.1" rand = "0.7.3" -strum = "0.18.0" -strum_macros = "0.18.0" +strum = "0.19.2" +strum_macros = "0.19.2" jsonwebtoken = "7.0.1" lazy_static = "1.3.0" rss = "1.9.0" @@ -53,7 +53,7 @@ futures = "0.3.5" itertools = "0.9.0" uuid = { version = "0.8", features = ["serde", "v4"] } sha2 = "0.9" -async-trait = "0.1.36" +async-trait = "0.1.40" captcha = "0.0.7" anyhow = "1.0.32" thiserror = "1.0.20" diff --git a/server/lemmy_api_structs/Cargo.toml b/server/lemmy_api_structs/Cargo.toml index 3778422be..b442f697d 100644 --- a/server/lemmy_api_structs/Cargo.toml +++ b/server/lemmy_api_structs/Cargo.toml @@ -10,4 +10,8 @@ path = "src/lib.rs" [dependencies] lemmy_db = { path = "../lemmy_db" } +lemmy_utils = { path = "../lemmy_utils" } serde = { version = "1.0.105", features = ["derive"] } +log = "0.4.0" +diesel = "1.4.4" +actix-web = { version = "3.0.0-beta.3", features = ["rustls"] } diff --git a/server/lemmy_api_structs/src/lib.rs b/server/lemmy_api_structs/src/lib.rs index b3576a0d6..e94a286f7 100644 --- a/server/lemmy_api_structs/src/lib.rs +++ b/server/lemmy_api_structs/src/lib.rs @@ -1,7 +1,159 @@ extern crate serde; +extern crate log; +extern crate diesel; +extern crate actix_web; pub mod comment; pub mod community; pub mod post; pub mod site; pub mod user; + +use lemmy_db::comment::Comment; +use lemmy_db::user::User_; +use lemmy_db::post::Post; +use lemmy_db::user_mention::{UserMentionForm, UserMention}; +use log::error; +use lemmy_db::{Crud, DbPool}; +use lemmy_utils::utils::MentionData; +use lemmy_utils::settings::Settings; +use lemmy_utils::email::send_email; +use diesel::PgConnection; +use lemmy_utils::LemmyError; + +pub async fn blocking(pool: &DbPool, f: F) -> Result + where + F: FnOnce(&diesel::PgConnection) -> T + Send + 'static, + T: Send + 'static, +{ + let pool = pool.clone(); + let res = actix_web::web::block(move || { + let conn = pool.get()?; + let res = (f)(&conn); + Ok(res) as Result<_, LemmyError> + }) + .await?; + + Ok(res) +} + +pub async fn send_local_notifs( + mentions: Vec, + comment: Comment, + user: &User_, + post: Post, + pool: &DbPool, + do_send_email: bool, +) -> Result, LemmyError> { + let user2 = user.clone(); + let ids = blocking(pool, move |conn| { + do_send_local_notifs(conn, &mentions, &comment, &user2, &post, do_send_email) + }) + .await?; + + Ok(ids) +} + +fn do_send_local_notifs( + conn: &PgConnection, + mentions: &[MentionData], + comment: &Comment, + user: &User_, + post: &Post, + do_send_email: bool, +) -> Vec { + let mut recipient_ids = Vec::new(); + let hostname = &format!("https://{}", Settings::get().hostname); + + // Send the local mentions + for mention in mentions + .iter() + .filter(|m| m.is_local() && m.name.ne(&user.name)) + .collect::>() + { + if let Ok(mention_user) = User_::read_from_name(&conn, &mention.name) { + // TODO + // At some point, make it so you can't tag the parent creator either + // This can cause two notifications, one for reply and the other for mention + recipient_ids.push(mention_user.id); + + let user_mention_form = UserMentionForm { + recipient_id: mention_user.id, + comment_id: comment.id, + read: None, + }; + + // Allow this to fail softly, since comment edits might re-update or replace it + // Let the uniqueness handle this fail + match UserMention::create(&conn, &user_mention_form) { + Ok(_mention) => (), + Err(_e) => error!("{}", &_e), + }; + + // Send an email to those users that have notifications on + if do_send_email && mention_user.send_notifications_to_email { + if let Some(mention_email) = mention_user.email { + let subject = &format!("{} - Mentioned by {}", Settings::get().hostname, user.name,); + let html = &format!( + "

User Mention


{} - {}

inbox", + user.name, comment.content, hostname + ); + match send_email(subject, &mention_email, &mention_user.name, html) { + Ok(_o) => _o, + Err(e) => error!("{}", e), + }; + } + } + } + } + + // Send notifs to the parent commenter / poster + match comment.parent_id { + Some(parent_id) => { + if let Ok(parent_comment) = Comment::read(&conn, parent_id) { + if parent_comment.creator_id != user.id { + if let Ok(parent_user) = User_::read(&conn, parent_comment.creator_id) { + recipient_ids.push(parent_user.id); + + if do_send_email && parent_user.send_notifications_to_email { + if let Some(comment_reply_email) = parent_user.email { + let subject = &format!("{} - Reply from {}", Settings::get().hostname, user.name,); + let html = &format!( + "

Comment Reply


{} - {}

inbox", + user.name, comment.content, hostname + ); + match send_email(subject, &comment_reply_email, &parent_user.name, html) { + Ok(_o) => _o, + Err(e) => error!("{}", e), + }; + } + } + } + } + } + } + // Its a post + None => { + if post.creator_id != user.id { + if let Ok(parent_user) = User_::read(&conn, post.creator_id) { + recipient_ids.push(parent_user.id); + + if do_send_email && parent_user.send_notifications_to_email { + if let Some(post_reply_email) = parent_user.email { + let subject = &format!("{} - Reply from {}", Settings::get().hostname, user.name,); + let html = &format!( + "

Post Reply


{} - {}

inbox", + user.name, comment.content, hostname + ); + match send_email(subject, &post_reply_email, &parent_user.name, html) { + Ok(_o) => _o, + Err(e) => error!("{}", e), + }; + } + } + } + } + } + }; + recipient_ids +} diff --git a/server/lemmy_db/Cargo.toml b/server/lemmy_db/Cargo.toml index 36d9d9a77..1ae42a8bd 100644 --- a/server/lemmy_db/Cargo.toml +++ b/server/lemmy_db/Cargo.toml @@ -12,11 +12,11 @@ diesel = { version = "1.4.4", features = ["postgres","chrono","r2d2","64-column- chrono = { version = "0.4.7", features = ["serde"] } serde = { version = "1.0.105", features = ["derive"] } serde_json = { version = "1.0.52", features = ["preserve_order"]} -strum = "0.18.0" -strum_macros = "0.18.0" +strum = "0.19.2" +strum_macros = "0.19.2" log = "0.4.0" sha2 = "0.9" -bcrypt = "0.8.0" +bcrypt = "0.8.2" url = { version = "2.1.1", features = ["serde"] } lazy_static = "1.3.0" regex = "1.3.5" diff --git a/server/lemmy_db/src/activity.rs b/server/lemmy_db/src/activity.rs index 6a5beb3bb..79b185fb7 100644 --- a/server/lemmy_db/src/activity.rs +++ b/server/lemmy_db/src/activity.rs @@ -1,7 +1,7 @@ use crate::{schema::activity, Crud}; use diesel::{dsl::*, result::Error, *}; use log::debug; -use serde::{Serialize}; +use serde::Serialize; use serde_json::Value; use std::{ fmt::Debug, diff --git a/server/lemmy_db/src/category.rs b/server/lemmy_db/src/category.rs index c4165c22f..36beb9ff6 100644 --- a/server/lemmy_db/src/category.rs +++ b/server/lemmy_db/src/category.rs @@ -3,7 +3,7 @@ use crate::{ Crud, }; use diesel::{dsl::*, result::Error, *}; -use serde::{Serialize}; +use serde::Serialize; #[derive(Queryable, Identifiable, PartialEq, Debug, Serialize)] #[table_name = "category"] diff --git a/server/lemmy_db/src/comment_view.rs b/server/lemmy_db/src/comment_view.rs index d7e5c08d0..cb121cf88 100644 --- a/server/lemmy_db/src/comment_view.rs +++ b/server/lemmy_db/src/comment_view.rs @@ -84,9 +84,7 @@ table! { } } -#[derive( - Queryable, Identifiable, PartialEq, Debug, Serialize, QueryableByName, Clone, -)] +#[derive(Queryable, Identifiable, PartialEq, Debug, Serialize, QueryableByName, Clone)] #[table_name = "comment_fast_view"] pub struct CommentView { pub id: i32, diff --git a/server/lemmy_db/src/community_view.rs b/server/lemmy_db/src/community_view.rs index 1ecca0a1f..a6355504d 100644 --- a/server/lemmy_db/src/community_view.rs +++ b/server/lemmy_db/src/community_view.rs @@ -123,9 +123,7 @@ table! { } } -#[derive( - Queryable, Identifiable, PartialEq, Debug, Serialize, QueryableByName, Clone, -)] +#[derive(Queryable, Identifiable, PartialEq, Debug, Serialize, QueryableByName, Clone)] #[table_name = "community_fast_view"] pub struct CommunityView { pub id: i32, diff --git a/server/lemmy_db/src/lib.rs b/server/lemmy_db/src/lib.rs index ed6e1dfb9..fc660208d 100644 --- a/server/lemmy_db/src/lib.rs +++ b/server/lemmy_db/src/lib.rs @@ -40,6 +40,8 @@ pub mod user_mention; pub mod user_mention_view; pub mod user_view; +pub type DbPool = diesel::r2d2::Pool>; + pub trait Crud { fn create(conn: &PgConnection, form: &T) -> Result where diff --git a/server/lemmy_db/src/moderator_views.rs b/server/lemmy_db/src/moderator_views.rs index 7054a05a0..efa949a4a 100644 --- a/server/lemmy_db/src/moderator_views.rs +++ b/server/lemmy_db/src/moderator_views.rs @@ -1,6 +1,6 @@ use crate::limit_and_offset; use diesel::{result::Error, *}; -use serde::{Serialize}; +use serde::Serialize; table! { mod_remove_post_view (id) { @@ -17,9 +17,7 @@ table! { } } -#[derive( - Queryable, Identifiable, PartialEq, Debug, Serialize, QueryableByName, Clone, -)] +#[derive(Queryable, Identifiable, PartialEq, Debug, Serialize, QueryableByName, Clone)] #[table_name = "mod_remove_post_view"] pub struct ModRemovePostView { pub id: i32, @@ -77,9 +75,7 @@ table! { } } -#[derive( - Queryable, Identifiable, PartialEq, Debug, Serialize, QueryableByName, Clone, -)] +#[derive(Queryable, Identifiable, PartialEq, Debug, Serialize, QueryableByName, Clone)] #[table_name = "mod_lock_post_view"] pub struct ModLockPostView { pub id: i32, @@ -136,9 +132,7 @@ table! { } } -#[derive( - Queryable, Identifiable, PartialEq, Debug, Serialize, QueryableByName, Clone, -)] +#[derive(Queryable, Identifiable, PartialEq, Debug, Serialize, QueryableByName, Clone)] #[table_name = "mod_sticky_post_view"] pub struct ModStickyPostView { pub id: i32, @@ -200,9 +194,7 @@ table! { } } -#[derive( - Queryable, Identifiable, PartialEq, Debug, Serialize, QueryableByName, Clone, -)] +#[derive(Queryable, Identifiable, PartialEq, Debug, Serialize, QueryableByName, Clone)] #[table_name = "mod_remove_comment_view"] pub struct ModRemoveCommentView { pub id: i32, @@ -264,9 +256,7 @@ table! { } } -#[derive( - Queryable, Identifiable, PartialEq, Debug, Serialize, QueryableByName, Clone, -)] +#[derive(Queryable, Identifiable, PartialEq, Debug, Serialize, QueryableByName, Clone)] #[table_name = "mod_remove_community_view"] pub struct ModRemoveCommunityView { pub id: i32, @@ -320,9 +310,7 @@ table! { } } -#[derive( - Queryable, Identifiable, PartialEq, Debug, Serialize, QueryableByName, Clone, -)] +#[derive(Queryable, Identifiable, PartialEq, Debug, Serialize, QueryableByName, Clone)] #[table_name = "mod_ban_from_community_view"] pub struct ModBanFromCommunityView { pub id: i32, @@ -381,9 +369,7 @@ table! { } } -#[derive( - Queryable, Identifiable, PartialEq, Debug, Serialize, QueryableByName, Clone, -)] +#[derive(Queryable, Identifiable, PartialEq, Debug, Serialize, QueryableByName, Clone)] #[table_name = "mod_ban_view"] pub struct ModBanView { pub id: i32, @@ -435,9 +421,7 @@ table! { } } -#[derive( - Queryable, Identifiable, PartialEq, Debug, Serialize, QueryableByName, Clone, -)] +#[derive(Queryable, Identifiable, PartialEq, Debug, Serialize, QueryableByName, Clone)] #[table_name = "mod_add_community_view"] pub struct ModAddCommunityView { pub id: i32, @@ -492,9 +476,7 @@ table! { } } -#[derive( - Queryable, Identifiable, PartialEq, Debug, Serialize, QueryableByName, Clone, -)] +#[derive(Queryable, Identifiable, PartialEq, Debug, Serialize, QueryableByName, Clone)] #[table_name = "mod_add_view"] pub struct ModAddView { pub id: i32, diff --git a/server/lemmy_db/src/post_view.rs b/server/lemmy_db/src/post_view.rs index 3d3afb709..e88a80e30 100644 --- a/server/lemmy_db/src/post_view.rs +++ b/server/lemmy_db/src/post_view.rs @@ -1,7 +1,7 @@ use super::post_view::post_fast_view::BoxedQuery; use crate::{fuzzy_search, limit_and_offset, ListingType, MaybeOptional, SortType}; use diesel::{dsl::*, pg::Pg, result::Error, *}; -use serde::{Serialize}; +use serde::Serialize; // The faked schema since diesel doesn't do views table! { @@ -106,9 +106,7 @@ table! { } } -#[derive( - Queryable, Identifiable, PartialEq, Debug, Serialize, QueryableByName, Clone, -)] +#[derive(Queryable, Identifiable, PartialEq, Debug, Serialize, QueryableByName, Clone)] #[table_name = "post_fast_view"] pub struct PostView { pub id: i32, diff --git a/server/lemmy_db/src/private_message_view.rs b/server/lemmy_db/src/private_message_view.rs index 39fe66efa..68f7df42c 100644 --- a/server/lemmy_db/src/private_message_view.rs +++ b/server/lemmy_db/src/private_message_view.rs @@ -1,6 +1,6 @@ use crate::{limit_and_offset, MaybeOptional}; use diesel::{pg::Pg, result::Error, *}; -use serde::{Serialize}; +use serde::Serialize; // The faked schema since diesel doesn't do views table! { @@ -28,9 +28,7 @@ table! { } } -#[derive( - Queryable, Identifiable, PartialEq, Debug, Serialize, QueryableByName, Clone, -)] +#[derive(Queryable, Identifiable, PartialEq, Debug, Serialize, QueryableByName, Clone)] #[table_name = "private_message_view"] pub struct PrivateMessageView { pub id: i32, diff --git a/server/lemmy_db/src/schema.rs b/server/lemmy_db/src/schema.rs index c446edd9f..a189dbced 100644 --- a/server/lemmy_db/src/schema.rs +++ b/server/lemmy_db/src/schema.rs @@ -523,36 +523,36 @@ joinable!(user_mention -> comment (comment_id)); joinable!(user_mention -> user_ (recipient_id)); allow_tables_to_appear_in_same_query!( - activity, - category, - comment, - comment_aggregates_fast, - comment_like, - comment_saved, - community, - community_aggregates_fast, - community_follower, - community_moderator, - community_user_ban, - mod_add, - mod_add_community, - mod_ban, - mod_ban_from_community, - mod_lock_post, - mod_remove_comment, - mod_remove_community, - mod_remove_post, - mod_sticky_post, - password_reset_request, - post, - post_aggregates_fast, - post_like, - post_read, - post_saved, - private_message, - site, - user_, - user_ban, - user_fast, - user_mention, + activity, + category, + comment, + comment_aggregates_fast, + comment_like, + comment_saved, + community, + community_aggregates_fast, + community_follower, + community_moderator, + community_user_ban, + mod_add, + mod_add_community, + mod_ban, + mod_ban_from_community, + mod_lock_post, + mod_remove_comment, + mod_remove_community, + mod_remove_post, + mod_sticky_post, + password_reset_request, + post, + post_aggregates_fast, + post_like, + post_read, + post_saved, + private_message, + site, + user_, + user_ban, + user_fast, + user_mention, ); diff --git a/server/lemmy_db/src/site_view.rs b/server/lemmy_db/src/site_view.rs index 0edfd2b70..fd15ac778 100644 --- a/server/lemmy_db/src/site_view.rs +++ b/server/lemmy_db/src/site_view.rs @@ -1,5 +1,5 @@ use diesel::{result::Error, *}; -use serde::{Serialize}; +use serde::Serialize; table! { site_view (id) { @@ -24,9 +24,7 @@ table! { } } -#[derive( - Queryable, Identifiable, PartialEq, Debug, Serialize, QueryableByName, Clone, -)] +#[derive(Queryable, Identifiable, PartialEq, Debug, Serialize, QueryableByName, Clone)] #[table_name = "site_view"] pub struct SiteView { pub id: i32, diff --git a/server/lemmy_db/src/user.rs b/server/lemmy_db/src/user.rs index 247b5b360..986701c43 100644 --- a/server/lemmy_db/src/user.rs +++ b/server/lemmy_db/src/user.rs @@ -6,7 +6,7 @@ use crate::{ }; use bcrypt::{hash, DEFAULT_COST}; use diesel::{dsl::*, result::Error, *}; -use serde::{Serialize}; +use serde::Serialize; #[derive(Clone, Queryable, Identifiable, PartialEq, Debug, Serialize)] #[table_name = "user_"] diff --git a/server/lemmy_db/src/user_mention_view.rs b/server/lemmy_db/src/user_mention_view.rs index c35ad3613..d1ce5ebd0 100644 --- a/server/lemmy_db/src/user_mention_view.rs +++ b/server/lemmy_db/src/user_mention_view.rs @@ -1,6 +1,6 @@ use crate::{limit_and_offset, MaybeOptional, SortType}; use diesel::{dsl::*, pg::Pg, result::Error, *}; -use serde::{Serialize}; +use serde::Serialize; // The faked schema since diesel doesn't do views table! { @@ -83,9 +83,7 @@ table! { } } -#[derive( - Queryable, Identifiable, PartialEq, Debug, Serialize, QueryableByName, Clone, -)] +#[derive(Queryable, Identifiable, PartialEq, Debug, Serialize, QueryableByName, Clone)] #[table_name = "user_mention_fast_view"] pub struct UserMentionView { pub id: i32, diff --git a/server/lemmy_db/src/user_view.rs b/server/lemmy_db/src/user_view.rs index a4f6f47ae..b0c28d31c 100644 --- a/server/lemmy_db/src/user_view.rs +++ b/server/lemmy_db/src/user_view.rs @@ -1,7 +1,7 @@ use super::user_view::user_fast::BoxedQuery; use crate::{fuzzy_search, limit_and_offset, MaybeOptional, SortType}; use diesel::{dsl::*, pg::Pg, result::Error, *}; -use serde::{Serialize}; +use serde::Serialize; table! { user_view (id) { @@ -51,9 +51,7 @@ table! { } } -#[derive( - Queryable, Identifiable, PartialEq, Debug, Serialize, QueryableByName, Clone, -)] +#[derive(Queryable, Identifiable, PartialEq, Debug, Serialize, QueryableByName, Clone)] #[table_name = "user_fast"] pub struct UserView { pub id: i32, diff --git a/server/lemmy_rate_limit/Cargo.toml b/server/lemmy_rate_limit/Cargo.toml index dc86523ff..c5daaf7cd 100644 --- a/server/lemmy_rate_limit/Cargo.toml +++ b/server/lemmy_rate_limit/Cargo.toml @@ -11,8 +11,8 @@ path = "src/lib.rs" [dependencies] lemmy_utils = { path = "../lemmy_utils" } tokio = "0.2.21" -strum = "0.18.0" -strum_macros = "0.18.0" +strum = "0.19.2" +strum_macros = "0.19.2" futures = "0.3.5" actix-web = { version = "3.0.0", default-features = false, features = ["rustls"] } log = "0.4.0" diff --git a/server/lemmy_rate_limit/src/lib.rs b/server/lemmy_rate_limit/src/lib.rs index 260f7e2cb..8f962bbe6 100644 --- a/server/lemmy_rate_limit/src/lib.rs +++ b/server/lemmy_rate_limit/src/lib.rs @@ -8,8 +8,8 @@ extern crate tokio; use actix_web::dev::{Service, ServiceRequest, ServiceResponse, Transform}; use futures::future::{ok, Ready}; use lemmy_utils::{ - get_ip, settings::{RateLimitConfig, Settings}, + utils::get_ip, LemmyError, }; use rate_limiter::{RateLimitType, RateLimiter}; diff --git a/server/lemmy_utils/Cargo.toml b/server/lemmy_utils/Cargo.toml index be6cb6535..58d0893e0 100644 --- a/server/lemmy_utils/Cargo.toml +++ b/server/lemmy_utils/Cargo.toml @@ -12,16 +12,16 @@ path = "src/lib.rs" [dependencies] regex = "1.3.5" config = { version = "0.10.1", default-features = false, features = ["hjson"] } -chrono = { version = "0.4.7", features = ["serde"] } +chrono = { version = "0.4.15", features = ["serde"] } lettre = "0.9.3" lettre_email = "0.9.4" log = "0.4.0" itertools = "0.9.0" rand = "0.7.3" -serde = { version = "1.0.105", features = ["derive"] } +serde = { version = "1.0.115", features = ["derive"] } serde_json = { version = "1.0.52", features = ["preserve_order"]} thiserror = "1.0.20" -comrak = "0.7" +comrak = "0.8.2" lazy_static = "1.3.0" openssl = "0.10" url = { version = "2.1.1", features = ["serde"] } diff --git a/server/lemmy_utils/src/apub.rs b/server/lemmy_utils/src/apub.rs new file mode 100644 index 000000000..08e7a4491 --- /dev/null +++ b/server/lemmy_utils/src/apub.rs @@ -0,0 +1,64 @@ +use crate::settings::Settings; +use openssl::{pkey::PKey, rsa::Rsa}; +use std::io::{Error, ErrorKind}; +use url::Url; + +pub struct Keypair { + pub private_key: String, + pub public_key: String, +} + +/// Generate the asymmetric keypair for ActivityPub HTTP signatures. +pub fn generate_actor_keypair() -> Result { + let rsa = Rsa::generate(2048)?; + let pkey = PKey::from_rsa(rsa)?; + let public_key = pkey.public_key_to_pem()?; + let private_key = pkey.private_key_to_pem_pkcs8()?; + let key_to_string = |key| match String::from_utf8(key) { + Ok(s) => Ok(s), + Err(e) => Err(Error::new( + ErrorKind::Other, + format!("Failed converting key to string: {}", e), + )), + }; + Ok(Keypair { + private_key: key_to_string(private_key)?, + public_key: key_to_string(public_key)?, + }) +} + +pub enum EndpointType { + Community, + User, + Post, + Comment, + PrivateMessage, +} + +pub fn get_apub_protocol_string() -> &'static str { + if Settings::get().federation.tls_enabled { + "https" + } else { + "http" + } +} + +/// Generates the ActivityPub ID for a given object type and ID. +pub fn make_apub_endpoint(endpoint_type: EndpointType, name: &str) -> Url { + let point = match endpoint_type { + EndpointType::Community => "c", + EndpointType::User => "u", + EndpointType::Post => "post", + EndpointType::Comment => "comment", + EndpointType::PrivateMessage => "private_message", + }; + + Url::parse(&format!( + "{}://{}/{}/{}", + get_apub_protocol_string(), + Settings::get().hostname, + point, + name + )) + .unwrap() +} diff --git a/server/lemmy_utils/src/email.rs b/server/lemmy_utils/src/email.rs new file mode 100644 index 000000000..43b880ebf --- /dev/null +++ b/server/lemmy_utils/src/email.rs @@ -0,0 +1,55 @@ +use crate::settings::Settings; +use lettre::{ + smtp::{ + authentication::{Credentials, Mechanism}, + extension::ClientId, + ConnectionReuseParameters, + }, + ClientSecurity, + SmtpClient, + Transport, +}; +use lettre_email::Email; + +pub fn send_email( + subject: &str, + to_email: &str, + to_username: &str, + html: &str, +) -> Result<(), String> { + let email_config = Settings::get().email.ok_or("no_email_setup")?; + + let email = Email::builder() + .to((to_email, to_username)) + .from(email_config.smtp_from_address.to_owned()) + .subject(subject) + .html(html) + .build() + .unwrap(); + + let mailer = if email_config.use_tls { + SmtpClient::new_simple(&email_config.smtp_server).unwrap() + } else { + SmtpClient::new(&email_config.smtp_server, ClientSecurity::None).unwrap() + } + .hello_name(ClientId::Domain(Settings::get().hostname)) + .smtp_utf8(true) + .authentication_mechanism(Mechanism::Plain) + .connection_reuse(ConnectionReuseParameters::ReuseUnlimited); + let mailer = if let (Some(login), Some(password)) = + (&email_config.smtp_login, &email_config.smtp_password) + { + mailer.credentials(Credentials::new(login.to_owned(), password.to_owned())) + } else { + mailer + }; + + let mut transport = mailer.transport(); + let result = transport.send(email.into()); + transport.close(); + + match result { + Ok(_) => Ok(()), + Err(e) => Err(e.to_string()), + } +} diff --git a/server/lemmy_utils/src/lib.rs b/server/lemmy_utils/src/lib.rs index d28218282..9cc1fe025 100644 --- a/server/lemmy_utils/src/lib.rs +++ b/server/lemmy_utils/src/lib.rs @@ -12,29 +12,16 @@ extern crate serde_json; extern crate thiserror; extern crate url; +pub mod apub; +pub mod email; pub mod settings; +#[cfg(test)] +mod test; +pub mod utils; use crate::settings::Settings; -use actix_web::dev::ConnectionInfo; -use chrono::{DateTime, FixedOffset, Local, NaiveDateTime}; -use itertools::Itertools; -use lettre::{ - smtp::{ - authentication::{Credentials, Mechanism}, - extension::ClientId, - ConnectionReuseParameters, - }, - ClientSecurity, - SmtpClient, - Transport, -}; -use lettre_email::Email; -use openssl::{pkey::PKey, rsa::Rsa}; -use rand::{distributions::Alphanumeric, thread_rng, Rng}; -use regex::{Regex, RegexBuilder}; -use std::io::{Error, ErrorKind}; +use regex::Regex; use thiserror::Error; -use url::Url; pub type ConnectionId = usize; pub type PostId = i32; @@ -90,236 +77,7 @@ impl std::fmt::Display for LemmyError { impl actix_web::error::ResponseError for LemmyError {} -pub fn naive_from_unix(time: i64) -> NaiveDateTime { - NaiveDateTime::from_timestamp(time, 0) -} - -pub fn convert_datetime(datetime: NaiveDateTime) -> DateTime { - let now = Local::now(); - DateTime::::from_utc(datetime, *now.offset()) -} - -pub fn remove_slurs(test: &str) -> String { - SLUR_REGEX.replace_all(test, "*removed*").to_string() -} - -pub fn slur_check(test: &str) -> Result<(), Vec<&str>> { - let mut matches: Vec<&str> = SLUR_REGEX.find_iter(test).map(|mat| mat.as_str()).collect(); - - // Unique - matches.sort_unstable(); - matches.dedup(); - - if matches.is_empty() { - Ok(()) - } else { - Err(matches) - } -} - -pub fn slurs_vec_to_str(slurs: Vec<&str>) -> String { - let start = "No slurs - "; - let combined = &slurs.join(", "); - [start, combined].concat() -} - -pub fn generate_random_string() -> String { - thread_rng().sample_iter(&Alphanumeric).take(30).collect() -} - -pub fn send_email( - subject: &str, - to_email: &str, - to_username: &str, - html: &str, -) -> Result<(), String> { - let email_config = Settings::get().email.ok_or("no_email_setup")?; - - let email = Email::builder() - .to((to_email, to_username)) - .from(email_config.smtp_from_address.to_owned()) - .subject(subject) - .html(html) - .build() - .unwrap(); - - let mailer = if email_config.use_tls { - SmtpClient::new_simple(&email_config.smtp_server).unwrap() - } else { - SmtpClient::new(&email_config.smtp_server, ClientSecurity::None).unwrap() - } - .hello_name(ClientId::Domain(Settings::get().hostname)) - .smtp_utf8(true) - .authentication_mechanism(Mechanism::Plain) - .connection_reuse(ConnectionReuseParameters::ReuseUnlimited); - let mailer = if let (Some(login), Some(password)) = - (&email_config.smtp_login, &email_config.smtp_password) - { - mailer.credentials(Credentials::new(login.to_owned(), password.to_owned())) - } else { - mailer - }; - - let mut transport = mailer.transport(); - let result = transport.send(email.into()); - transport.close(); - - match result { - Ok(_) => Ok(()), - Err(e) => Err(e.to_string()), - } -} - -pub fn markdown_to_html(text: &str) -> String { - comrak::markdown_to_html(text, &comrak::ComrakOptions::default()) -} - -// TODO nothing is done with community / group webfingers yet, so just ignore those for now -#[derive(Clone, PartialEq, Eq, Hash)] -pub struct MentionData { - pub name: String, - pub domain: String, -} - -impl MentionData { - pub fn is_local(&self) -> bool { - Settings::get().hostname.eq(&self.domain) - } - pub fn full_name(&self) -> String { - format!("@{}@{}", &self.name, &self.domain) - } -} - -pub fn scrape_text_for_mentions(text: &str) -> Vec { - let mut out: Vec = Vec::new(); - for caps in MENTIONS_REGEX.captures_iter(text) { - out.push(MentionData { - name: caps["name"].to_string(), - domain: caps["domain"].to_string(), - }); - } - out.into_iter().unique().collect() -} - -pub fn is_valid_username(name: &str) -> bool { - VALID_USERNAME_REGEX.is_match(name) -} - -// Can't do a regex here, reverse lookarounds not supported -pub fn is_valid_preferred_username(preferred_username: &str) -> bool { - !preferred_username.starts_with('@') - && preferred_username.len() >= 3 - && preferred_username.len() <= 20 -} - -pub fn is_valid_community_name(name: &str) -> bool { - VALID_COMMUNITY_NAME_REGEX.is_match(name) -} - -pub fn is_valid_post_title(title: &str) -> bool { - VALID_POST_TITLE_REGEX.is_match(title) -} - -#[cfg(test)] -mod tests { - use crate::{ - is_valid_community_name, - is_valid_post_title, - is_valid_preferred_username, - is_valid_username, - remove_slurs, - scrape_text_for_mentions, - slur_check, - slurs_vec_to_str, - }; - - #[test] - fn test_mentions_regex() { - let text = "Just read a great blog post by [@tedu@honk.teduangst.com](/u/test). And another by !test_community@fish.teduangst.com . Another [@lemmy@lemmy-alpha:8540](/u/fish)"; - let mentions = scrape_text_for_mentions(text); - - assert_eq!(mentions[0].name, "tedu".to_string()); - assert_eq!(mentions[0].domain, "honk.teduangst.com".to_string()); - assert_eq!(mentions[1].domain, "lemmy-alpha:8540".to_string()); - } - - #[test] - fn test_valid_register_username() { - assert!(is_valid_username("Hello_98")); - assert!(is_valid_username("ten")); - assert!(!is_valid_username("Hello-98")); - assert!(!is_valid_username("a")); - assert!(!is_valid_username("")); - } - - #[test] - fn test_valid_preferred_username() { - assert!(is_valid_preferred_username("hello @there")); - assert!(!is_valid_preferred_username("@hello there")); - } - - #[test] - fn test_valid_community_name() { - assert!(is_valid_community_name("example")); - assert!(is_valid_community_name("example_community")); - assert!(!is_valid_community_name("Example")); - assert!(!is_valid_community_name("Ex")); - assert!(!is_valid_community_name("")); - } - - #[test] - fn test_valid_post_title() { - assert!(is_valid_post_title("Post Title")); - assert!(is_valid_post_title(" POST TITLE 😃😃😃😃😃")); - assert!(!is_valid_post_title("\n \n \n \n ")); // tabs/spaces/newlines - } - - #[test] - fn test_slur_filter() { - let test = - "coons test dindu ladyboy tranny retardeds. Capitalized Niggerz. This is a bunch of other safe text."; - let slur_free = "No slurs here"; - assert_eq!( - remove_slurs(&test), - "*removed* test *removed* *removed* *removed* *removed*. Capitalized *removed*. This is a bunch of other safe text." - .to_string() - ); - - let has_slurs_vec = vec![ - "Niggerz", - "coons", - "dindu", - "ladyboy", - "retardeds", - "tranny", - ]; - let has_slurs_err_str = "No slurs - Niggerz, coons, dindu, ladyboy, retardeds, tranny"; - - assert_eq!(slur_check(test), Err(has_slurs_vec)); - assert_eq!(slur_check(slur_free), Ok(())); - if let Err(slur_vec) = slur_check(test) { - assert_eq!(&slurs_vec_to_str(slur_vec), has_slurs_err_str); - } - } - - // These helped with testing - // #[test] - // fn test_send_email() { - // let result = send_email("not a subject", "test_email@gmail.com", "ur user", "

HI there

"); - // assert!(result.is_ok()); - // } -} - lazy_static! { - static ref EMAIL_REGEX: Regex = Regex::new(r"^[a-zA-Z0-9.!#$%&’*+/=?^_`{|}~-]+@[a-zA-Z0-9-]+(?:\.[a-zA-Z0-9-]+)*$").unwrap(); - static ref SLUR_REGEX: Regex = RegexBuilder::new(r"(fag(g|got|tard)?|maricos?|cock\s?sucker(s|ing)?|\bn(i|1)g(\b|g?(a|er)?(s|z)?)\b|dindu(s?)|mudslime?s?|kikes?|mongoloids?|towel\s*heads?|\bspi(c|k)s?\b|\bchinks?|niglets?|beaners?|\bnips?\b|\bcoons?\b|jungle\s*bunn(y|ies?)|jigg?aboo?s?|\bpakis?\b|rag\s*heads?|gooks?|cunts?|bitch(es|ing|y)?|puss(y|ies?)|twats?|feminazis?|whor(es?|ing)|\bslut(s|t?y)?|\btr(a|@)nn?(y|ies?)|ladyboy(s?)|\b(b|re|r)tard(ed)?s?)").case_insensitive(true).build().unwrap(); - static ref USERNAME_MATCHES_REGEX: Regex = Regex::new(r"/u/[a-zA-Z][0-9a-zA-Z_]*").unwrap(); - // TODO keep this old one, it didn't work with port well tho - // static ref MENTIONS_REGEX: Regex = Regex::new(r"@(?P[\w.]+)@(?P[a-zA-Z0-9._-]+\.[a-zA-Z0-9_-]+)").unwrap(); - static ref MENTIONS_REGEX: Regex = Regex::new(r"@(?P[\w.]+)@(?P[a-zA-Z0-9._:-]+)").unwrap(); - static ref VALID_USERNAME_REGEX: Regex = Regex::new(r"^[a-zA-Z0-9_]{3,20}$").unwrap(); - static ref VALID_COMMUNITY_NAME_REGEX: Regex = Regex::new(r"^[a-z0-9_]{3,20}$").unwrap(); - static ref VALID_POST_TITLE_REGEX: Regex = Regex::new(r".*\S.*").unwrap(); pub static ref WEBFINGER_COMMUNITY_REGEX: Regex = Regex::new(&format!( "^group:([a-z0-9_]{{3, 20}})@{}$", Settings::get().hostname @@ -333,73 +91,3 @@ lazy_static! { pub static ref CACHE_CONTROL_REGEX: Regex = Regex::new("^((text|image)/.+|application/javascript)$").unwrap(); } - -pub struct Keypair { - pub private_key: String, - pub public_key: String, -} - -/// Generate the asymmetric keypair for ActivityPub HTTP signatures. -pub fn generate_actor_keypair() -> Result { - let rsa = Rsa::generate(2048)?; - let pkey = PKey::from_rsa(rsa)?; - let public_key = pkey.public_key_to_pem()?; - let private_key = pkey.private_key_to_pem_pkcs8()?; - let key_to_string = |key| match String::from_utf8(key) { - Ok(s) => Ok(s), - Err(e) => Err(Error::new( - ErrorKind::Other, - format!("Failed converting key to string: {}", e), - )), - }; - Ok(Keypair { - private_key: key_to_string(private_key)?, - public_key: key_to_string(public_key)?, - }) -} - -pub enum EndpointType { - Community, - User, - Post, - Comment, - PrivateMessage, -} - -pub fn get_apub_protocol_string() -> &'static str { - if Settings::get().federation.tls_enabled { - "https" - } else { - "http" - } -} - -/// Generates the ActivityPub ID for a given object type and ID. -pub fn make_apub_endpoint(endpoint_type: EndpointType, name: &str) -> Url { - let point = match endpoint_type { - EndpointType::Community => "c", - EndpointType::User => "u", - EndpointType::Post => "post", - EndpointType::Comment => "comment", - EndpointType::PrivateMessage => "private_message", - }; - - Url::parse(&format!( - "{}://{}/{}/{}", - get_apub_protocol_string(), - Settings::get().hostname, - point, - name - )) - .unwrap() -} - -pub fn get_ip(conn_info: &ConnectionInfo) -> String { - conn_info - .realip_remote_addr() - .unwrap_or("127.0.0.1:12345") - .split(':') - .next() - .unwrap_or("127.0.0.1") - .to_string() -} diff --git a/server/lemmy_utils/src/test.rs b/server/lemmy_utils/src/test.rs new file mode 100644 index 000000000..fdca384f5 --- /dev/null +++ b/server/lemmy_utils/src/test.rs @@ -0,0 +1,86 @@ +use crate::utils::{ + is_valid_community_name, + is_valid_post_title, + is_valid_preferred_username, + is_valid_username, + remove_slurs, + scrape_text_for_mentions, + slur_check, + slurs_vec_to_str, +}; + +#[test] +fn test_mentions_regex() { + let text = "Just read a great blog post by [@tedu@honk.teduangst.com](/u/test). And another by !test_community@fish.teduangst.com . Another [@lemmy@lemmy-alpha:8540](/u/fish)"; + let mentions = scrape_text_for_mentions(text); + + assert_eq!(mentions[0].name, "tedu".to_string()); + assert_eq!(mentions[0].domain, "honk.teduangst.com".to_string()); + assert_eq!(mentions[1].domain, "lemmy-alpha:8540".to_string()); +} + +#[test] +fn test_valid_register_username() { + assert!(is_valid_username("Hello_98")); + assert!(is_valid_username("ten")); + assert!(!is_valid_username("Hello-98")); + assert!(!is_valid_username("a")); + assert!(!is_valid_username("")); +} + +#[test] +fn test_valid_preferred_username() { + assert!(is_valid_preferred_username("hello @there")); + assert!(!is_valid_preferred_username("@hello there")); +} + +#[test] +fn test_valid_community_name() { + assert!(is_valid_community_name("example")); + assert!(is_valid_community_name("example_community")); + assert!(!is_valid_community_name("Example")); + assert!(!is_valid_community_name("Ex")); + assert!(!is_valid_community_name("")); +} + +#[test] +fn test_valid_post_title() { + assert!(is_valid_post_title("Post Title")); + assert!(is_valid_post_title(" POST TITLE 😃😃😃😃😃")); + assert!(!is_valid_post_title("\n \n \n \n ")); // tabs/spaces/newlines +} + +#[test] +fn test_slur_filter() { + let test = + "coons test dindu ladyboy tranny retardeds. Capitalized Niggerz. This is a bunch of other safe text."; + let slur_free = "No slurs here"; + assert_eq!( + remove_slurs(&test), + "*removed* test *removed* *removed* *removed* *removed*. Capitalized *removed*. This is a bunch of other safe text." + .to_string() + ); + + let has_slurs_vec = vec![ + "Niggerz", + "coons", + "dindu", + "ladyboy", + "retardeds", + "tranny", + ]; + let has_slurs_err_str = "No slurs - Niggerz, coons, dindu, ladyboy, retardeds, tranny"; + + assert_eq!(slur_check(test), Err(has_slurs_vec)); + assert_eq!(slur_check(slur_free), Ok(())); + if let Err(slur_vec) = slur_check(test) { + assert_eq!(&slurs_vec_to_str(slur_vec), has_slurs_err_str); + } +} + +// These helped with testing +// #[test] +// fn test_send_email() { +// let result = send_email("not a subject", "test_email@gmail.com", "ur user", "

HI there

"); +// assert!(result.is_ok()); +// } diff --git a/server/lemmy_utils/src/utils.rs b/server/lemmy_utils/src/utils.rs new file mode 100644 index 000000000..87aad574a --- /dev/null +++ b/server/lemmy_utils/src/utils.rs @@ -0,0 +1,130 @@ +use crate::{settings::Settings, APIError}; +use actix_web::dev::ConnectionInfo; +use chrono::{DateTime, FixedOffset, Local, NaiveDateTime}; +use itertools::Itertools; +use rand::{distributions::Alphanumeric, thread_rng, Rng}; +use regex::{Regex, RegexBuilder}; + +lazy_static! { +static ref EMAIL_REGEX: Regex = Regex::new(r"^[a-zA-Z0-9.!#$%&’*+/=?^_`{|}~-]+@[a-zA-Z0-9-]+(?:\.[a-zA-Z0-9-]+)*$").unwrap(); +static ref SLUR_REGEX: Regex = RegexBuilder::new(r"(fag(g|got|tard)?|maricos?|cock\s?sucker(s|ing)?|\bn(i|1)g(\b|g?(a|er)?(s|z)?)\b|dindu(s?)|mudslime?s?|kikes?|mongoloids?|towel\s*heads?|\bspi(c|k)s?\b|\bchinks?|niglets?|beaners?|\bnips?\b|\bcoons?\b|jungle\s*bunn(y|ies?)|jigg?aboo?s?|\bpakis?\b|rag\s*heads?|gooks?|cunts?|bitch(es|ing|y)?|puss(y|ies?)|twats?|feminazis?|whor(es?|ing)|\bslut(s|t?y)?|\btr(a|@)nn?(y|ies?)|ladyboy(s?)|\b(b|re|r)tard(ed)?s?)").case_insensitive(true).build().unwrap(); +static ref USERNAME_MATCHES_REGEX: Regex = Regex::new(r"/u/[a-zA-Z][0-9a-zA-Z_]*").unwrap(); +// TODO keep this old one, it didn't work with port well tho +// static ref MENTIONS_REGEX: Regex = Regex::new(r"@(?P[\w.]+)@(?P[a-zA-Z0-9._-]+\.[a-zA-Z0-9_-]+)").unwrap(); +static ref MENTIONS_REGEX: Regex = Regex::new(r"@(?P[\w.]+)@(?P[a-zA-Z0-9._:-]+)").unwrap(); +static ref VALID_USERNAME_REGEX: Regex = Regex::new(r"^[a-zA-Z0-9_]{3,20}$").unwrap(); +static ref VALID_COMMUNITY_NAME_REGEX: Regex = Regex::new(r"^[a-z0-9_]{3,20}$").unwrap(); +static ref VALID_POST_TITLE_REGEX: Regex = Regex::new(r".*\S.*").unwrap(); +} + +pub fn naive_from_unix(time: i64) -> NaiveDateTime { + NaiveDateTime::from_timestamp(time, 0) +} + +pub fn convert_datetime(datetime: NaiveDateTime) -> DateTime { + let now = Local::now(); + DateTime::::from_utc(datetime, *now.offset()) +} + +pub fn remove_slurs(test: &str) -> String { + SLUR_REGEX.replace_all(test, "*removed*").to_string() +} + +pub(crate) fn slur_check(test: &str) -> Result<(), Vec<&str>> { + let mut matches: Vec<&str> = SLUR_REGEX.find_iter(test).map(|mat| mat.as_str()).collect(); + + // Unique + matches.sort_unstable(); + matches.dedup(); + + if matches.is_empty() { + Ok(()) + } else { + Err(matches) + } +} + +pub fn check_slurs(text: &str) -> Result<(), APIError> { + if let Err(slurs) = slur_check(text) { + Err(APIError::err(&slurs_vec_to_str(slurs))) + } else { + Ok(()) + } +} + +pub fn check_slurs_opt(text: &Option) -> Result<(), APIError> { + match text { + Some(t) => check_slurs(t), + None => Ok(()), + } +} + +pub(crate) fn slurs_vec_to_str(slurs: Vec<&str>) -> String { + let start = "No slurs - "; + let combined = &slurs.join(", "); + [start, combined].concat() +} + +pub fn generate_random_string() -> String { + thread_rng().sample_iter(&Alphanumeric).take(30).collect() +} + +pub fn markdown_to_html(text: &str) -> String { + comrak::markdown_to_html(text, &comrak::ComrakOptions::default()) +} + +// TODO nothing is done with community / group webfingers yet, so just ignore those for now +#[derive(Clone, PartialEq, Eq, Hash)] +pub struct MentionData { + pub name: String, + pub domain: String, +} + +impl MentionData { + pub fn is_local(&self) -> bool { + Settings::get().hostname.eq(&self.domain) + } + pub fn full_name(&self) -> String { + format!("@{}@{}", &self.name, &self.domain) + } +} + +pub fn scrape_text_for_mentions(text: &str) -> Vec { + let mut out: Vec = Vec::new(); + for caps in MENTIONS_REGEX.captures_iter(text) { + out.push(MentionData { + name: caps["name"].to_string(), + domain: caps["domain"].to_string(), + }); + } + out.into_iter().unique().collect() +} + +pub fn is_valid_username(name: &str) -> bool { + VALID_USERNAME_REGEX.is_match(name) +} + +// Can't do a regex here, reverse lookarounds not supported +pub fn is_valid_preferred_username(preferred_username: &str) -> bool { + !preferred_username.starts_with('@') + && preferred_username.len() >= 3 + && preferred_username.len() <= 20 +} + +pub fn is_valid_community_name(name: &str) -> bool { + VALID_COMMUNITY_NAME_REGEX.is_match(name) +} + +pub fn is_valid_post_title(title: &str) -> bool { + VALID_POST_TITLE_REGEX.is_match(title) +} + +pub fn get_ip(conn_info: &ConnectionInfo) -> String { + conn_info + .realip_remote_addr() + .unwrap_or("127.0.0.1:12345") + .split(':') + .next() + .unwrap_or("127.0.0.1") + .to_string() +} diff --git a/server/src/api/comment.rs b/server/src/api/comment.rs index f0e56810e..dd1970393 100644 --- a/server/src/api/comment.rs +++ b/server/src/api/comment.rs @@ -8,16 +8,14 @@ use crate::{ Perform, }, apub::{ApubLikeableType, ApubObjectType}, - blocking, websocket::{ messages::{JoinCommunityRoom, SendComment}, UserOperation, }, - DbPool, LemmyContext, }; use actix_web::web::Data; -use lemmy_api_structs::comment::*; +use lemmy_api_structs::{blocking, comment::*, send_local_notifs}; use lemmy_db::{ comment::*, comment_view::*, @@ -25,7 +23,6 @@ use lemmy_db::{ post::*, site_view::*, user::*, - user_mention::*, Crud, Likeable, ListingType, @@ -33,18 +30,12 @@ use lemmy_db::{ SortType, }; use lemmy_utils::{ - make_apub_endpoint, - remove_slurs, - scrape_text_for_mentions, - send_email, - settings::Settings, + apub::{make_apub_endpoint, EndpointType}, + utils::{remove_slurs, scrape_text_for_mentions}, APIError, ConnectionId, - EndpointType, LemmyError, - MentionData, }; -use log::error; use std::str::FromStr; #[async_trait::async_trait(?Send)] @@ -707,124 +698,3 @@ impl Perform for GetComments { Ok(GetCommentsResponse { comments }) } } - -pub async fn send_local_notifs( - mentions: Vec, - comment: Comment, - user: &User_, - post: Post, - pool: &DbPool, - do_send_email: bool, -) -> Result, LemmyError> { - let user2 = user.clone(); - let ids = blocking(pool, move |conn| { - do_send_local_notifs(conn, &mentions, &comment, &user2, &post, do_send_email) - }) - .await?; - - Ok(ids) -} - -fn do_send_local_notifs( - conn: &diesel::PgConnection, - mentions: &[MentionData], - comment: &Comment, - user: &User_, - post: &Post, - do_send_email: bool, -) -> Vec { - let mut recipient_ids = Vec::new(); - let hostname = &format!("https://{}", Settings::get().hostname); - - // Send the local mentions - for mention in mentions - .iter() - .filter(|m| m.is_local() && m.name.ne(&user.name)) - .collect::>() - { - if let Ok(mention_user) = User_::read_from_name(&conn, &mention.name) { - // TODO - // At some point, make it so you can't tag the parent creator either - // This can cause two notifications, one for reply and the other for mention - recipient_ids.push(mention_user.id); - - let user_mention_form = UserMentionForm { - recipient_id: mention_user.id, - comment_id: comment.id, - read: None, - }; - - // Allow this to fail softly, since comment edits might re-update or replace it - // Let the uniqueness handle this fail - match UserMention::create(&conn, &user_mention_form) { - Ok(_mention) => (), - Err(_e) => error!("{}", &_e), - }; - - // Send an email to those users that have notifications on - if do_send_email && mention_user.send_notifications_to_email { - if let Some(mention_email) = mention_user.email { - let subject = &format!("{} - Mentioned by {}", Settings::get().hostname, user.name,); - let html = &format!( - "

User Mention


{} - {}

inbox", - user.name, comment.content, hostname - ); - match send_email(subject, &mention_email, &mention_user.name, html) { - Ok(_o) => _o, - Err(e) => error!("{}", e), - }; - } - } - } - } - - // Send notifs to the parent commenter / poster - match comment.parent_id { - Some(parent_id) => { - if let Ok(parent_comment) = Comment::read(&conn, parent_id) { - if parent_comment.creator_id != user.id { - if let Ok(parent_user) = User_::read(&conn, parent_comment.creator_id) { - recipient_ids.push(parent_user.id); - - if do_send_email && parent_user.send_notifications_to_email { - if let Some(comment_reply_email) = parent_user.email { - let subject = &format!("{} - Reply from {}", Settings::get().hostname, user.name,); - let html = &format!( - "

Comment Reply


{} - {}

inbox", - user.name, comment.content, hostname - ); - match send_email(subject, &comment_reply_email, &parent_user.name, html) { - Ok(_o) => _o, - Err(e) => error!("{}", e), - }; - } - } - } - } - } - } - // Its a post - None => { - if post.creator_id != user.id { - if let Ok(parent_user) = User_::read(&conn, post.creator_id) { - recipient_ids.push(parent_user.id); - - if do_send_email && parent_user.send_notifications_to_email { - if let Some(post_reply_email) = parent_user.email { - let subject = &format!("{} - Reply from {}", Settings::get().hostname, user.name,); - let html = &format!( - "

Post Reply


{} - {}

inbox", - user.name, comment.content, hostname - ); - match send_email(subject, &post_reply_email, &parent_user.name, html) { - Ok(_o) => _o, - Err(e) => error!("{}", e), - }; - } - } - } - } - } - }; - recipient_ids -} diff --git a/server/src/api/community.rs b/server/src/api/community.rs index 6fb67b330..8486861c4 100644 --- a/server/src/api/community.rs +++ b/server/src/api/community.rs @@ -1,15 +1,6 @@ use crate::{ - api::{ - check_slurs, - check_slurs_opt, - get_user_from_jwt, - get_user_from_jwt_opt, - is_admin, - is_mod_or_admin, - Perform, - }, + api::{get_user_from_jwt, get_user_from_jwt_opt, is_admin, is_mod_or_admin, Perform}, apub::ActorType, - blocking, websocket::{ messages::{GetCommunityUsersOnline, JoinCommunityRoom, SendCommunityRoomMessage}, UserOperation, @@ -18,7 +9,7 @@ use crate::{ }; use actix_web::web::Data; use anyhow::Context; -use lemmy_api_structs::community::*; +use lemmy_api_structs::{blocking, community::*}; use lemmy_db::{ comment::Comment, comment_view::CommentQueryBuilder, @@ -37,14 +28,11 @@ use lemmy_db::{ SortType, }; use lemmy_utils::{ - generate_actor_keypair, - is_valid_community_name, + apub::{generate_actor_keypair, make_apub_endpoint, EndpointType}, location_info, - make_apub_endpoint, - naive_from_unix, + utils::{check_slurs, check_slurs_opt, is_valid_community_name, naive_from_unix}, APIError, ConnectionId, - EndpointType, LemmyError, }; use std::str::FromStr; diff --git a/server/src/api/mod.rs b/server/src/api/mod.rs index a1b8c188d..1aff5d106 100644 --- a/server/src/api/mod.rs +++ b/server/src/api/mod.rs @@ -1,5 +1,6 @@ -use crate::{api::claims::Claims, blocking, DbPool, LemmyContext}; +use crate::{api::claims::Claims, DbPool, LemmyContext}; use actix_web::web::Data; +use lemmy_api_structs::blocking; use lemmy_db::{ community::Community, community_view::CommunityUserBanView, @@ -7,7 +8,7 @@ use lemmy_db::{ user::User_, Crud, }; -use lemmy_utils::{slur_check, slurs_vec_to_str, APIError, ConnectionId, LemmyError}; +use lemmy_utils::{APIError, ConnectionId, LemmyError}; pub mod claims; pub mod comment; @@ -83,19 +84,6 @@ pub(in crate::api) async fn get_user_from_jwt_opt( } } -pub(in crate) fn check_slurs(text: &str) -> Result<(), APIError> { - if let Err(slurs) = slur_check(text) { - Err(APIError::err(&slurs_vec_to_str(slurs))) - } else { - Ok(()) - } -} -pub(in crate) fn check_slurs_opt(text: &Option) -> Result<(), APIError> { - match text { - Some(t) => check_slurs(t), - None => Ok(()), - } -} pub(in crate::api) async fn check_community_ban( user_id: i32, community_id: i32, diff --git a/server/src/api/post.rs b/server/src/api/post.rs index 9e1e97d51..a1801d0f2 100644 --- a/server/src/api/post.rs +++ b/server/src/api/post.rs @@ -1,15 +1,6 @@ use crate::{ - api::{ - check_community_ban, - check_slurs, - check_slurs_opt, - get_user_from_jwt, - get_user_from_jwt_opt, - is_mod_or_admin, - Perform, - }, + api::{check_community_ban, get_user_from_jwt, get_user_from_jwt_opt, is_mod_or_admin, Perform}, apub::{ApubLikeableType, ApubObjectType}, - blocking, fetch_iframely_and_pictrs_data, websocket::{ messages::{GetPostUsersOnline, JoinCommunityRoom, JoinPostRoom, SendPost}, @@ -18,7 +9,7 @@ use crate::{ LemmyContext, }; use actix_web::web::Data; -use lemmy_api_structs::post::*; +use lemmy_api_structs::{blocking, post::*}; use lemmy_db::{ comment_view::*, community_view::*, @@ -34,11 +25,10 @@ use lemmy_db::{ SortType, }; use lemmy_utils::{ - is_valid_post_title, - make_apub_endpoint, + apub::{make_apub_endpoint, EndpointType}, + utils::{check_slurs, check_slurs_opt, is_valid_post_title}, APIError, ConnectionId, - EndpointType, LemmyError, }; use std::str::FromStr; diff --git a/server/src/api/site.rs b/server/src/api/site.rs index 8bcc4b77a..727078e27 100644 --- a/server/src/api/site.rs +++ b/server/src/api/site.rs @@ -1,14 +1,6 @@ use crate::{ - api::{ - check_slurs, - check_slurs_opt, - get_user_from_jwt, - get_user_from_jwt_opt, - is_admin, - Perform, - }, + api::{get_user_from_jwt, get_user_from_jwt_opt, is_admin, Perform}, apub::fetcher::search_by_apub_id, - blocking, version, websocket::{ messages::{GetUsersOnline, SendAllMessage}, @@ -18,7 +10,7 @@ use crate::{ }; use actix_web::web::Data; use anyhow::Context; -use lemmy_api_structs::{site::*, user::Register}; +use lemmy_api_structs::{blocking, site::*, user::Register}; use lemmy_db::{ category::*, comment_view::*, @@ -35,7 +27,14 @@ use lemmy_db::{ SearchType, SortType, }; -use lemmy_utils::{location_info, settings::Settings, APIError, ConnectionId, LemmyError}; +use lemmy_utils::{ + location_info, + settings::Settings, + utils::{check_slurs, check_slurs_opt}, + APIError, + ConnectionId, + LemmyError, +}; use log::{debug, info}; use std::str::FromStr; diff --git a/server/src/api/user.rs b/server/src/api/user.rs index 1a71b3aa0..778a225fa 100644 --- a/server/src/api/user.rs +++ b/server/src/api/user.rs @@ -1,7 +1,6 @@ use crate::{ - api::{check_slurs, claims::Claims, get_user_from_jwt, get_user_from_jwt_opt, is_admin, Perform}, + api::{claims::Claims, get_user_from_jwt, get_user_from_jwt_opt, is_admin, Perform}, apub::ApubObjectType, - blocking, captcha_espeak_wav_base64, websocket::{ messages::{CaptchaItem, CheckCaptcha, JoinUserRoom, SendAllMessage, SendUserRoomMessage}, @@ -14,7 +13,7 @@ use anyhow::Context; use bcrypt::verify; use captcha::{gen, Difficulty}; use chrono::Duration; -use lemmy_api_structs::user::*; +use lemmy_api_structs::{blocking, user::*}; use lemmy_db::{ comment::*, comment_view::*, @@ -41,19 +40,20 @@ use lemmy_db::{ SortType, }; use lemmy_utils::{ - generate_actor_keypair, - generate_random_string, - is_valid_preferred_username, - is_valid_username, + apub::{generate_actor_keypair, make_apub_endpoint, EndpointType}, + email::send_email, location_info, - make_apub_endpoint, - naive_from_unix, - remove_slurs, - send_email, settings::Settings, + utils::{ + check_slurs, + generate_random_string, + is_valid_preferred_username, + is_valid_username, + naive_from_unix, + remove_slurs, + }, APIError, ConnectionId, - EndpointType, LemmyError, }; use log::error; diff --git a/server/src/apub/activities.rs b/server/src/apub/activities.rs index 0ba8f8735..dc1892693 100644 --- a/server/src/apub/activities.rs +++ b/server/src/apub/activities.rs @@ -7,7 +7,7 @@ use activitystreams::{ object::AsObject, }; use lemmy_db::{community::Community, user::User_}; -use lemmy_utils::{get_apub_protocol_string, settings::Settings, LemmyError}; +use lemmy_utils::{apub::get_apub_protocol_string, settings::Settings, LemmyError}; use serde::{export::fmt::Debug, Serialize}; use url::{ParseError, Url}; use uuid::Uuid; diff --git a/server/src/apub/comment.rs b/server/src/apub/comment.rs index 8c416369f..78fe6f8ab 100644 --- a/server/src/apub/comment.rs +++ b/server/src/apub/comment.rs @@ -17,7 +17,6 @@ use crate::{ FromApub, ToApub, }, - blocking, DbPool, LemmyContext, }; @@ -41,6 +40,7 @@ use activitystreams::{ use actix_web::{body::Body, web, web::Path, HttpResponse}; use anyhow::Context; use itertools::Itertools; +use lemmy_api_structs::blocking; use lemmy_db::{ comment::{Comment, CommentForm}, community::Community, @@ -49,12 +49,9 @@ use lemmy_db::{ Crud, }; use lemmy_utils::{ - convert_datetime, location_info, - remove_slurs, - scrape_text_for_mentions, + utils::{convert_datetime, remove_slurs, scrape_text_for_mentions, MentionData}, LemmyError, - MentionData, }; use log::debug; use serde::Deserialize; diff --git a/server/src/apub/community.rs b/server/src/apub/community.rs index 8d9b4dc1c..89819f453 100644 --- a/server/src/apub/community.rs +++ b/server/src/apub/community.rs @@ -1,5 +1,4 @@ use crate::{ - api::{check_slurs, check_slurs_opt}, apub::{ activities::generate_activity_id, activity_queue::send_activity, @@ -15,7 +14,6 @@ use crate::{ GroupExt, ToApub, }, - blocking, DbPool, LemmyContext, }; @@ -40,6 +38,7 @@ use activitystreams_ext::Ext2; use actix_web::{body::Body, web, HttpResponse}; use anyhow::Context; use itertools::Itertools; +use lemmy_api_structs::blocking; use lemmy_db::{ community::{Community, CommunityForm}, community_view::{CommunityFollowerView, CommunityModeratorView}, @@ -47,7 +46,12 @@ use lemmy_db::{ post::Post, user::User_, }; -use lemmy_utils::{convert_datetime, get_apub_protocol_string, location_info, LemmyError}; +use lemmy_utils::{ + apub::get_apub_protocol_string, + location_info, + utils::{check_slurs, check_slurs_opt, convert_datetime}, + LemmyError, +}; use serde::Deserialize; use url::Url; diff --git a/server/src/apub/fetcher.rs b/server/src/apub/fetcher.rs index d45165777..4891a9705 100644 --- a/server/src/apub/fetcher.rs +++ b/server/src/apub/fetcher.rs @@ -8,7 +8,6 @@ use crate::{ PersonExt, APUB_JSON_CONTENT_TYPE, }, - blocking, request::{retry, RecvError}, LemmyContext, }; @@ -16,7 +15,7 @@ use activitystreams::{base::BaseExt, collection::OrderedCollection, object::Note use anyhow::{anyhow, Context}; use chrono::NaiveDateTime; use diesel::result::Error::NotFound; -use lemmy_api_structs::site::SearchResponse; +use lemmy_api_structs::{blocking, site::SearchResponse}; use lemmy_db::{ comment::{Comment, CommentForm}, comment_view::CommentView, @@ -31,7 +30,7 @@ use lemmy_db::{ Joinable, SearchType, }; -use lemmy_utils::{get_apub_protocol_string, location_info, LemmyError}; +use lemmy_utils::{apub::get_apub_protocol_string, location_info, LemmyError}; use log::debug; use reqwest::Client; use serde::Deserialize; diff --git a/server/src/apub/inbox/activities/create.rs b/server/src/apub/inbox/activities/create.rs index a39152047..66286ce17 100644 --- a/server/src/apub/inbox/activities/create.rs +++ b/server/src/apub/inbox/activities/create.rs @@ -1,5 +1,4 @@ use crate::{ - api::comment::send_local_notifs, apub::{ inbox::shared_inbox::{ announce_if_community_is_local, @@ -10,7 +9,6 @@ use crate::{ FromApub, PageExt, }, - blocking, websocket::{ messages::{SendComment, SendPost}, UserOperation, @@ -20,14 +18,19 @@ use crate::{ use activitystreams::{activity::Create, base::AnyBase, object::Note, prelude::*}; use actix_web::HttpResponse; use anyhow::Context; -use lemmy_api_structs::{comment::CommentResponse, post::PostResponse}; +use lemmy_api_structs::{ + blocking, + comment::CommentResponse, + post::PostResponse, + send_local_notifs, +}; use lemmy_db::{ comment::{Comment, CommentForm}, comment_view::CommentView, post::{Post, PostForm}, post_view::PostView, }; -use lemmy_utils::{location_info, scrape_text_for_mentions, LemmyError}; +use lemmy_utils::{location_info, utils::scrape_text_for_mentions, LemmyError}; pub async fn receive_create( activity: AnyBase, diff --git a/server/src/apub/inbox/activities/delete.rs b/server/src/apub/inbox/activities/delete.rs index 70bf37616..4adc7c33c 100644 --- a/server/src/apub/inbox/activities/delete.rs +++ b/server/src/apub/inbox/activities/delete.rs @@ -11,7 +11,6 @@ use crate::{ GroupExt, PageExt, }, - blocking, websocket::{ messages::{SendComment, SendCommunityRoomMessage, SendPost}, UserOperation, @@ -22,6 +21,7 @@ use activitystreams::{activity::Delete, base::AnyBase, object::Note, prelude::*} use actix_web::HttpResponse; use anyhow::Context; use lemmy_api_structs::{ + blocking, comment::CommentResponse, community::CommunityResponse, post::PostResponse, diff --git a/server/src/apub/inbox/activities/dislike.rs b/server/src/apub/inbox/activities/dislike.rs index 599389b18..9b63e82d8 100644 --- a/server/src/apub/inbox/activities/dislike.rs +++ b/server/src/apub/inbox/activities/dislike.rs @@ -9,7 +9,6 @@ use crate::{ FromApub, PageExt, }, - blocking, websocket::{ messages::{SendComment, SendPost}, UserOperation, @@ -19,7 +18,7 @@ use crate::{ use activitystreams::{activity::Dislike, base::AnyBase, object::Note, prelude::*}; use actix_web::HttpResponse; use anyhow::Context; -use lemmy_api_structs::{comment::CommentResponse, post::PostResponse}; +use lemmy_api_structs::{blocking, comment::CommentResponse, post::PostResponse}; use lemmy_db::{ comment::{CommentForm, CommentLike, CommentLikeForm}, comment_view::CommentView, diff --git a/server/src/apub/inbox/activities/like.rs b/server/src/apub/inbox/activities/like.rs index 2cb95521e..e6f865b93 100644 --- a/server/src/apub/inbox/activities/like.rs +++ b/server/src/apub/inbox/activities/like.rs @@ -9,7 +9,6 @@ use crate::{ FromApub, PageExt, }, - blocking, websocket::{ messages::{SendComment, SendPost}, UserOperation, @@ -19,7 +18,7 @@ use crate::{ use activitystreams::{activity::Like, base::AnyBase, object::Note, prelude::*}; use actix_web::HttpResponse; use anyhow::Context; -use lemmy_api_structs::{comment::CommentResponse, post::PostResponse}; +use lemmy_api_structs::{blocking, comment::CommentResponse, post::PostResponse}; use lemmy_db::{ comment::{CommentForm, CommentLike, CommentLikeForm}, comment_view::CommentView, diff --git a/server/src/apub/inbox/activities/remove.rs b/server/src/apub/inbox/activities/remove.rs index 846842d4e..011f57b95 100644 --- a/server/src/apub/inbox/activities/remove.rs +++ b/server/src/apub/inbox/activities/remove.rs @@ -12,7 +12,6 @@ use crate::{ GroupExt, PageExt, }, - blocking, websocket::{ messages::{SendComment, SendCommunityRoomMessage, SendPost}, UserOperation, @@ -23,6 +22,7 @@ use activitystreams::{activity::Remove, base::AnyBase, object::Note, prelude::*} use actix_web::HttpResponse; use anyhow::{anyhow, Context}; use lemmy_api_structs::{ + blocking, comment::CommentResponse, community::CommunityResponse, post::PostResponse, diff --git a/server/src/apub/inbox/activities/undo.rs b/server/src/apub/inbox/activities/undo.rs index 0b695d32e..bf2073860 100644 --- a/server/src/apub/inbox/activities/undo.rs +++ b/server/src/apub/inbox/activities/undo.rs @@ -11,7 +11,6 @@ use crate::{ GroupExt, PageExt, }, - blocking, websocket::{ messages::{SendComment, SendCommunityRoomMessage, SendPost}, UserOperation, @@ -27,6 +26,7 @@ use activitystreams::{ use actix_web::HttpResponse; use anyhow::{anyhow, Context}; use lemmy_api_structs::{ + blocking, comment::CommentResponse, community::CommunityResponse, post::PostResponse, diff --git a/server/src/apub/inbox/activities/update.rs b/server/src/apub/inbox/activities/update.rs index eb1b67f1d..f078e67aa 100644 --- a/server/src/apub/inbox/activities/update.rs +++ b/server/src/apub/inbox/activities/update.rs @@ -1,5 +1,4 @@ use crate::{ - api::comment::send_local_notifs, apub::{ fetcher::{get_or_fetch_and_insert_comment, get_or_fetch_and_insert_post}, inbox::shared_inbox::{ @@ -11,7 +10,6 @@ use crate::{ FromApub, PageExt, }, - blocking, websocket::{ messages::{SendComment, SendPost}, UserOperation, @@ -21,7 +19,12 @@ use crate::{ use activitystreams::{activity::Update, base::AnyBase, object::Note, prelude::*}; use actix_web::HttpResponse; use anyhow::Context; -use lemmy_api_structs::{comment::CommentResponse, post::PostResponse}; +use lemmy_api_structs::{ + blocking, + comment::CommentResponse, + post::PostResponse, + send_local_notifs, +}; use lemmy_db::{ comment::{Comment, CommentForm}, comment_view::CommentView, @@ -29,7 +32,7 @@ use lemmy_db::{ post_view::PostView, Crud, }; -use lemmy_utils::{location_info, scrape_text_for_mentions, LemmyError}; +use lemmy_utils::{location_info, utils::scrape_text_for_mentions, LemmyError}; pub async fn receive_update( activity: AnyBase, diff --git a/server/src/apub/inbox/community_inbox.rs b/server/src/apub/inbox/community_inbox.rs index 0631f9391..b27e6b6a9 100644 --- a/server/src/apub/inbox/community_inbox.rs +++ b/server/src/apub/inbox/community_inbox.rs @@ -6,7 +6,6 @@ use crate::{ insert_activity, ActorType, }, - blocking, LemmyContext, }; use activitystreams::{ @@ -16,6 +15,7 @@ use activitystreams::{ }; use actix_web::{web, HttpRequest, HttpResponse}; use anyhow::{anyhow, Context}; +use lemmy_api_structs::blocking; use lemmy_db::{ community::{Community, CommunityFollower, CommunityFollowerForm}, user::User_, diff --git a/server/src/apub/inbox/user_inbox.rs b/server/src/apub/inbox/user_inbox.rs index 7ea95833e..ccc4d105a 100644 --- a/server/src/apub/inbox/user_inbox.rs +++ b/server/src/apub/inbox/user_inbox.rs @@ -6,7 +6,6 @@ use crate::{ insert_activity, FromApub, }, - blocking, websocket::{messages::SendUserRoomMessage, UserOperation}, LemmyContext, }; @@ -18,7 +17,7 @@ use activitystreams::{ }; use actix_web::{web, HttpRequest, HttpResponse}; use anyhow::Context; -use lemmy_api_structs::user::PrivateMessageResponse; +use lemmy_api_structs::{blocking, user::PrivateMessageResponse}; use lemmy_db::{ community::{CommunityFollower, CommunityFollowerForm}, naive_now, diff --git a/server/src/apub/mod.rs b/server/src/apub/mod.rs index b3b161c7d..e9184c33a 100644 --- a/server/src/apub/mod.rs +++ b/server/src/apub/mod.rs @@ -15,7 +15,6 @@ use crate::{ page_extension::PageExtension, signatures::{PublicKey, PublicKeyExtension}, }, - blocking, request::{retry, RecvError}, routes::webfinger::WebFingerResponse, DbPool, @@ -33,14 +32,14 @@ use activitystreams_ext::{Ext1, Ext2}; use actix_web::{body::Body, HttpResponse}; use anyhow::{anyhow, Context}; use chrono::NaiveDateTime; +use lemmy_api_structs::blocking; use lemmy_db::{activity::do_insert_activity, user::User_}; use lemmy_utils::{ - convert_datetime, - get_apub_protocol_string, + apub::get_apub_protocol_string, location_info, settings::Settings, + utils::{convert_datetime, MentionData}, LemmyError, - MentionData, }; use log::debug; use reqwest::Client; diff --git a/server/src/apub/post.rs b/server/src/apub/post.rs index a54b8f6a3..86ac3e600 100644 --- a/server/src/apub/post.rs +++ b/server/src/apub/post.rs @@ -1,5 +1,4 @@ use crate::{ - api::check_slurs, apub::{ activities::{generate_activity_id, send_activity_to_community}, check_actor_domain, @@ -15,7 +14,6 @@ use crate::{ PageExt, ToApub, }, - blocking, DbPool, LemmyContext, }; @@ -37,13 +35,18 @@ use activitystreams::{ use activitystreams_ext::Ext1; use actix_web::{body::Body, web, HttpResponse}; use anyhow::Context; +use lemmy_api_structs::blocking; use lemmy_db::{ community::Community, post::{Post, PostForm}, user::User_, Crud, }; -use lemmy_utils::{convert_datetime, location_info, remove_slurs, LemmyError}; +use lemmy_utils::{ + location_info, + utils::{check_slurs, convert_datetime, remove_slurs}, + LemmyError, +}; use serde::Deserialize; use url::Url; diff --git a/server/src/apub/private_message.rs b/server/src/apub/private_message.rs index 5563aef36..6ae30a3f4 100644 --- a/server/src/apub/private_message.rs +++ b/server/src/apub/private_message.rs @@ -12,7 +12,6 @@ use crate::{ FromApub, ToApub, }, - blocking, DbPool, LemmyContext, }; @@ -28,12 +27,13 @@ use activitystreams::{ prelude::*, }; use anyhow::Context; +use lemmy_api_structs::blocking; use lemmy_db::{ private_message::{PrivateMessage, PrivateMessageForm}, user::User_, Crud, }; -use lemmy_utils::{convert_datetime, location_info, LemmyError}; +use lemmy_utils::{location_info, utils::convert_datetime, LemmyError}; use url::Url; #[async_trait::async_trait(?Send)] diff --git a/server/src/apub/user.rs b/server/src/apub/user.rs index 5522a3413..54daadb96 100644 --- a/server/src/apub/user.rs +++ b/server/src/apub/user.rs @@ -1,5 +1,4 @@ use crate::{ - api::{check_slurs, check_slurs_opt}, apub::{ activities::generate_activity_id, activity_queue::send_activity, @@ -12,7 +11,6 @@ use crate::{ PersonExt, ToApub, }, - blocking, DbPool, LemmyContext, }; @@ -29,11 +27,16 @@ use activitystreams::{ use activitystreams_ext::Ext1; use actix_web::{body::Body, web, HttpResponse}; use anyhow::Context; +use lemmy_api_structs::blocking; use lemmy_db::{ naive_now, user::{UserForm, User_}, }; -use lemmy_utils::{convert_datetime, location_info, LemmyError}; +use lemmy_utils::{ + location_info, + utils::{check_slurs, check_slurs_opt, convert_datetime}, + LemmyError, +}; use serde::Deserialize; use url::Url; diff --git a/server/src/code_migrations.rs b/server/src/code_migrations.rs index e59aa88c7..47d046696 100644 --- a/server/src/code_migrations.rs +++ b/server/src/code_migrations.rs @@ -13,11 +13,8 @@ use lemmy_db::{ Crud, }; use lemmy_utils::{ - generate_actor_keypair, - get_apub_protocol_string, - make_apub_endpoint, + apub::{generate_actor_keypair, get_apub_protocol_string, make_apub_endpoint, EndpointType}, settings::Settings, - EndpointType, LemmyError, }; use log::info; diff --git a/server/src/lib.rs b/server/src/lib.rs index d811d908a..11b8df7cb 100644 --- a/server/src/lib.rs +++ b/server/src/lib.rs @@ -36,15 +36,14 @@ use crate::{ use actix::Addr; use anyhow::anyhow; use background_jobs::QueueHandle; -use lemmy_utils::{get_apub_protocol_string, settings::Settings, LemmyError}; +use lemmy_db::DbPool; +use lemmy_utils::{apub::get_apub_protocol_string, settings::Settings, LemmyError}; use log::error; use percent_encoding::{utf8_percent_encode, NON_ALPHANUMERIC}; use reqwest::Client; use serde::Deserialize; use std::process::Command; -pub type DbPool = diesel::r2d2::Pool>; - pub struct LemmyContext { pub pool: DbPool, pub chat_server: Addr, @@ -224,22 +223,6 @@ pub async fn is_image_content_type(client: &Client, test: &str) -> Result<(), Le } } -pub async fn blocking(pool: &DbPool, f: F) -> Result -where - F: FnOnce(&diesel::PgConnection) -> T + Send + 'static, - T: Send + 'static, -{ - let pool = pool.clone(); - let res = actix_web::web::block(move || { - let conn = pool.get()?; - let res = (f)(&conn); - Ok(res) as Result<_, LemmyError> - }) - .await?; - - Ok(res) -} - pub fn captcha_espeak_wav_base64(captcha: &str) -> Result { let mut built_text = String::new(); diff --git a/server/src/main.rs b/server/src/main.rs index 688e1aa1f..8445a68ba 100644 --- a/server/src/main.rs +++ b/server/src/main.rs @@ -16,11 +16,11 @@ use diesel::{ PgConnection, }; use lazy_static::lazy_static; +use lemmy_api_structs::blocking; use lemmy_db::get_database_url_from_env; use lemmy_rate_limit::{rate_limiter::RateLimiter, RateLimit}; use lemmy_server::{ apub::activity_queue::create_activity_queue, - blocking, code_migrations::run_advanced_migrations, routes::*, websocket::chat_server::ChatServer, diff --git a/server/src/routes/feeds.rs b/server/src/routes/feeds.rs index 317c10307..1d8641490 100644 --- a/server/src/routes/feeds.rs +++ b/server/src/routes/feeds.rs @@ -1,8 +1,9 @@ -use crate::{api::claims::Claims, blocking, LemmyContext}; +use crate::{api::claims::Claims, LemmyContext}; use actix_web::{error::ErrorBadRequest, *}; use anyhow::anyhow; use chrono::{DateTime, NaiveDateTime, Utc}; use diesel::PgConnection; +use lemmy_api_structs::blocking; use lemmy_db::{ comment_view::{ReplyQueryBuilder, ReplyView}, community::Community, @@ -13,7 +14,7 @@ use lemmy_db::{ ListingType, SortType, }; -use lemmy_utils::{markdown_to_html, settings::Settings, LemmyError}; +use lemmy_utils::{settings::Settings, utils::markdown_to_html, LemmyError}; use rss::{CategoryBuilder, ChannelBuilder, GuidBuilder, Item, ItemBuilder}; use serde::Deserialize; use std::str::FromStr; diff --git a/server/src/routes/nodeinfo.rs b/server/src/routes/nodeinfo.rs index 1390b21ec..2273fae54 100644 --- a/server/src/routes/nodeinfo.rs +++ b/server/src/routes/nodeinfo.rs @@ -1,8 +1,9 @@ -use crate::{blocking, version, LemmyContext}; +use crate::{version, LemmyContext}; use actix_web::{body::Body, error::ErrorBadRequest, *}; use anyhow::anyhow; +use lemmy_api_structs::blocking; use lemmy_db::site_view::SiteView; -use lemmy_utils::{get_apub_protocol_string, settings::Settings, LemmyError}; +use lemmy_utils::{apub::get_apub_protocol_string, settings::Settings, LemmyError}; use serde::{Deserialize, Serialize}; use url::Url; diff --git a/server/src/routes/webfinger.rs b/server/src/routes/webfinger.rs index 22861434c..458fa0f37 100644 --- a/server/src/routes/webfinger.rs +++ b/server/src/routes/webfinger.rs @@ -1,6 +1,7 @@ -use crate::{blocking, LemmyContext}; +use crate::LemmyContext; use actix_web::{error::ErrorBadRequest, web::Query, *}; use anyhow::anyhow; +use lemmy_api_structs::blocking; use lemmy_db::{community::Community, user::User_}; use lemmy_utils::{ settings::Settings, diff --git a/server/src/routes/websocket.rs b/server/src/routes/websocket.rs index 465974c00..540af34fa 100644 --- a/server/src/routes/websocket.rs +++ b/server/src/routes/websocket.rs @@ -8,7 +8,7 @@ use crate::{ use actix::prelude::*; use actix_web::*; use actix_web_actors::ws; -use lemmy_utils::get_ip; +use lemmy_utils::utils::get_ip; use log::{debug, error, info}; use std::time::{Duration, Instant}; diff --git a/server/test.sh b/server/test.sh index 9a8e445b6..beb499bdf 100755 --- a/server/test.sh +++ b/server/test.sh @@ -2,4 +2,4 @@ export DATABASE_URL=postgres://lemmy:password@localhost:5432/lemmy diesel migration run export LEMMY_DATABASE_URL=postgres://lemmy:password@localhost:5432/lemmy -RUST_TEST_THREADS=1 cargo test --workspace +RUST_TEST_THREADS=1 cargo test --workspace --no-fail-fast