diff --git a/.travis.yml b/.travis.yml index 602a8613d..9541afaae 100644 --- a/.travis.yml +++ b/.travis.yml @@ -24,10 +24,11 @@ script: - cargo clippy -- -D clippy::style -D clippy::correctness -D clippy::complexity -D clippy::perf - cargo install diesel_cli --no-default-features --features postgres --force - diesel migration run - - cargo test + - cargo test --workspace env: global: - DATABASE_URL=postgres://lemmy:password@localhost:5432/lemmy + - LEMMY_DATABASE_URL=postgres://lemmy:password@localhost:5432/lemmy - RUST_TEST_THREADS=1 addons: diff --git a/ansible/VERSION b/ansible/VERSION index 3a952db02..e72bb64f4 100644 --- a/ansible/VERSION +++ b/ansible/VERSION @@ -1 +1 @@ -v0.7.13 +v0.7.19 diff --git a/docker/dev/Dockerfile b/docker/dev/Dockerfile index 82a03f3c9..b86618d85 100644 --- a/docker/dev/Dockerfile +++ b/docker/dev/Dockerfile @@ -18,11 +18,12 @@ RUN sudo chown -R rust:rust . RUN USER=root cargo new server WORKDIR /app/server COPY server/Cargo.toml server/Cargo.lock ./ -RUN sudo chown -R rust:rust . +COPY server/lemmy_db ./lemmy_db +COPY server/lemmy_utils ./lemmy_utils RUN mkdir -p ./src/bin \ - && echo 'fn main() { println!("Dummy") }' > ./src/bin/main.rs + && echo 'fn main() { println!("Dummy") }' > ./src/bin/main.rs RUN cargo build -RUN rm -f ./target/x86_64-unknown-linux-musl/release/deps/lemmy_server* +RUN find target/debug -type f -name "$(echo "lemmy_server" | tr '-' '_')*" -exec touch -t 200001010000 {} + COPY server/src ./src/ COPY server/migrations ./migrations/ diff --git a/docker/prod/Dockerfile b/docker/prod/Dockerfile index 54485a37e..774387404 100644 --- a/docker/prod/Dockerfile +++ b/docker/prod/Dockerfile @@ -10,14 +10,15 @@ WORKDIR /app RUN sudo chown -R rust:rust . RUN USER=root cargo new server WORKDIR /app/server -COPY --chown=rust:rust server/Cargo.toml server/Cargo.lock ./ -#RUN sudo chown -R rust:rust . +COPY server/Cargo.toml server/Cargo.lock ./ +COPY server/lemmy_db ./lemmy_db +COPY server/lemmy_utils ./lemmy_utils RUN mkdir -p ./src/bin \ && echo 'fn main() { println!("Dummy") }' > ./src/bin/main.rs RUN cargo build --release -RUN rm -f ./target/$CARGO_BUILD_TARGET/$RUSTRELEASEDIR/deps/lemmy_server* -COPY --chown=rust:rust server/src ./src/ -COPY --chown=rust:rust server/migrations ./migrations/ +RUN find target/$CARGO_BUILD_TARGET/$RUSTRELEASEDIR -type f -name "$(echo "lemmy_server" | tr '-' '_')*" -exec touch -t 200001010000 {} + +COPY server/src ./src/ +COPY server/migrations ./migrations/ # build for release # workaround for https://github.com/rust-lang/rust/issues/62896 diff --git a/docker/prod/docker-compose.yml b/docker/prod/docker-compose.yml index f05a9b384..ad00d6416 100644 --- a/docker/prod/docker-compose.yml +++ b/docker/prod/docker-compose.yml @@ -12,7 +12,7 @@ services: restart: always lemmy: - image: dessalines/lemmy:v0.7.13 + image: dessalines/lemmy:v0.7.19 ports: - "127.0.0.1:8536:8536" restart: always diff --git a/docs/src/administration_configuration.md b/docs/src/administration_configuration.md index 56448de46..cc4c56898 100644 --- a/docs/src/administration_configuration.md +++ b/docs/src/administration_configuration.md @@ -5,6 +5,8 @@ The configuration is based on the file This file also contains documentation for all the available options. To override the defaults, you can copy the options you want to change into your local `config.hjson` file. +To use a different `config.hjson` location than the current directory, set the environment variable `LEMMY_CONFIG_LOCATION`. + Additionally, you can override any config files with environment variables. These have the same name as the config options, and are prefixed with `LEMMY_`. For example, you can override the `database.password` with `LEMMY_DATABASE__POOL_SIZE=10`. diff --git a/docs/src/contributing_tests.md b/docs/src/contributing_tests.md index 13e5d1222..d4168e190 100644 --- a/docs/src/contributing_tests.md +++ b/docs/src/contributing_tests.md @@ -7,9 +7,7 @@ following commands in the `server` subfolder: ```bash psql -U lemmy -c "DROP SCHEMA public CASCADE; CREATE SCHEMA public;" -export DATABASE_URL=postgres://lemmy:password@localhost:5432/lemmy -diesel migration run -RUST_TEST_THREADS=1 cargo test +./test.sh ``` ### Federation diff --git a/docs/src/contributing_websocket_http_api.md b/docs/src/contributing_websocket_http_api.md index 567f674cd..6ed25b98e 100644 --- a/docs/src/contributing_websocket_http_api.md +++ b/docs/src/contributing_websocket_http_api.md @@ -1149,6 +1149,7 @@ Post listing types are `All, Subscribed, Community` page: Option, limit: Option, community_id: Option, + community_name: Option, auth: Option } } diff --git a/install.sh b/install.sh index fb42b26d1..19b847b1c 100755 --- a/install.sh +++ b/install.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash set -e # Set the database variable to the default first. diff --git a/server/Cargo.lock b/server/Cargo.lock index 6d6364d4e..d90b96799 100644 --- a/server/Cargo.lock +++ b/server/Cargo.lock @@ -1399,12 +1399,6 @@ dependencies = [ "winapi 0.3.9", ] -[[package]] -name = "htmlescape" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9025058dae765dee5070ec375f591e2ba14638c63feff74f13805a72e523163" - [[package]] name = "http" version = "0.2.1" @@ -1428,9 +1422,9 @@ dependencies = [ [[package]] name = "http-signature-normalization-actix" -version = "0.4.0-alpha.0" +version = "0.4.0-alpha.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09afff6987c7edbed101d1cddd2185786fb0af0dd9c06b654aca73a0a763680f" +checksum = "131fc982391a6b37847888b568cbe0e9cd302f1b0015f4f6f4a50234bebd049c" dependencies = [ "actix-http", "actix-web", @@ -1572,6 +1566,21 @@ version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" +[[package]] +name = "lemmy_db" +version = "0.1.0" +dependencies = [ + "bcrypt", + "chrono", + "diesel", + "log", + "serde 1.0.114", + "serde_json", + "sha2", + "strum", + "strum_macros", +] + [[package]] name = "lemmy_server" version = "0.0.1" @@ -1589,27 +1598,23 @@ dependencies = [ "base64 0.12.3", "bcrypt", "chrono", - "comrak", - "config", "diesel", "diesel_migrations", "dotenv", "env_logger", "failure", "futures", - "htmlescape", "http", "http-signature-normalization-actix", "itertools", "jsonwebtoken", "lazy_static", - "lettre", - "lettre_email", + "lemmy_db", + "lemmy_utils", "log", "openssl", "percent-encoding", "rand 0.7.3", - "regex", "rss", "serde 1.0.114", "serde_json", @@ -1621,6 +1626,26 @@ dependencies = [ "uuid 0.8.1", ] +[[package]] +name = "lemmy_utils" +version = "0.1.0" +dependencies = [ + "chrono", + "comrak", + "config", + "itertools", + "lazy_static", + "lettre", + "lettre_email", + "log", + "openssl", + "rand 0.7.3", + "regex", + "serde 1.0.114", + "serde_json", + "url", +] + [[package]] name = "lettre" version = "0.9.3" diff --git a/server/Cargo.toml b/server/Cargo.toml index 8daf72c4a..2aa3c139b 100644 --- a/server/Cargo.toml +++ b/server/Cargo.toml @@ -1,14 +1,21 @@ [package] name = "lemmy_server" version = "0.0.1" -authors = ["Dessalines "] edition = "2018" [profile.release] lto = true +[workspace] +members = [ + "lemmy_utils", + "lemmy_db" +] + [dependencies] -diesel = { version = "1.4.4", features = ["postgres","chrono","r2d2","64-column-tables","serde_json"] } +lemmy_utils = { path = "./lemmy_utils" } +lemmy_db = { path = "./lemmy_db" } +diesel = "1.4.4" diesel_migrations = "1.4.0" dotenv = "0.15.0" activitystreams = "0.6.2" @@ -31,19 +38,13 @@ rand = "0.7.3" strum = "0.18.0" strum_macros = "0.18.0" jsonwebtoken = "7.0.1" -regex = "1.3.5" lazy_static = "1.3.0" -lettre = "0.9.3" -lettre_email = "0.9.4" rss = "1.9.0" -htmlescape = "0.3.1" url = { version = "2.1.1", features = ["serde"] } -config = {version = "0.10.1", default-features = false, features = ["hjson"] } percent-encoding = "2.1.0" -comrak = "0.7" openssl = "0.10" http = "0.2.1" -http-signature-normalization-actix = { version = "0.4.0-alpha.0", default-features = false, features = ["sha-2"] } +http-signature-normalization-actix = { version = "0.4.0-alpha.2", default-features = false, features = ["sha-2"] } base64 = "0.12.1" tokio = "0.2.21" futures = "0.3.5" diff --git a/server/db-init.sh b/server/db-init.sh index a2ad77b59..ccecb7de7 100755 --- a/server/db-init.sh +++ b/server/db-init.sh @@ -1,4 +1,5 @@ -#!/bin/sh +#!/bin/bash +set -e # Default configurations username=lemmy diff --git a/server/diesel.toml b/server/diesel.toml index 92267c829..1644558f1 100644 --- a/server/diesel.toml +++ b/server/diesel.toml @@ -2,4 +2,4 @@ # see diesel.rs/guides/configuring-diesel-cli [print_schema] -file = "src/schema.rs" +file = "lemmy_db/src/schema.rs" diff --git a/server/lemmy_db/Cargo.toml b/server/lemmy_db/Cargo.toml new file mode 100644 index 000000000..d94cf5fc6 --- /dev/null +++ b/server/lemmy_db/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "lemmy_db" +version = "0.1.0" +edition = "2018" + +[dependencies] +diesel = { version = "1.4.4", features = ["postgres","chrono","r2d2","64-column-tables","serde_json"] } +chrono = { version = "0.4.7", features = ["serde"] } +serde = { version = "1.0.105", features = ["derive"] } +serde_json = { version = "1.0.52", features = ["preserve_order"]} +strum = "0.18.0" +strum_macros = "0.18.0" +log = "0.4.0" +sha2 = "0.9" +bcrypt = "0.8.0" \ No newline at end of file diff --git a/server/src/db/activity.rs b/server/lemmy_db/src/activity.rs similarity index 84% rename from server/src/db/activity.rs rename to server/lemmy_db/src/activity.rs index 8c2b0c742..83f85ca1e 100644 --- a/server/src/db/activity.rs +++ b/server/lemmy_db/src/activity.rs @@ -1,9 +1,12 @@ -use crate::{blocking, db::Crud, schema::activity, DbPool, LemmyError}; +use crate::{schema::activity, Crud}; use diesel::{dsl::*, result::Error, *}; use log::debug; use serde::{Deserialize, Serialize}; use serde_json::Value; -use std::fmt::Debug; +use std::{ + fmt::Debug, + io::{Error as IoError, ErrorKind}, +}; #[derive(Queryable, Identifiable, PartialEq, Debug, Serialize, Deserialize)] #[table_name = "activity"] @@ -55,46 +58,43 @@ impl Crud for Activity { } } -pub async fn insert_activity( - user_id: i32, - data: T, - local: bool, - pool: &DbPool, -) -> Result<(), LemmyError> -where - T: Serialize + Debug + Send + 'static, -{ - blocking(pool, move |conn| { - do_insert_activity(conn, user_id, &data, local) - }) - .await??; - Ok(()) -} - -fn do_insert_activity( +pub fn do_insert_activity( conn: &PgConnection, user_id: i32, data: &T, local: bool, -) -> Result<(), LemmyError> +) -> Result where T: Serialize + Debug, { + debug!("inserting activity for user {}, data {:?}", user_id, &data); let activity_form = ActivityForm { user_id, data: serde_json::to_value(&data)?, local, updated: None, }; - debug!("inserting activity for user {}, data {:?}", user_id, data); - Activity::create(&conn, &activity_form)?; - Ok(()) + let result = Activity::create(&conn, &activity_form); + match result { + Ok(s) => Ok(s), + Err(e) => Err(IoError::new( + ErrorKind::Other, + format!("Failed to insert activity into database: {}", e), + )), + } } #[cfg(test)] mod tests { - use super::{super::user::*, *}; - use crate::db::{establish_unpooled_connection, Crud, ListingType, SortType}; + use crate::{ + activity::{Activity, ActivityForm}, + tests::establish_unpooled_connection, + user::{UserForm, User_}, + Crud, + ListingType, + SortType, + }; + use serde_json::Value; #[test] fn test_crud() { diff --git a/server/src/db/category.rs b/server/lemmy_db/src/category.rs similarity index 95% rename from server/src/db/category.rs rename to server/lemmy_db/src/category.rs index ff49bbbee..ec2efc7b7 100644 --- a/server/src/db/category.rs +++ b/server/lemmy_db/src/category.rs @@ -1,6 +1,6 @@ use crate::{ - db::Crud, schema::{category, category::dsl::*}, + Crud, }; use diesel::{dsl::*, result::Error, *}; use serde::{Deserialize, Serialize}; @@ -52,8 +52,7 @@ impl Category { #[cfg(test)] mod tests { - use super::*; - use crate::db::establish_unpooled_connection; + use crate::{category::Category, tests::establish_unpooled_connection}; #[test] fn test_crud() { diff --git a/server/src/db/comment.rs b/server/lemmy_db/src/comment.rs similarity index 96% rename from server/src/db/comment.rs rename to server/lemmy_db/src/comment.rs index 7e76770f6..602070d51 100644 --- a/server/src/db/comment.rs +++ b/server/lemmy_db/src/comment.rs @@ -1,9 +1,5 @@ use super::{post::Post, *}; -use crate::{ - apub::{make_apub_endpoint, EndpointType}, - naive_now, - schema::{comment, comment_like, comment_saved}, -}; +use crate::schema::{comment, comment_like, comment_saved}; // WITH RECURSIVE MyTree AS ( // SELECT * FROM comment WHERE parent_id IS NULL @@ -77,12 +73,15 @@ impl Crud for Comment { } impl Comment { - pub fn update_ap_id(conn: &PgConnection, comment_id: i32) -> Result { + pub fn update_ap_id( + conn: &PgConnection, + comment_id: i32, + apub_id: String, + ) -> Result { use crate::schema::comment::dsl::*; - let apid = make_apub_endpoint(EndpointType::Comment, &comment_id.to_string()).to_string(); diesel::update(comment.find(comment_id)) - .set(ap_id.eq(apid)) + .set(ap_id.eq(apub_id)) .get_result::(conn) } @@ -204,10 +203,8 @@ impl Saveable for CommentSaved { #[cfg(test)] mod tests { - use super::{ - super::{community::*, post::*, user::*}, - *, - }; + use crate::{comment::*, community::*, post::*, tests::establish_unpooled_connection, user::*}; + #[test] fn test_crud() { let conn = establish_unpooled_connection(); diff --git a/server/src/db/comment_view.rs b/server/lemmy_db/src/comment_view.rs similarity index 95% rename from server/src/db/comment_view.rs rename to server/lemmy_db/src/comment_view.rs index d1b27a3c8..4af13c2d9 100644 --- a/server/src/db/comment_view.rs +++ b/server/lemmy_db/src/comment_view.rs @@ -1,5 +1,5 @@ // TODO, remove the cross join here, just join to user directly -use crate::db::{fuzzy_search, limit_and_offset, ListingType, MaybeOptional, SortType}; +use crate::{fuzzy_search, limit_and_offset, ListingType, MaybeOptional, SortType}; use diesel::{dsl::*, pg::Pg, result::Error, *}; use serde::{Deserialize, Serialize}; @@ -9,6 +9,7 @@ table! { id -> Int4, creator_id -> Int4, post_id -> Int4, + post_name -> Varchar, parent_id -> Nullable, content -> Text, removed -> Bool, @@ -27,6 +28,7 @@ table! { creator_actor_id -> Text, creator_local -> Bool, creator_name -> Varchar, + creator_published -> Timestamp, creator_avatar -> Nullable, score -> BigInt, upvotes -> BigInt, @@ -44,6 +46,7 @@ table! { id -> Int4, creator_id -> Int4, post_id -> Int4, + post_name -> Varchar, parent_id -> Nullable, content -> Text, removed -> Bool, @@ -62,6 +65,7 @@ table! { creator_actor_id -> Text, creator_local -> Bool, creator_name -> Varchar, + creator_published -> Timestamp, creator_avatar -> Nullable, score -> BigInt, upvotes -> BigInt, @@ -82,6 +86,7 @@ pub struct CommentView { pub id: i32, pub creator_id: i32, pub post_id: i32, + pub post_name: String, pub parent_id: Option, pub content: String, pub removed: bool, @@ -100,6 +105,7 @@ pub struct CommentView { pub creator_actor_id: String, pub creator_local: bool, pub creator_name: String, + pub creator_published: chrono::NaiveDateTime, pub creator_avatar: Option, pub score: i64, pub upvotes: i64, @@ -295,6 +301,7 @@ table! { id -> Int4, creator_id -> Int4, post_id -> Int4, + post_name -> Varchar, parent_id -> Nullable, content -> Text, removed -> Bool, @@ -314,6 +321,7 @@ table! { creator_local -> Bool, creator_name -> Varchar, creator_avatar -> Nullable, + creator_published -> Timestamp, score -> BigInt, upvotes -> BigInt, downvotes -> BigInt, @@ -334,6 +342,7 @@ pub struct ReplyView { pub id: i32, pub creator_id: i32, pub post_id: i32, + pub post_name: String, pub parent_id: Option, pub content: String, pub removed: bool, @@ -353,6 +362,7 @@ pub struct ReplyView { pub creator_local: bool, pub creator_name: String, pub creator_avatar: Option, + pub creator_published: chrono::NaiveDateTime, pub score: i64, pub upvotes: i64, pub downvotes: i64, @@ -455,11 +465,17 @@ impl<'a> ReplyQueryBuilder<'a> { #[cfg(test)] mod tests { - use super::{ - super::{comment::*, community::*, post::*, user::*}, + use crate::{ + comment::*, + comment_view::*, + community::*, + post::*, + tests::establish_unpooled_connection, + user::*, + Crud, + Likeable, *, }; - use crate::db::{establish_unpooled_connection, Crud, Likeable}; #[test] fn test_crud() { @@ -565,6 +581,7 @@ mod tests { content: "A test comment 32".into(), creator_id: inserted_user.id, post_id: inserted_post.id, + post_name: inserted_post.name.to_owned(), community_id: inserted_community.id, community_name: inserted_community.name.to_owned(), parent_id: None, @@ -576,6 +593,7 @@ mod tests { published: inserted_comment.published, updated: None, creator_name: inserted_user.name.to_owned(), + creator_published: inserted_user.published, creator_avatar: None, score: 1, downvotes: 0, @@ -598,6 +616,7 @@ mod tests { content: "A test comment 32".into(), creator_id: inserted_user.id, post_id: inserted_post.id, + post_name: inserted_post.name.to_owned(), community_id: inserted_community.id, community_name: inserted_community.name.to_owned(), parent_id: None, @@ -609,6 +628,7 @@ mod tests { published: inserted_comment.published, updated: None, creator_name: inserted_user.name.to_owned(), + creator_published: inserted_user.published, creator_avatar: None, score: 1, downvotes: 0, diff --git a/server/src/db/community.rs b/server/lemmy_db/src/community.rs similarity index 98% rename from server/src/db/community.rs rename to server/lemmy_db/src/community.rs index 461ba473a..607520803 100644 --- a/server/src/db/community.rs +++ b/server/lemmy_db/src/community.rs @@ -1,6 +1,9 @@ use crate::{ - db::{Bannable, Crud, Followable, Joinable}, schema::{community, community_follower, community_moderator, community_user_ban}, + Bannable, + Crud, + Followable, + Joinable, }; use diesel::{dsl::*, result::Error, *}; use serde::{Deserialize, Serialize}; @@ -232,8 +235,7 @@ impl Followable for CommunityFollower { #[cfg(test)] mod tests { - use super::{super::user::*, *}; - use crate::db::{establish_unpooled_connection, ListingType, SortType}; + use crate::{community::*, tests::establish_unpooled_connection, user::*, ListingType, SortType}; #[test] fn test_crud() { diff --git a/server/src/db/community_view.rs b/server/lemmy_db/src/community_view.rs similarity index 98% rename from server/src/db/community_view.rs rename to server/lemmy_db/src/community_view.rs index 4ec839acf..5c6bd81a1 100644 --- a/server/src/db/community_view.rs +++ b/server/lemmy_db/src/community_view.rs @@ -1,5 +1,5 @@ use super::community_view::community_fast_view::BoxedQuery; -use crate::db::{fuzzy_search, limit_and_offset, MaybeOptional, SortType}; +use crate::{fuzzy_search, limit_and_offset, MaybeOptional, SortType}; use diesel::{pg::Pg, result::Error, *}; use serde::{Deserialize, Serialize}; @@ -299,6 +299,7 @@ impl CommunityModeratorView { use super::community_view::community_moderator_view::dsl::*; community_moderator_view .filter(community_id.eq(from_community_id)) + .order_by(published) .load::(conn) } @@ -306,6 +307,7 @@ impl CommunityModeratorView { use super::community_view::community_moderator_view::dsl::*; community_moderator_view .filter(user_id.eq(from_user_id)) + .order_by(published) .load::(conn) } } diff --git a/server/src/db/mod.rs b/server/lemmy_db/src/lib.rs similarity index 78% rename from server/src/db/mod.rs rename to server/lemmy_db/src/lib.rs index da69f8dcd..5cf20bb3c 100644 --- a/server/src/db/mod.rs +++ b/server/lemmy_db/src/lib.rs @@ -1,10 +1,22 @@ -use crate::settings::Settings; +#[macro_use] +pub extern crate diesel; +#[macro_use] +pub extern crate strum_macros; +pub extern crate bcrypt; +pub extern crate chrono; +pub extern crate log; +pub extern crate serde; +pub extern crate serde_json; +pub extern crate sha2; +pub extern crate strum; + +use chrono::NaiveDateTime; use diesel::{dsl::*, result::Error, *}; use serde::{Deserialize, Serialize}; +use std::{env, env::VarError}; pub mod activity; pub mod category; -pub mod code_migrations; pub mod comment; pub mod comment_view; pub mod community; @@ -16,6 +28,7 @@ pub mod post; pub mod post_view; pub mod private_message; pub mod private_message_view; +pub mod schema; pub mod site; pub mod site_view; pub mod user; @@ -111,9 +124,8 @@ impl MaybeOptional for Option { } } -pub fn establish_unpooled_connection() -> PgConnection { - let db_url = Settings::get().get_database_url(); - PgConnection::establish(&db_url).unwrap_or_else(|_| panic!("Error connecting to {}", db_url)) +pub fn get_database_url_from_env() -> Result { + env::var("LEMMY_DATABASE_URL") } #[derive(EnumString, ToString, Debug, Serialize, Deserialize)] @@ -155,9 +167,25 @@ pub fn limit_and_offset(page: Option, limit: Option) -> (i64, i64) { let offset = limit * (page - 1); (limit, offset) } + +pub fn naive_now() -> NaiveDateTime { + chrono::prelude::Utc::now().naive_utc() +} + #[cfg(test)] mod tests { use super::fuzzy_search; + use crate::get_database_url_from_env; + use diesel::{Connection, PgConnection}; + + pub fn establish_unpooled_connection() -> PgConnection { + let db_url = match get_database_url_from_env() { + Ok(url) => url, + Err(e) => panic!("Failed to read database URL from env var LEMMY_DATABASE_URL: {}", e), + }; + PgConnection::establish(&db_url).unwrap_or_else(|_| panic!("Error connecting to {}", db_url)) + } + #[test] fn test_fuzzy_search() { let test = "This is a fuzzy search"; diff --git a/server/src/db/moderator.rs b/server/lemmy_db/src/moderator.rs similarity index 99% rename from server/src/db/moderator.rs rename to server/lemmy_db/src/moderator.rs index 44b04ec63..f5d33d967 100644 --- a/server/src/db/moderator.rs +++ b/server/lemmy_db/src/moderator.rs @@ -1,5 +1,4 @@ use crate::{ - db::Crud, schema::{ mod_add, mod_add_community, @@ -11,6 +10,7 @@ use crate::{ mod_remove_post, mod_sticky_post, }, + Crud, }; use diesel::{dsl::*, result::Error, *}; use serde::{Deserialize, Serialize}; @@ -437,11 +437,16 @@ impl Crud for ModAdd { #[cfg(test)] mod tests { - use super::{ - super::{comment::*, community::*, post::*, user::*}, - *, + use crate::{ + comment::*, + community::*, + moderator::*, + post::*, + tests::establish_unpooled_connection, + user::*, + ListingType, + SortType, }; - use crate::db::{establish_unpooled_connection, ListingType, SortType}; // use Crud; #[test] diff --git a/server/src/db/moderator_views.rs b/server/lemmy_db/src/moderator_views.rs similarity index 99% rename from server/src/db/moderator_views.rs rename to server/lemmy_db/src/moderator_views.rs index f5b109fe6..024907c39 100644 --- a/server/src/db/moderator_views.rs +++ b/server/lemmy_db/src/moderator_views.rs @@ -1,4 +1,4 @@ -use crate::db::limit_and_offset; +use crate::limit_and_offset; use diesel::{result::Error, *}; use serde::{Deserialize, Serialize}; diff --git a/server/src/db/password_reset_request.rs b/server/lemmy_db/src/password_reset_request.rs similarity index 98% rename from server/src/db/password_reset_request.rs rename to server/lemmy_db/src/password_reset_request.rs index 4a071f078..a2692add8 100644 --- a/server/src/db/password_reset_request.rs +++ b/server/lemmy_db/src/password_reset_request.rs @@ -1,6 +1,6 @@ use crate::{ - db::Crud, schema::{password_reset_request, password_reset_request::dsl::*}, + Crud, }; use diesel::{dsl::*, result::Error, *}; use sha2::{Digest, Sha256}; @@ -82,7 +82,7 @@ impl PasswordResetRequest { #[cfg(test)] mod tests { use super::{super::user::*, *}; - use crate::db::{establish_unpooled_connection, ListingType, SortType}; + use crate::{tests::establish_unpooled_connection, ListingType, SortType}; #[test] fn test_crud() { diff --git a/server/src/db/post.rs b/server/lemmy_db/src/post.rs similarity index 96% rename from server/src/db/post.rs rename to server/lemmy_db/src/post.rs index 91c1dcbff..1525a675f 100644 --- a/server/src/db/post.rs +++ b/server/lemmy_db/src/post.rs @@ -1,8 +1,10 @@ use crate::{ - apub::{make_apub_endpoint, EndpointType}, - db::{Crud, Likeable, Readable, Saveable}, naive_now, schema::{post, post_like, post_read, post_saved}, + Crud, + Likeable, + Readable, + Saveable, }; use diesel::{dsl::*, result::Error, *}; use serde::{Deserialize, Serialize}; @@ -75,12 +77,11 @@ impl Post { post.filter(ap_id.eq(object_id)).first::(conn) } - pub fn update_ap_id(conn: &PgConnection, post_id: i32) -> Result { + pub fn update_ap_id(conn: &PgConnection, post_id: i32, apub_id: String) -> Result { use crate::schema::post::dsl::*; - let apid = make_apub_endpoint(EndpointType::Post, &post_id.to_string()).to_string(); diesel::update(post.find(post_id)) - .set(ap_id.eq(apid)) + .set(ap_id.eq(apub_id)) .get_result::(conn) } @@ -241,11 +242,14 @@ impl Readable for PostRead { #[cfg(test)] mod tests { - use super::{ - super::{community::*, user::*}, - *, + use crate::{ + community::*, + post::*, + tests::establish_unpooled_connection, + user::*, + ListingType, + SortType, }; - use crate::db::{establish_unpooled_connection, ListingType, SortType}; #[test] fn test_crud() { diff --git a/server/src/db/post_view.rs b/server/lemmy_db/src/post_view.rs similarity index 94% rename from server/src/db/post_view.rs rename to server/lemmy_db/src/post_view.rs index 808cf28c4..3e9f87376 100644 --- a/server/src/db/post_view.rs +++ b/server/lemmy_db/src/post_view.rs @@ -1,5 +1,5 @@ use super::post_view::post_fast_view::BoxedQuery; -use crate::db::{fuzzy_search, limit_and_offset, ListingType, MaybeOptional, SortType}; +use crate::{fuzzy_search, limit_and_offset, ListingType, MaybeOptional, SortType}; use diesel::{dsl::*, pg::Pg, result::Error, *}; use serde::{Deserialize, Serialize}; @@ -28,6 +28,7 @@ table! { creator_actor_id -> Text, creator_local -> Bool, creator_name -> Varchar, + creator_published -> Timestamp, creator_avatar -> Nullable, banned -> Bool, banned_from_community -> Bool, @@ -75,6 +76,7 @@ table! { creator_actor_id -> Text, creator_local -> Bool, creator_name -> Varchar, + creator_published -> Timestamp, creator_avatar -> Nullable, banned -> Bool, banned_from_community -> Bool, @@ -125,6 +127,7 @@ pub struct PostView { pub creator_actor_id: String, pub creator_local: bool, pub creator_name: String, + pub creator_published: chrono::NaiveDateTime, pub creator_avatar: Option, pub banned: bool, pub banned_from_community: bool, @@ -155,6 +158,7 @@ pub struct PostQueryBuilder<'a> { my_user_id: Option, for_creator_id: Option, for_community_id: Option, + for_community_name: Option, search_term: Option, url_search: Option, show_nsfw: bool, @@ -178,6 +182,7 @@ impl<'a> PostQueryBuilder<'a> { my_user_id: None, for_creator_id: None, for_community_id: None, + for_community_name: None, search_term: None, url_search: None, show_nsfw: true, @@ -203,6 +208,11 @@ impl<'a> PostQueryBuilder<'a> { self } + pub fn for_community_name>(mut self, for_community_name: T) -> Self { + self.for_community_name = for_community_name.get_optional(); + self + } + pub fn for_creator_id>(mut self, for_creator_id: T) -> Self { self.for_creator_id = for_creator_id.get_optional(); self @@ -262,6 +272,11 @@ impl<'a> PostQueryBuilder<'a> { query = query.then_order_by(stickied.desc()); } + if let Some(for_community_name) = self.for_community_name { + query = query.filter(community_name.eq(for_community_name)); + query = query.then_order_by(stickied.desc()); + } + if let Some(url_search) = self.url_search { query = query.filter(url.eq(url_search)); } @@ -364,11 +379,16 @@ impl PostView { #[cfg(test)] mod tests { - use super::{ - super::{community::*, post::*, user::*}, + use crate::{ + community::*, + post::*, + post_view::*, + tests::establish_unpooled_connection, + user::*, + Crud, + Likeable, *, }; - use crate::db::{establish_unpooled_connection, Crud, Likeable}; #[test] fn test_crud() { @@ -499,6 +519,7 @@ mod tests { body: None, creator_id: inserted_user.id, creator_name: user_name.to_owned(), + creator_published: inserted_user.published, creator_avatar: None, banned: false, banned_from_community: false, @@ -548,6 +569,7 @@ mod tests { stickied: false, creator_id: inserted_user.id, creator_name: user_name, + creator_published: inserted_user.published, creator_avatar: None, banned: false, banned_from_community: false, diff --git a/server/src/db/private_message.rs b/server/lemmy_db/src/private_message.rs similarity index 92% rename from server/src/db/private_message.rs rename to server/lemmy_db/src/private_message.rs index 9d362bbf1..1c0b455f3 100644 --- a/server/src/db/private_message.rs +++ b/server/lemmy_db/src/private_message.rs @@ -1,8 +1,4 @@ -use crate::{ - apub::{make_apub_endpoint, EndpointType}, - db::Crud, - schema::private_message, -}; +use crate::{schema::private_message, Crud}; use diesel::{dsl::*, result::Error, *}; use serde::{Deserialize, Serialize}; @@ -66,16 +62,15 @@ impl Crud for PrivateMessage { } impl PrivateMessage { - pub fn update_ap_id(conn: &PgConnection, private_message_id: i32) -> Result { + pub fn update_ap_id( + conn: &PgConnection, + private_message_id: i32, + apub_id: String, + ) -> Result { use crate::schema::private_message::dsl::*; - let apid = make_apub_endpoint( - EndpointType::PrivateMessage, - &private_message_id.to_string(), - ) - .to_string(); diesel::update(private_message.find(private_message_id)) - .set(ap_id.eq(apid)) + .set(ap_id.eq(apub_id)) .get_result::(conn) } @@ -89,8 +84,13 @@ impl PrivateMessage { #[cfg(test)] mod tests { - use super::{super::user::*, *}; - use crate::db::{establish_unpooled_connection, ListingType, SortType}; + use crate::{ + private_message::*, + tests::establish_unpooled_connection, + user::*, + ListingType, + SortType, + }; #[test] fn test_crud() { diff --git a/server/src/db/private_message_view.rs b/server/lemmy_db/src/private_message_view.rs similarity index 98% rename from server/src/db/private_message_view.rs rename to server/lemmy_db/src/private_message_view.rs index 899a1084d..dfb11c444 100644 --- a/server/src/db/private_message_view.rs +++ b/server/lemmy_db/src/private_message_view.rs @@ -1,4 +1,4 @@ -use crate::db::{limit_and_offset, MaybeOptional}; +use crate::{limit_and_offset, MaybeOptional}; use diesel::{pg::Pg, result::Error, *}; use serde::{Deserialize, Serialize}; diff --git a/server/src/schema.rs b/server/lemmy_db/src/schema.rs similarity index 98% rename from server/src/schema.rs rename to server/lemmy_db/src/schema.rs index 0367c7506..9608fb7d4 100644 --- a/server/src/schema.rs +++ b/server/lemmy_db/src/schema.rs @@ -47,6 +47,7 @@ table! { deleted -> Nullable, ap_id -> Nullable, local -> Nullable, + post_name -> Nullable, community_id -> Nullable, community_actor_id -> Nullable, community_local -> Nullable, @@ -56,6 +57,7 @@ table! { creator_actor_id -> Nullable, creator_local -> Nullable, creator_name -> Nullable, + creator_published -> Nullable, creator_avatar -> Nullable, score -> Nullable, upvotes -> Nullable, @@ -317,6 +319,7 @@ table! { creator_actor_id -> Nullable, creator_local -> Nullable, creator_name -> Nullable, + creator_published -> Nullable, creator_avatar -> Nullable, banned -> Nullable, banned_from_community -> Nullable, diff --git a/server/src/db/site.rs b/server/lemmy_db/src/site.rs similarity index 97% rename from server/src/db/site.rs rename to server/lemmy_db/src/site.rs index c752bfe79..066ae0b1a 100644 --- a/server/src/db/site.rs +++ b/server/lemmy_db/src/site.rs @@ -1,4 +1,4 @@ -use crate::{db::Crud, schema::site}; +use crate::{schema::site, Crud}; use diesel::{dsl::*, result::Error, *}; use serde::{Deserialize, Serialize}; diff --git a/server/src/db/site_view.rs b/server/lemmy_db/src/site_view.rs similarity index 100% rename from server/src/db/site_view.rs rename to server/lemmy_db/src/site_view.rs diff --git a/server/src/db/user.rs b/server/lemmy_db/src/user.rs similarity index 73% rename from server/src/db/user.rs rename to server/lemmy_db/src/user.rs index 4ca0a0419..556fc1a75 100644 --- a/server/src/db/user.rs +++ b/server/lemmy_db/src/user.rs @@ -1,14 +1,10 @@ use crate::{ - db::Crud, - is_email_regex, naive_now, schema::{user_, user_::dsl::*}, - settings::Settings, + Crud, }; use bcrypt::{hash, DEFAULT_COST}; use diesel::{dsl::*, result::Error, *}; -use jsonwebtoken::{decode, encode, DecodingKey, EncodingKey, Header, TokenData, Validation}; -use serde::{Deserialize, Serialize}; #[derive(Clone, Queryable, Identifiable, PartialEq, Debug)] #[table_name = "user_"] @@ -131,90 +127,23 @@ impl User_ { } } -#[derive(Debug, Serialize, Deserialize)] -pub struct Claims { - pub id: i32, - pub username: String, - pub iss: String, - pub show_nsfw: bool, - pub theme: String, - pub default_sort_type: i16, - pub default_listing_type: i16, - pub lang: String, - pub avatar: Option, - pub show_avatars: bool, -} - -impl Claims { - pub fn decode(jwt: &str) -> Result, jsonwebtoken::errors::Error> { - let v = Validation { - validate_exp: false, - ..Validation::default() - }; - decode::( - &jwt, - &DecodingKey::from_secret(Settings::get().jwt_secret.as_ref()), - &v, - ) - } -} - -type Jwt = String; impl User_ { - pub fn jwt(&self) -> Jwt { - let my_claims = Claims { - id: self.id, - username: self.name.to_owned(), - iss: Settings::get().hostname, - show_nsfw: self.show_nsfw, - theme: self.theme.to_owned(), - default_sort_type: self.default_sort_type, - default_listing_type: self.default_listing_type, - lang: self.lang.to_owned(), - avatar: self.avatar.to_owned(), - show_avatars: self.show_avatars.to_owned(), - }; - encode( - &Header::default(), - &my_claims, - &EncodingKey::from_secret(Settings::get().jwt_secret.as_ref()), - ) - .unwrap() - } - - pub fn find_by_username(conn: &PgConnection, username: &str) -> Result { + pub fn find_by_username(conn: &PgConnection, username: &str) -> Result { user_.filter(name.eq(username)).first::(conn) } - pub fn find_by_email(conn: &PgConnection, from_email: &str) -> Result { + pub fn find_by_email(conn: &PgConnection, from_email: &str) -> Result { user_.filter(email.eq(from_email)).first::(conn) } - pub fn find_by_email_or_username( - conn: &PgConnection, - username_or_email: &str, - ) -> Result { - if is_email_regex(username_or_email) { - User_::find_by_email(conn, username_or_email) - } else { - User_::find_by_username(conn, username_or_email) - } - } - - pub fn get_profile_url(&self) -> String { - format!("https://{}/u/{}", Settings::get().hostname, self.name) - } - - pub fn find_by_jwt(conn: &PgConnection, jwt: &str) -> Result { - let claims: Claims = Claims::decode(&jwt).expect("Invalid token").claims; - Self::read(&conn, claims.id) + pub fn get_profile_url(&self, hostname: &str) -> String { + format!("https://{}/u/{}", hostname, self.name) } } #[cfg(test)] mod tests { - use super::{User_, *}; - use crate::db::{establish_unpooled_connection, ListingType, SortType}; + use crate::{tests::establish_unpooled_connection, user::*, ListingType, SortType}; #[test] fn test_crud() { diff --git a/server/src/db/user_mention.rs b/server/lemmy_db/src/user_mention.rs similarity index 96% rename from server/src/db/user_mention.rs rename to server/lemmy_db/src/user_mention.rs index 1d54fa988..9f23f4410 100644 --- a/server/src/db/user_mention.rs +++ b/server/lemmy_db/src/user_mention.rs @@ -1,5 +1,5 @@ use super::comment::Comment; -use crate::{db::Crud, schema::user_mention}; +use crate::{schema::user_mention, Crud}; use diesel::{dsl::*, result::Error, *}; use serde::{Deserialize, Serialize}; @@ -54,11 +54,16 @@ impl Crud for UserMention { #[cfg(test)] mod tests { - use super::{ - super::{comment::*, community::*, post::*, user::*}, - *, + use crate::{ + comment::*, + community::*, + post::*, + tests::establish_unpooled_connection, + user::*, + user_mention::*, + ListingType, + SortType, }; - use crate::db::{establish_unpooled_connection, ListingType, SortType}; #[test] fn test_crud() { diff --git a/server/src/db/user_mention_view.rs b/server/lemmy_db/src/user_mention_view.rs similarity index 97% rename from server/src/db/user_mention_view.rs rename to server/lemmy_db/src/user_mention_view.rs index 59aefb200..359f166d6 100644 --- a/server/src/db/user_mention_view.rs +++ b/server/lemmy_db/src/user_mention_view.rs @@ -1,4 +1,4 @@ -use crate::db::{limit_and_offset, MaybeOptional, SortType}; +use crate::{limit_and_offset, MaybeOptional, SortType}; use diesel::{dsl::*, pg::Pg, result::Error, *}; use serde::{Deserialize, Serialize}; @@ -11,6 +11,7 @@ table! { creator_actor_id -> Text, creator_local -> Bool, post_id -> Int4, + post_name -> Varchar, parent_id -> Nullable, content -> Text, removed -> Bool, @@ -47,6 +48,7 @@ table! { creator_actor_id -> Text, creator_local -> Bool, post_id -> Int4, + post_name -> Varchar, parent_id -> Nullable, content -> Text, removed -> Bool, @@ -86,6 +88,7 @@ pub struct UserMentionView { pub creator_actor_id: String, pub creator_local: bool, pub post_id: i32, + pub post_name: String, pub parent_id: Option, pub content: String, pub removed: bool, diff --git a/server/src/db/user_view.rs b/server/lemmy_db/src/user_view.rs similarity index 96% rename from server/src/db/user_view.rs rename to server/lemmy_db/src/user_view.rs index 490521721..f2ac47422 100644 --- a/server/src/db/user_view.rs +++ b/server/lemmy_db/src/user_view.rs @@ -1,5 +1,5 @@ use super::user_view::user_fast::BoxedQuery; -use crate::db::{fuzzy_search, limit_and_offset, MaybeOptional, SortType}; +use crate::{fuzzy_search, limit_and_offset, MaybeOptional, SortType}; use diesel::{dsl::*, pg::Pg, result::Error, *}; use serde::{Deserialize, Serialize}; @@ -157,7 +157,10 @@ impl UserView { pub fn admins(conn: &PgConnection) -> Result, Error> { use super::user_view::user_fast::dsl::*; - user_fast.filter(admin.eq(true)).load::(conn) + user_fast + .filter(admin.eq(true)) + .order_by(published) + .load::(conn) } pub fn banned(conn: &PgConnection) -> Result, Error> { diff --git a/server/lemmy_utils/Cargo.toml b/server/lemmy_utils/Cargo.toml new file mode 100644 index 000000000..fed22f585 --- /dev/null +++ b/server/lemmy_utils/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "lemmy_utils" +version = "0.1.0" +edition = "2018" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +regex = "1.3.5" +config = { version = "0.10.1", default-features = false, features = ["hjson"] } +chrono = { version = "0.4.7", features = ["serde"] } +lettre = "0.9.3" +lettre_email = "0.9.4" +log = "0.4.0" +itertools = "0.9.0" +rand = "0.7.3" +serde = { version = "1.0.105", features = ["derive"] } +serde_json = { version = "1.0.52", features = ["preserve_order"]} +comrak = "0.7" +lazy_static = "1.3.0" +openssl = "0.10" +url = { version = "2.1.1", features = ["serde"] } \ No newline at end of file diff --git a/server/lemmy_utils/src/lib.rs b/server/lemmy_utils/src/lib.rs new file mode 100644 index 000000000..d88335e2d --- /dev/null +++ b/server/lemmy_utils/src/lib.rs @@ -0,0 +1,337 @@ +#[macro_use] +pub extern crate lazy_static; +pub extern crate comrak; +pub extern crate lettre; +pub extern crate lettre_email; +pub extern crate openssl; +pub extern crate rand; +pub extern crate regex; +pub extern crate serde_json; +pub extern crate url; + +pub mod settings; + +use crate::settings::Settings; +use chrono::{DateTime, FixedOffset, Local, NaiveDateTime, Utc}; +use itertools::Itertools; +use lettre::{ + smtp::{ + authentication::{Credentials, Mechanism}, + extension::ClientId, + ConnectionReuseParameters, + }, + ClientSecurity, + SmtpClient, + Transport, +}; +use lettre_email::Email; +use openssl::{pkey::PKey, rsa::Rsa}; +use rand::{distributions::Alphanumeric, thread_rng, Rng}; +use regex::{Regex, RegexBuilder}; +use std::io::{Error, ErrorKind}; +use url::Url; + +pub fn to_datetime_utc(ndt: NaiveDateTime) -> DateTime { + DateTime::::from_utc(ndt, Utc) +} + +pub fn naive_from_unix(time: i64) -> NaiveDateTime { + NaiveDateTime::from_timestamp(time, 0) +} + +pub fn convert_datetime(datetime: NaiveDateTime) -> DateTime { + let now = Local::now(); + DateTime::::from_utc(datetime, *now.offset()) +} + +pub fn is_email_regex(test: &str) -> bool { + EMAIL_REGEX.is_match(test) +} + +pub fn remove_slurs(test: &str) -> String { + SLUR_REGEX.replace_all(test, "*removed*").to_string() +} + +pub fn slur_check(test: &str) -> Result<(), Vec<&str>> { + let mut matches: Vec<&str> = SLUR_REGEX.find_iter(test).map(|mat| mat.as_str()).collect(); + + // Unique + matches.sort_unstable(); + matches.dedup(); + + if matches.is_empty() { + Ok(()) + } else { + Err(matches) + } +} + +pub fn slurs_vec_to_str(slurs: Vec<&str>) -> String { + let start = "No slurs - "; + let combined = &slurs.join(", "); + [start, combined].concat() +} + +pub fn generate_random_string() -> String { + thread_rng().sample_iter(&Alphanumeric).take(30).collect() +} + +pub fn send_email( + subject: &str, + to_email: &str, + to_username: &str, + html: &str, +) -> Result<(), String> { + let email_config = Settings::get().email.ok_or("no_email_setup")?; + + let email = Email::builder() + .to((to_email, to_username)) + .from(email_config.smtp_from_address.to_owned()) + .subject(subject) + .html(html) + .build() + .unwrap(); + + let mailer = if email_config.use_tls { + SmtpClient::new_simple(&email_config.smtp_server).unwrap() + } else { + SmtpClient::new(&email_config.smtp_server, ClientSecurity::None).unwrap() + } + .hello_name(ClientId::Domain(Settings::get().hostname)) + .smtp_utf8(true) + .authentication_mechanism(Mechanism::Plain) + .connection_reuse(ConnectionReuseParameters::ReuseUnlimited); + let mailer = if let (Some(login), Some(password)) = + (&email_config.smtp_login, &email_config.smtp_password) + { + mailer.credentials(Credentials::new(login.to_owned(), password.to_owned())) + } else { + mailer + }; + + let mut transport = mailer.transport(); + let result = transport.send(email.into()); + transport.close(); + + match result { + Ok(_) => Ok(()), + Err(e) => Err(e.to_string()), + } +} + +pub fn markdown_to_html(text: &str) -> String { + comrak::markdown_to_html(text, &comrak::ComrakOptions::default()) +} + +// TODO nothing is done with community / group webfingers yet, so just ignore those for now +#[derive(Clone, PartialEq, Eq, Hash)] +pub struct MentionData { + pub name: String, + pub domain: String, +} + +impl MentionData { + pub fn is_local(&self) -> bool { + Settings::get().hostname.eq(&self.domain) + } + pub fn full_name(&self) -> String { + format!("@{}@{}", &self.name, &self.domain) + } +} + +pub fn scrape_text_for_mentions(text: &str) -> Vec { + let mut out: Vec = Vec::new(); + for caps in MENTIONS_REGEX.captures_iter(text) { + out.push(MentionData { + name: caps["name"].to_string(), + domain: caps["domain"].to_string(), + }); + } + out.into_iter().unique().collect() +} + +pub fn is_valid_username(name: &str) -> bool { + VALID_USERNAME_REGEX.is_match(name) +} + +pub fn is_valid_community_name(name: &str) -> bool { + VALID_COMMUNITY_NAME_REGEX.is_match(name) +} + +pub fn is_valid_post_title(title: &str) -> bool { + VALID_POST_TITLE_REGEX.is_match(title) +} + +#[cfg(test)] +mod tests { + use crate::{ + is_email_regex, + is_valid_community_name, + is_valid_post_title, + is_valid_username, + remove_slurs, + scrape_text_for_mentions, + slur_check, + slurs_vec_to_str, + }; + + #[test] + fn test_mentions_regex() { + let text = "Just read a great blog post by [@tedu@honk.teduangst.com](/u/test). And another by !test_community@fish.teduangst.com . Another [@lemmy@lemmy-alpha:8540](/u/fish)"; + let mentions = scrape_text_for_mentions(text); + + assert_eq!(mentions[0].name, "tedu".to_string()); + assert_eq!(mentions[0].domain, "honk.teduangst.com".to_string()); + assert_eq!(mentions[1].domain, "lemmy-alpha:8540".to_string()); + } + + #[test] + fn test_email() { + assert!(is_email_regex("gush@gmail.com")); + assert!(!is_email_regex("nada_neutho")); + } + + #[test] + fn test_valid_register_username() { + assert!(is_valid_username("Hello_98")); + assert!(is_valid_username("ten")); + assert!(!is_valid_username("Hello-98")); + assert!(!is_valid_username("a")); + assert!(!is_valid_username("")); + } + + #[test] + fn test_valid_community_name() { + assert!(is_valid_community_name("example")); + assert!(is_valid_community_name("example_community")); + assert!(!is_valid_community_name("Example")); + assert!(!is_valid_community_name("Ex")); + assert!(!is_valid_community_name("")); + } + + #[test] + fn test_valid_post_title() { + assert!(is_valid_post_title("Post Title")); + assert!(is_valid_post_title(" POST TITLE 😃😃😃😃😃")); + assert!(!is_valid_post_title("\n \n \n \n ")); // tabs/spaces/newlines + } + + #[test] + fn test_slur_filter() { + let test = + "coons test dindu ladyboy tranny retardeds. Capitalized Niggerz. This is a bunch of other safe text."; + let slur_free = "No slurs here"; + assert_eq!( + remove_slurs(&test), + "*removed* test *removed* *removed* *removed* *removed*. Capitalized *removed*. This is a bunch of other safe text." + .to_string() + ); + + let has_slurs_vec = vec![ + "Niggerz", + "coons", + "dindu", + "ladyboy", + "retardeds", + "tranny", + ]; + let has_slurs_err_str = "No slurs - Niggerz, coons, dindu, ladyboy, retardeds, tranny"; + + assert_eq!(slur_check(test), Err(has_slurs_vec)); + assert_eq!(slur_check(slur_free), Ok(())); + if let Err(slur_vec) = slur_check(test) { + assert_eq!(&slurs_vec_to_str(slur_vec), has_slurs_err_str); + } + } + + // These helped with testing + // #[test] + // fn test_send_email() { + // let result = send_email("not a subject", "test_email@gmail.com", "ur user", "

HI there

"); + // assert!(result.is_ok()); + // } +} + +lazy_static! { + static ref EMAIL_REGEX: Regex = Regex::new(r"^[a-zA-Z0-9.!#$%&’*+/=?^_`{|}~-]+@[a-zA-Z0-9-]+(?:\.[a-zA-Z0-9-]+)*$").unwrap(); + static ref SLUR_REGEX: Regex = RegexBuilder::new(r"(fag(g|got|tard)?|maricos?|cock\s?sucker(s|ing)?|\bn(i|1)g(\b|g?(a|er)?(s|z)?)\b|dindu(s?)|mudslime?s?|kikes?|mongoloids?|towel\s*heads?|\bspi(c|k)s?\b|\bchinks?|niglets?|beaners?|\bnips?\b|\bcoons?\b|jungle\s*bunn(y|ies?)|jigg?aboo?s?|\bpakis?\b|rag\s*heads?|gooks?|cunts?|bitch(es|ing|y)?|puss(y|ies?)|twats?|feminazis?|whor(es?|ing)|\bslut(s|t?y)?|\btr(a|@)nn?(y|ies?)|ladyboy(s?)|\b(b|re|r)tard(ed)?s?)").case_insensitive(true).build().unwrap(); + static ref USERNAME_MATCHES_REGEX: Regex = Regex::new(r"/u/[a-zA-Z][0-9a-zA-Z_]*").unwrap(); + // TODO keep this old one, it didn't work with port well tho + // static ref MENTIONS_REGEX: Regex = Regex::new(r"@(?P[\w.]+)@(?P[a-zA-Z0-9._-]+\.[a-zA-Z0-9_-]+)").unwrap(); + static ref MENTIONS_REGEX: Regex = Regex::new(r"@(?P[\w.]+)@(?P[a-zA-Z0-9._:-]+)").unwrap(); + static ref VALID_USERNAME_REGEX: Regex = Regex::new(r"^[a-zA-Z0-9_]{3,20}$").unwrap(); + static ref VALID_COMMUNITY_NAME_REGEX: Regex = Regex::new(r"^[a-z0-9_]{3,20}$").unwrap(); + static ref VALID_POST_TITLE_REGEX: Regex = Regex::new(r".*\S.*").unwrap(); + pub static ref WEBFINGER_COMMUNITY_REGEX: Regex = Regex::new(&format!( + "^group:([a-z0-9_]{{3, 20}})@{}$", + Settings::get().hostname + )) + .unwrap(); + pub static ref WEBFINGER_USER_REGEX: Regex = Regex::new(&format!( + "^acct:([a-z0-9_]{{3, 20}})@{}$", + Settings::get().hostname + )) + .unwrap(); + pub static ref CACHE_CONTROL_REGEX: Regex = + Regex::new("^((text|image)/.+|application/javascript)$").unwrap(); +} + +pub struct Keypair { + pub private_key: String, + pub public_key: String, +} + +/// Generate the asymmetric keypair for ActivityPub HTTP signatures. +pub fn generate_actor_keypair() -> Result { + let rsa = Rsa::generate(2048)?; + let pkey = PKey::from_rsa(rsa)?; + let public_key = pkey.public_key_to_pem()?; + let private_key = pkey.private_key_to_pem_pkcs8()?; + let key_to_string = |key| match String::from_utf8(key) { + Ok(s) => Ok(s), + Err(e) => Err(Error::new( + ErrorKind::Other, + format!("Failed converting key to string: {}", e), + )), + }; + Ok(Keypair { + private_key: key_to_string(private_key)?, + public_key: key_to_string(public_key)?, + }) +} + +pub enum EndpointType { + Community, + User, + Post, + Comment, + PrivateMessage, +} + +pub fn get_apub_protocol_string() -> &'static str { + if Settings::get().federation.tls_enabled { + "https" + } else { + "http" + } +} + +/// Generates the ActivityPub ID for a given object type and ID. +pub fn make_apub_endpoint(endpoint_type: EndpointType, name: &str) -> Url { + let point = match endpoint_type { + EndpointType::Community => "c", + EndpointType::User => "u", + EndpointType::Post => "post", + EndpointType::Comment => "comment", + EndpointType::PrivateMessage => "private_message", + }; + + Url::parse(&format!( + "{}://{}/{}/{}", + get_apub_protocol_string(), + Settings::get().hostname, + point, + name + )) + .unwrap() +} diff --git a/server/src/settings.rs b/server/lemmy_utils/src/settings.rs similarity index 78% rename from server/src/settings.rs rename to server/lemmy_utils/src/settings.rs index 12ffaceab..0607974f9 100644 --- a/server/src/settings.rs +++ b/server/lemmy_utils/src/settings.rs @@ -1,7 +1,7 @@ -use crate::LemmyError; use config::{Config, ConfigError, Environment, File}; use serde::Deserialize; -use std::{env, fs, net::IpAddr, sync::RwLock}; +use std::{fs, io::Error, net::IpAddr, sync::RwLock}; +use std::env; static CONFIG_FILE_DEFAULTS: &str = "config/defaults.hjson"; static CONFIG_FILE: &str = "config/config.hjson"; @@ -76,12 +76,15 @@ impl Settings { /// First, defaults are loaded from CONFIG_FILE_DEFAULTS, then these values can be overwritten /// from CONFIG_FILE (optional). Finally, values from the environment (with prefix LEMMY) are /// added to the config. + /// + /// Note: The env var `LEMMY_DATABASE_URL` is parsed in + /// `server/lemmy_db/src/lib.rs::get_database_url_from_env()` fn init() -> Result { let mut s = Config::new(); s.merge(File::with_name(CONFIG_FILE_DEFAULTS))?; - s.merge(File::with_name(CONFIG_FILE).required(false))?; + s.merge(File::with_name(&Self::get_config_location()).required(false))?; // Add in settings from the environment (with a prefix of LEMMY) // Eg.. `LEMMY_DEBUG=1 ./target/app` would set the `debug` key @@ -98,32 +101,31 @@ impl Settings { SETTINGS.read().unwrap().to_owned() } - /// Returns the postgres connection url. If LEMMY_DATABASE_URL is set, that is used, - /// otherwise the connection url is generated from the config. pub fn get_database_url(&self) -> String { - match env::var("LEMMY_DATABASE_URL") { - Ok(url) => url, - Err(_) => format!( - "postgres://{}:{}@{}:{}/{}", - self.database.user, - self.database.password, - self.database.host, - self.database.port, - self.database.database - ), - } + format!( + "postgres://{}:{}@{}:{}/{}", + self.database.user, + self.database.password, + self.database.host, + self.database.port, + self.database.database + ) } pub fn api_endpoint(&self) -> String { format!("{}/api/v1", self.hostname) } - pub fn read_config_file() -> Result { - Ok(fs::read_to_string(CONFIG_FILE)?) + pub fn get_config_location() -> String { + env::var("LEMMY_CONFIG_LOCATION").unwrap_or_else(|_| CONFIG_FILE.to_string()) } - pub fn save_config_file(data: &str) -> Result { - fs::write(CONFIG_FILE, data)?; + pub fn read_config_file() -> Result { + fs::read_to_string(Self::get_config_location()) + } + + pub fn save_config_file(data: &str) -> Result { + fs::write(Self::get_config_location(), data)?; // Reload the new settings // From https://stackoverflow.com/questions/29654927/how-do-i-assign-a-string-to-a-mutable-static-variable/47181804#47181804 diff --git a/server/migrations/2020-07-08-202609_add_creator_published/down.sql b/server/migrations/2020-07-08-202609_add_creator_published/down.sql new file mode 100644 index 000000000..b8e4452e3 --- /dev/null +++ b/server/migrations/2020-07-08-202609_add_creator_published/down.sql @@ -0,0 +1,388 @@ +drop view user_mention_view; +drop view reply_fast_view; +drop view comment_fast_view; +drop view comment_view; + +drop view user_mention_fast_view; +drop table comment_aggregates_fast; +drop view comment_aggregates_view; + +create view comment_aggregates_view as +select + ct.*, + -- community details + p.community_id, + c.actor_id as community_actor_id, + c."local" as community_local, + c."name" as community_name, + -- creator details + u.banned as banned, + coalesce(cb.id, 0)::bool as banned_from_community, + u.actor_id as creator_actor_id, + u.local as creator_local, + u.name as creator_name, + u.avatar as creator_avatar, + -- score details + coalesce(cl.total, 0) as score, + coalesce(cl.up, 0) as upvotes, + coalesce(cl.down, 0) as downvotes, + hot_rank(coalesce(cl.total, 0), ct.published) as hot_rank +from comment ct +left join post p on ct.post_id = p.id +left join community c on p.community_id = c.id +left join user_ u on ct.creator_id = u.id +left join community_user_ban cb on ct.creator_id = cb.user_id and p.id = ct.post_id and p.community_id = cb.community_id +left join ( + select + l.comment_id as id, + sum(l.score) as total, + count(case when l.score = 1 then 1 else null end) as up, + count(case when l.score = -1 then 1 else null end) as down + from comment_like l + group by comment_id +) as cl on cl.id = ct.id; + +create or replace view comment_view as ( +select + cav.*, + us.user_id as user_id, + us.my_vote as my_vote, + us.is_subbed::bool as subscribed, + us.is_saved::bool as saved +from comment_aggregates_view cav +cross join lateral ( + select + u.id as user_id, + coalesce(cl.score, 0) as my_vote, + coalesce(cf.id, 0) as is_subbed, + coalesce(cs.id, 0) as is_saved + from user_ u + left join comment_like cl on u.id = cl.user_id and cav.id = cl.comment_id + left join comment_saved cs on u.id = cs.user_id and cs.comment_id = cav.id + left join community_follower cf on u.id = cf.user_id and cav.community_id = cf.community_id +) as us + +union all + +select + cav.*, + null as user_id, + null as my_vote, + null as subscribed, + null as saved +from comment_aggregates_view cav +); + +create table comment_aggregates_fast as select * from comment_aggregates_view; +alter table comment_aggregates_fast add primary key (id); + +create view comment_fast_view as +select + cav.*, + us.user_id as user_id, + us.my_vote as my_vote, + us.is_subbed::bool as subscribed, + us.is_saved::bool as saved +from comment_aggregates_fast cav +cross join lateral ( + select + u.id as user_id, + coalesce(cl.score, 0) as my_vote, + coalesce(cf.id, 0) as is_subbed, + coalesce(cs.id, 0) as is_saved + from user_ u + left join comment_like cl on u.id = cl.user_id and cav.id = cl.comment_id + left join comment_saved cs on u.id = cs.user_id and cs.comment_id = cav.id + left join community_follower cf on u.id = cf.user_id and cav.community_id = cf.community_id +) as us + +union all + +select + cav.*, + null as user_id, + null as my_vote, + null as subscribed, + null as saved +from comment_aggregates_fast cav; + +create view user_mention_view as +select + c.id, + um.id as user_mention_id, + c.creator_id, + c.creator_actor_id, + c.creator_local, + c.post_id, + c.parent_id, + c.content, + c.removed, + um.read, + c.published, + c.updated, + c.deleted, + c.community_id, + c.community_actor_id, + c.community_local, + c.community_name, + c.banned, + c.banned_from_community, + c.creator_name, + c.creator_avatar, + c.score, + c.upvotes, + c.downvotes, + c.hot_rank, + c.user_id, + c.my_vote, + c.saved, + um.recipient_id, + (select actor_id from user_ u where u.id = um.recipient_id) as recipient_actor_id, + (select local from user_ u where u.id = um.recipient_id) as recipient_local +from user_mention um, comment_view c +where um.comment_id = c.id; + +create view user_mention_fast_view as +select + ac.id, + um.id as user_mention_id, + ac.creator_id, + ac.creator_actor_id, + ac.creator_local, + ac.post_id, + ac.parent_id, + ac.content, + ac.removed, + um.read, + ac.published, + ac.updated, + ac.deleted, + ac.community_id, + ac.community_actor_id, + ac.community_local, + ac.community_name, + ac.banned, + ac.banned_from_community, + ac.creator_name, + ac.creator_avatar, + ac.score, + ac.upvotes, + ac.downvotes, + ac.hot_rank, + u.id as user_id, + coalesce(cl.score, 0) as my_vote, + (select cs.id::bool from comment_saved cs where u.id = cs.user_id and cs.comment_id = ac.id) as saved, + um.recipient_id, + (select actor_id from user_ u where u.id = um.recipient_id) as recipient_actor_id, + (select local from user_ u where u.id = um.recipient_id) as recipient_local +from user_ u +cross join ( + select + ca.* + from comment_aggregates_fast ca +) ac +left join comment_like cl on u.id = cl.user_id and ac.id = cl.comment_id +left join user_mention um on um.comment_id = ac.id + +union all + +select + ac.id, + um.id as user_mention_id, + ac.creator_id, + ac.creator_actor_id, + ac.creator_local, + ac.post_id, + ac.parent_id, + ac.content, + ac.removed, + um.read, + ac.published, + ac.updated, + ac.deleted, + ac.community_id, + ac.community_actor_id, + ac.community_local, + ac.community_name, + ac.banned, + ac.banned_from_community, + ac.creator_name, + ac.creator_avatar, + ac.score, + ac.upvotes, + ac.downvotes, + ac.hot_rank, + null as user_id, + null as my_vote, + null as saved, + um.recipient_id, + (select actor_id from user_ u where u.id = um.recipient_id) as recipient_actor_id, + (select local from user_ u where u.id = um.recipient_id) as recipient_local +from comment_aggregates_fast ac +left join user_mention um on um.comment_id = ac.id +; + +-- Do the reply_view referencing the comment_fast_view +create view reply_fast_view as +with closereply as ( + select + c2.id, + c2.creator_id as sender_id, + c.creator_id as recipient_id + from comment c + inner join comment c2 on c.id = c2.parent_id + where c2.creator_id != c.creator_id + -- Do union where post is null + union + select + c.id, + c.creator_id as sender_id, + p.creator_id as recipient_id + from comment c, post p + where c.post_id = p.id and c.parent_id is null and c.creator_id != p.creator_id +) +select cv.*, +closereply.recipient_id +from comment_fast_view cv, closereply +where closereply.id = cv.id +; + +-- add creator_published to the post view +drop view post_fast_view; +drop table post_aggregates_fast; +drop view post_view; +drop view post_aggregates_view; + +create view post_aggregates_view as +select + p.*, + -- creator details + u.actor_id as creator_actor_id, + u."local" as creator_local, + u."name" as creator_name, + u.avatar as creator_avatar, + u.banned as banned, + cb.id::bool as banned_from_community, + -- community details + c.actor_id as community_actor_id, + c."local" as community_local, + c."name" as community_name, + c.removed as community_removed, + c.deleted as community_deleted, + c.nsfw as community_nsfw, + -- post score data/comment count + coalesce(ct.comments, 0) as number_of_comments, + coalesce(pl.score, 0) as score, + coalesce(pl.upvotes, 0) as upvotes, + coalesce(pl.downvotes, 0) as downvotes, + hot_rank( + coalesce(pl.score , 0), ( + case + when (p.published < ('now'::timestamp - '1 month'::interval)) + then p.published + else greatest(ct.recent_comment_time, p.published) + end + ) + ) as hot_rank, + ( + case + when (p.published < ('now'::timestamp - '1 month'::interval)) + then p.published + else greatest(ct.recent_comment_time, p.published) + end + ) as newest_activity_time +from post p +left join user_ u on p.creator_id = u.id +left join community_user_ban cb on p.creator_id = cb.user_id and p.community_id = cb.community_id +left join community c on p.community_id = c.id +left join ( + select + post_id, + count(*) as comments, + max(published) as recent_comment_time + from comment + group by post_id +) ct on ct.post_id = p.id +left join ( + select + post_id, + sum(score) as score, + sum(score) filter (where score = 1) as upvotes, + -sum(score) filter (where score = -1) as downvotes + from post_like + group by post_id +) pl on pl.post_id = p.id +order by p.id; + +create view post_view as +select + pav.*, + us.id as user_id, + us.user_vote as my_vote, + us.is_subbed::bool as subscribed, + us.is_read::bool as read, + us.is_saved::bool as saved +from post_aggregates_view pav +cross join lateral ( + select + u.id, + coalesce(cf.community_id, 0) as is_subbed, + coalesce(pr.post_id, 0) as is_read, + coalesce(ps.post_id, 0) as is_saved, + coalesce(pl.score, 0) as user_vote + from user_ u + left join community_user_ban cb on u.id = cb.user_id and cb.community_id = pav.community_id + left join community_follower cf on u.id = cf.user_id and cf.community_id = pav.community_id + left join post_read pr on u.id = pr.user_id and pr.post_id = pav.id + left join post_saved ps on u.id = ps.user_id and ps.post_id = pav.id + left join post_like pl on u.id = pl.user_id and pav.id = pl.post_id +) as us + +union all + +select +pav.*, +null as user_id, +null as my_vote, +null as subscribed, +null as read, +null as saved +from post_aggregates_view pav; + +create table post_aggregates_fast as select * from post_aggregates_view; +alter table post_aggregates_fast add primary key (id); + +create view post_fast_view as +select + pav.*, + us.id as user_id, + us.user_vote as my_vote, + us.is_subbed::bool as subscribed, + us.is_read::bool as read, + us.is_saved::bool as saved +from post_aggregates_fast pav +cross join lateral ( + select + u.id, + coalesce(cf.community_id, 0) as is_subbed, + coalesce(pr.post_id, 0) as is_read, + coalesce(ps.post_id, 0) as is_saved, + coalesce(pl.score, 0) as user_vote + from user_ u + left join community_user_ban cb on u.id = cb.user_id and cb.community_id = pav.community_id + left join community_follower cf on u.id = cf.user_id and cf.community_id = pav.community_id + left join post_read pr on u.id = pr.user_id and pr.post_id = pav.id + left join post_saved ps on u.id = ps.user_id and ps.post_id = pav.id + left join post_like pl on u.id = pl.user_id and pav.id = pl.post_id +) as us + +union all + +select +pav.*, +null as user_id, +null as my_vote, +null as subscribed, +null as read, +null as saved +from post_aggregates_fast pav; \ No newline at end of file diff --git a/server/migrations/2020-07-08-202609_add_creator_published/up.sql b/server/migrations/2020-07-08-202609_add_creator_published/up.sql new file mode 100644 index 000000000..1f2b59ea9 --- /dev/null +++ b/server/migrations/2020-07-08-202609_add_creator_published/up.sql @@ -0,0 +1,390 @@ +drop view user_mention_view; +drop view reply_fast_view; +drop view comment_fast_view; +drop view comment_view; + +drop view user_mention_fast_view; +drop table comment_aggregates_fast; +drop view comment_aggregates_view; + +create view comment_aggregates_view as +select + ct.*, + -- community details + p.community_id, + c.actor_id as community_actor_id, + c."local" as community_local, + c."name" as community_name, + -- creator details + u.banned as banned, + coalesce(cb.id, 0)::bool as banned_from_community, + u.actor_id as creator_actor_id, + u.local as creator_local, + u.name as creator_name, + u.published as creator_published, + u.avatar as creator_avatar, + -- score details + coalesce(cl.total, 0) as score, + coalesce(cl.up, 0) as upvotes, + coalesce(cl.down, 0) as downvotes, + hot_rank(coalesce(cl.total, 0), ct.published) as hot_rank +from comment ct +left join post p on ct.post_id = p.id +left join community c on p.community_id = c.id +left join user_ u on ct.creator_id = u.id +left join community_user_ban cb on ct.creator_id = cb.user_id and p.id = ct.post_id and p.community_id = cb.community_id +left join ( + select + l.comment_id as id, + sum(l.score) as total, + count(case when l.score = 1 then 1 else null end) as up, + count(case when l.score = -1 then 1 else null end) as down + from comment_like l + group by comment_id +) as cl on cl.id = ct.id; + +create or replace view comment_view as ( +select + cav.*, + us.user_id as user_id, + us.my_vote as my_vote, + us.is_subbed::bool as subscribed, + us.is_saved::bool as saved +from comment_aggregates_view cav +cross join lateral ( + select + u.id as user_id, + coalesce(cl.score, 0) as my_vote, + coalesce(cf.id, 0) as is_subbed, + coalesce(cs.id, 0) as is_saved + from user_ u + left join comment_like cl on u.id = cl.user_id and cav.id = cl.comment_id + left join comment_saved cs on u.id = cs.user_id and cs.comment_id = cav.id + left join community_follower cf on u.id = cf.user_id and cav.community_id = cf.community_id +) as us + +union all + +select + cav.*, + null as user_id, + null as my_vote, + null as subscribed, + null as saved +from comment_aggregates_view cav +); + +create table comment_aggregates_fast as select * from comment_aggregates_view; +alter table comment_aggregates_fast add primary key (id); + +create view comment_fast_view as +select + cav.*, + us.user_id as user_id, + us.my_vote as my_vote, + us.is_subbed::bool as subscribed, + us.is_saved::bool as saved +from comment_aggregates_fast cav +cross join lateral ( + select + u.id as user_id, + coalesce(cl.score, 0) as my_vote, + coalesce(cf.id, 0) as is_subbed, + coalesce(cs.id, 0) as is_saved + from user_ u + left join comment_like cl on u.id = cl.user_id and cav.id = cl.comment_id + left join comment_saved cs on u.id = cs.user_id and cs.comment_id = cav.id + left join community_follower cf on u.id = cf.user_id and cav.community_id = cf.community_id +) as us + +union all + +select + cav.*, + null as user_id, + null as my_vote, + null as subscribed, + null as saved +from comment_aggregates_fast cav; + +create view user_mention_view as +select + c.id, + um.id as user_mention_id, + c.creator_id, + c.creator_actor_id, + c.creator_local, + c.post_id, + c.parent_id, + c.content, + c.removed, + um.read, + c.published, + c.updated, + c.deleted, + c.community_id, + c.community_actor_id, + c.community_local, + c.community_name, + c.banned, + c.banned_from_community, + c.creator_name, + c.creator_avatar, + c.score, + c.upvotes, + c.downvotes, + c.hot_rank, + c.user_id, + c.my_vote, + c.saved, + um.recipient_id, + (select actor_id from user_ u where u.id = um.recipient_id) as recipient_actor_id, + (select local from user_ u where u.id = um.recipient_id) as recipient_local +from user_mention um, comment_view c +where um.comment_id = c.id; + +create view user_mention_fast_view as +select + ac.id, + um.id as user_mention_id, + ac.creator_id, + ac.creator_actor_id, + ac.creator_local, + ac.post_id, + ac.parent_id, + ac.content, + ac.removed, + um.read, + ac.published, + ac.updated, + ac.deleted, + ac.community_id, + ac.community_actor_id, + ac.community_local, + ac.community_name, + ac.banned, + ac.banned_from_community, + ac.creator_name, + ac.creator_avatar, + ac.score, + ac.upvotes, + ac.downvotes, + ac.hot_rank, + u.id as user_id, + coalesce(cl.score, 0) as my_vote, + (select cs.id::bool from comment_saved cs where u.id = cs.user_id and cs.comment_id = ac.id) as saved, + um.recipient_id, + (select actor_id from user_ u where u.id = um.recipient_id) as recipient_actor_id, + (select local from user_ u where u.id = um.recipient_id) as recipient_local +from user_ u +cross join ( + select + ca.* + from comment_aggregates_fast ca +) ac +left join comment_like cl on u.id = cl.user_id and ac.id = cl.comment_id +left join user_mention um on um.comment_id = ac.id + +union all + +select + ac.id, + um.id as user_mention_id, + ac.creator_id, + ac.creator_actor_id, + ac.creator_local, + ac.post_id, + ac.parent_id, + ac.content, + ac.removed, + um.read, + ac.published, + ac.updated, + ac.deleted, + ac.community_id, + ac.community_actor_id, + ac.community_local, + ac.community_name, + ac.banned, + ac.banned_from_community, + ac.creator_name, + ac.creator_avatar, + ac.score, + ac.upvotes, + ac.downvotes, + ac.hot_rank, + null as user_id, + null as my_vote, + null as saved, + um.recipient_id, + (select actor_id from user_ u where u.id = um.recipient_id) as recipient_actor_id, + (select local from user_ u where u.id = um.recipient_id) as recipient_local +from comment_aggregates_fast ac +left join user_mention um on um.comment_id = ac.id +; + +-- Do the reply_view referencing the comment_fast_view +create view reply_fast_view as +with closereply as ( + select + c2.id, + c2.creator_id as sender_id, + c.creator_id as recipient_id + from comment c + inner join comment c2 on c.id = c2.parent_id + where c2.creator_id != c.creator_id + -- Do union where post is null + union + select + c.id, + c.creator_id as sender_id, + p.creator_id as recipient_id + from comment c, post p + where c.post_id = p.id and c.parent_id is null and c.creator_id != p.creator_id +) +select cv.*, +closereply.recipient_id +from comment_fast_view cv, closereply +where closereply.id = cv.id +; + +-- add creator_published to the post view +drop view post_fast_view; +drop table post_aggregates_fast; +drop view post_view; +drop view post_aggregates_view; + +create view post_aggregates_view as +select + p.*, + -- creator details + u.actor_id as creator_actor_id, + u."local" as creator_local, + u."name" as creator_name, + u.published as creator_published, + u.avatar as creator_avatar, + u.banned as banned, + cb.id::bool as banned_from_community, + -- community details + c.actor_id as community_actor_id, + c."local" as community_local, + c."name" as community_name, + c.removed as community_removed, + c.deleted as community_deleted, + c.nsfw as community_nsfw, + -- post score data/comment count + coalesce(ct.comments, 0) as number_of_comments, + coalesce(pl.score, 0) as score, + coalesce(pl.upvotes, 0) as upvotes, + coalesce(pl.downvotes, 0) as downvotes, + hot_rank( + coalesce(pl.score , 0), ( + case + when (p.published < ('now'::timestamp - '1 month'::interval)) + then p.published + else greatest(ct.recent_comment_time, p.published) + end + ) + ) as hot_rank, + ( + case + when (p.published < ('now'::timestamp - '1 month'::interval)) + then p.published + else greatest(ct.recent_comment_time, p.published) + end + ) as newest_activity_time +from post p +left join user_ u on p.creator_id = u.id +left join community_user_ban cb on p.creator_id = cb.user_id and p.community_id = cb.community_id +left join community c on p.community_id = c.id +left join ( + select + post_id, + count(*) as comments, + max(published) as recent_comment_time + from comment + group by post_id +) ct on ct.post_id = p.id +left join ( + select + post_id, + sum(score) as score, + sum(score) filter (where score = 1) as upvotes, + -sum(score) filter (where score = -1) as downvotes + from post_like + group by post_id +) pl on pl.post_id = p.id +order by p.id; + +create view post_view as +select + pav.*, + us.id as user_id, + us.user_vote as my_vote, + us.is_subbed::bool as subscribed, + us.is_read::bool as read, + us.is_saved::bool as saved +from post_aggregates_view pav +cross join lateral ( + select + u.id, + coalesce(cf.community_id, 0) as is_subbed, + coalesce(pr.post_id, 0) as is_read, + coalesce(ps.post_id, 0) as is_saved, + coalesce(pl.score, 0) as user_vote + from user_ u + left join community_user_ban cb on u.id = cb.user_id and cb.community_id = pav.community_id + left join community_follower cf on u.id = cf.user_id and cf.community_id = pav.community_id + left join post_read pr on u.id = pr.user_id and pr.post_id = pav.id + left join post_saved ps on u.id = ps.user_id and ps.post_id = pav.id + left join post_like pl on u.id = pl.user_id and pav.id = pl.post_id +) as us + +union all + +select +pav.*, +null as user_id, +null as my_vote, +null as subscribed, +null as read, +null as saved +from post_aggregates_view pav; + +create table post_aggregates_fast as select * from post_aggregates_view; +alter table post_aggregates_fast add primary key (id); + +create view post_fast_view as +select + pav.*, + us.id as user_id, + us.user_vote as my_vote, + us.is_subbed::bool as subscribed, + us.is_read::bool as read, + us.is_saved::bool as saved +from post_aggregates_fast pav +cross join lateral ( + select + u.id, + coalesce(cf.community_id, 0) as is_subbed, + coalesce(pr.post_id, 0) as is_read, + coalesce(ps.post_id, 0) as is_saved, + coalesce(pl.score, 0) as user_vote + from user_ u + left join community_user_ban cb on u.id = cb.user_id and cb.community_id = pav.community_id + left join community_follower cf on u.id = cf.user_id and cf.community_id = pav.community_id + left join post_read pr on u.id = pr.user_id and pr.post_id = pav.id + left join post_saved ps on u.id = ps.user_id and ps.post_id = pav.id + left join post_like pl on u.id = pl.user_id and pav.id = pl.post_id +) as us + +union all + +select +pav.*, +null as user_id, +null as my_vote, +null as subscribed, +null as read, +null as saved +from post_aggregates_fast pav; \ No newline at end of file diff --git a/server/migrations/2020-07-12-100442_add_post_title_to_comments_view/down.sql b/server/migrations/2020-07-12-100442_add_post_title_to_comments_view/down.sql new file mode 100644 index 000000000..b7c9d51ef --- /dev/null +++ b/server/migrations/2020-07-12-100442_add_post_title_to_comments_view/down.sql @@ -0,0 +1,249 @@ +drop view user_mention_view; +drop view reply_fast_view; +drop view comment_fast_view; +drop view comment_view; + +drop view user_mention_fast_view; +drop table comment_aggregates_fast; +drop view comment_aggregates_view; + +create view comment_aggregates_view as +select + ct.*, + -- community details + p.community_id, + c.actor_id as community_actor_id, + c."local" as community_local, + c."name" as community_name, + -- creator details + u.banned as banned, + coalesce(cb.id, 0)::bool as banned_from_community, + u.actor_id as creator_actor_id, + u.local as creator_local, + u.name as creator_name, + u.published as creator_published, + u.avatar as creator_avatar, + -- score details + coalesce(cl.total, 0) as score, + coalesce(cl.up, 0) as upvotes, + coalesce(cl.down, 0) as downvotes, + hot_rank(coalesce(cl.total, 0), ct.published) as hot_rank +from comment ct +left join post p on ct.post_id = p.id +left join community c on p.community_id = c.id +left join user_ u on ct.creator_id = u.id +left join community_user_ban cb on ct.creator_id = cb.user_id and p.id = ct.post_id and p.community_id = cb.community_id +left join ( + select + l.comment_id as id, + sum(l.score) as total, + count(case when l.score = 1 then 1 else null end) as up, + count(case when l.score = -1 then 1 else null end) as down + from comment_like l + group by comment_id +) as cl on cl.id = ct.id; + +create or replace view comment_view as ( +select + cav.*, + us.user_id as user_id, + us.my_vote as my_vote, + us.is_subbed::bool as subscribed, + us.is_saved::bool as saved +from comment_aggregates_view cav +cross join lateral ( + select + u.id as user_id, + coalesce(cl.score, 0) as my_vote, + coalesce(cf.id, 0) as is_subbed, + coalesce(cs.id, 0) as is_saved + from user_ u + left join comment_like cl on u.id = cl.user_id and cav.id = cl.comment_id + left join comment_saved cs on u.id = cs.user_id and cs.comment_id = cav.id + left join community_follower cf on u.id = cf.user_id and cav.community_id = cf.community_id +) as us + +union all + +select + cav.*, + null as user_id, + null as my_vote, + null as subscribed, + null as saved +from comment_aggregates_view cav +); + +create table comment_aggregates_fast as select * from comment_aggregates_view; +alter table comment_aggregates_fast add primary key (id); + +create view comment_fast_view as +select + cav.*, + us.user_id as user_id, + us.my_vote as my_vote, + us.is_subbed::bool as subscribed, + us.is_saved::bool as saved +from comment_aggregates_fast cav +cross join lateral ( + select + u.id as user_id, + coalesce(cl.score, 0) as my_vote, + coalesce(cf.id, 0) as is_subbed, + coalesce(cs.id, 0) as is_saved + from user_ u + left join comment_like cl on u.id = cl.user_id and cav.id = cl.comment_id + left join comment_saved cs on u.id = cs.user_id and cs.comment_id = cav.id + left join community_follower cf on u.id = cf.user_id and cav.community_id = cf.community_id +) as us + +union all + +select + cav.*, + null as user_id, + null as my_vote, + null as subscribed, + null as saved +from comment_aggregates_fast cav; + +create view user_mention_view as +select + c.id, + um.id as user_mention_id, + c.creator_id, + c.creator_actor_id, + c.creator_local, + c.post_id, + c.parent_id, + c.content, + c.removed, + um.read, + c.published, + c.updated, + c.deleted, + c.community_id, + c.community_actor_id, + c.community_local, + c.community_name, + c.banned, + c.banned_from_community, + c.creator_name, + c.creator_avatar, + c.score, + c.upvotes, + c.downvotes, + c.hot_rank, + c.user_id, + c.my_vote, + c.saved, + um.recipient_id, + (select actor_id from user_ u where u.id = um.recipient_id) as recipient_actor_id, + (select local from user_ u where u.id = um.recipient_id) as recipient_local +from user_mention um, comment_view c +where um.comment_id = c.id; + +create view user_mention_fast_view as +select + ac.id, + um.id as user_mention_id, + ac.creator_id, + ac.creator_actor_id, + ac.creator_local, + ac.post_id, + ac.parent_id, + ac.content, + ac.removed, + um.read, + ac.published, + ac.updated, + ac.deleted, + ac.community_id, + ac.community_actor_id, + ac.community_local, + ac.community_name, + ac.banned, + ac.banned_from_community, + ac.creator_name, + ac.creator_avatar, + ac.score, + ac.upvotes, + ac.downvotes, + ac.hot_rank, + u.id as user_id, + coalesce(cl.score, 0) as my_vote, + (select cs.id::bool from comment_saved cs where u.id = cs.user_id and cs.comment_id = ac.id) as saved, + um.recipient_id, + (select actor_id from user_ u where u.id = um.recipient_id) as recipient_actor_id, + (select local from user_ u where u.id = um.recipient_id) as recipient_local +from user_ u +cross join ( + select + ca.* + from comment_aggregates_fast ca +) ac +left join comment_like cl on u.id = cl.user_id and ac.id = cl.comment_id +left join user_mention um on um.comment_id = ac.id + +union all + +select + ac.id, + um.id as user_mention_id, + ac.creator_id, + ac.creator_actor_id, + ac.creator_local, + ac.post_id, + ac.parent_id, + ac.content, + ac.removed, + um.read, + ac.published, + ac.updated, + ac.deleted, + ac.community_id, + ac.community_actor_id, + ac.community_local, + ac.community_name, + ac.banned, + ac.banned_from_community, + ac.creator_name, + ac.creator_avatar, + ac.score, + ac.upvotes, + ac.downvotes, + ac.hot_rank, + null as user_id, + null as my_vote, + null as saved, + um.recipient_id, + (select actor_id from user_ u where u.id = um.recipient_id) as recipient_actor_id, + (select local from user_ u where u.id = um.recipient_id) as recipient_local +from comment_aggregates_fast ac +left join user_mention um on um.comment_id = ac.id +; + +-- Do the reply_view referencing the comment_fast_view +create view reply_fast_view as +with closereply as ( + select + c2.id, + c2.creator_id as sender_id, + c.creator_id as recipient_id + from comment c + inner join comment c2 on c.id = c2.parent_id + where c2.creator_id != c.creator_id + -- Do union where post is null + union + select + c.id, + c.creator_id as sender_id, + p.creator_id as recipient_id + from comment c, post p + where c.post_id = p.id and c.parent_id is null and c.creator_id != p.creator_id +) +select cv.*, +closereply.recipient_id +from comment_fast_view cv, closereply +where closereply.id = cv.id +; \ No newline at end of file diff --git a/server/migrations/2020-07-12-100442_add_post_title_to_comments_view/up.sql b/server/migrations/2020-07-12-100442_add_post_title_to_comments_view/up.sql new file mode 100644 index 000000000..4cfa7edbc --- /dev/null +++ b/server/migrations/2020-07-12-100442_add_post_title_to_comments_view/up.sql @@ -0,0 +1,254 @@ +drop view user_mention_view; +drop view reply_fast_view; +drop view comment_fast_view; +drop view comment_view; + +drop view user_mention_fast_view; +drop table comment_aggregates_fast; +drop view comment_aggregates_view; + +create view comment_aggregates_view as +select + ct.*, + -- post details + p."name" as post_name, + p.community_id, + -- community details + c.actor_id as community_actor_id, + c."local" as community_local, + c."name" as community_name, + -- creator details + u.banned as banned, + coalesce(cb.id, 0)::bool as banned_from_community, + u.actor_id as creator_actor_id, + u.local as creator_local, + u.name as creator_name, + u.published as creator_published, + u.avatar as creator_avatar, + -- score details + coalesce(cl.total, 0) as score, + coalesce(cl.up, 0) as upvotes, + coalesce(cl.down, 0) as downvotes, + hot_rank(coalesce(cl.total, 0), ct.published) as hot_rank +from comment ct +left join post p on ct.post_id = p.id +left join community c on p.community_id = c.id +left join user_ u on ct.creator_id = u.id +left join community_user_ban cb on ct.creator_id = cb.user_id and p.id = ct.post_id and p.community_id = cb.community_id +left join ( + select + l.comment_id as id, + sum(l.score) as total, + count(case when l.score = 1 then 1 else null end) as up, + count(case when l.score = -1 then 1 else null end) as down + from comment_like l + group by comment_id +) as cl on cl.id = ct.id; + +create or replace view comment_view as ( +select + cav.*, + us.user_id as user_id, + us.my_vote as my_vote, + us.is_subbed::bool as subscribed, + us.is_saved::bool as saved +from comment_aggregates_view cav +cross join lateral ( + select + u.id as user_id, + coalesce(cl.score, 0) as my_vote, + coalesce(cf.id, 0) as is_subbed, + coalesce(cs.id, 0) as is_saved + from user_ u + left join comment_like cl on u.id = cl.user_id and cav.id = cl.comment_id + left join comment_saved cs on u.id = cs.user_id and cs.comment_id = cav.id + left join community_follower cf on u.id = cf.user_id and cav.community_id = cf.community_id +) as us + +union all + +select + cav.*, + null as user_id, + null as my_vote, + null as subscribed, + null as saved +from comment_aggregates_view cav +); + +create table comment_aggregates_fast as select * from comment_aggregates_view; +alter table comment_aggregates_fast add primary key (id); + +create view comment_fast_view as +select + cav.*, + us.user_id as user_id, + us.my_vote as my_vote, + us.is_subbed::bool as subscribed, + us.is_saved::bool as saved +from comment_aggregates_fast cav +cross join lateral ( + select + u.id as user_id, + coalesce(cl.score, 0) as my_vote, + coalesce(cf.id, 0) as is_subbed, + coalesce(cs.id, 0) as is_saved + from user_ u + left join comment_like cl on u.id = cl.user_id and cav.id = cl.comment_id + left join comment_saved cs on u.id = cs.user_id and cs.comment_id = cav.id + left join community_follower cf on u.id = cf.user_id and cav.community_id = cf.community_id +) as us + +union all + +select + cav.*, + null as user_id, + null as my_vote, + null as subscribed, + null as saved +from comment_aggregates_fast cav; + +create view user_mention_view as +select + c.id, + um.id as user_mention_id, + c.creator_id, + c.creator_actor_id, + c.creator_local, + c.post_id, + c.post_name, + c.parent_id, + c.content, + c.removed, + um.read, + c.published, + c.updated, + c.deleted, + c.community_id, + c.community_actor_id, + c.community_local, + c.community_name, + c.banned, + c.banned_from_community, + c.creator_name, + c.creator_avatar, + c.score, + c.upvotes, + c.downvotes, + c.hot_rank, + c.user_id, + c.my_vote, + c.saved, + um.recipient_id, + (select actor_id from user_ u where u.id = um.recipient_id) as recipient_actor_id, + (select local from user_ u where u.id = um.recipient_id) as recipient_local +from user_mention um, comment_view c +where um.comment_id = c.id; + +create view user_mention_fast_view as +select + ac.id, + um.id as user_mention_id, + ac.creator_id, + ac.creator_actor_id, + ac.creator_local, + ac.post_id, + ac.post_name, + ac.parent_id, + ac.content, + ac.removed, + um.read, + ac.published, + ac.updated, + ac.deleted, + ac.community_id, + ac.community_actor_id, + ac.community_local, + ac.community_name, + ac.banned, + ac.banned_from_community, + ac.creator_name, + ac.creator_avatar, + ac.score, + ac.upvotes, + ac.downvotes, + ac.hot_rank, + u.id as user_id, + coalesce(cl.score, 0) as my_vote, + (select cs.id::bool from comment_saved cs where u.id = cs.user_id and cs.comment_id = ac.id) as saved, + um.recipient_id, + (select actor_id from user_ u where u.id = um.recipient_id) as recipient_actor_id, + (select local from user_ u where u.id = um.recipient_id) as recipient_local +from user_ u +cross join ( + select + ca.* + from comment_aggregates_fast ca +) ac +left join comment_like cl on u.id = cl.user_id and ac.id = cl.comment_id +left join user_mention um on um.comment_id = ac.id + +union all + +select + ac.id, + um.id as user_mention_id, + ac.creator_id, + ac.creator_actor_id, + ac.creator_local, + ac.post_id, + ac.post_name, + ac.parent_id, + ac.content, + ac.removed, + um.read, + ac.published, + ac.updated, + ac.deleted, + ac.community_id, + ac.community_actor_id, + ac.community_local, + ac.community_name, + ac.banned, + ac.banned_from_community, + ac.creator_name, + ac.creator_avatar, + ac.score, + ac.upvotes, + ac.downvotes, + ac.hot_rank, + null as user_id, + null as my_vote, + null as saved, + um.recipient_id, + (select actor_id from user_ u where u.id = um.recipient_id) as recipient_actor_id, + (select local from user_ u where u.id = um.recipient_id) as recipient_local +from comment_aggregates_fast ac +left join user_mention um on um.comment_id = ac.id +; + +-- Do the reply_view referencing the comment_fast_view +create view reply_fast_view as +with closereply as ( + select + c2.id, + c2.creator_id as sender_id, + c.creator_id as recipient_id + from comment c + inner join comment c2 on c.id = c2.parent_id + where c2.creator_id != c.creator_id + -- Do union where post is null + union + select + c.id, + c.creator_id as sender_id, + p.creator_id as recipient_id + from comment c, post p + where c.post_id = p.id and c.parent_id is null and c.creator_id != p.creator_id +) +select cv.*, +closereply.recipient_id +from comment_fast_view cv, closereply +where closereply.id = cv.id +; \ No newline at end of file diff --git a/server/src/api/claims.rs b/server/src/api/claims.rs new file mode 100644 index 000000000..eec9d1a71 --- /dev/null +++ b/server/src/api/claims.rs @@ -0,0 +1,73 @@ +use diesel::{result::Error, PgConnection}; +use jsonwebtoken::{decode, encode, DecodingKey, EncodingKey, Header, TokenData, Validation}; +use lemmy_db::{user::User_, Crud}; +use lemmy_utils::{is_email_regex, settings::Settings}; +use serde::{Deserialize, Serialize}; + +type Jwt = String; + +#[derive(Debug, Serialize, Deserialize)] +pub struct Claims { + pub id: i32, + pub username: String, + pub iss: String, + pub show_nsfw: bool, + pub theme: String, + pub default_sort_type: i16, + pub default_listing_type: i16, + pub lang: String, + pub avatar: Option, + pub show_avatars: bool, +} + +impl Claims { + pub fn decode(jwt: &str) -> Result, jsonwebtoken::errors::Error> { + let v = Validation { + validate_exp: false, + ..Validation::default() + }; + decode::( + &jwt, + &DecodingKey::from_secret(Settings::get().jwt_secret.as_ref()), + &v, + ) + } + + pub fn jwt(user: User_, hostname: String) -> Jwt { + let my_claims = Claims { + id: user.id, + username: user.name.to_owned(), + iss: hostname, + show_nsfw: user.show_nsfw, + theme: user.theme.to_owned(), + default_sort_type: user.default_sort_type, + default_listing_type: user.default_listing_type, + lang: user.lang.to_owned(), + avatar: user.avatar.to_owned(), + show_avatars: user.show_avatars.to_owned(), + }; + encode( + &Header::default(), + &my_claims, + &EncodingKey::from_secret(Settings::get().jwt_secret.as_ref()), + ) + .unwrap() + } + + // TODO: move these into user? + pub fn find_by_email_or_username( + conn: &PgConnection, + username_or_email: &str, + ) -> Result { + if is_email_regex(username_or_email) { + User_::find_by_email(conn, username_or_email) + } else { + User_::find_by_username(conn, username_or_email) + } + } + + pub fn find_by_jwt(conn: &PgConnection, jwt: &str) -> Result { + let claims: Claims = Claims::decode(&jwt).expect("Invalid token").claims; + User_::read(&conn, claims.id) + } +} diff --git a/server/src/api/comment.rs b/server/src/api/comment.rs index c7406b370..2007542fa 100644 --- a/server/src/api/comment.rs +++ b/server/src/api/comment.rs @@ -1,28 +1,7 @@ use crate::{ - api::{APIError, Oper, Perform}, + api::{claims::Claims, APIError, Oper, Perform}, apub::{ApubLikeableType, ApubObjectType}, blocking, - db::{ - comment::*, - comment_view::*, - community_view::*, - moderator::*, - post::*, - site_view::*, - user::*, - user_mention::*, - user_view::*, - Crud, - Likeable, - ListingType, - Saveable, - SortType, - }, - naive_now, - remove_slurs, - scrape_text_for_mentions, - send_email, - settings::Settings, websocket::{ server::{JoinCommunityRoom, SendComment}, UserOperation, @@ -30,6 +9,31 @@ use crate::{ }, DbPool, LemmyError, +}; +use lemmy_db::{ + comment::*, + comment_view::*, + community_view::*, + moderator::*, + naive_now, + post::*, + site_view::*, + user::*, + user_mention::*, + user_view::*, + Crud, + Likeable, + ListingType, + Saveable, + SortType, +}; +use lemmy_utils::{ + make_apub_endpoint, + remove_slurs, + scrape_text_for_mentions, + send_email, + settings::Settings, + EndpointType, MentionData, }; use log::error; @@ -155,7 +159,9 @@ impl Perform for Oper { let inserted_comment_id = inserted_comment.id; let updated_comment: Comment = match blocking(pool, move |conn| { - Comment::update_ap_id(&conn, inserted_comment_id) + let apub_id = + make_apub_endpoint(EndpointType::Comment, &inserted_comment_id.to_string()).to_string(); + Comment::update_ap_id(&conn, inserted_comment_id, apub_id) }) .await? { diff --git a/server/src/api/community.rs b/server/src/api/community.rs index 02071c577..e703dcf41 100644 --- a/server/src/api/community.rs +++ b/server/src/api/community.rs @@ -1,26 +1,24 @@ use super::*; use crate::{ - api::{APIError, Oper, Perform}, - apub::{ - extensions::signatures::generate_actor_keypair, - make_apub_endpoint, - ActorType, - EndpointType, - }, + api::{claims::Claims, APIError, Oper, Perform}, + apub::ActorType, blocking, - db::{Bannable, Crud, Followable, Joinable, SortType}, - is_valid_community_name, - naive_from_unix, - naive_now, - slur_check, - slurs_vec_to_str, websocket::{ server::{JoinCommunityRoom, SendCommunityRoomMessage}, UserOperation, WebsocketInfo, }, DbPool, - LemmyError, +}; +use lemmy_db::{naive_now, Bannable, Crud, Followable, Joinable, SortType}; +use lemmy_utils::{ + generate_actor_keypair, + is_valid_community_name, + make_apub_endpoint, + naive_from_unix, + slur_check, + slurs_vec_to_str, + EndpointType, }; use serde::{Deserialize, Serialize}; use std::str::FromStr; diff --git a/server/src/api/mod.rs b/server/src/api/mod.rs index 6df9909c5..bb65815ad 100644 --- a/server/src/api/mod.rs +++ b/server/src/api/mod.rs @@ -1,11 +1,8 @@ -use crate::{ - db::{community::*, community_view::*, moderator::*, site::*, user::*, user_view::*}, - websocket::WebsocketInfo, - DbPool, - LemmyError, -}; +use crate::{websocket::WebsocketInfo, DbPool, LemmyError}; use actix_web::client::Client; +use lemmy_db::{community::*, community_view::*, moderator::*, site::*, user::*, user_view::*}; +pub mod claims; pub mod comment; pub mod community; pub mod post; diff --git a/server/src/api/post.rs b/server/src/api/post.rs index 840f15305..6710a2cdf 100644 --- a/server/src/api/post.rs +++ b/server/src/api/post.rs @@ -1,27 +1,8 @@ use crate::{ - api::{APIError, Oper, Perform}, + api::{claims::Claims, APIError, Oper, Perform}, apub::{ApubLikeableType, ApubObjectType}, blocking, - db::{ - comment_view::*, - community_view::*, - moderator::*, - post::*, - post_view::*, - site::*, - site_view::*, - user::*, - user_view::*, - Crud, - Likeable, - ListingType, - Saveable, - SortType, - }, fetch_iframely_and_pictrs_data, - naive_now, - slur_check, - slurs_vec_to_str, websocket::{ server::{JoinCommunityRoom, JoinPostRoom, SendPost}, UserOperation, @@ -30,6 +11,30 @@ use crate::{ DbPool, LemmyError, }; +use lemmy_db::{ + comment_view::*, + community_view::*, + moderator::*, + naive_now, + post::*, + post_view::*, + site::*, + site_view::*, + user::*, + user_view::*, + Crud, + Likeable, + ListingType, + Saveable, + SortType, +}; +use lemmy_utils::{ + is_valid_post_title, + make_apub_endpoint, + slur_check, + slurs_vec_to_str, + EndpointType, +}; use serde::{Deserialize, Serialize}; use std::str::FromStr; @@ -71,6 +76,7 @@ pub struct GetPosts { page: Option, limit: Option, pub community_id: Option, + pub community_name: Option, auth: Option, } @@ -136,6 +142,10 @@ impl Perform for Oper { } } + if !is_valid_post_title(&data.name) { + return Err(APIError::err("invalid_post_title").into()); + } + let user_id = claims.id; // Check for a community ban @@ -157,7 +167,7 @@ impl Perform for Oper { fetch_iframely_and_pictrs_data(&self.client, data.url.to_owned()).await; let post_form = PostForm { - name: data.name.to_owned(), + name: data.name.trim().to_owned(), url: data.url.to_owned(), body: data.body.to_owned(), community_id: data.community_id, @@ -191,11 +201,16 @@ impl Perform for Oper { }; let inserted_post_id = inserted_post.id; - let updated_post = - match blocking(pool, move |conn| Post::update_ap_id(conn, inserted_post_id)).await? { - Ok(post) => post, - Err(_e) => return Err(APIError::err("couldnt_create_post").into()), - }; + let updated_post = match blocking(pool, move |conn| { + let apub_id = + make_apub_endpoint(EndpointType::Post, &inserted_post_id.to_string()).to_string(); + Post::update_ap_id(conn, inserted_post_id, apub_id) + }) + .await? + { + Ok(post) => post, + Err(_e) => return Err(APIError::err("couldnt_create_post").into()), + }; updated_post.send_create(&user, &self.client, pool).await?; @@ -361,12 +376,14 @@ impl Perform for Oper { let page = data.page; let limit = data.limit; let community_id = data.community_id; + let community_name = data.community_name.to_owned(); let posts = match blocking(pool, move |conn| { PostQueryBuilder::create(conn) .listing_type(type_) .sort(&sort) .show_nsfw(show_nsfw) .for_community_id(community_id) + .for_community_name(community_name) .my_user_id(user_id) .page(page) .limit(limit) @@ -512,6 +529,10 @@ impl Perform for Oper { } } + if !is_valid_post_title(&data.name) { + return Err(APIError::err("invalid_post_title").into()); + } + let claims = match Claims::decode(&data.auth) { Ok(claims) => claims.claims, Err(_e) => return Err(APIError::err("not_logged_in").into()), @@ -561,7 +582,7 @@ impl Perform for Oper { let read_post = blocking(pool, move |conn| Post::read(conn, edit_id)).await??; let post_form = PostForm { - name: data.name.to_owned(), + name: data.name.trim().to_owned(), url: data.url.to_owned(), body: data.body.to_owned(), creator_id: data.creator_id.to_owned(), diff --git a/server/src/api/site.rs b/server/src/api/site.rs index f45561a82..241a80e31 100644 --- a/server/src/api/site.rs +++ b/server/src/api/site.rs @@ -1,31 +1,28 @@ use super::user::Register; use crate::{ - api::{APIError, Oper, Perform}, + api::{claims::Claims, APIError, Oper, Perform}, apub::fetcher::search_by_apub_id, blocking, - db::{ - category::*, - comment_view::*, - community_view::*, - moderator::*, - moderator_views::*, - post_view::*, - site::*, - site_view::*, - user::*, - user_view::*, - Crud, - SearchType, - SortType, - }, - naive_now, - settings::Settings, - slur_check, - slurs_vec_to_str, websocket::{server::SendAllMessage, UserOperation, WebsocketInfo}, DbPool, LemmyError, }; +use lemmy_db::{ + category::*, + comment_view::*, + community_view::*, + moderator::*, + moderator_views::*, + naive_now, + post_view::*, + site::*, + site_view::*, + user_view::*, + Crud, + SearchType, + SortType, +}; +use lemmy_utils::{settings::Settings, slur_check, slurs_vec_to_str}; use log::{debug, info}; use serde::{Deserialize, Serialize}; use std::str::FromStr; diff --git a/server/src/api/user.rs b/server/src/api/user.rs index 9b72a9199..9f33843f6 100644 --- a/server/src/api/user.rs +++ b/server/src/api/user.rs @@ -1,53 +1,53 @@ use crate::{ - api::{APIError, Oper, Perform}, - apub::{ - extensions::signatures::generate_actor_keypair, - make_apub_endpoint, - ApubObjectType, - EndpointType, - }, + api::{claims::Claims, APIError, Oper, Perform}, + apub::ApubObjectType, blocking, - db::{ - comment::*, - comment_view::*, - community::*, - community_view::*, - moderator::*, - password_reset_request::*, - post::*, - post_view::*, - private_message::*, - private_message_view::*, - site::*, - site_view::*, - user::*, - user_mention::*, - user_mention_view::*, - user_view::*, - Crud, - Followable, - Joinable, - ListingType, - SortType, + websocket::{ + server::{JoinUserRoom, SendAllMessage, SendUserRoomMessage}, + UserOperation, + WebsocketInfo, }, + DbPool, + LemmyError, +}; +use bcrypt::verify; +use lemmy_db::{ + comment::*, + comment_view::*, + community::*, + community_view::*, + moderator::*, + naive_now, + password_reset_request::*, + post::*, + post_view::*, + private_message::*, + private_message_view::*, + site::*, + site_view::*, + user::*, + user_mention::*, + user_mention_view::*, + user_view::*, + Crud, + Followable, + Joinable, + ListingType, + SortType, +}; +use lemmy_utils::{ + generate_actor_keypair, generate_random_string, is_valid_username, + make_apub_endpoint, naive_from_unix, - naive_now, remove_slurs, send_email, settings::Settings, slur_check, slurs_vec_to_str, - websocket::{ - server::{JoinUserRoom, SendAllMessage, SendUserRoomMessage}, - UserOperation, - WebsocketInfo, - }, - DbPool, - LemmyError, + EndpointType, }; -use bcrypt::verify; use log::error; use serde::{Deserialize, Serialize}; use std::str::FromStr; @@ -264,7 +264,7 @@ impl Perform for Oper { // Fetch that username / email let username_or_email = data.username_or_email.clone(); let user = match blocking(pool, move |conn| { - User_::find_by_email_or_username(conn, &username_or_email) + Claims::find_by_email_or_username(conn, &username_or_email) }) .await? { @@ -279,7 +279,9 @@ impl Perform for Oper { } // Return the jwt - Ok(LoginResponse { jwt: user.jwt() }) + Ok(LoginResponse { + jwt: Claims::jwt(user, Settings::get().hostname), + }) } } @@ -421,7 +423,7 @@ impl Perform for Oper { // Return the jwt Ok(LoginResponse { - jwt: inserted_user.jwt(), + jwt: Claims::jwt(inserted_user, Settings::get().hostname), }) } } @@ -451,6 +453,11 @@ impl Perform for Oper { None => read_user.email, }; + let avatar = match &data.avatar { + Some(avatar) => Some(avatar.to_owned()), + None => read_user.avatar, + }; + let password_encrypted = match &data.new_password { Some(new_password) => { match &data.new_password_verify { @@ -488,7 +495,7 @@ impl Perform for Oper { name: read_user.name, email, matrix_user_id: data.matrix_user_id.to_owned(), - avatar: data.avatar.to_owned(), + avatar, password_encrypted, preferred_username: read_user.preferred_username, updated: Some(naive_now()), @@ -527,7 +534,7 @@ impl Perform for Oper { // Return the jwt Ok(LoginResponse { - jwt: updated_user.jwt(), + jwt: Claims::jwt(updated_user, Settings::get().hostname), }) } } @@ -1150,7 +1157,7 @@ impl Perform for Oper { // Return the jwt Ok(LoginResponse { - jwt: updated_user.jwt(), + jwt: Claims::jwt(updated_user, Settings::get().hostname), }) } } @@ -1208,7 +1215,12 @@ impl Perform for Oper { let inserted_private_message_id = inserted_private_message.id; let updated_private_message = match blocking(pool, move |conn| { - PrivateMessage::update_ap_id(&conn, inserted_private_message_id) + let apub_id = make_apub_endpoint( + EndpointType::PrivateMessage, + &inserted_private_message_id.to_string(), + ) + .to_string(); + PrivateMessage::update_ap_id(&conn, inserted_private_message_id, apub_id) }) .await? { diff --git a/server/src/apub/activities.rs b/server/src/apub/activities.rs index e5dc70457..204a380d3 100644 --- a/server/src/apub/activities.rs +++ b/server/src/apub/activities.rs @@ -1,12 +1,18 @@ use crate::{ - apub::{extensions::signatures::sign, is_apub_id_valid, ActorType}, - db::{activity::insert_activity, community::Community, user::User_}, + apub::{ + community::do_announce, + extensions::signatures::sign, + insert_activity, + is_apub_id_valid, + ActorType, + }, request::retry_custom, DbPool, LemmyError, }; use activitystreams::{context, object::properties::ObjectProperties, public, Activity, Base}; use actix_web::client::Client; +use lemmy_db::{community::Community, user::User_}; use log::debug; use serde::Serialize; use std::fmt::Debug; @@ -43,7 +49,7 @@ where // if this is a local community, we need to do an announce from the community instead if community.local { - Community::do_announce(activity, &community, creator, client, pool).await?; + do_announce(activity, &community, creator, client, pool).await?; } else { send_activity(client, &activity, creator, to).await?; } diff --git a/server/src/apub/comment.rs b/server/src/apub/comment.rs index dbc15909e..84d11275d 100644 --- a/server/src/apub/comment.rs +++ b/server/src/apub/comment.rs @@ -1,35 +1,16 @@ use crate::{ apub::{ activities::{populate_object_props, send_activity_to_community}, - create_apub_response, - create_apub_tombstone_response, - create_tombstone, - fetch_webfinger_url, + create_apub_response, create_apub_tombstone_response, create_tombstone, fetch_webfinger_url, fetcher::{ - get_or_fetch_and_insert_remote_comment, - get_or_fetch_and_insert_remote_post, + get_or_fetch_and_insert_remote_comment, get_or_fetch_and_insert_remote_post, get_or_fetch_and_upsert_remote_user, }, - ActorType, - ApubLikeableType, - ApubObjectType, - FromApub, - ToApub, + ActorType, ApubLikeableType, ApubObjectType, FromApub, ToApub, }, blocking, - convert_datetime, - db::{ - comment::{Comment, CommentForm}, - community::Community, - post::Post, - user::User_, - Crud, - }, routes::DbPoolParam, - scrape_text_for_mentions, - DbPool, - LemmyError, - MentionData, + DbPool, LemmyError, }; use activitystreams::{ activity::{Create, Delete, Dislike, Like, Remove, Undo, Update}, @@ -40,6 +21,14 @@ use activitystreams::{ use activitystreams_new::object::Tombstone; use actix_web::{body::Body, client::Client, web::Path, HttpResponse}; use itertools::Itertools; +use lemmy_db::{ + comment::{Comment, CommentForm}, + community::Community, + post::Post, + user::User_, + Crud, +}; +use lemmy_utils::{convert_datetime, scrape_text_for_mentions, MentionData}; use log::debug; use serde::Deserialize; @@ -123,7 +112,7 @@ impl FromApub for CommentForm { /// Parse an ActivityPub note received from another instance into a Lemmy comment async fn from_apub( - note: &mut Note, + note: &Note, client: &Client, pool: &DbPool, ) -> Result { diff --git a/server/src/apub/community.rs b/server/src/apub/community.rs index bfc896af6..587977a33 100644 --- a/server/src/apub/community.rs +++ b/server/src/apub/community.rs @@ -1,35 +1,18 @@ use crate::{ apub::{ activities::{populate_object_props, send_activity}, - create_apub_response, - create_apub_tombstone_response, - create_tombstone, + create_apub_response, create_apub_tombstone_response, create_tombstone, extensions::group_extensions::GroupExtension, fetcher::get_or_fetch_and_upsert_remote_user, - get_shared_inbox, - ActorType, - FromApub, - GroupExt, - ToApub, + get_shared_inbox, insert_activity, ActorType, FromApub, GroupExt, ToApub, }, blocking, - convert_datetime, - db::{ - activity::insert_activity, - community::{Community, CommunityForm}, - community_view::{CommunityFollowerView, CommunityModeratorView}, - user::User_, - }, - naive_now, routes::DbPoolParam, - DbPool, - LemmyError, + DbPool, LemmyError, }; use activitystreams::{ activity::{Accept, Announce, Delete, Remove, Undo}, - Activity, - Base, - BaseBox, + Activity, Base, BaseBox, }; use activitystreams_ext::Ext2; use activitystreams_new::{ @@ -44,6 +27,13 @@ use activitystreams_new::{ }; use actix_web::{body::Body, client::Client, web, HttpResponse}; use itertools::Itertools; +use lemmy_db::{ + community::{Community, CommunityForm}, + community_view::{CommunityFollowerView, CommunityModeratorView}, + naive_now, + user::User_, +}; +use lemmy_utils::convert_datetime; use serde::{Deserialize, Serialize}; use std::{fmt::Debug, str::FromStr}; @@ -367,13 +357,8 @@ impl FromApub for CommunityForm { type ApubType = GroupExt; /// Parse an ActivityPub group received from another instance into a Lemmy community. - async fn from_apub( - group: &mut GroupExt, - client: &Client, - pool: &DbPool, - ) -> Result { - // TODO: this is probably gonna cause problems cause fetcher:292 also calls take_attributed_to() - let creator_and_moderator_uris = group.clone().take_attributed_to().unwrap(); + async fn from_apub(group: &GroupExt, client: &Client, pool: &DbPool) -> Result { + let creator_and_moderator_uris = group.attributed_to().unwrap(); let creator_uri = creator_and_moderator_uris .as_many() .unwrap() @@ -386,27 +371,20 @@ impl FromApub for CommunityForm { let creator = get_or_fetch_and_upsert_remote_user(creator_uri.as_str(), client, pool).await?; Ok(CommunityForm { - name: group - .take_name() - .unwrap() - .as_single_xsd_string() - .unwrap() - .into(), - title: group.inner.take_preferred_username().unwrap(), + name: group.name().unwrap().as_single_xsd_string().unwrap().into(), + title: group.inner.preferred_username().unwrap().to_string(), // TODO: should be parsed as html and tags like