diff --git a/Cargo.lock b/Cargo.lock index 61fcb5cd4..6d97f9aac 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2582,13 +2582,13 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "lemmy_api" -version = "0.19.4-beta.4" +version = "0.19.4-beta.5" dependencies = [ "activitypub_federation", "actix-web", "actix-web-httpauth", "anyhow", - "base64 0.21.7", + "base64 0.22.0", "bcrypt", "captcha", "chrono", @@ -2611,7 +2611,7 @@ dependencies = [ [[package]] name = "lemmy_api_common" -version = "0.19.4-beta.4" +version = "0.19.4-beta.5" dependencies = [ "activitypub_federation", "actix-web", @@ -2649,7 +2649,7 @@ dependencies = [ [[package]] name = "lemmy_api_crud" -version = "0.19.4-beta.4" +version = "0.19.4-beta.5" dependencies = [ "accept-language", "activitypub_federation", @@ -2672,7 +2672,7 @@ dependencies = [ [[package]] name = "lemmy_apub" -version = "0.19.4-beta.4" +version = "0.19.4-beta.5" dependencies = [ "activitypub_federation", "actix-web", @@ -2710,7 +2710,7 @@ dependencies = [ [[package]] name = "lemmy_db_perf" -version = "0.19.4-beta.4" +version = "0.19.4-beta.5" dependencies = [ "anyhow", "clap", @@ -2725,7 +2725,7 @@ dependencies = [ [[package]] name = "lemmy_db_schema" -version = "0.19.4-beta.4" +version = "0.19.4-beta.5" dependencies = [ "activitypub_federation", "anyhow", @@ -2765,7 +2765,7 @@ dependencies = [ [[package]] name = "lemmy_db_views" -version = "0.19.4-beta.4" +version = "0.19.4-beta.5" dependencies = [ "actix-web", "chrono", @@ -2787,7 +2787,7 @@ dependencies = [ [[package]] name = "lemmy_db_views_actor" -version = "0.19.4-beta.4" +version = "0.19.4-beta.5" dependencies = [ "chrono", "diesel", @@ -2807,7 +2807,7 @@ dependencies = [ [[package]] name = "lemmy_db_views_moderator" -version = "0.19.4-beta.4" +version = "0.19.4-beta.5" dependencies = [ "diesel", "diesel-async", @@ -2819,7 +2819,7 @@ dependencies = [ [[package]] name = "lemmy_federate" -version = "0.19.4-beta.4" +version = "0.19.4-beta.5" dependencies = [ "activitypub_federation", "anyhow", @@ -2842,7 +2842,7 @@ dependencies = [ [[package]] name = "lemmy_routes" -version = "0.19.4-beta.4" +version = "0.19.4-beta.5" dependencies = [ "activitypub_federation", "actix-web", @@ -2867,7 +2867,7 @@ dependencies = [ [[package]] name = "lemmy_server" -version = "0.19.4-beta.4" +version = "0.19.4-beta.5" dependencies = [ "activitypub_federation", "actix-cors", @@ -2910,7 +2910,7 @@ dependencies = [ [[package]] name = "lemmy_utils" -version = "0.19.4-beta.4" +version = "0.19.4-beta.5" dependencies = [ "actix-web", "anyhow", diff --git a/Cargo.toml b/Cargo.toml index b8c029752..05b97924c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,5 @@ [workspace.package] -version = "0.19.4-beta.4" +version = "0.19.4-beta.5" edition = "2021" description = "A link aggregator for the fediverse" license = "AGPL-3.0" @@ -88,17 +88,17 @@ unused_self = "deny" unwrap_used = "deny" [workspace.dependencies] -lemmy_api = { version = "=0.19.4-beta.4", path = "./crates/api" } -lemmy_api_crud = { version = "=0.19.4-beta.4", path = "./crates/api_crud" } -lemmy_apub = { version = "=0.19.4-beta.4", path = "./crates/apub" } -lemmy_utils = { version = "=0.19.4-beta.4", path = "./crates/utils", default-features = false } -lemmy_db_schema = { version = "=0.19.4-beta.4", path = "./crates/db_schema" } -lemmy_api_common = { version = "=0.19.4-beta.4", path = "./crates/api_common" } -lemmy_routes = { version = "=0.19.4-beta.4", path = "./crates/routes" } -lemmy_db_views = { version = "=0.19.4-beta.4", path = "./crates/db_views" } -lemmy_db_views_actor = { version = "=0.19.4-beta.4", path = "./crates/db_views_actor" } -lemmy_db_views_moderator = { version = "=0.19.4-beta.4", path = "./crates/db_views_moderator" } -lemmy_federate = { version = "=0.19.4-beta.4", path = "./crates/federate" } +lemmy_api = { version = "=0.19.4-beta.5", path = "./crates/api" } +lemmy_api_crud = { version = "=0.19.4-beta.5", path = "./crates/api_crud" } +lemmy_apub = { version = "=0.19.4-beta.5", path = "./crates/apub" } +lemmy_utils = { version = "=0.19.4-beta.5", path = "./crates/utils", default-features = false } +lemmy_db_schema = { version = "=0.19.4-beta.5", path = "./crates/db_schema" } +lemmy_api_common = { version = "=0.19.4-beta.5", path = "./crates/api_common" } +lemmy_routes = { version = "=0.19.4-beta.5", path = "./crates/routes" } +lemmy_db_views = { version = "=0.19.4-beta.5", path = "./crates/db_views" } +lemmy_db_views_actor = { version = "=0.19.4-beta.5", path = "./crates/db_views_actor" } +lemmy_db_views_moderator = { version = "=0.19.4-beta.5", path = "./crates/db_views_moderator" } +lemmy_federate = { version = "=0.19.4-beta.5", path = "./crates/federate" } activitypub_federation = { version = "0.5.4", default-features = false, features = [ "actix-web", ] } diff --git a/crates/api/src/community/add_mod.rs b/crates/api/src/community/add_mod.rs index df6b3fbe4..5245f592e 100644 --- a/crates/api/src/community/add_mod.rs +++ b/crates/api/src/community/add_mod.rs @@ -36,8 +36,20 @@ pub async fn add_mod_to_community( let community = Community::read(&mut context.pool(), community_id) .await? .ok_or(LemmyErrorType::CouldntFindCommunity)?; + + // If user is admin and community is remote, explicitly check that he is a + // moderator. This is necessary because otherwise the action would be rejected + // by the community's home instance. if local_user_view.local_user.admin && !community.local { - Err(LemmyErrorType::NotAModerator)? + let is_mod = CommunityModeratorView::is_community_moderator( + &mut context.pool(), + community.id, + local_user_view.person.id, + ) + .await?; + if !is_mod { + Err(LemmyErrorType::NotAModerator)? + } } // Update in local database diff --git a/crates/api_common/src/request.rs b/crates/api_common/src/request.rs index ddb2a4551..abf99f670 100644 --- a/crates/api_common/src/request.rs +++ b/crates/api_common/src/request.rs @@ -105,7 +105,11 @@ pub fn generate_post_link_metadata( } // Generate local thumbnail if allowed else if allow_generate_thumbnail { - match post.url.or(metadata.opengraph_data.image) { + match post + .url + .filter(|_| is_image_post) + .or(metadata.opengraph_data.image) + { Some(url) => generate_pictrs_thumbnail(&url, &context).await.ok(), None => None, } diff --git a/crates/api_common/src/utils.rs b/crates/api_common/src/utils.rs index 3cd7902e1..4ab587928 100644 --- a/crates/api_common/src/utils.rs +++ b/crates/api_common/src/utils.rs @@ -536,25 +536,8 @@ pub async fn get_url_blocklist(context: &LemmyContext) -> LemmyResult .try_get_with::<_, LemmyError>((), async { let urls = LocalSiteUrlBlocklist::get_all(&mut context.pool()).await?; - let regexes = urls.iter().map(|url| { - let url = &url.url; - let parsed = Url::parse(url).expect("Coundln't parse URL."); - if url.ends_with('/') { - format!( - "({}://)?{}{}?", - parsed.scheme(), - escape(parsed.domain().expect("No domain.")), - escape(parsed.path()) - ) - } else { - format!( - "({}://)?{}{}", - parsed.scheme(), - escape(parsed.domain().expect("No domain.")), - escape(parsed.path()) - ) - } - }); + // The urls are already validated on saving, so just escape them. + let regexes = urls.iter().map(|url| escape(&url.url)); let set = RegexSet::new(regexes)?; Ok(set) diff --git a/crates/db_schema/src/impls/actor_language.rs b/crates/db_schema/src/impls/actor_language.rs index fe21cf003..682be2ed0 100644 --- a/crates/db_schema/src/impls/actor_language.rs +++ b/crates/db_schema/src/impls/actor_language.rs @@ -43,20 +43,13 @@ impl LocalUserLanguage { }; let conn = &mut get_conn(pool).await?; - conn - .build_transaction() - .run(|conn| { - Box::pin(async move { - let langs = local_user_language - .filter(local_user_id.eq(for_local_user_id)) - .order(language_id) - .select(language_id) - .get_results(conn) - .await?; - convert_read_languages(conn, langs).await - }) as _ - }) - .await + let langs = local_user_language + .filter(local_user_id.eq(for_local_user_id)) + .order(language_id) + .select(language_id) + .get_results(conn) + .await?; + convert_read_languages(conn, langs).await } /// Update the user's languages. @@ -90,24 +83,33 @@ impl LocalUserLanguage { .build_transaction() .run(|conn| { Box::pin(async move { - use crate::schema::local_user_language::dsl::{local_user_id, local_user_language}; - // Clear the current user languages - delete(local_user_language.filter(local_user_id.eq(for_local_user_id))) - .execute(conn) - .await?; + use crate::schema::local_user_language::dsl::{ + language_id, + local_user_id, + local_user_language, + }; + // Delete old languages, not including new languages + let delete_old = delete(local_user_language) + .filter(local_user_id.eq(for_local_user_id)) + .filter(language_id.ne_all(&lang_ids)) + .execute(conn); let forms = lang_ids - .into_iter() - .map(|l| LocalUserLanguageForm { + .iter() + .map(|&l| LocalUserLanguageForm { local_user_id: for_local_user_id, language_id: l, }) .collect::>(); - insert_into(local_user_language) + // Insert new languages + let insert_new = insert_into(local_user_language) .values(forms) - .execute(conn) - .await?; + .on_conflict((language_id, local_user_id)) + .do_nothing() + .execute(conn); + + tokio::try_join!(delete_old, insert_new)?; Ok(()) }) as _ }) @@ -159,25 +161,30 @@ impl SiteLanguage { .build_transaction() .run(|conn| { Box::pin(async move { - use crate::schema::site_language::dsl::{site_id, site_language}; + use crate::schema::site_language::dsl::{language_id, site_id, site_language}; - // Clear the current languages - delete(site_language.filter(site_id.eq(for_site_id))) - .execute(conn) - .await?; + // Delete old languages, not including new languages + let delete_old = delete(site_language) + .filter(site_id.eq(for_site_id)) + .filter(language_id.ne_all(&lang_ids)) + .execute(conn); let forms = lang_ids - .into_iter() - .map(|l| SiteLanguageForm { + .iter() + .map(|&l| SiteLanguageForm { site_id: for_site_id, language_id: l, }) .collect::>(); - insert_into(site_language) + // Insert new languages + let insert_new = insert_into(site_language) .values(forms) - .get_result::(conn) - .await?; + .on_conflict((site_id, language_id)) + .do_nothing() + .execute(conn); + + tokio::try_join!(delete_old, insert_new)?; CommunityLanguage::limit_languages(conn, instance_id).await?; @@ -278,8 +285,8 @@ impl CommunityLanguage { } let form = lang_ids - .into_iter() - .map(|language_id| CommunityLanguageForm { + .iter() + .map(|&language_id| CommunityLanguageForm { community_id: for_community_id, language_id, }) @@ -289,25 +296,25 @@ impl CommunityLanguage { .build_transaction() .run(|conn| { Box::pin(async move { - use crate::schema::community_language::dsl::{community_id, community_language}; - use diesel::result::DatabaseErrorKind::UniqueViolation; - // Clear the current languages - delete(community_language.filter(community_id.eq(for_community_id))) - .execute(conn) - .await?; - - let insert_res = insert_into(community_language) + use crate::schema::community_language::dsl::{ + community_id, + community_language, + language_id, + }; + // Delete old languages, not including new languages + let delete_old = delete(community_language) + .filter(community_id.eq(for_community_id)) + .filter(language_id.ne_all(&lang_ids)) + .execute(conn); + + // Insert new languages + let insert_new = insert_into(community_language) .values(form) - .get_result::(conn) - .await; - - if let Err(Error::DatabaseError(UniqueViolation, _info)) = insert_res { - // race condition: this function was probably called simultaneously from another caller. ignore error - // tracing::warn!("unique error: {_info:#?}"); - // _info.constraint_name() should be = "community_language_community_id_language_id_key" - return Ok(()); - } - insert_res?; + .on_conflict((community_id, language_id)) + .do_nothing() + .execute(conn); + + tokio::try_join!(delete_old, insert_new)?; Ok(()) }) as _ diff --git a/crates/db_views/src/comment_view.rs b/crates/db_views/src/comment_view.rs index e8957e1e9..910c37406 100644 --- a/crates/db_views/src/comment_view.rs +++ b/crates/db_views/src/comment_view.rs @@ -215,9 +215,14 @@ fn queries<'a>() -> Queries< if let Some(parent_path) = options.parent_path.as_ref() { query = query.filter(comment::path.contained_by(parent_path)); }; - + //filtering out removed and deleted comments from search if let Some(search_term) = options.search_term { - query = query.filter(comment::content.ilike(fuzzy_search(&search_term))); + query = query.filter( + comment::content + .ilike(fuzzy_search(&search_term)) + .and(comment::removed.eq(false)) + .and(comment::deleted.eq(false)), + ); }; if let Some(community_id) = options.community_id { diff --git a/crates/db_views_moderator/src/admin_purge_comment_view.rs b/crates/db_views_moderator/src/admin_purge_comment_view.rs index f62fe0f22..4c650b6fa 100644 --- a/crates/db_views_moderator/src/admin_purge_comment_view.rs +++ b/crates/db_views_moderator/src/admin_purge_comment_view.rs @@ -40,6 +40,11 @@ impl AdminPurgeCommentView { query = query.filter(admin_purge_comment::admin_person_id.eq(admin_person_id)); }; + // If a post or comment ID is given, then don't find any results + if params.post_id.is_some() || params.comment_id.is_some() { + return Ok(vec![]); + } + let (limit, offset) = limit_and_offset(params.page, params.limit)?; query diff --git a/crates/db_views_moderator/src/admin_purge_community_view.rs b/crates/db_views_moderator/src/admin_purge_community_view.rs index 23967ee3b..5eadb8985 100644 --- a/crates/db_views_moderator/src/admin_purge_community_view.rs +++ b/crates/db_views_moderator/src/admin_purge_community_view.rs @@ -38,6 +38,11 @@ impl AdminPurgeCommunityView { query = query.filter(admin_purge_community::admin_person_id.eq(admin_person_id)); }; + // If a post or comment ID is given, then don't find any results + if params.post_id.is_some() || params.comment_id.is_some() { + return Ok(vec![]); + } + let (limit, offset) = limit_and_offset(params.page, params.limit)?; query diff --git a/crates/db_views_moderator/src/admin_purge_person_view.rs b/crates/db_views_moderator/src/admin_purge_person_view.rs index 097785d25..b6dd834c5 100644 --- a/crates/db_views_moderator/src/admin_purge_person_view.rs +++ b/crates/db_views_moderator/src/admin_purge_person_view.rs @@ -38,6 +38,11 @@ impl AdminPurgePersonView { query = query.filter(admin_purge_person::admin_person_id.eq(admin_person_id)); }; + // If a post or comment ID is given, then don't find any results + if params.post_id.is_some() || params.comment_id.is_some() { + return Ok(vec![]); + } + let (limit, offset) = limit_and_offset(params.page, params.limit)?; query diff --git a/crates/db_views_moderator/src/admin_purge_post_view.rs b/crates/db_views_moderator/src/admin_purge_post_view.rs index 8f5eb3a14..b77493c25 100644 --- a/crates/db_views_moderator/src/admin_purge_post_view.rs +++ b/crates/db_views_moderator/src/admin_purge_post_view.rs @@ -40,6 +40,11 @@ impl AdminPurgePostView { query = query.filter(admin_purge_post::admin_person_id.eq(admin_person_id)); }; + // If a post or comment ID is given, then don't find any results + if params.post_id.is_some() || params.comment_id.is_some() { + return Ok(vec![]); + } + let (limit, offset) = limit_and_offset(params.page, params.limit)?; query diff --git a/crates/db_views_moderator/src/mod_add_community_view.rs b/crates/db_views_moderator/src/mod_add_community_view.rs index f96a9b80b..1068aba75 100644 --- a/crates/db_views_moderator/src/mod_add_community_view.rs +++ b/crates/db_views_moderator/src/mod_add_community_view.rs @@ -52,6 +52,11 @@ impl ModAddCommunityView { query = query.filter(person_alias_1.field(person::id).eq(other_person_id)); }; + // If a post or comment ID is given, then don't find any results + if params.post_id.is_some() || params.comment_id.is_some() { + return Ok(vec![]); + } + let (limit, offset) = limit_and_offset(params.page, params.limit)?; query diff --git a/crates/db_views_moderator/src/mod_add_view.rs b/crates/db_views_moderator/src/mod_add_view.rs index 28fb0a2b6..c5612c4ad 100644 --- a/crates/db_views_moderator/src/mod_add_view.rs +++ b/crates/db_views_moderator/src/mod_add_view.rs @@ -44,6 +44,11 @@ impl ModAddView { query = query.filter(person_alias_1.field(person::id).eq(other_person_id)); }; + // If a post or comment ID is given, then don't find any results + if params.post_id.is_some() || params.comment_id.is_some() { + return Ok(vec![]); + } + let (limit, offset) = limit_and_offset(params.page, params.limit)?; query diff --git a/crates/db_views_moderator/src/mod_ban_from_community_view.rs b/crates/db_views_moderator/src/mod_ban_from_community_view.rs index 02f18099c..d2d6038f3 100644 --- a/crates/db_views_moderator/src/mod_ban_from_community_view.rs +++ b/crates/db_views_moderator/src/mod_ban_from_community_view.rs @@ -54,6 +54,11 @@ impl ModBanFromCommunityView { query = query.filter(mod_ban_from_community::other_person_id.eq(other_person_id)); }; + // If a post or comment ID is given, then don't find any results + if params.post_id.is_some() || params.comment_id.is_some() { + return Ok(vec![]); + } + let (limit, offset) = limit_and_offset(params.page, params.limit)?; query diff --git a/crates/db_views_moderator/src/mod_ban_view.rs b/crates/db_views_moderator/src/mod_ban_view.rs index 94ac360db..ca0723e83 100644 --- a/crates/db_views_moderator/src/mod_ban_view.rs +++ b/crates/db_views_moderator/src/mod_ban_view.rs @@ -44,6 +44,11 @@ impl ModBanView { query = query.filter(person_alias_1.field(person::id).eq(other_person_id)); }; + // If a post or comment ID is given, then don't find any results + if params.post_id.is_some() || params.comment_id.is_some() { + return Ok(vec![]); + } + let (limit, offset) = limit_and_offset(params.page, params.limit)?; query diff --git a/crates/db_views_moderator/src/mod_feature_post_view.rs b/crates/db_views_moderator/src/mod_feature_post_view.rs index 1bf83688d..4c0fdb4f7 100644 --- a/crates/db_views_moderator/src/mod_feature_post_view.rs +++ b/crates/db_views_moderator/src/mod_feature_post_view.rs @@ -55,6 +55,11 @@ impl ModFeaturePostView { query = query.filter(post::id.eq(post_id)); } + // If a comment ID is given, then don't find any results + if params.comment_id.is_some() { + return Ok(vec![]); + } + let (limit, offset) = limit_and_offset(params.page, params.limit)?; query diff --git a/crates/db_views_moderator/src/mod_hide_community_view.rs b/crates/db_views_moderator/src/mod_hide_community_view.rs index 36b549814..3c8a7e627 100644 --- a/crates/db_views_moderator/src/mod_hide_community_view.rs +++ b/crates/db_views_moderator/src/mod_hide_community_view.rs @@ -45,6 +45,11 @@ impl ModHideCommunityView { query = query.filter(mod_hide_community::mod_person_id.eq(admin_id)); }; + // If a post or comment ID is given, then don't find any results + if params.post_id.is_some() || params.comment_id.is_some() { + return Ok(vec![]); + } + let (limit, offset) = limit_and_offset(params.page, params.limit)?; query diff --git a/crates/db_views_moderator/src/mod_lock_post_view.rs b/crates/db_views_moderator/src/mod_lock_post_view.rs index 6f7e83ec7..5a6c753d9 100644 --- a/crates/db_views_moderator/src/mod_lock_post_view.rs +++ b/crates/db_views_moderator/src/mod_lock_post_view.rs @@ -56,6 +56,11 @@ impl ModLockPostView { query = query.filter(post::id.eq(post_id)); } + // If a comment ID is given, then don't find any results + if params.comment_id.is_some() { + return Ok(vec![]); + } + let (limit, offset) = limit_and_offset(params.page, params.limit)?; query diff --git a/crates/db_views_moderator/src/mod_remove_comment_view.rs b/crates/db_views_moderator/src/mod_remove_comment_view.rs index e7d695a5c..cf0ed325c 100644 --- a/crates/db_views_moderator/src/mod_remove_comment_view.rs +++ b/crates/db_views_moderator/src/mod_remove_comment_view.rs @@ -58,6 +58,11 @@ impl ModRemoveCommentView { query = query.filter(comment::id.eq(comment_id)); } + // If a post ID is given, then don't find any results + if params.post_id.is_some() { + return Ok(vec![]); + } + let (limit, offset) = limit_and_offset(params.page, params.limit)?; query diff --git a/crates/db_views_moderator/src/mod_remove_community_view.rs b/crates/db_views_moderator/src/mod_remove_community_view.rs index 2bc92acc8..ac620ebdb 100644 --- a/crates/db_views_moderator/src/mod_remove_community_view.rs +++ b/crates/db_views_moderator/src/mod_remove_community_view.rs @@ -39,6 +39,11 @@ impl ModRemoveCommunityView { query = query.filter(mod_remove_community::mod_person_id.eq(mod_person_id)); }; + // If a post or comment ID is given, then don't find any results + if params.post_id.is_some() || params.comment_id.is_some() { + return Ok(vec![]); + } + let (limit, offset) = limit_and_offset(params.page, params.limit)?; query diff --git a/crates/db_views_moderator/src/mod_remove_post_view.rs b/crates/db_views_moderator/src/mod_remove_post_view.rs index fb0b3e6c1..98504a8e7 100644 --- a/crates/db_views_moderator/src/mod_remove_post_view.rs +++ b/crates/db_views_moderator/src/mod_remove_post_view.rs @@ -56,6 +56,11 @@ impl ModRemovePostView { query = query.filter(post::id.eq(post_id)); } + // If a comment ID is given, then don't find any results + if params.comment_id.is_some() { + return Ok(vec![]); + } + let (limit, offset) = limit_and_offset(params.page, params.limit)?; query diff --git a/crates/db_views_moderator/src/mod_transfer_community_view.rs b/crates/db_views_moderator/src/mod_transfer_community_view.rs index 3d48b0f67..6d62d347a 100644 --- a/crates/db_views_moderator/src/mod_transfer_community_view.rs +++ b/crates/db_views_moderator/src/mod_transfer_community_view.rs @@ -54,6 +54,11 @@ impl ModTransferCommunityView { query = query.filter(person_alias_1.field(person::id).eq(other_person_id)); }; + // If a post or comment ID is given, then don't find any results + if params.post_id.is_some() || params.comment_id.is_some() { + return Ok(vec![]); + } + let (limit, offset) = limit_and_offset(params.page, params.limit)?; query diff --git a/crates/utils/src/utils/validation.rs b/crates/utils/src/utils/validation.rs index 93d581327..a6539f1b1 100644 --- a/crates/utils/src/utils/validation.rs +++ b/crates/utils/src/utils/validation.rs @@ -309,21 +309,44 @@ pub fn is_url_blocked(url: &Option, blocklist: &RegexSet) -> LemmyResult<() Ok(()) } +/// Check that urls are valid, and also remove the scheme, and uniques pub fn check_urls_are_valid(urls: &Vec) -> LemmyResult> { let mut parsed_urls = vec![]; for url in urls { - let url = Url::parse(url).or_else(|e| { - if e == ParseError::RelativeUrlWithoutBase { - Url::parse(&format!("https://{url}")) - } else { - Err(e) - } - })?; + parsed_urls.push(build_url_str_without_scheme(url)?); + } - parsed_urls.push(url.to_string()); + let unique_urls = parsed_urls.into_iter().unique().collect(); + Ok(unique_urls) +} + +pub fn build_url_str_without_scheme(url_str: &str) -> LemmyResult { + // Parse and check for errors + let mut url = Url::parse(url_str).or_else(|e| { + if e == ParseError::RelativeUrlWithoutBase { + Url::parse(&format!("http://{url_str}")) + } else { + Err(e) + } + })?; + + // Set the scheme to http, then remove the http:// part + url + .set_scheme("http") + .map_err(|_| LemmyErrorType::InvalidUrl)?; + + let mut out = url + .to_string() + .get(7..) + .ok_or(LemmyErrorType::InvalidUrl)? + .to_string(); + + // Remove trailing / if necessary + if out.ends_with('/') { + out.pop(); } - Ok(parsed_urls) + Ok(out) } #[cfg(test)] @@ -600,17 +623,21 @@ mod tests { #[test] fn test_url_parsed() { + // Make sure the scheme is removed, and uniques also assert_eq!( - vec![String::from("https://example.com/")], - check_urls_are_valid(&vec![String::from("example.com")]).unwrap() + &check_urls_are_valid(&vec![ + "example.com".to_string(), + "http://example.com".to_string(), + "https://example.com".to_string(), + "https://example.com/test?q=test2&q2=test3#test4".to_string(), + ]) + .unwrap(), + &vec![ + "example.com".to_string(), + "example.com/test?q=test2&q2=test3#test4".to_string() + ], ); - assert!(check_urls_are_valid(&vec![ - String::from("example.com"), - String::from("https://example.blog") - ]) - .is_ok()); - - assert!(check_urls_are_valid(&vec![String::from("https://example .com"),]).is_err()); + assert!(check_urls_are_valid(&vec!["https://example .com".to_string()]).is_err()); } } diff --git a/crates/utils/translations b/crates/utils/translations index c88dd1e3b..866e40566 160000 --- a/crates/utils/translations +++ b/crates/utils/translations @@ -1 +1 @@ -Subproject commit c88dd1e3b36ee1617f1b86acf94c1b7946e97cd4 +Subproject commit 866e4056656755f7b31e20094b46391e6931e3e7 diff --git a/migrations/2024-04-23-020604_add_post_id_index/down.sql b/migrations/2024-04-23-020604_add_post_id_index/down.sql new file mode 100644 index 000000000..fb4c92f6c --- /dev/null +++ b/migrations/2024-04-23-020604_add_post_id_index/down.sql @@ -0,0 +1,120 @@ +DROP INDEX idx_post_aggregates_community_active; + +DROP INDEX idx_post_aggregates_community_controversy; + +DROP INDEX idx_post_aggregates_community_hot; + +DROP INDEX idx_post_aggregates_community_most_comments; + +DROP INDEX idx_post_aggregates_community_newest_comment_time; + +DROP INDEX idx_post_aggregates_community_newest_comment_time_necro; + +DROP INDEX idx_post_aggregates_community_published; + +DROP INDEX idx_post_aggregates_community_published_asc; + +DROP INDEX idx_post_aggregates_community_scaled; + +DROP INDEX idx_post_aggregates_community_score; + +DROP INDEX idx_post_aggregates_featured_community_active; + +DROP INDEX idx_post_aggregates_featured_community_controversy; + +DROP INDEX idx_post_aggregates_featured_community_hot; + +DROP INDEX idx_post_aggregates_featured_community_most_comments; + +DROP INDEX idx_post_aggregates_featured_community_newest_comment_time; + +DROP INDEX idx_post_aggregates_featured_community_newest_comment_time_necr; + +DROP INDEX idx_post_aggregates_featured_community_published; + +DROP INDEX idx_post_aggregates_featured_community_published_asc; + +DROP INDEX idx_post_aggregates_featured_community_scaled; + +DROP INDEX idx_post_aggregates_featured_community_score; + +DROP INDEX idx_post_aggregates_featured_local_active; + +DROP INDEX idx_post_aggregates_featured_local_controversy; + +DROP INDEX idx_post_aggregates_featured_local_hot; + +DROP INDEX idx_post_aggregates_featured_local_most_comments; + +DROP INDEX idx_post_aggregates_featured_local_newest_comment_time; + +DROP INDEX idx_post_aggregates_featured_local_newest_comment_time_necro; + +DROP INDEX idx_post_aggregates_featured_local_published; + +DROP INDEX idx_post_aggregates_featured_local_published_asc; + +DROP INDEX idx_post_aggregates_featured_local_scaled; + +DROP INDEX idx_post_aggregates_featured_local_score; + +CREATE INDEX idx_post_aggregates_community_active ON public.post_aggregates USING btree (community_id, featured_local DESC, hot_rank_active DESC, published DESC); + +CREATE INDEX idx_post_aggregates_community_controversy ON public.post_aggregates USING btree (community_id, featured_local DESC, controversy_rank DESC); + +CREATE INDEX idx_post_aggregates_community_hot ON public.post_aggregates USING btree (community_id, featured_local DESC, hot_rank DESC, published DESC); + +CREATE INDEX idx_post_aggregates_community_most_comments ON public.post_aggregates USING btree (community_id, featured_local DESC, comments DESC, published DESC); + +CREATE INDEX idx_post_aggregates_community_newest_comment_time ON public.post_aggregates USING btree (community_id, featured_local DESC, newest_comment_time DESC); + +CREATE INDEX idx_post_aggregates_community_newest_comment_time_necro ON public.post_aggregates USING btree (community_id, featured_local DESC, newest_comment_time_necro DESC); + +CREATE INDEX idx_post_aggregates_community_published ON public.post_aggregates USING btree (community_id, featured_local DESC, published DESC); + +CREATE INDEX idx_post_aggregates_community_published_asc ON public.post_aggregates USING btree (community_id, featured_local DESC, public.reverse_timestamp_sort (published) DESC); + +CREATE INDEX idx_post_aggregates_community_scaled ON public.post_aggregates USING btree (community_id, featured_local DESC, scaled_rank DESC, published DESC); + +CREATE INDEX idx_post_aggregates_community_score ON public.post_aggregates USING btree (community_id, featured_local DESC, score DESC, published DESC); + +CREATE INDEX idx_post_aggregates_featured_community_active ON public.post_aggregates USING btree (community_id, featured_community DESC, hot_rank_active DESC, published DESC); + +CREATE INDEX idx_post_aggregates_featured_community_controversy ON public.post_aggregates USING btree (community_id, featured_community DESC, controversy_rank DESC); + +CREATE INDEX idx_post_aggregates_featured_community_hot ON public.post_aggregates USING btree (community_id, featured_community DESC, hot_rank DESC, published DESC); + +CREATE INDEX idx_post_aggregates_featured_community_most_comments ON public.post_aggregates USING btree (community_id, featured_community DESC, comments DESC, published DESC); + +CREATE INDEX idx_post_aggregates_featured_community_newest_comment_time ON public.post_aggregates USING btree (community_id, featured_community DESC, newest_comment_time DESC); + +CREATE INDEX idx_post_aggregates_featured_community_newest_comment_time_necr ON public.post_aggregates USING btree (community_id, featured_community DESC, newest_comment_time_necro DESC); + +CREATE INDEX idx_post_aggregates_featured_community_published ON public.post_aggregates USING btree (community_id, featured_community DESC, published DESC); + +CREATE INDEX idx_post_aggregates_featured_community_published_asc ON public.post_aggregates USING btree (community_id, featured_community DESC, public.reverse_timestamp_sort (published) DESC); + +CREATE INDEX idx_post_aggregates_featured_community_scaled ON public.post_aggregates USING btree (community_id, featured_community DESC, scaled_rank DESC, published DESC); + +CREATE INDEX idx_post_aggregates_featured_community_score ON public.post_aggregates USING btree (community_id, featured_community DESC, score DESC, published DESC); + +CREATE INDEX idx_post_aggregates_featured_local_active ON public.post_aggregates USING btree (featured_local DESC, hot_rank_active DESC, published DESC); + +CREATE INDEX idx_post_aggregates_featured_local_controversy ON public.post_aggregates USING btree (featured_local DESC, controversy_rank DESC); + +CREATE INDEX idx_post_aggregates_featured_local_hot ON public.post_aggregates USING btree (featured_local DESC, hot_rank DESC, published DESC); + +CREATE INDEX idx_post_aggregates_featured_local_most_comments ON public.post_aggregates USING btree (featured_local DESC, comments DESC, published DESC); + +CREATE INDEX idx_post_aggregates_featured_local_newest_comment_time ON public.post_aggregates USING btree (featured_local DESC, newest_comment_time DESC); + +CREATE INDEX idx_post_aggregates_featured_local_newest_comment_time_necro ON public.post_aggregates USING btree (featured_local DESC, newest_comment_time_necro DESC); + +CREATE INDEX idx_post_aggregates_featured_local_published ON public.post_aggregates USING btree (featured_local DESC, published DESC); + +CREATE INDEX idx_post_aggregates_featured_local_published_asc ON public.post_aggregates USING btree (featured_local DESC, public.reverse_timestamp_sort (published) DESC); + +CREATE INDEX idx_post_aggregates_featured_local_scaled ON public.post_aggregates USING btree (featured_local DESC, scaled_rank DESC, published DESC); + +CREATE INDEX idx_post_aggregates_featured_local_score ON public.post_aggregates USING btree (featured_local DESC, score DESC, published DESC); + diff --git a/migrations/2024-04-23-020604_add_post_id_index/up.sql b/migrations/2024-04-23-020604_add_post_id_index/up.sql new file mode 100644 index 000000000..8581d7169 --- /dev/null +++ b/migrations/2024-04-23-020604_add_post_id_index/up.sql @@ -0,0 +1,121 @@ +-- Add , post_id DESC to all these +DROP INDEX idx_post_aggregates_community_active; + +DROP INDEX idx_post_aggregates_community_controversy; + +DROP INDEX idx_post_aggregates_community_hot; + +DROP INDEX idx_post_aggregates_community_most_comments; + +DROP INDEX idx_post_aggregates_community_newest_comment_time; + +DROP INDEX idx_post_aggregates_community_newest_comment_time_necro; + +DROP INDEX idx_post_aggregates_community_published; + +DROP INDEX idx_post_aggregates_community_published_asc; + +DROP INDEX idx_post_aggregates_community_scaled; + +DROP INDEX idx_post_aggregates_community_score; + +DROP INDEX idx_post_aggregates_featured_community_active; + +DROP INDEX idx_post_aggregates_featured_community_controversy; + +DROP INDEX idx_post_aggregates_featured_community_hot; + +DROP INDEX idx_post_aggregates_featured_community_most_comments; + +DROP INDEX idx_post_aggregates_featured_community_newest_comment_time; + +DROP INDEX idx_post_aggregates_featured_community_newest_comment_time_necr; + +DROP INDEX idx_post_aggregates_featured_community_published; + +DROP INDEX idx_post_aggregates_featured_community_published_asc; + +DROP INDEX idx_post_aggregates_featured_community_scaled; + +DROP INDEX idx_post_aggregates_featured_community_score; + +DROP INDEX idx_post_aggregates_featured_local_active; + +DROP INDEX idx_post_aggregates_featured_local_controversy; + +DROP INDEX idx_post_aggregates_featured_local_hot; + +DROP INDEX idx_post_aggregates_featured_local_most_comments; + +DROP INDEX idx_post_aggregates_featured_local_newest_comment_time; + +DROP INDEX idx_post_aggregates_featured_local_newest_comment_time_necro; + +DROP INDEX idx_post_aggregates_featured_local_published; + +DROP INDEX idx_post_aggregates_featured_local_published_asc; + +DROP INDEX idx_post_aggregates_featured_local_scaled; + +DROP INDEX idx_post_aggregates_featured_local_score; + +CREATE INDEX idx_post_aggregates_community_active ON public.post_aggregates USING btree (community_id, featured_local DESC, hot_rank_active DESC, published DESC, post_id DESC); + +CREATE INDEX idx_post_aggregates_community_controversy ON public.post_aggregates USING btree (community_id, featured_local DESC, controversy_rank DESC, post_id DESC); + +CREATE INDEX idx_post_aggregates_community_hot ON public.post_aggregates USING btree (community_id, featured_local DESC, hot_rank DESC, published DESC, post_id DESC); + +CREATE INDEX idx_post_aggregates_community_most_comments ON public.post_aggregates USING btree (community_id, featured_local DESC, comments DESC, published DESC, post_id DESC); + +CREATE INDEX idx_post_aggregates_community_newest_comment_time ON public.post_aggregates USING btree (community_id, featured_local DESC, newest_comment_time DESC, post_id DESC); + +CREATE INDEX idx_post_aggregates_community_newest_comment_time_necro ON public.post_aggregates USING btree (community_id, featured_local DESC, newest_comment_time_necro DESC, post_id DESC); + +CREATE INDEX idx_post_aggregates_community_published ON public.post_aggregates USING btree (community_id, featured_local DESC, published DESC, post_id DESC); + +CREATE INDEX idx_post_aggregates_community_published_asc ON public.post_aggregates USING btree (community_id, featured_local DESC, public.reverse_timestamp_sort (published) DESC, post_id DESC); + +CREATE INDEX idx_post_aggregates_community_scaled ON public.post_aggregates USING btree (community_id, featured_local DESC, scaled_rank DESC, published DESC, post_id DESC); + +CREATE INDEX idx_post_aggregates_community_score ON public.post_aggregates USING btree (community_id, featured_local DESC, score DESC, published DESC, post_id DESC); + +CREATE INDEX idx_post_aggregates_featured_community_active ON public.post_aggregates USING btree (community_id, featured_community DESC, hot_rank_active DESC, published DESC, post_id DESC); + +CREATE INDEX idx_post_aggregates_featured_community_controversy ON public.post_aggregates USING btree (community_id, featured_community DESC, controversy_rank DESC, post_id DESC); + +CREATE INDEX idx_post_aggregates_featured_community_hot ON public.post_aggregates USING btree (community_id, featured_community DESC, hot_rank DESC, published DESC, post_id DESC); + +CREATE INDEX idx_post_aggregates_featured_community_most_comments ON public.post_aggregates USING btree (community_id, featured_community DESC, comments DESC, published DESC, post_id DESC); + +CREATE INDEX idx_post_aggregates_featured_community_newest_comment_time ON public.post_aggregates USING btree (community_id, featured_community DESC, newest_comment_time DESC, post_id DESC); + +CREATE INDEX idx_post_aggregates_featured_community_newest_comment_time_necr ON public.post_aggregates USING btree (community_id, featured_community DESC, newest_comment_time_necro DESC, post_id DESC); + +CREATE INDEX idx_post_aggregates_featured_community_published ON public.post_aggregates USING btree (community_id, featured_community DESC, published DESC, post_id DESC); + +CREATE INDEX idx_post_aggregates_featured_community_published_asc ON public.post_aggregates USING btree (community_id, featured_community DESC, public.reverse_timestamp_sort (published) DESC, post_id DESC); + +CREATE INDEX idx_post_aggregates_featured_community_scaled ON public.post_aggregates USING btree (community_id, featured_community DESC, scaled_rank DESC, published DESC, post_id DESC); + +CREATE INDEX idx_post_aggregates_featured_community_score ON public.post_aggregates USING btree (community_id, featured_community DESC, score DESC, published DESC, post_id DESC); + +CREATE INDEX idx_post_aggregates_featured_local_active ON public.post_aggregates USING btree (featured_local DESC, hot_rank_active DESC, published DESC, post_id DESC); + +CREATE INDEX idx_post_aggregates_featured_local_controversy ON public.post_aggregates USING btree (featured_local DESC, controversy_rank DESC, post_id DESC); + +CREATE INDEX idx_post_aggregates_featured_local_hot ON public.post_aggregates USING btree (featured_local DESC, hot_rank DESC, published DESC, post_id DESC); + +CREATE INDEX idx_post_aggregates_featured_local_most_comments ON public.post_aggregates USING btree (featured_local DESC, comments DESC, published DESC, post_id DESC); + +CREATE INDEX idx_post_aggregates_featured_local_newest_comment_time ON public.post_aggregates USING btree (featured_local DESC, newest_comment_time DESC, post_id DESC); + +CREATE INDEX idx_post_aggregates_featured_local_newest_comment_time_necro ON public.post_aggregates USING btree (featured_local DESC, newest_comment_time_necro DESC, post_id DESC); + +CREATE INDEX idx_post_aggregates_featured_local_published ON public.post_aggregates USING btree (featured_local DESC, published DESC, post_id DESC); + +CREATE INDEX idx_post_aggregates_featured_local_published_asc ON public.post_aggregates USING btree (featured_local DESC, public.reverse_timestamp_sort (published) DESC, post_id DESC); + +CREATE INDEX idx_post_aggregates_featured_local_scaled ON public.post_aggregates USING btree (featured_local DESC, scaled_rank DESC, published DESC, post_id DESC); + +CREATE INDEX idx_post_aggregates_featured_local_score ON public.post_aggregates USING btree (featured_local DESC, score DESC, published DESC, post_id DESC); +