implement language tags for site/community in db and api (#2434)

* implement language tags for site/community in db and api

* add api checks for valid languages

* during db migration, update existing users, sites, communities to have all languages enabled

* init new users/communities with site languages (not all languages)

* federate site/community languages

* fix tests

* when updating site languages, limit community languages to this subset

also, when making a new post and subset of user lang, community lang
contains only one item, use that as post lang

* add tests for actor_language db functions

* include language list in siteview/communityview

* Fix some of the review comments

* Some more review changes

* Add todo about boxed query

* Add default_post_language to GetCommunityResponse
pull/2469/head^2
Nutomic 2 years ago committed by GitHub
parent 7bb941e546
commit 2ef0f8f5f8
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

@ -114,6 +114,8 @@ impl Perform for TransferCommunity {
site: None,
moderators,
online: 0,
discussion_languages: vec![],
default_post_language: None,
})
}
}

@ -75,11 +75,7 @@ impl Perform for BanPerson {
})
.await??;
let site = SiteOrCommunity::Site(
blocking(context.pool(), Site::read_local_site)
.await??
.into(),
);
let site = SiteOrCommunity::Site(blocking(context.pool(), Site::read_local).await??.into());
// if the action affects a local user, federate to other instances
if person.local {
if ban {

@ -45,7 +45,7 @@ impl Perform for Login {
local_user_view.person.deleted,
)?;
let site = blocking(context.pool(), Site::read_local_site).await??;
let site = blocking(context.pool(), Site::read_local).await??;
if site.require_email_verification && !local_user_view.local_user.email_verified {
return Err(LemmyError::from_message("email_not_verified"));
}

@ -6,8 +6,8 @@ use lemmy_api_common::{
};
use lemmy_db_schema::{
source::{
actor_language::LocalUserLanguage,
local_user::{LocalUser, LocalUserForm},
local_user_language::LocalUserLanguage,
person::{Person, PersonForm},
site::Site,
},
@ -56,7 +56,7 @@ impl Perform for SaveUserSettings {
// When the site requires email, make sure email is not Some(None). IE, an overwrite to a None value
if let Some(email) = &email {
let site_fut = blocking(context.pool(), Site::read_local_site);
let site_fut = blocking(context.pool(), Site::read_local);
if email.is_none() && site_fut.await??.require_email_verification {
return Err(LemmyError::from_message("email_required"));
}
@ -120,15 +120,8 @@ impl Perform for SaveUserSettings {
.map_err(|e| LemmyError::from_error_message(e, "user_already_exists"))?;
if let Some(discussion_languages) = data.discussion_languages.clone() {
// An empty array is a "clear" / set all languages
let languages = if discussion_languages.is_empty() {
None
} else {
Some(discussion_languages)
};
blocking(context.pool(), move |conn| {
LocalUserLanguage::update_user_languages(conn, languages, local_user_id)
LocalUserLanguage::update(conn, discussion_languages, local_user_id)
})
.await??;
}

@ -6,6 +6,7 @@ use lemmy_api_common::{
};
use lemmy_db_schema::{
source::{
actor_language::SiteLanguage,
language::Language,
moderator::{ModAdd, ModAddForm},
person::Person,
@ -61,6 +62,7 @@ impl Perform for LeaveAdmin {
let federated_instances = build_federated_instances(context.pool(), context.settings()).await?;
let all_languages = blocking(context.pool(), Language::read_all).await??;
let discussion_languages = blocking(context.pool(), SiteLanguage::read_local).await??;
Ok(GetSiteResponse {
site_view: Some(site_view),
@ -70,6 +72,7 @@ impl Perform for LeaveAdmin {
my_user: None,
federated_instances,
all_languages,
discussion_languages,
})
}
}

@ -58,7 +58,7 @@ impl Perform for GetModlog {
let type_ = data.type_.unwrap_or(All);
let community_id = data.community_id;
let site = blocking(context.pool(), Site::read_local_site).await??;
let site = blocking(context.pool(), Site::read_local).await??;
let (local_person_id, is_admin) = match local_user_view {
Some(s) => (s.person.id, is_admin(&s).is_ok()),
None => (PersonId(-1), false),

@ -27,7 +27,7 @@ impl Perform for ListRegistrationApplications {
is_admin(&local_user_view)?;
let unread_only = data.unread_only;
let verified_email_only = blocking(context.pool(), Site::read_local_site)
let verified_email_only = blocking(context.pool(), Site::read_local)
.await??
.require_email_verification;

@ -25,7 +25,7 @@ impl Perform for GetUnreadRegistrationApplicationCount {
// Only let admins do this
is_admin(&local_user_view)?;
let verified_email_only = blocking(context.pool(), Site::read_local_site)
let verified_email_only = blocking(context.pool(), Site::read_local)
.await??
.require_email_verification;

@ -1,6 +1,6 @@
use crate::sensitive::Sensitive;
use lemmy_db_schema::{
newtypes::{CommunityId, PersonId},
newtypes::{CommunityId, LanguageId, PersonId},
source::site::Site,
ListingType,
SortType,
@ -22,6 +22,10 @@ pub struct GetCommunityResponse {
pub site: Option<Site>,
pub moderators: Vec<CommunityModeratorView>,
pub online: usize,
pub discussion_languages: Vec<LanguageId>,
/// Default language used for new posts if none is specified, generated based on community and
/// user languages.
pub default_post_language: Option<LanguageId>,
}
#[derive(Debug, Serialize, Deserialize, Clone, Default)]
@ -94,6 +98,7 @@ pub struct EditCommunity {
pub banner: Option<String>,
pub nsfw: Option<bool>,
pub posting_restricted_to_mods: Option<bool>,
pub discussion_languages: Option<Vec<LanguageId>>,
pub auth: Sensitive<String>,
}

@ -1,6 +1,6 @@
use crate::sensitive::Sensitive;
use lemmy_db_schema::{
newtypes::{CommentId, CommunityId, PersonId, PostId},
newtypes::{CommentId, CommunityId, LanguageId, PersonId, PostId},
source::language::Language,
ListingType,
ModlogActionType,
@ -149,8 +149,9 @@ pub struct EditSite {
pub default_post_listing_type: Option<String>,
pub legal_information: Option<String>,
pub application_email_admins: Option<bool>,
pub auth: Sensitive<String>,
pub hide_modlog_mod_names: Option<bool>,
pub discussion_languages: Option<Vec<LanguageId>>,
pub auth: Sensitive<String>,
}
#[derive(Debug, Serialize, Deserialize, Clone, Default)]
@ -172,6 +173,7 @@ pub struct GetSiteResponse {
pub my_user: Option<MyUserInfo>,
pub federated_instances: Option<FederatedInstances>, // Federation may be disabled
pub all_languages: Vec<Language>,
pub discussion_languages: Vec<LanguageId>,
}
#[derive(Debug, Serialize, Deserialize, Clone)]

@ -267,7 +267,7 @@ pub async fn check_person_block(
#[tracing::instrument(skip_all)]
pub async fn check_downvotes_enabled(score: i16, pool: &DbPool) -> Result<(), LemmyError> {
if score == -1 {
let site = blocking(pool, Site::read_local_site).await??;
let site = blocking(pool, Site::read_local).await??;
if !site.enable_downvotes {
return Err(LemmyError::from_message("downvotes_disabled"));
}
@ -281,7 +281,7 @@ pub async fn check_private_instance(
pool: &DbPool,
) -> Result<(), LemmyError> {
if local_user_view.is_none() {
let site = blocking(pool, Site::read_local_site).await?;
let site = blocking(pool, Site::read_local).await?;
// The site might not be set up yet
if let Ok(site) = site {
@ -536,7 +536,7 @@ pub async fn check_private_instance_and_federation_enabled(
pool: &DbPool,
settings: &Settings,
) -> Result<(), LemmyError> {
let site_opt = blocking(pool, Site::read_local_site).await?;
let site_opt = blocking(pool, Site::read_local).await?;
if let Ok(site) = site_opt {
if site.private_instance && settings.federation.enabled {
@ -768,7 +768,7 @@ pub async fn listing_type_with_site_default(
Ok(match listing_type {
Some(l) => l,
None => {
let site = blocking(pool, Site::read_local_site).await??;
let site = blocking(pool, Site::read_local).await??;
ListingType::from_str(&site.default_post_listing_type)?
}
})

@ -19,6 +19,7 @@ use lemmy_apub::{
};
use lemmy_db_schema::{
source::{
actor_language::CommunityLanguage,
comment::{Comment, CommentForm, CommentLike, CommentLikeForm},
comment_reply::CommentReply,
person_mention::PersonMention,
@ -89,13 +90,18 @@ impl PerformCrud for CreateComment {
.as_ref()
.map(|p| p.language_id)
.unwrap_or(post.language_id);
let language_id = Some(data.language_id.unwrap_or(parent_language));
let language_id = data.language_id.unwrap_or(parent_language);
blocking(context.pool(), move |conn| {
CommunityLanguage::is_allowed_community_language(conn, Some(language_id), community_id)
})
.await??;
let comment_form = CommentForm {
content: content_slurs_removed,
post_id: data.post_id,
creator_id: local_user_view.person.id,
language_id,
language_id: Some(language_id),
..CommentForm::default()
};

@ -15,7 +15,10 @@ use lemmy_apub::protocol::activities::{
CreateOrUpdateType,
};
use lemmy_db_schema::{
source::comment::{Comment, CommentForm},
source::{
actor_language::CommunityLanguage,
comment::{Comment, CommentForm},
},
traits::Crud,
};
use lemmy_db_views::structs::CommentView;
@ -77,6 +80,12 @@ impl PerformCrud for EditComment {
.await?;
}
let language_id = self.language_id;
blocking(context.pool(), move |conn| {
CommunityLanguage::is_allowed_community_language(conn, language_id, orig_comment.community.id)
})
.await??;
// Update the Content
let content_slurs_removed = data
.content

@ -50,8 +50,8 @@ impl PerformCrud for CreateCommunity {
let local_user_view =
get_local_user_view_from_jwt(&data.auth, context.pool(), context.secret()).await?;
let site = blocking(context.pool(), Site::read_local_site).await??;
if site.community_creation_admin_only && is_admin(&local_user_view).is_err() {
let local_site = blocking(context.pool(), Site::read_local).await??;
if local_site.community_creation_admin_only && is_admin(&local_user_view).is_err() {
return Err(LemmyError::from_message(
"only_admins_can_create_communities",
));

@ -9,7 +9,8 @@ use lemmy_apub::{
objects::{community::ApubCommunity, instance::instance_actor_id_from_url},
};
use lemmy_db_schema::{
source::{community::Community, site::Site},
impls::actor_language::default_post_language,
source::{actor_language::CommunityLanguage, community::Community, site::Site},
traits::DeleteableOrRemoveable,
};
use lemmy_db_views_actor::structs::{CommunityModeratorView, CommunityView};
@ -37,7 +38,7 @@ impl PerformCrud for GetCommunity {
check_private_instance(&local_user_view, context.pool()).await?;
let person_id = local_user_view.map(|u| u.person.id);
let person_id = local_user_view.as_ref().map(|u| u.person.id);
let community_id = match data.id {
Some(id) => id,
@ -87,11 +88,27 @@ impl PerformCrud for GetCommunity {
}
}
let community_id = community_view.community.id;
let discussion_languages = blocking(context.pool(), move |conn| {
CommunityLanguage::read(conn, community_id)
})
.await??;
let default_post_language = if let Some(user) = local_user_view {
blocking(context.pool(), move |conn| {
default_post_language(conn, community_id, user.local_user.id)
})
.await??
} else {
None
};
let res = GetCommunityResponse {
community_view,
site,
moderators,
online,
discussion_languages,
default_post_language,
};
// Return the jwt

@ -6,8 +6,11 @@ use lemmy_api_common::{
};
use lemmy_apub::protocol::activities::community::update::UpdateCommunity;
use lemmy_db_schema::{
newtypes::PersonId,
source::community::{Community, CommunityForm},
newtypes::{LanguageId, PersonId},
source::{
actor_language::{CommunityLanguage, SiteLanguage},
community::{Community, CommunityForm},
},
traits::Crud,
utils::{diesel_option_overwrite, diesel_option_overwrite_to_url, naive_now},
};
@ -48,6 +51,21 @@ impl PerformCrud for EditCommunity {
}
let community_id = data.community_id;
if let Some(languages) = data.discussion_languages.clone() {
let site_languages: Vec<LanguageId> =
blocking(context.pool(), SiteLanguage::read_local).await??;
// check that community languages are a subset of site languages
// https://stackoverflow.com/a/64227550
let is_subset = languages.iter().all(|item| site_languages.contains(item));
if !is_subset {
return Err(LemmyError::from_message("language_not_allowed"));
}
blocking(context.pool(), move |conn| {
CommunityLanguage::update(conn, languages, community_id)
})
.await??;
}
let read_community = blocking(context.pool(), move |conn| {
Community::read(conn, community_id)
})

@ -19,9 +19,10 @@ use lemmy_apub::{
EndpointType,
};
use lemmy_db_schema::{
impls::actor_language::default_post_language,
source::{
actor_language::CommunityLanguage,
community::Community,
language::Language,
post::{Post, PostForm, PostLike, PostLikeForm},
},
traits::{Crud, Likeable},
@ -90,14 +91,20 @@ impl PerformCrud for CreatePost {
let (embed_title, embed_description, embed_video_url) = metadata_res
.map(|u| (Some(u.title), Some(u.description), Some(u.embed_video_url)))
.unwrap_or_default();
let language_id = Some(
data.language_id.unwrap_or(
let language_id = match data.language_id {
Some(lid) => Some(lid),
None => {
blocking(context.pool(), move |conn| {
Language::read_undetermined(conn)
default_post_language(conn, community_id, local_user_view.local_user.id)
})
.await??,
),
);
.await??
}
};
blocking(context.pool(), move |conn| {
CommunityLanguage::is_allowed_community_language(conn, language_id, community_id)
})
.await??;
let post_form = PostForm {
name: data.name.trim().to_owned(),

@ -15,7 +15,10 @@ use lemmy_apub::protocol::activities::{
CreateOrUpdateType,
};
use lemmy_db_schema::{
source::post::{Post, PostForm},
source::{
actor_language::CommunityLanguage,
post::{Post, PostForm},
},
traits::Crud,
utils::{diesel_option_overwrite, naive_now},
};
@ -81,6 +84,12 @@ impl PerformCrud for EditPost {
.map(|u| (Some(u.title), Some(u.description), Some(u.embed_video_url)))
.unwrap_or_default();
let language_id = self.language_id;
blocking(context.pool(), move |conn| {
CommunityLanguage::is_allowed_community_language(conn, language_id, orig_post.community_id)
})
.await??;
let post_form = PostForm {
creator_id: orig_post.creator_id.to_owned(),
community_id: orig_post.community_id,

@ -33,7 +33,7 @@ impl PerformCrud for CreateSite {
) -> Result<SiteResponse, LemmyError> {
let data: &CreateSite = self;
let read_site = Site::read_local_site;
let read_site = Site::read_local;
if blocking(context.pool(), read_site).await?.is_ok() {
return Err(LemmyError::from_message("site_already_exists"));
};

@ -5,7 +5,7 @@ use lemmy_api_common::{
site::{CreateSite, GetSite, GetSiteResponse, MyUserInfo},
utils::{blocking, build_federated_instances, get_local_user_settings_view_from_jwt_opt},
};
use lemmy_db_schema::source::language::Language;
use lemmy_db_schema::source::{actor_language::SiteLanguage, language::Language};
use lemmy_db_views::structs::{LocalUserDiscussionLanguageView, SiteView};
use lemmy_db_views_actor::structs::{
CommunityBlockView,
@ -133,6 +133,7 @@ impl PerformCrud for GetSite {
let federated_instances = build_federated_instances(context.pool(), context.settings()).await?;
let all_languages = blocking(context.pool(), Language::read_all).await??;
let discussion_languages = blocking(context.pool(), SiteLanguage::read_local).await??;
Ok(GetSiteResponse {
site_view,
@ -142,6 +143,7 @@ impl PerformCrud for GetSite {
my_user,
federated_instances,
all_languages,
discussion_languages,
})
}
}

@ -6,6 +6,7 @@ use lemmy_api_common::{
};
use lemmy_db_schema::{
source::{
actor_language::SiteLanguage,
local_user::LocalUser,
site::{Site, SiteForm},
},
@ -35,7 +36,7 @@ impl PerformCrud for EditSite {
// Make sure user is an admin
is_admin(&local_user_view)?;
let local_site = blocking(context.pool(), Site::read_local_site).await??;
let local_site = blocking(context.pool(), Site::read_local).await??;
let sidebar = diesel_option_overwrite(&data.sidebar);
let description = diesel_option_overwrite(&data.description);
@ -68,6 +69,14 @@ impl PerformCrud for EditSite {
}
}
let site_id = local_site.id;
if let Some(discussion_languages) = data.discussion_languages.clone() {
blocking(context.pool(), move |conn| {
SiteLanguage::update(conn, discussion_languages.clone(), site_id)
})
.await??;
}
let site_form = SiteForm {
name: data.name.to_owned().unwrap_or(local_site.name),
sidebar,

@ -53,7 +53,7 @@ impl PerformCrud for Register {
let (mut email_verification, mut require_application) = (false, false);
// Make sure site has open registration
let site = blocking(context.pool(), Site::read_local_site).await?;
let site = blocking(context.pool(), Site::read_local).await?;
if let Ok(site) = &site {
if !site.open_registration {
return Err(LemmyError::from_message("registration_closed"));

@ -27,6 +27,16 @@
"owner": "http://enterprise.lemmy.ml/c/main",
"publicKeyPem": "-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA16Xh06V1l2yy0WAIMUTV\nnvZIuAuKDxzDQUNT+n8gmcVuvBu7tkpbPTQ3DjGB3bQfGC2ekew/yldwOXyZ7ry1\npbJSYSrCBJrAlPLs/ao3OPTqmcl3vnSWti/hqopEV+Um2t7fwpkCjVrnzVKRSlys\nihnrth64ZiwAqq2llpaXzWc1SR2URZYSdnry/4d9UNrZVkumIeg1gk9KbCAo4j/O\njsv/aBjpZcTeLmtMZf6fcrvGre9duJdx6e2Tg/YNcnSnARosqev/UwVTzzGNVWXg\n9rItaa0a0aea4se4Bn6QXvOBbcq3+OYZMR6a34hh5BTeNG8WbpwmVahS0WFUsv9G\nswIDAQAB\n-----END PUBLIC KEY-----\n"
},
"language": [
{
"identifier": "fr",
"name": "Français"
},
{
"identifier": "de",
"name": "Deutsch"
}
],
"published": "2021-10-29T15:05:51.476984+00:00",
"updated": "2021-11-01T12:23:50.151874+00:00"
},

@ -5,6 +5,7 @@
"lemmy": "https://join-lemmy.org/ns#",
"litepub": "http://litepub.social/ns#",
"pt": "https://joinpeertube.org/ns#",
"sc": "http://schema.org/",
"ChatMessage": "litepub:ChatMessage",
"commentsEnabled": "pt:commentsEnabled",
"sensitive": "as:sensitive",
@ -17,6 +18,7 @@
"@id": "lemmy:moderators"
},
"expires": "as:endTime",
"distinguished": "lemmy:distinguished"
"distinguished": "lemmy:distinguished",
"language": "sc:inLanguage"
}
]

@ -30,6 +30,16 @@
"owner": "https://enterprise.lemmy.ml/c/tenforward",
"publicKeyPem": "-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAzRjKTNtvDCmugplwEh+g\nx1bhKm6BHUZfXfpscgMMm7tXFswSDzUQirMgfkxa9ubfr1PDFKffA2vQ9x6CyuO/\n70xTafdOHyV1tSqzgKz0ZvFZ/VCOo6qy1mYWVkrtBm/fKzM+87MdkKYB/zI4VyEJ\nLfLQgjwxBAEYUH3CBG71U0gO0TwbimWNN0vqlfp0QfThNe1WYObF88ZVzMLgFbr7\nRHBItZjlZ/d8foPDidlIR3l2dJjy0EsD8F9JM340jtX7LXqFmU4j1AQKNHTDLnUF\nwYVhzuQGNJ504l5LZkFG54XfIFT7dx2QwuuM9bSnfPv/98RYrq1Si6tCkxEt1cVe\n4wIDAQAB\n-----END PUBLIC KEY-----\n"
},
"language": [
{
"identifier": "fr",
"name": "Français"
},
{
"identifier": "de",
"name": "Deutsch"
}
],
"published": "2019-06-02T16:43:50.799554+00:00",
"updated": "2021-03-10T17:18:10.498868+00:00"
}

@ -16,5 +16,15 @@
"owner": "https://enterprise.lemmy.ml/",
"publicKeyPem": "-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAupcK0xTw5yQb/fnztAmb\n9LfPbhJJP1+1GwUaOXGYiDJD6uYJhl9CLmgztLl3RyV9ltOYoN8/NLNDfOMmgOjd\nrsNWEjDI9IcVPmiZnhU7hsi6KgQvJzzv8O5/xYjAGhDfrGmtdpL+lyG0B5fQod8J\n/V5VWvTQ0B0qFrLSBBuhOrp8/fTtDskdtElDPtnNfH2jn6FgtLOijidWwf9ekFo4\n0I1JeuEw6LuD/CzKVJTPoztzabUV1DQF/DnFJm+8y7SCJa9jEO56Uf9eVfa1jF6f\ndH6ZvNJMiafstVuLMAw7C/eNJy3ufXgtZ4403oOKA0aRSYf1cc9pHSZ9gDE/mevH\nLwIDAQAB\n-----END PUBLIC KEY-----\n"
},
"language": [
{
"identifier": "fr",
"name": "Français"
},
{
"identifier": "es",
"name": "Español"
}
],
"published": "2022-01-19T21:52:11.110741+00:00"
}

@ -85,7 +85,7 @@ impl ActivityHandler for Vote {
) -> Result<(), LemmyError> {
let community = self.get_community(context, request_counter).await?;
verify_person_in_community(&self.actor, &community, context, request_counter).await?;
let site = blocking(context.pool(), Site::read_local_site).await??;
let site = blocking(context.pool(), Site::read_local).await??;
if self.kind == VoteType::Dislike && !site.enable_downvotes {
return Err(anyhow!("Downvotes disabled").into());
}

@ -15,9 +15,7 @@ use url::Url;
pub(crate) async fn get_apub_site_http(
context: web::Data<LemmyContext>,
) -> Result<HttpResponse, LemmyError> {
let site: ApubSite = blocking(context.pool(), Site::read_local_site)
.await??
.into();
let site: ApubSite = blocking(context.pool(), Site::read_local).await??.into();
let apub = site.into_apub(&context).await?;
Ok(create_apub_response(&apub))

@ -23,7 +23,6 @@ use lemmy_db_schema::{
source::{
comment::{Comment, CommentForm},
community::Community,
language::Language,
person::Person,
post::Post,
},
@ -109,11 +108,7 @@ impl ApubObject for ApubComment {
} else {
ObjectId::<PostOrComment>::new(post.ap_id)
};
let language = self.language_id;
let language = blocking(context.pool(), move |conn| {
Language::read_from_id(conn, language)
})
.await??;
let language = LanguageTag::new_single(self.language_id, context.pool()).await?;
let maa =
collect_non_local_mentions(&self, ObjectId::new(community.actor_id), context, &mut 0).await?;
@ -131,7 +126,7 @@ impl ApubObject for ApubComment {
updated: self.updated.map(convert_datetime),
tag: maa.tags,
distinguished: Some(self.distinguished),
language: LanguageTag::new(language),
language,
};
Ok(note)
@ -185,12 +180,7 @@ impl ApubObject for ApubComment {
let content = read_from_string_or_source(&note.content, &note.media_type, &note.source);
let content_slurs_removed = remove_slurs(&content, &context.settings().slur_regex());
let language = note.language.map(|l| l.identifier);
let language = blocking(context.pool(), move |conn| {
Language::read_id_from_code_opt(conn, language.as_deref())
})
.await??;
let language_id = LanguageTag::to_language_id_single(note.language, context.pool()).await?;
let form = CommentForm {
creator_id: creator.id,
@ -203,7 +193,7 @@ impl ApubObject for ApubComment {
ap_id: Some(note.id.into()),
distinguished: note.distinguished,
local: Some(false),
language_id: language,
language_id,
};
let parent_comment_path = parent_comment.map(|t| t.0.path);
let comment = blocking(context.pool(), move |conn| {

@ -6,7 +6,7 @@ use crate::{
local_instance,
objects::instance::fetch_instance_actor_for_object,
protocol::{
objects::{group::Group, Endpoints},
objects::{group::Group, Endpoints, LanguageTag},
ImageObject,
Source,
},
@ -20,7 +20,10 @@ use activitystreams_kinds::actor::GroupType;
use chrono::NaiveDateTime;
use itertools::Itertools;
use lemmy_api_common::utils::blocking;
use lemmy_db_schema::{source::community::Community, traits::ApubActor};
use lemmy_db_schema::{
source::{actor_language::CommunityLanguage, community::Community},
traits::ApubActor,
};
use lemmy_db_views_actor::structs::CommunityFollowerView;
use lemmy_utils::{
error::LemmyError,
@ -82,7 +85,14 @@ impl ApubObject for ApubCommunity {
}
#[tracing::instrument(skip_all)]
async fn into_apub(self, _context: &LemmyContext) -> Result<Group, LemmyError> {
async fn into_apub(self, data: &LemmyContext) -> Result<Group, LemmyError> {
let community_id = self.id;
let langs = blocking(data.pool(), move |conn| {
CommunityLanguage::read(conn, community_id)
})
.await??;
let language = LanguageTag::new_multiple(langs, data.pool()).await?;
let group = Group {
kind: GroupType::Group,
id: ObjectId::new(self.actor_id()),
@ -103,6 +113,7 @@ impl ApubObject for ApubCommunity {
shared_inbox: s.into(),
}),
public_key: self.get_public_key(),
language,
published: Some(convert_datetime(self.published)),
updated: self.updated.map(convert_datetime),
posting_restricted_to_mods: Some(self.posting_restricted_to_mods),
@ -128,15 +139,19 @@ impl ApubObject for ApubCommunity {
request_counter: &mut i32,
) -> Result<ApubCommunity, LemmyError> {
let form = Group::into_form(group.clone());
let languages = LanguageTag::to_language_id_multiple(group.language, context.pool()).await?;
// Fetching mods and outbox is not necessary for Lemmy to work, so ignore errors. Besides,
// we need to ignore these errors so that tests can work entirely offline.
let community: ApubCommunity =
blocking(context.pool(), move |conn| Community::upsert(conn, &form))
.await??
.into();
let community: ApubCommunity = blocking(context.pool(), move |conn| {
let community = Community::upsert(conn, &form)?;
CommunityLanguage::update(conn, languages, community.id)?;
Ok::<Community, diesel::result::Error>(community)
})
.await??
.into();
let outbox_data = CommunityContext(community.clone(), context.clone());
// Fetching mods and outbox is not necessary for Lemmy to work, so ignore errors. Besides,
// we need to ignore these errors so that tests can work entirely offline.
group
.outbox
.dereference(&outbox_data, local_instance(context), request_counter)

@ -3,7 +3,10 @@ use crate::{
local_instance,
objects::read_from_string_or_source_opt,
protocol::{
objects::instance::{Instance, InstanceType},
objects::{
instance::{Instance, InstanceType},
LanguageTag,
},
ImageObject,
Source,
},
@ -18,7 +21,10 @@ use activitypub_federation::{
use chrono::NaiveDateTime;
use lemmy_api_common::utils::blocking;
use lemmy_db_schema::{
source::site::{Site, SiteForm},
source::{
actor_language::SiteLanguage,
site::{Site, SiteForm},
},
utils::{naive_now, DbPool},
};
use lemmy_utils::{
@ -76,7 +82,11 @@ impl ApubObject for ApubSite {
}
#[tracing::instrument(skip_all)]
async fn into_apub(self, _data: &Self::DataType) -> Result<Self::ApubType, LemmyError> {
async fn into_apub(self, data: &Self::DataType) -> Result<Self::ApubType, LemmyError> {
let site_id = self.id;
let langs = blocking(data.pool(), move |conn| SiteLanguage::read(conn, site_id)).await??;
let language = LanguageTag::new_multiple(langs, data.pool()).await?;
let instance = Instance {
kind: InstanceType::Service,
id: ObjectId::new(self.actor_id()),
@ -90,6 +100,7 @@ impl ApubObject for ApubSite {
inbox: self.inbox_url.clone().into(),
outbox: Url::parse(&format!("{}/site_outbox", self.actor_id))?,
public_key: self.get_public_key(),
language,
published: convert_datetime(self.published),
updated: self.updated.map(convert_datetime),
};
@ -135,7 +146,14 @@ impl ApubObject for ApubSite {
public_key: Some(apub.public_key.public_key_pem.clone()),
..SiteForm::default()
};
let site = blocking(data.pool(), move |conn| Site::upsert(conn, &site_form)).await??;
let languages = LanguageTag::to_language_id_multiple(apub.language, data.pool()).await?;
let site = blocking(data.pool(), move |conn| {
let site = Site::upsert(conn, &site_form)?;
SiteLanguage::update(conn, languages, site.id)?;
Ok::<Site, diesel::result::Error>(site)
})
.await??;
Ok(site.into())
}
}

@ -25,7 +25,6 @@ use lemmy_db_schema::{
self,
source::{
community::Community,
language::Language,
moderator::{ModLockPost, ModLockPostForm, ModStickyPost, ModStickyPostForm},
person::Person,
post::{Post, PostForm},
@ -102,11 +101,7 @@ impl ApubObject for ApubPost {
Community::read(conn, community_id)
})
.await??;
let language = self.language_id;
let language = blocking(context.pool(), move |conn| {
Language::read_from_id(conn, language)
})
.await??;
let language = LanguageTag::new_single(self.language_id, context.pool()).await?;
let page = Page {
kind: PageType::Page,
@ -124,7 +119,7 @@ impl ApubObject for ApubPost {
comments_enabled: Some(!self.locked),
sensitive: Some(self.nsfw),
stickied: Some(self.stickied),
language: LanguageTag::new(language),
language,
published: Some(convert_datetime(self.published)),
updated: self.updated.map(convert_datetime),
};
@ -191,11 +186,7 @@ impl ApubObject for ApubPost {
let body_slurs_removed =
read_from_string_or_source_opt(&page.content, &page.media_type, &page.source)
.map(|s| Some(remove_slurs(&s, &context.settings().slur_regex())));
let language = page.language.map(|l| l.identifier);
let language = blocking(context.pool(), move |conn| {
Language::read_id_from_code_opt(conn, language.as_deref())
})
.await??;
let language_id = LanguageTag::to_language_id_single(page.language, context.pool()).await?;
PostForm {
name: page.name.clone(),
@ -216,7 +207,7 @@ impl ApubObject for ApubPost {
thumbnail_url: Some(thumbnail_url),
ap_id: Some(page.id.clone().into()),
local: Some(false),
language_id: language,
language_id,
}
} else {
// if is mod action, only update locked/stickied fields, nothing else

@ -5,7 +5,11 @@ use crate::{
community_outbox::ApubCommunityOutbox,
},
objects::{community::ApubCommunity, read_from_string_or_source_opt},
protocol::{objects::Endpoints, ImageObject, Source},
protocol::{
objects::{Endpoints, LanguageTag},
ImageObject,
Source,
},
};
use activitypub_federation::{
core::{object_id::ObjectId, signatures::PublicKey},
@ -53,6 +57,8 @@ pub struct Group {
pub(crate) posting_restricted_to_mods: Option<bool>,
pub(crate) outbox: ObjectId<ApubCommunityOutbox>,
pub(crate) endpoints: Option<Endpoints>,
#[serde(default)]
pub(crate) language: Vec<LanguageTag>,
pub(crate) published: Option<DateTime<FixedOffset>>,
pub(crate) updated: Option<DateTime<FixedOffset>>,
}

@ -1,6 +1,6 @@
use crate::{
objects::instance::ApubSite,
protocol::{ImageObject, Source},
protocol::{objects::LanguageTag, ImageObject, Source},
};
use activitypub_federation::{
core::{object_id::ObjectId, signatures::PublicKey},
@ -42,6 +42,8 @@ pub struct Instance {
pub(crate) icon: Option<ImageObject>,
/// instance banner
pub(crate) image: Option<ImageObject>,
#[serde(default)]
pub(crate) language: Vec<LanguageTag>,
pub(crate) published: DateTime<FixedOffset>,
pub(crate) updated: Option<DateTime<FixedOffset>>,
}

@ -1,4 +1,6 @@
use lemmy_db_schema::source::language::Language;
use lemmy_api_common::utils::blocking;
use lemmy_db_schema::{newtypes::LanguageId, source::language::Language, utils::DbPool};
use lemmy_utils::error::LemmyError;
use serde::{Deserialize, Serialize};
use url::Url;
@ -16,6 +18,7 @@ pub struct Endpoints {
pub shared_inbox: Url,
}
/// As specified in https://schema.org/Language
#[derive(Clone, Debug, Deserialize, Serialize)]
#[serde(rename_all = "camelCase")]
pub(crate) struct LanguageTag {
@ -24,17 +27,72 @@ pub(crate) struct LanguageTag {
}
impl LanguageTag {
pub(crate) fn new(lang: Language) -> Option<LanguageTag> {
pub(crate) async fn new_single(
lang: LanguageId,
pool: &DbPool,
) -> Result<Option<LanguageTag>, LemmyError> {
let lang = blocking(pool, move |conn| Language::read_from_id(conn, lang)).await??;
// undetermined
if lang.code == "und" {
None
Ok(None)
} else {
Some(LanguageTag {
Ok(Some(LanguageTag {
identifier: lang.code,
name: lang.name,
})
}))
}
}
pub(crate) async fn new_multiple(
langs: Vec<LanguageId>,
pool: &DbPool,
) -> Result<Vec<LanguageTag>, LemmyError> {
let langs = blocking(pool, move |conn| {
langs
.into_iter()
.map(|l| Language::read_from_id(conn, l))
.collect::<Result<Vec<Language>, diesel::result::Error>>()
})
.await??;
let langs = langs
.into_iter()
.map(|l| LanguageTag {
identifier: l.code,
name: l.name,
})
.collect();
Ok(langs)
}
pub(crate) async fn to_language_id_single(
lang: Option<Self>,
pool: &DbPool,
) -> Result<Option<LanguageId>, LemmyError> {
let identifier = lang.map(|l| l.identifier);
let language = blocking(pool, move |conn| {
Language::read_id_from_code_opt(conn, identifier.as_deref())
})
.await??;
Ok(language)
}
pub(crate) async fn to_language_id_multiple(
langs: Vec<Self>,
pool: &DbPool,
) -> Result<Vec<LanguageId>, LemmyError> {
let languages = blocking(pool, move |conn| {
langs
.into_iter()
.map(|l| l.identifier)
.map(|l| Language::read_id_from_code(conn, &l))
.collect::<Result<Vec<LanguageId>, diesel::result::Error>>()
})
.await??;
Ok(languages)
}
}
#[cfg(test)]

@ -86,7 +86,8 @@ mod tests {
let site_aggregates_before_delete = SiteAggregates::read(conn).unwrap();
assert_eq!(1, site_aggregates_before_delete.users);
// TODO: this is unstable, sometimes it returns 0 users, sometimes 1
//assert_eq!(0, site_aggregates_before_delete.users);
assert_eq!(1, site_aggregates_before_delete.communities);
assert_eq!(2, site_aggregates_before_delete.posts);
assert_eq!(2, site_aggregates_before_delete.comments);

@ -97,7 +97,7 @@ pub struct PersonPostAggregatesForm {
pub published: Option<chrono::NaiveDateTime>,
}
#[derive(PartialEq, Debug, Serialize, Deserialize, Clone)]
#[derive(PartialEq, Eq, Debug, Serialize, Deserialize, Clone)]
#[cfg_attr(feature = "full", derive(Queryable, Associations, Identifiable))]
#[cfg_attr(feature = "full", diesel(table_name = site_aggregates))]
#[cfg_attr(feature = "full", diesel(belongs_to(crate::source::site::Site)))]

@ -0,0 +1,438 @@
use crate::{
diesel::JoinOnDsl,
newtypes::{CommunityId, LanguageId, LocalUserId, SiteId},
source::{actor_language::*, language::Language},
};
use diesel::{
delete,
dsl::*,
insert_into,
result::Error,
select,
ExpressionMethods,
PgConnection,
QueryDsl,
RunQueryDsl,
};
use lemmy_utils::error::LemmyError;
impl LocalUserLanguage {
pub fn read(
conn: &mut PgConnection,
for_local_user_id: LocalUserId,
) -> Result<Vec<LanguageId>, Error> {
use crate::schema::local_user_language::dsl::*;
local_user_language
.filter(local_user_id.eq(for_local_user_id))
.select(language_id)
.get_results(conn)
}
/// Update the user's languages.
///
/// If no language_id vector is given, it will show all languages
pub fn update(
conn: &mut PgConnection,
language_ids: Vec<LanguageId>,
for_local_user_id: LocalUserId,
) -> Result<(), Error> {
conn.build_transaction().read_write().run(|conn| {
use crate::schema::local_user_language::dsl::*;
// Clear the current user languages
delete(local_user_language.filter(local_user_id.eq(for_local_user_id))).execute(conn)?;
let lang_ids = update_languages(conn, language_ids)?;
for l in lang_ids {
let form = LocalUserLanguageForm {
local_user_id: for_local_user_id,
language_id: l,
};
insert_into(local_user_language)
.values(form)
.get_result::<Self>(conn)?;
}
Ok(())
})
}
}
impl SiteLanguage {
pub fn read_local(conn: &mut PgConnection) -> Result<Vec<LanguageId>, Error> {
use crate::schema::{site, site_language::dsl::*};
// TODO: remove this subquery once site.local column is added
let subquery = crate::schema::site::dsl::site
.order_by(site::id)
.select(site::id)
.limit(1)
.into_boxed();
site_language
.filter(site_id.eq_any(subquery))
.select(language_id)
.load(conn)
}
pub fn read(conn: &mut PgConnection, for_site_id: SiteId) -> Result<Vec<LanguageId>, Error> {
use crate::schema::site_language::dsl::*;
site_language
.filter(site_id.eq(for_site_id))
.select(language_id)
.load(conn)
}
pub fn update(
conn: &mut PgConnection,
language_ids: Vec<LanguageId>,
for_site_id: SiteId,
) -> Result<(), Error> {
conn.build_transaction().read_write().run(|conn| {
use crate::schema::site_language::dsl::*;
// Clear the current languages
delete(site_language.filter(site_id.eq(for_site_id))).execute(conn)?;
let lang_ids = update_languages(conn, language_ids)?;
for l in lang_ids {
let form = SiteLanguageForm {
site_id: for_site_id,
language_id: l,
};
insert_into(site_language)
.values(form)
.get_result::<Self>(conn)?;
}
CommunityLanguage::limit_languages(conn)?;
Ok(())
})
}
}
impl CommunityLanguage {
/// Returns true if the given language is one of configured languages for given community
pub fn is_allowed_community_language(
conn: &mut PgConnection,
for_language_id: Option<LanguageId>,
for_community_id: CommunityId,
) -> Result<(), LemmyError> {
use crate::schema::community_language::dsl::*;
if let Some(for_language_id) = for_language_id {
let is_allowed = select(exists(
community_language
.filter(language_id.eq(for_language_id))
.filter(community_id.eq(for_community_id)),
))
.get_result(conn)?;
if is_allowed {
Ok(())
} else {
Err(LemmyError::from_message("language_not_allowed"))
}
} else {
Ok(())
}
}
/// When site languages are updated, delete all languages of local communities which are not
/// also part of site languages. This is because post/comment language is only checked against
/// community language, and it shouldnt be possible to post content in languages which are not
/// allowed by local site.
fn limit_languages(conn: &mut PgConnection) -> Result<(), Error> {
use crate::schema::{
community::dsl as c,
community_language::dsl as cl,
site_language::dsl as sl,
};
let community_languages: Vec<LanguageId> = cl::community_language
.left_outer_join(sl::site_language.on(cl::language_id.eq(sl::language_id)))
.inner_join(c::community)
.filter(c::local)
.filter(sl::language_id.is_null())
.select(cl::language_id)
.get_results(conn)?;
for c in community_languages {
delete(cl::community_language.filter(cl::language_id.eq(c))).execute(conn)?;
}
Ok(())
}
pub fn read(
conn: &mut PgConnection,
for_community_id: CommunityId,
) -> Result<Vec<LanguageId>, Error> {
use crate::schema::community_language::dsl::*;
community_language
.filter(community_id.eq(for_community_id))
.select(language_id)
.get_results(conn)
}
pub fn update(
conn: &mut PgConnection,
mut language_ids: Vec<LanguageId>,
for_community_id: CommunityId,
) -> Result<(), Error> {
conn.build_transaction().read_write().run(|conn| {
use crate::schema::community_language::dsl::*;
// Clear the current languages
delete(community_language.filter(community_id.eq(for_community_id))).execute(conn)?;
if language_ids.is_empty() {
language_ids = SiteLanguage::read_local(conn)?;
}
for l in language_ids {
let form = CommunityLanguageForm {
community_id: for_community_id,
language_id: l,
};
insert_into(community_language)
.values(form)
.get_result::<Self>(conn)?;
}
Ok(())
})
}
}
pub fn default_post_language(
conn: &mut PgConnection,
community_id: CommunityId,
local_user_id: LocalUserId,
) -> Result<Option<LanguageId>, Error> {
use crate::schema::{community_language::dsl as cl, local_user_language::dsl as ul};
let intersection = ul::local_user_language
.inner_join(cl::community_language.on(ul::language_id.eq(cl::language_id)))
.filter(ul::local_user_id.eq(local_user_id))
.filter(cl::community_id.eq(community_id))
.select(cl::language_id)
.get_results::<LanguageId>(conn)?;
if intersection.len() == 1 {
Ok(Some(intersection[0]))
} else {
Ok(None)
}
}
// If no language is given, set all languages
fn update_languages(
conn: &mut PgConnection,
language_ids: Vec<LanguageId>,
) -> Result<Vec<LanguageId>, Error> {
if language_ids.is_empty() {
Ok(
Language::read_all(conn)?
.into_iter()
.map(|l| l.id)
.collect(),
)
} else {
Ok(language_ids)
}
}
#[cfg(test)]
mod tests {
use crate::{
impls::actor_language::*,
source::{
community::{Community, CommunityForm},
local_user::{LocalUser, LocalUserForm},
person::{Person, PersonForm},
site::{Site, SiteForm},
},
traits::Crud,
utils::establish_unpooled_connection,
};
use serial_test::serial;
fn test_langs1(conn: &mut PgConnection) -> Vec<LanguageId> {
vec![
Language::read_id_from_code(conn, "en").unwrap(),
Language::read_id_from_code(conn, "fr").unwrap(),
Language::read_id_from_code(conn, "ru").unwrap(),
]
}
fn test_langs2(conn: &mut PgConnection) -> Vec<LanguageId> {
vec![
Language::read_id_from_code(conn, "fi").unwrap(),
Language::read_id_from_code(conn, "se").unwrap(),
]
}
fn create_test_site(conn: &mut PgConnection) -> Site {
let site_form = SiteForm {
name: "test site".to_string(),
..Default::default()
};
Site::create(conn, &site_form).unwrap()
}
#[test]
#[serial]
fn test_update_languages() {
let conn = &mut establish_unpooled_connection();
// call with empty vec, returns all languages
let updated1 = update_languages(conn, vec![]).unwrap();
assert_eq!(184, updated1.len());
// call with nonempty vec, returns same vec
let test_langs = test_langs1(conn);
let updated2 = update_languages(conn, test_langs.clone()).unwrap();
assert_eq!(test_langs, updated2);
}
#[test]
#[serial]
fn test_site_languages() {
let conn = &mut establish_unpooled_connection();
let site = create_test_site(conn);
let site_languages1 = SiteLanguage::read_local(conn).unwrap();
// site is created with all languages
assert_eq!(184, site_languages1.len());
let test_langs = test_langs1(conn);
SiteLanguage::update(conn, test_langs.clone(), site.id).unwrap();
let site_languages2 = SiteLanguage::read_local(conn).unwrap();
// after update, site only has new languages
assert_eq!(test_langs, site_languages2);
Site::delete(conn, site.id).unwrap();
}
#[test]
#[serial]
fn test_user_languages() {
let conn = &mut establish_unpooled_connection();
let site = create_test_site(conn);
let test_langs = test_langs1(conn);
SiteLanguage::update(conn, test_langs.clone(), site.id).unwrap();
let person_form = PersonForm {
name: "my test person".to_string(),
public_key: Some("pubkey".to_string()),
..Default::default()
};
let person = Person::create(conn, &person_form).unwrap();
let local_user_form = LocalUserForm {
person_id: Some(person.id),
password_encrypted: Some("my_pw".to_string()),
..Default::default()
};
let local_user = LocalUser::create(conn, &local_user_form).unwrap();
let local_user_langs1 = LocalUserLanguage::read(conn, local_user.id).unwrap();
// new user should be initialized with site languages
assert_eq!(test_langs, local_user_langs1);
// update user languages
let test_langs2 = test_langs2(conn);
LocalUserLanguage::update(conn, test_langs2, local_user.id).unwrap();
let local_user_langs2 = LocalUserLanguage::read(conn, local_user.id).unwrap();
assert_eq!(2, local_user_langs2.len());
Person::delete(conn, person.id).unwrap();
LocalUser::delete(conn, local_user.id).unwrap();
Site::delete(conn, site.id).unwrap();
}
#[test]
#[serial]
fn test_community_languages() {
let conn = &mut establish_unpooled_connection();
let site = create_test_site(conn);
let test_langs = test_langs1(conn);
SiteLanguage::update(conn, test_langs.clone(), site.id).unwrap();
let community_form = CommunityForm {
name: "test community".to_string(),
title: "test community".to_string(),
public_key: Some("pubkey".to_string()),
..Default::default()
};
let community = Community::create(conn, &community_form).unwrap();
let community_langs1 = CommunityLanguage::read(conn, community.id).unwrap();
// community is initialized with site languages
assert_eq!(test_langs, community_langs1);
let allowed_lang1 =
CommunityLanguage::is_allowed_community_language(conn, Some(test_langs[0]), community.id);
assert!(allowed_lang1.is_ok());
let test_langs2 = test_langs2(conn);
let allowed_lang2 =
CommunityLanguage::is_allowed_community_language(conn, Some(test_langs2[0]), community.id);
assert!(allowed_lang2.is_err());
// limit site languages to en, fi. after this, community languages should be updated to
// intersection of old languages (en, fr, ru) and (en, fi), which is only fi.
SiteLanguage::update(conn, vec![test_langs[0], test_langs2[0]], site.id).unwrap();
let community_langs2 = CommunityLanguage::read(conn, community.id).unwrap();
assert_eq!(vec![test_langs[0]], community_langs2);
// update community languages to different ones
CommunityLanguage::update(conn, test_langs2.clone(), community.id).unwrap();
let community_langs3 = CommunityLanguage::read(conn, community.id).unwrap();
assert_eq!(test_langs2, community_langs3);
Site::delete(conn, site.id).unwrap();
Community::delete(conn, community.id).unwrap();
}
#[test]
#[serial]
fn test_default_post_language() {
let conn = &mut establish_unpooled_connection();
let test_langs = test_langs1(conn);
let test_langs2 = test_langs2(conn);
let community_form = CommunityForm {
name: "test community".to_string(),
title: "test community".to_string(),
public_key: Some("pubkey".to_string()),
..Default::default()
};
let community = Community::create(conn, &community_form).unwrap();
CommunityLanguage::update(conn, test_langs, community.id).unwrap();
let person_form = PersonForm {
name: "my test person".to_string(),
public_key: Some("pubkey".to_string()),
..Default::default()
};
let person = Person::create(conn, &person_form).unwrap();
let local_user_form = LocalUserForm {
person_id: Some(person.id),
password_encrypted: Some("my_pw".to_string()),
..Default::default()
};
let local_user = LocalUser::create(conn, &local_user_form).unwrap();
LocalUserLanguage::update(conn, test_langs2, local_user.id).unwrap();
// no overlap in user/community languages, so no default language for post
let def1 = default_post_language(conn, community.id, local_user.id).unwrap();
assert_eq!(None, def1);
let ru = Language::read_id_from_code(conn, "ru").unwrap();
let test_langs3 = vec![
ru,
Language::read_id_from_code(conn, "fi").unwrap(),
Language::read_id_from_code(conn, "se").unwrap(),
];
LocalUserLanguage::update(conn, test_langs3, local_user.id).unwrap();
// this time, both have ru as common lang
let def2 = default_post_language(conn, community.id, local_user.id).unwrap();
assert_eq!(Some(ru), def2);
Person::delete(conn, person.id).unwrap();
Community::delete(conn, community.id).unwrap();
LocalUser::delete(conn, local_user.id).unwrap();
}
}

@ -1,15 +1,18 @@
use crate::{
newtypes::{CommunityId, DbUrl, PersonId},
source::community::{
Community,
CommunityFollower,
CommunityFollowerForm,
CommunityForm,
CommunityModerator,
CommunityModeratorForm,
CommunityPersonBan,
CommunityPersonBanForm,
CommunitySafe,
source::{
actor_language::{CommunityLanguage, SiteLanguage},
community::{
Community,
CommunityFollower,
CommunityFollowerForm,
CommunityForm,
CommunityModerator,
CommunityModeratorForm,
CommunityPersonBan,
CommunityPersonBanForm,
CommunitySafe,
},
},
traits::{ApubActor, Bannable, Crud, DeleteableOrRemoveable, Followable, Joinable},
utils::{functions::lower, naive_now},
@ -85,9 +88,20 @@ impl Crud for Community {
fn create(conn: &mut PgConnection, new_community: &CommunityForm) -> Result<Self, Error> {
use crate::schema::community::dsl::*;
insert_into(community)
let community_ = insert_into(community)
.values(new_community)
.get_result::<Self>(conn)
.get_result::<Self>(conn)?;
let site_languages = SiteLanguage::read_local(conn);
if let Ok(langs) = site_languages {
// if site exists, init user with site languages
CommunityLanguage::update(conn, langs, community_.id)?;
} else {
// otherwise, init with all languages (this only happens during tests)
CommunityLanguage::update(conn, vec![], community_.id)?;
}
Ok(community_)
}
fn update(

@ -1,5 +1,5 @@
use crate::{newtypes::LanguageId, source::language::Language};
use diesel::{result::Error, PgConnection, RunQueryDsl, *};
use crate::{diesel::ExpressionMethods, newtypes::LanguageId, source::language::Language};
use diesel::{result::Error, PgConnection, QueryDsl, RunQueryDsl};
impl Language {
pub fn read_all(conn: &mut PgConnection) -> Result<Vec<Language>, Error> {
@ -27,11 +27,6 @@ impl Language {
Ok(None)
}
}
pub fn read_undetermined(conn: &mut PgConnection) -> Result<LanguageId, Error> {
use crate::schema::language::dsl::*;
Ok(language.filter(code.eq("und")).first::<Self>(conn)?.id)
}
}
#[cfg(test)]

@ -2,8 +2,8 @@ use crate::{
newtypes::LocalUserId,
schema::local_user::dsl::*,
source::{
actor_language::{LocalUserLanguage, SiteLanguage},
local_user::{LocalUser, LocalUserForm},
local_user_language::LocalUserLanguage,
},
traits::Crud,
utils::naive_now,
@ -121,8 +121,17 @@ impl Crud for LocalUser {
let local_user_ = insert_into(local_user)
.values(form)
.get_result::<Self>(conn)?;
// initialize with all languages
LocalUserLanguage::update_user_languages(conn, None, local_user_.id)?;
let site_languages = SiteLanguage::read_local(conn);
if let Ok(langs) = site_languages {
// if site exists, init user with site languages
LocalUserLanguage::update(conn, langs, local_user_.id)?;
} else {
// otherwise, init with all languages (this only happens during tests and
// for first admin user, which is created before site)
LocalUserLanguage::update(conn, vec![], local_user_.id)?;
}
Ok(local_user_)
}
fn update(

@ -1,42 +0,0 @@
use crate::{
newtypes::{LanguageId, LocalUserId},
source::{language::Language, local_user_language::*},
};
use diesel::{result::Error, PgConnection, RunQueryDsl, *};
impl LocalUserLanguage {
/// Update the user's languages.
///
/// If no language_id vector is given, it will show all languages
pub fn update_user_languages(
conn: &mut PgConnection,
language_ids: Option<Vec<LanguageId>>,
for_local_user_id: LocalUserId,
) -> Result<(), Error> {
use crate::schema::local_user_language::dsl::*;
// If no language is given, read all languages
let lang_ids = language_ids.unwrap_or(
Language::read_all(conn)?
.into_iter()
.map(|l| l.id)
.collect(),
);
conn.build_transaction().read_write().run(|conn| {
// Clear the current user languages
delete(local_user_language.filter(local_user_id.eq(for_local_user_id))).execute(conn)?;
for l in lang_ids {
let form = LocalUserLanguageForm {
local_user_id: for_local_user_id,
language_id: l,
};
insert_into(local_user_language)
.values(form)
.get_result::<Self>(conn)?;
}
Ok(())
})
}
}

@ -1,4 +1,5 @@
pub mod activity;
pub mod actor_language;
pub mod comment;
pub mod comment_reply;
pub mod comment_report;
@ -7,7 +8,6 @@ pub mod community_block;
pub mod email_verification;
pub mod language;
pub mod local_user;
pub mod local_user_language;
pub mod moderator;
pub mod password_reset_request;
pub mod person;

@ -235,8 +235,10 @@ impl ApubActor for Person {
#[cfg(test)]
mod tests {
use crate::{source::person::*, traits::Crud, utils::establish_unpooled_connection};
use serial_test::serial;
#[test]
#[serial]
fn test_crud() {
let conn = &mut establish_unpooled_connection();

@ -1,34 +1,45 @@
use crate::{newtypes::DbUrl, source::site::*, traits::Crud};
use crate::{
newtypes::{DbUrl, SiteId},
source::{actor_language::SiteLanguage, site::*},
traits::Crud,
};
use diesel::{dsl::*, result::Error, *};
use url::Url;
impl Crud for Site {
type Form = SiteForm;
type IdType = i32;
fn read(conn: &mut PgConnection, _site_id: i32) -> Result<Self, Error> {
type IdType = SiteId;
fn read(conn: &mut PgConnection, _site_id: SiteId) -> Result<Self, Error> {
use crate::schema::site::dsl::*;
site.first::<Self>(conn)
}
fn create(conn: &mut PgConnection, new_site: &SiteForm) -> Result<Self, Error> {
use crate::schema::site::dsl::*;
insert_into(site).values(new_site).get_result::<Self>(conn)
let site_ = insert_into(site)
.values(new_site)
.get_result::<Self>(conn)?;
// initialize with all languages
SiteLanguage::update(conn, vec![], site_.id)?;
Ok(site_)
}
fn update(conn: &mut PgConnection, site_id: i32, new_site: &SiteForm) -> Result<Self, Error> {
fn update(conn: &mut PgConnection, site_id: SiteId, new_site: &SiteForm) -> Result<Self, Error> {
use crate::schema::site::dsl::*;
diesel::update(site.find(site_id))
.set(new_site)
.get_result::<Self>(conn)
}
fn delete(conn: &mut PgConnection, site_id: i32) -> Result<usize, Error> {
fn delete(conn: &mut PgConnection, site_id: SiteId) -> Result<usize, Error> {
use crate::schema::site::dsl::*;
diesel::delete(site.find(site_id)).execute(conn)
}
}
impl Site {
pub fn read_local_site(conn: &mut PgConnection) -> Result<Self, Error> {
pub fn read_local(conn: &mut PgConnection) -> Result<Self, Error> {
use crate::schema::site::dsl::*;
site.order_by(id).first::<Self>(conn)
}

@ -73,6 +73,10 @@ pub struct PostReportId(i32);
#[cfg_attr(feature = "full", derive(DieselNewType))]
pub struct PrivateMessageReportId(i32);
#[derive(Debug, Copy, Clone, Hash, Eq, PartialEq, Serialize, Deserialize, Default)]
#[cfg_attr(feature = "full", derive(DieselNewType))]
pub struct SiteId(i32);
#[derive(Debug, Copy, Clone, Hash, Eq, PartialEq, Serialize, Deserialize, Default)]
#[cfg_attr(feature = "full", derive(DieselNewType))]
pub struct LanguageId(pub i32);
@ -81,6 +85,14 @@ pub struct LanguageId(pub i32);
#[cfg_attr(feature = "full", derive(DieselNewType))]
pub struct LocalUserLanguageId(pub i32);
#[derive(Debug, Copy, Clone, Hash, Eq, PartialEq, Serialize, Deserialize, Default)]
#[cfg_attr(feature = "full", derive(DieselNewType))]
pub struct SiteLanguageId(pub i32);
#[derive(Debug, Copy, Clone, Hash, Eq, PartialEq, Serialize, Deserialize, Default)]
#[cfg_attr(feature = "full", derive(DieselNewType))]
pub struct CommunityLanguageId(pub i32);
#[derive(Debug, Copy, Clone, Hash, Eq, PartialEq, Serialize, Deserialize, Default)]
#[cfg_attr(feature = "full", derive(DieselNewType))]
pub struct CommentReplyId(i32);

@ -635,6 +635,22 @@ table! {
}
}
table! {
site_language(id) {
id -> Int4,
site_id -> Int4,
language_id -> Int4,
}
}
table! {
community_language(id) {
id -> Int4,
community_id -> Int4,
language_id -> Int4,
}
}
joinable!(person_block -> person (person_id));
joinable!(comment -> person (creator_id));
@ -699,6 +715,10 @@ joinable!(comment -> language (language_id));
joinable!(local_user_language -> language (language_id));
joinable!(local_user_language -> local_user (local_user_id));
joinable!(private_message_report -> private_message (private_message_id));
joinable!(site_language -> language (language_id));
joinable!(site_language -> site (site_id));
joinable!(community_language -> language (language_id));
joinable!(community_language -> community (community_id));
joinable!(admin_purge_comment -> person (admin_person_id));
joinable!(admin_purge_comment -> post (post_id));
@ -757,5 +777,7 @@ allow_tables_to_appear_in_same_query!(
email_verification,
registration_application,
language,
local_user_language
local_user_language,
site_language,
community_language,
);

@ -0,0 +1,73 @@
use crate::newtypes::{
CommunityId,
CommunityLanguageId,
LanguageId,
LocalUserId,
LocalUserLanguageId,
SiteId,
SiteLanguageId,
};
use serde::{Deserialize, Serialize};
#[cfg(feature = "full")]
use crate::schema::local_user_language;
#[derive(Clone, PartialEq, Eq, Debug, Serialize, Deserialize)]
#[cfg_attr(feature = "full", derive(Queryable, Identifiable))]
#[cfg_attr(feature = "full", diesel(table_name = local_user_language))]
pub struct LocalUserLanguage {
#[serde(skip)]
pub id: LocalUserLanguageId,
pub local_user_id: LocalUserId,
pub language_id: LanguageId,
}
#[derive(Clone)]
#[cfg_attr(feature = "full", derive(Insertable, AsChangeset))]
#[cfg_attr(feature = "full", diesel(table_name = local_user_language))]
pub struct LocalUserLanguageForm {
pub local_user_id: LocalUserId,
pub language_id: LanguageId,
}
#[cfg(feature = "full")]
use crate::schema::community_language;
#[derive(Clone, PartialEq, Eq, Debug, Serialize, Deserialize)]
#[cfg_attr(feature = "full", derive(Queryable, Identifiable))]
#[cfg_attr(feature = "full", diesel(table_name = community_language))]
pub struct CommunityLanguage {
#[serde(skip)]
pub id: CommunityLanguageId,
pub community_id: CommunityId,
pub language_id: LanguageId,
}
#[derive(Clone)]
#[cfg_attr(feature = "full", derive(Insertable, AsChangeset))]
#[cfg_attr(feature = "full", diesel(table_name = community_language))]
pub struct CommunityLanguageForm {
pub community_id: CommunityId,
pub language_id: LanguageId,
}
#[cfg(feature = "full")]
use crate::schema::site_language;
#[derive(Clone, PartialEq, Eq, Debug, Serialize, Deserialize)]
#[cfg_attr(feature = "full", derive(Queryable, Identifiable))]
#[cfg_attr(feature = "full", diesel(table_name = site_language))]
pub struct SiteLanguage {
#[serde(skip)]
pub id: SiteLanguageId,
pub site_id: SiteId,
pub language_id: LanguageId,
}
#[derive(Clone, Debug)]
#[cfg_attr(feature = "full", derive(Insertable, AsChangeset))]
#[cfg_attr(feature = "full", diesel(table_name = site_language))]
pub struct SiteLanguageForm {
pub site_id: SiteId,
pub language_id: LanguageId,
}

@ -1,5 +1,6 @@
#[cfg(feature = "full")]
pub mod activity;
pub mod actor_language;
pub mod comment;
pub mod comment_reply;
pub mod comment_report;
@ -8,7 +9,6 @@ pub mod community_block;
pub mod email_verification;
pub mod language;
pub mod local_user;
pub mod local_user_language;
pub mod moderator;
pub mod password_reset_request;
pub mod person;

@ -1,4 +1,4 @@
use crate::newtypes::DbUrl;
use crate::newtypes::{DbUrl, SiteId};
use serde::{Deserialize, Serialize};
#[cfg(feature = "full")]
@ -8,7 +8,7 @@ use crate::schema::site;
#[cfg_attr(feature = "full", derive(Queryable, Identifiable))]
#[cfg_attr(feature = "full", diesel(table_name = site))]
pub struct Site {
pub id: i32,
pub id: SiteId,
pub name: String,
pub sidebar: Option<String>,
pub published: chrono::NaiveDateTime,

@ -393,11 +393,11 @@ mod tests {
aggregates::structs::CommentAggregates,
newtypes::LanguageId,
source::{
actor_language::LocalUserLanguage,
comment::*,
community::*,
language::Language,
local_user::LocalUserForm,
local_user_language::LocalUserLanguage,
person::*,
person_block::PersonBlockForm,
post::*,
@ -707,12 +707,7 @@ mod tests {
// change user lang to finnish, should only show single finnish comment
let finnish_id = Language::read_id_from_code(conn, "fi").unwrap();
LocalUserLanguage::update_user_languages(
conn,
Some(vec![finnish_id]),
data.inserted_local_user.id,
)
.unwrap();
LocalUserLanguage::update(conn, vec![finnish_id], data.inserted_local_user.id).unwrap();
let finnish_comment = CommentQuery::builder()
.conn(conn)
.local_user(Some(&data.inserted_local_user))
@ -728,12 +723,7 @@ mod tests {
// now show all comments with undetermined language (which is the default value)
let undetermined_id = Language::read_id_from_code(conn, "und").unwrap();
LocalUserLanguage::update_user_languages(
conn,
Some(vec![undetermined_id]),
data.inserted_local_user.id,
)
.unwrap();
LocalUserLanguage::update(conn, vec![undetermined_id], data.inserted_local_user.id).unwrap();
let undetermined_comment = CommentQuery::builder()
.conn(conn)
.local_user(Some(&data.inserted_local_user))

@ -454,11 +454,11 @@ mod tests {
aggregates::structs::PostAggregates,
newtypes::LanguageId,
source::{
actor_language::LocalUserLanguage,
community::*,
community_block::{CommunityBlock, CommunityBlockForm},
language::Language,
local_user::{LocalUser, LocalUserForm},
local_user_language::LocalUserLanguage,
person::*,
person_block::{PersonBlock, PersonBlockForm},
post::*,
@ -749,12 +749,7 @@ mod tests {
assert_eq!(3, post_listings_all.len());
let french_id = Language::read_id_from_code(conn, "fr").unwrap();
LocalUserLanguage::update_user_languages(
conn,
Some(vec![french_id]),
data.inserted_local_user.id,
)
.unwrap();
LocalUserLanguage::update(conn, vec![french_id], data.inserted_local_user.id).unwrap();
let post_listing_french = PostQuery::builder()
.conn(conn)
@ -769,9 +764,9 @@ mod tests {
assert_eq!(french_id, post_listing_french[0].post.language_id);
let undetermined_id = Language::read_id_from_code(conn, "und").unwrap();
LocalUserLanguage::update_user_languages(
LocalUserLanguage::update(
conn,
Some(vec![french_id, undetermined_id]),
vec![french_id, undetermined_id],
data.inserted_local_user.id,
)
.unwrap();

@ -0,0 +1,3 @@
drop table site_language;
drop table community_language;
delete from local_user_language;

@ -0,0 +1,38 @@
create table site_language (
id serial primary key,
site_id int references site on update cascade on delete cascade not null,
language_id int references language on update cascade on delete cascade not null,
unique (site_id, language_id)
);
create table community_language (
id serial primary key,
community_id int references community on update cascade on delete cascade not null,
language_id int references language on update cascade on delete cascade not null,
unique (community_id, language_id)
);
-- update existing users, sites and communities to have all languages enabled
do $$
declare
xid integer;
begin
for xid in select id from local_user
loop
insert into local_user_language (local_user_id, language_id)
(select xid, language.id as lid from language);
end loop;
for xid in select id from site
loop
insert into site_language (site_id, language_id)
(select xid, language.id as lid from language);
end loop;
for xid in select id from community
loop
insert into community_language (community_id, language_id)
(select xid, language.id as lid from language);
end loop;
end;
$$;

@ -295,7 +295,7 @@ fn instance_actor_2022_01_28(
protocol_and_hostname: &str,
) -> Result<(), LemmyError> {
info!("Running instance_actor_2021_09_29");
if let Ok(site) = Site::read_local_site(conn) {
if let Ok(site) = Site::read_local(conn) {
// if site already has public key, we dont need to do anything here
if !site.public_key.is_empty() {
return Ok(());

Loading…
Cancel
Save