Extend sitemap span (#4231)

* Extend sitemap span

* Keep cargo_fmt happy

* Add FETCH_LIMIT_SITEMAP

* Use FETCH_LIMIT_SITEMAP

* Keep cargo_fmt happy

* Update utils.rs

* Use SITEMAP_DAYS

* Keep cargo_fmt happy

* Sitemap

* Keep cargo_fmt happy

* Sitemap

* Sitemap

* Increase to 31 days

crates/api/src/sitemap.rs

@@ -26,7 +26,7 @@ async fn generate_urlset(
 }
 
 pub async fn get_sitemap(context: Data<LemmyContext>) -> LemmyResult<HttpResponse> {
-  info!("Generating sitemap with posts from last {} hours...", 24);
+  info!("Generating sitemap...",);
   let posts = Post::list_for_sitemap(&mut context.pool()).await?;
   info!("Loaded latest {} posts", posts.len());
 
@@ -36,7 +36,7 @@ pub async fn get_sitemap(context: Data<LemmyContext>) -> LemmyResult<HttpRespons
   Ok(
     HttpResponse::Ok()
       .content_type("application/xml")
-      .insert_header(header::CacheControl(vec![CacheDirective::MaxAge(86_400)])) // 24 h
+      .insert_header(header::CacheControl(vec![CacheDirective::MaxAge(3_600)])) // 1 h
       .body(buf),
   )
 }

crates/db_schema/src/impls/post.rs

@@ -29,7 +29,15 @@ use crate::{
     PostUpdateForm,
   },
   traits::{Crud, Likeable, Saveable},
-  utils::{get_conn, naive_now, DbPool, DELETED_REPLACEMENT_TEXT, FETCH_LIMIT_MAX},
+  utils::{
+    get_conn,
+    naive_now,
+    DbPool,
+    DELETED_REPLACEMENT_TEXT,
+    FETCH_LIMIT_MAX,
+    SITEMAP_DAYS,
+    SITEMAP_LIMIT,
+  },
 };
 use ::url::Url;
 use chrono::{Duration, Utc};
@@ -109,8 +117,9 @@ impl Post {
       .filter(local.eq(true))
       .filter(deleted.eq(false))
       .filter(removed.eq(false))
-      .filter(published.ge(Utc::now().naive_utc() - Duration::days(1)))
+      .filter(published.ge(Utc::now().naive_utc() - Duration::days(SITEMAP_DAYS)))
       .order(published.desc())
+      .limit(SITEMAP_LIMIT)
       .load::<(DbUrl, chrono::DateTime<Utc>)>(conn)
       .await
   }

crates/db_schema/src/utils.rs

@@ -49,6 +49,8 @@ use url::Url;
 
 const FETCH_LIMIT_DEFAULT: i64 = 10;
 pub const FETCH_LIMIT_MAX: i64 = 50;
+pub const SITEMAP_LIMIT: i64 = 50000;
+pub const SITEMAP_DAYS: i64 = 31;
 const POOL_TIMEOUT: Option<Duration> = Some(Duration::from_secs(5));
 pub const RANK_DEFAULT: f64 = 0.0001;