diff --git a/crates/apub/src/activities/voting/vote.rs b/crates/apub/src/activities/voting/vote.rs
index e15c3882b..5625ea6ba 100644
--- a/crates/apub/src/activities/voting/vote.rs
+++ b/crates/apub/src/activities/voting/vote.rs
@@ -2,7 +2,7 @@ use crate::{
   activities::{
     generate_activity_id,
     verify_person_in_community,
-    voting::{vote_comment, vote_post},
+    voting::{undo_vote_comment, undo_vote_post, vote_comment, vote_post},
   },
   insert_received_activity,
   objects::{community::ApubCommunity, person::ApubPerson},
@@ -17,7 +17,6 @@ use activitypub_federation::{
   fetch::object_id::ObjectId,
   traits::{ActivityHandler, Actor},
 };
-use anyhow::anyhow;
 use lemmy_api_common::{context::LemmyContext, utils::check_bot_account};
 use lemmy_db_schema::source::local_site::LocalSite;
 use lemmy_utils::error::LemmyError;
@@ -58,15 +57,7 @@ impl ActivityHandler for Vote {
   async fn verify(&self, context: &Data<LemmyContext>) -> Result<(), LemmyError> {
     let community = self.community(context).await?;
     verify_person_in_community(&self.actor, &community, context).await?;
-    let enable_downvotes = LocalSite::read(&mut context.pool())
-      .await
-      .map(|l| l.enable_downvotes)
-      .unwrap_or(true);
-    if self.kind == VoteType::Dislike && !enable_downvotes {
-      Err(anyhow!("Downvotes disabled").into())
-    } else {
-      Ok(())
-    }
+    Ok(())
   }
 
   #[tracing::instrument(skip_all)]
@@ -77,9 +68,22 @@ impl ActivityHandler for Vote {
 
     check_bot_account(&actor.0)?;
 
-    match object {
-      PostOrComment::Post(p) => vote_post(&self.kind, actor, &p, context).await,
-      PostOrComment::Comment(c) => vote_comment(&self.kind, actor, &c, context).await,
+    let enable_downvotes = LocalSite::read(&mut context.pool())
+      .await
+      .map(|l| l.enable_downvotes)
+      .unwrap_or(true);
+    if self.kind == VoteType::Dislike && !enable_downvotes {
+      // If this is a downvote but downvotes are ignored, only undo any existing vote
+      match object {
+        PostOrComment::Post(p) => undo_vote_post(actor, &p, context).await,
+        PostOrComment::Comment(c) => undo_vote_comment(actor, &c, context).await,
+      }
+    } else {
+      // Otherwise apply the vote normally
+      match object {
+        PostOrComment::Post(p) => vote_post(&self.kind, actor, &p, context).await,
+        PostOrComment::Comment(c) => vote_comment(&self.kind, actor, &c, context).await,
+      }
     }
   }
 }