diff --git a/src/viewport.cpp b/src/viewport.cpp
index 7f3a5078e3..ef78356c46 100644
--- a/src/viewport.cpp
+++ b/src/viewport.cpp
@@ -2839,18 +2839,16 @@ void SetObjectToPlaceWnd(CursorID icon, PaletteID pal, HighLightStyle mode, Wind
 
 void SetObjectToPlace(CursorID icon, PaletteID pal, HighLightStyle mode, WindowClass window_class, WindowNumber window_num)
 {
-	/* undo clicking on button and drag & drop */
-	if ((_thd.place_mode & ~HT_DIR_MASK) != HT_NONE || _special_mouse_mode == WSM_DRAGDROP) {
+	if (_thd.window_class != WC_INVALID) {
+		/* Undo clicking on button and drag & drop */
 		Window *w = FindWindowById(_thd.window_class, _thd.window_number);
-		if (w != NULL) {
-			/* Call the abort function, but set the window class to something
-			 * that will never be used to avoid infinite loops. Setting it to
-			 * the 'next' window class must not be done because recursion into
-			 * this function might in some cases reset the newly set object to
-			 * place or not properly reset the original selection. */
-			_thd.window_class = WC_INVALID;
-			w->OnPlaceObjectAbort();
-		}
+		/* Call the abort function, but set the window class to something
+		 * that will never be used to avoid infinite loops. Setting it to
+		 * the 'next' window class must not be done because recursion into
+		 * this function might in some cases reset the newly set object to
+		 * place or not properly reset the original selection. */
+		_thd.window_class = WC_INVALID;
+		if (w != NULL) w->OnPlaceObjectAbort();
 	}
 
 	SetTileSelectSize(1, 1);
diff --git a/src/window.cpp b/src/window.cpp
index e9b7e67484..ae9a683ac2 100644
--- a/src/window.cpp
+++ b/src/window.cpp
@@ -653,8 +653,7 @@ void Window::DeleteChildWindows(WindowClass wc) const
  */
 Window::~Window()
 {
-	if ((_thd.place_mode & ~HT_DIR_MASK) != HT_NONE &&
-			_thd.window_class == this->window_class &&
+	if (_thd.window_class == this->window_class &&
 			_thd.window_number == this->window_number) {
 		ResetObjectToPlace();
 	}