diff --git a/src/fontcache.cpp b/src/fontcache.cpp
index c2cc9fc3aa..7282eb3f05 100644
--- a/src/fontcache.cpp
+++ b/src/fontcache.cpp
@@ -128,6 +128,26 @@ extern void LoadWin32Font(FontSize fs);
 extern void LoadCoreTextFont(FontSize fs);
 #endif
 
+/**
+ * Test if a font setting uses the default font.
+ * @return true iff the font is not configured and no fallback font data is present.
+ */
+static bool IsDefaultFont(const FontCacheSubSetting &setting)
+{
+	return setting.font.empty() && setting.os_handle == nullptr;
+}
+
+/**
+ * Get the scalable font size to use for a FontSize.
+ * @param fs FontSize to get the scalable font size for.
+ * @return Scalable font size to use.
+ */
+uint GetFontCacheFontSize(FontSize fs)
+{
+	const FontCacheSubSetting &setting = *GetFontCacheSubSetting(fs);
+	return IsDefaultFont(setting) ? FontCache::GetDefaultFontHeight(fs) : setting.size;
+}
+
 #if defined(WITH_FREETYPE) || defined(_WIN32) || defined(WITH_COCOA)
 /**
  * Get name of default font file for a given font size.
diff --git a/src/fontcache.h b/src/fontcache.h
index bba371570b..9caa8abd44 100644
--- a/src/fontcache.h
+++ b/src/fontcache.h
@@ -228,6 +228,7 @@ inline FontCacheSubSetting *GetFontCacheSubSetting(FontSize fs)
 	}
 }
 
+uint GetFontCacheFontSize(FontSize fs);
 std::string GetFontCacheFontName(FontSize fs);
 void InitFontCache(bool monospace);
 void UninitFontCache();
diff --git a/src/fontcache/freetypefontcache.cpp b/src/fontcache/freetypefontcache.cpp
index 091913a301..8b0cb34af2 100644
--- a/src/fontcache/freetypefontcache.cpp
+++ b/src/fontcache/freetypefontcache.cpp
@@ -195,7 +195,7 @@ void LoadFreeTypeFont(FontSize fs)
 	if (error != FT_Err_Ok) error = GetFontByFaceName(font_name, &face);
 
 	if (error == FT_Err_Ok) {
-		error = LoadFont(fs, face, font_name, settings->size);
+		error = LoadFont(fs, face, font_name, GetFontCacheFontSize(fs));
 		if (error != FT_Err_Ok) {
 			ShowInfo("Unable to use '{}' for {} font, FreeType reported error 0x{:X}, using sprite font instead", font_name, FontSizeToName(fs), error);
 		}
diff --git a/src/os/macosx/font_osx.cpp b/src/os/macosx/font_osx.cpp
index a3138cf42f..8af574faf5 100644
--- a/src/os/macosx/font_osx.cpp
+++ b/src/os/macosx/font_osx.cpp
@@ -372,5 +372,5 @@ void LoadCoreTextFont(FontSize fs)
 		return;
 	}
 
-	new CoreTextFontCache(fs, std::move(font_ref), settings->size);
+	new CoreTextFontCache(fs, std::move(font_ref), GetFontCacheFontSize(fs));
 }
diff --git a/src/os/windows/font_win32.cpp b/src/os/windows/font_win32.cpp
index e0c8c3866b..7b34c6172c 100644
--- a/src/os/windows/font_win32.cpp
+++ b/src/os/windows/font_win32.cpp
@@ -379,5 +379,5 @@ void LoadWin32Font(FontSize fs)
 		convert_to_fs(font_name, logfont.lfFaceName, lengthof(logfont.lfFaceName));
 	}
 
-	LoadWin32Font(fs, logfont, settings->size, font_name);
+	LoadWin32Font(fs, logfont, GetFontCacheFontSize(fs), font_name);
 }