Use tagged pointers in the vehicle pool on supported (64 bit) platforms

Use to avoid needing to dereference each pointer to get the vehicle
type when doing per-vehicle type iteration

cmake/CompileFlags.cmake

@@ -85,8 +85,6 @@ macro(compile_flags)
             #  break anything. So disable strict-aliasing to make the
             #  compiler all happy.
             -fno-strict-aliasing
-
-
         )
 
         if(OPTION_TRIM_PATH_PREFIX)
@@ -137,6 +135,13 @@ macro(compile_flags)
                 # and of course they both warn when the other compiler is happy
                 "$<$<COMPILE_LANGUAGE:CXX>:-Wno-redundant-move>"
             )
+
+            if(CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL 11)
+                add_compile_options(
+                    # GCC >= 11 has false positives if operator new is inlined but operator delete isn't, or vice versa
+                    "-Wno-mismatched-new-delete"
+                )
+            endif()
         endif()
 
         if(CMAKE_CXX_COMPILER_ID STREQUAL "AppleClang")

src/core/endian_type.hpp

@@ -27,4 +27,12 @@
 #	error "TTD_ENDIAN is not defined; please set it to either TTD_LITTLE_ENDIAN or TTD_BIG_ENDIAN"
 #endif /* !TTD_ENDIAN */
 
+#if defined(__x86_64__) || defined(_M_X64) || defined(__aarch64__) || defined(_M_ARM64)
+	/** Tagged pointers (upper bits) may be used on this architecture */
+#	define OTTD_UPPER_TAGGED_PTR 1
+#else
+	/** Tagged pointers (upper bits) may not be used on this architecture */
+#	define OTTD_UPPER_TAGGED_PTR 0
+#endif
+
 #endif /* ENDIAN_TYPE_HPP */

src/core/pool_func.hpp

@@ -21,8 +21,8 @@
  * @param type The return type of the method.
  */
 #define DEFINE_POOL_METHOD(type) \
-	template <class Titem, typename Tindex, size_t Tgrowth_step, size_t Tmax_size, PoolType Tpool_type, bool Tcache, bool Tzero> \
-	type Pool<Titem, Tindex, Tgrowth_step, Tmax_size, Tpool_type, Tcache, Tzero>
+	template <class Titem, typename Tindex, size_t Tgrowth_step, size_t Tmax_size, PoolType Tpool_type, bool Tcache, bool Tzero, typename Tops> \
+	type Pool<Titem, Tindex, Tgrowth_step, Tmax_size, Tpool_type, Tcache, Tzero, Tops>
 
 /**
  * Create a clean pool.
@@ -107,9 +107,9 @@ DEFINE_POOL_METHOD(inline size_t)::FindFirstFree()
  * @pre index < this->size
  * @pre this->Get(index) == nullptr
  */
-DEFINE_POOL_METHOD(inline void *)::AllocateItem(size_t size, size_t index)
+DEFINE_POOL_METHOD(inline void *)::AllocateItem(size_t size, size_t index, Pool::ParamType param)
 {
-	dbg_assert(this->data[index] == nullptr);
+	dbg_assert(this->data[index] == Tops::NullValue());
 
 	this->first_unused = std::max(this->first_unused, index + 1);
 	this->items++;
@@ -129,7 +129,7 @@ DEFINE_POOL_METHOD(inline void *)::AllocateItem(size_t size, size_t index)
 	} else {
 		item = (Titem *)MallocT<byte>(size);
 	}
-	this->data[index] = item;
+	this->data[index] = Tops::PutPtr(item, param);
 	SetBit(this->free_bitmap[index / 64], index % 64);
 	item->index = (Tindex)(uint)index;
 	return item;
@@ -141,7 +141,7 @@ DEFINE_POOL_METHOD(inline void *)::AllocateItem(size_t size, size_t index)
  * @return pointer to allocated item
  * @note error() on failure! (no free item)
  */
-DEFINE_POOL_METHOD(void *)::GetNew(size_t size)
+DEFINE_POOL_METHOD(void *)::GetNew(size_t size, Pool::ParamType param)
 {
 	size_t index = this->FindFirstFree();
 
@@ -154,7 +154,7 @@ DEFINE_POOL_METHOD(void *)::GetNew(size_t size)
 	}
 
 	this->first_free = index + 1;
-	return this->AllocateItem(size, index);
+	return this->AllocateItem(size, index, param);
 }
 
 /**
@@ -164,7 +164,7 @@ DEFINE_POOL_METHOD(void *)::GetNew(size_t size)
  * @return pointer to allocated item
  * @note SlErrorCorruptFmt() on failure! (index out of range or already used)
  */
-DEFINE_POOL_METHOD(void *)::GetNew(size_t size, size_t index)
+DEFINE_POOL_METHOD(void *)::GetNew(size_t size, size_t index, Pool::ParamType param)
 {
 	[[noreturn]] extern void SlErrorCorruptFmt(const char *format, ...);
 
@@ -174,11 +174,11 @@ DEFINE_POOL_METHOD(void *)::GetNew(size_t size, size_t index)
 
 	if (index >= this->size) this->ResizeFor(index);
 
-	if (this->data[index] != nullptr) {
+	if (this->data[index] != Tops::NullValue()) {
 		SlErrorCorruptFmt("%s index " PRINTF_SIZE " already in use", this->name, index);
 	}
 
-	return this->AllocateItem(size, index);
+	return this->AllocateItem(size, index, param);
 }
 
 /**
@@ -190,15 +190,15 @@ DEFINE_POOL_METHOD(void *)::GetNew(size_t size, size_t index)
 DEFINE_POOL_METHOD(void)::FreeItem(size_t index)
 {
 	dbg_assert(index < this->size);
-	dbg_assert(this->data[index] != nullptr);
+	dbg_assert(this->data[index] != Tops::NullValue());
 	if (Tcache) {
 		AllocCache *ac = (AllocCache *)this->data[index];
 		ac->next = this->alloc_cache;
 		this->alloc_cache = ac;
 	} else {
-		free(this->data[index]);
+		free(Tops::GetPtr(this->data[index]));
 	}
-	this->data[index] = nullptr;
+	this->data[index] = Tops::NullValue();
 	ClrBit(this->free_bitmap[index / 64], index % 64);
 	this->first_free = std::min(this->first_free, index);
 	this->items--;
@@ -238,8 +238,8 @@ DEFINE_POOL_METHOD(void)::CleanPool()
  * forcefully instantiated.
  */
 #define INSTANTIATE_POOL_METHODS(name) \
-	template void * name ## Pool::GetNew(size_t size); \
-	template void * name ## Pool::GetNew(size_t size, size_t index); \
+	template void * name ## Pool::GetNew(size_t size, name ## Pool::ParamType param); \
+	template void * name ## Pool::GetNew(size_t size, size_t index, name ## Pool::ParamType param); \
 	template void name ## Pool::FreeItem(size_t index); \
 	template void name ## Pool::CleanPool();
 

src/core/pool_type.hpp

@@ -66,6 +66,19 @@ private:
 	PoolBase(const PoolBase &other);
 };
 
+struct DefaultPoolItemParam{};
+
+template <class Titem>
+struct DefaultPoolOps {
+	using Tptr = Titem *;
+	using Tparam_type = DefaultPoolItemParam;
+
+	static constexpr Titem *GetPtr(Titem *ptr) { return ptr; }
+	static constexpr Titem *PutPtr(Titem *ptr, DefaultPoolItemParam param) { return ptr; }
+	static constexpr Titem *NullValue() { return nullptr; }
+	static constexpr DefaultPoolItemParam DefaultItemParam() { return {}; }
+};
+
 /**
  * Base class for all pools.
  * @tparam Titem        Type of the class/struct that is going to be pooled
@@ -77,8 +90,11 @@ private:
  * @tparam Tzero        Whether to zero the memory
  * @warning when Tcache is enabled *all* instances of this pool's item must be of the same size.
  */
-template <class Titem, typename Tindex, size_t Tgrowth_step, size_t Tmax_size, PoolType Tpool_type = PT_NORMAL, bool Tcache = false, bool Tzero = true>
+template <class Titem, typename Tindex, size_t Tgrowth_step, size_t Tmax_size, PoolType Tpool_type = PT_NORMAL, bool Tcache = false, bool Tzero = true, typename Tops = DefaultPoolOps<Titem> >
 struct Pool : PoolBase {
+	using ParamType = typename Tops::Tparam_type;
+	using PtrType = typename Tops::Tptr;
+
 	/* Ensure Tmax_size is within the bounds of Tindex. */
 	static_assert((uint64_t)(Tmax_size - 1) >> 8 * sizeof(Tindex) == 0);
 
@@ -95,12 +111,18 @@ struct Pool : PoolBase {
 #endif /* WITH_ASSERT */
 	bool cleaning;       ///< True if cleaning pool (deleting all items)
 
-	Titem **data;        ///< Pointer to array of pointers to Titem
+	PtrType *data;       ///< Pointer to array of Tops::Tptr (by default: pointers to Titem)
 	uint64_t *free_bitmap; ///< Pointer to free bitmap
 
 	Pool(const char *name);
 	void CleanPool() override;
 
+	inline PtrType GetRaw(size_t index)
+	{
+		dbg_assert_msg(index < this->first_unused, "index: " PRINTF_SIZE ", first_unused: " PRINTF_SIZE ", name: %s", index, this->first_unused, this->name);
+		return this->data[index];
+	}
+
 	/**
 	 * Returns Titem with given index
 	 * @param index of item to get
@@ -109,8 +131,7 @@ struct Pool : PoolBase {
 	 */
 	inline Titem *Get(size_t index)
 	{
-		dbg_assert_msg(index < this->first_unused, "index: " PRINTF_SIZE ", first_unused: " PRINTF_SIZE ", name: %s", index, this->first_unused, this->name);
-		return this->data[index];
+		return Tops::GetPtr(this->GetRaw(index));
 	}
 
 	/**
@@ -120,7 +141,7 @@ struct Pool : PoolBase {
 	 */
 	inline bool IsValidID(size_t index)
 	{
-		return index < this->first_unused && this->Get(index) != nullptr;
+		return index < this->first_unused && this->GetRaw(index) != Tops::NullValue();
 	}
 
 	/**
@@ -231,13 +252,25 @@ struct Pool : PoolBase {
 	 * Base class for all PoolItems
 	 * @tparam Tpool The pool this item is going to be part of
 	 */
-	template <struct Pool<Titem, Tindex, Tgrowth_step, Tmax_size, Tpool_type, Tcache, Tzero> *Tpool>
+	template <struct Pool<Titem, Tindex, Tgrowth_step, Tmax_size, Tpool_type, Tcache, Tzero, Tops> *Tpool>
 	struct PoolItem {
 		Tindex index; ///< Index of this pool item
 
 		/** Type of the pool this item is going to be part of */
-		typedef struct Pool<Titem, Tindex, Tgrowth_step, Tmax_size, Tpool_type, Tcache, Tzero> Pool;
+		typedef struct Pool<Titem, Tindex, Tgrowth_step, Tmax_size, Tpool_type, Tcache, Tzero, Tops> Pool;
+
+protected:
+		static inline void *NewWithParam(size_t size, ParamType param)
+		{
+			return Tpool->GetNew(size, param);
+		}
+
+		static inline void *NewWithParam(size_t size, size_t index, ParamType param)
+		{
+			return Tpool->GetNew(size, index, param);
+		}
 
+public:
 		/**
 		 * Allocates space for new Titem
 		 * @param size size of Titem
@@ -246,7 +279,7 @@ struct Pool : PoolBase {
 		 */
 		inline void *operator new(size_t size)
 		{
-			return Tpool->GetNew(size);
+			return NewWithParam(size, Tops::DefaultItemParam());
 		}
 
 		/**
@@ -272,7 +305,7 @@ struct Pool : PoolBase {
 		 */
 		inline void *operator new(size_t size, size_t index)
 		{
-			return Tpool->GetNew(size, index);
+			return NewWithParam(size, index, Tops::DefaultItemParam());
 		}
 
 		/**
@@ -408,12 +441,12 @@ private:
 	/** Cache of freed pointers */
 	AllocCache *alloc_cache;
 
-	void *AllocateItem(size_t size, size_t index);
+	void *AllocateItem(size_t size, size_t index, ParamType param);
 	void ResizeFor(size_t index);
 	size_t FindFirstFree();
 
-	void *GetNew(size_t size);
-	void *GetNew(size_t size, size_t index);
+	void *GetNew(size_t size, ParamType param);
+	void *GetNew(size_t size, size_t index, ParamType param);
 
 	void FreeItem(size_t index);
 };

src/vehicle_base.h

@@ -25,6 +25,7 @@
 #include "landscape.h"
 #include "network/network.h"
 #include "core/mem_func.hpp"
+#include "core/endian_type.hpp"
 #include "sl/saveload_common.h"
 #include <list>
 #include <map>
@@ -234,7 +235,33 @@ struct PendingSpeedRestrictionChange {
 };
 
 /** A vehicle pool for a little over 1 million vehicles. */
+#if OTTD_UPPER_TAGGED_PTR
+struct VehiclePoolOps {
+	using Tptr = uintptr_t;
+	using Tparam_type = VehicleType;
+
+	static inline Vehicle *GetPtr(uintptr_t ptr) {
+		return reinterpret_cast<Vehicle *>(ptr & ((static_cast<uintptr_t>(1) << 60) - 1)); // GB can't be used here because its return type is limited to 32 bits
+	}
+
+	static inline uintptr_t PutPtr(Vehicle *v, VehicleType vtype)
+	{
+		uintptr_t ptr = reinterpret_cast<uintptr_t>(v);
+		SB(ptr, 60, 3, vtype & 7);
+		return ptr;
+	}
+
+	static constexpr uintptr_t NullValue() { return 0; }
+	static constexpr VehicleType DefaultItemParam() { return VEH_INVALID; }
+
+	static constexpr VehicleType GetVehicleType(uintptr_t ptr) { return static_cast<VehicleType>(GB(ptr, 60, 3)); }
+};
+
+typedef Pool<Vehicle, VehicleID, 512, 0xFF000, PT_NORMAL, false, true, VehiclePoolOps> VehiclePool;
+#else
 typedef Pool<Vehicle, VehicleID, 512, 0xFF000> VehiclePool;
+#endif
+
 extern VehiclePool _vehicle_pool;
 
 /* Some declarations of functions, so we can make them friendly */
@@ -1287,6 +1314,25 @@ struct SpecializedVehicle : public Vehicle {
 
 	typedef SpecializedVehicle<T, Type> SpecializedVehicleBase; ///< Our type
 
+#if OTTD_UPPER_TAGGED_PTR
+	inline void *operator new(size_t size)
+	{
+		return Vehicle::NewWithParam(size, Type);
+	}
+
+	inline void *operator new(size_t size, size_t index)
+	{
+		return Vehicle::NewWithParam(size, index, Type);
+	}
+
+	inline void operator delete(void *p)
+	{
+		Vehicle::operator delete(p);
+	}
+
+	void *operator new(size_t, void *ptr) = delete;
+#endif
+
 	/**
 	 * Set vehicle type correctly
 	 */
@@ -1376,7 +1422,11 @@ struct SpecializedVehicle : public Vehicle {
 	 */
 	static inline bool IsValidID(size_t index)
 	{
+#if OTTD_UPPER_TAGGED_PTR
+		return Vehicle::IsValidID(index) && VehiclePoolOps::GetVehicleType(_vehicle_pool.GetRaw(index)) == Type;
+#else
 		return Vehicle::IsValidID(index) && Vehicle::Get(index)->type == Type;
+#endif
 	}
 
 	/**