From b284d34277f1ffa6e23ec21c557e13897ab508a8 Mon Sep 17 00:00:00 2001 From: dP Date: Mon, 1 Mar 2021 01:05:50 +0300 Subject: [PATCH] Add: Support Zstandard(zstd) savegame compression (cherry picked from commit 6f0aeaf2c5436550c93205e704624957e9abc969) --- .github/workflows/ci-build.yml | 5 +- .github/workflows/release.yml | 6 +- CMakeLists.txt | 2 + COMPILING.md | 6 +- Doxyfile.in | 1 + cmake/FindZSTD.cmake | 89 +++++++++++++++++++++++ src/crashlog.cpp | 7 ++ src/saveload/saveload.cpp | 124 +++++++++++++++++++++++++++++++++ 8 files changed, 234 insertions(+), 6 deletions(-) create mode 100644 cmake/FindZSTD.cmake diff --git a/.github/workflows/ci-build.yml b/.github/workflows/ci-build.yml index 0f9ddf9ff3..0515abfced 100644 --- a/.github/workflows/ci-build.yml +++ b/.github/workflows/ci-build.yml @@ -97,6 +97,7 @@ jobs: libfontconfig-dev \ libicu-dev \ liblzma-dev \ + libzstd-dev \ liblzo2-dev \ ${{ matrix.libsdl }} \ zlib1g-dev \ @@ -161,7 +162,7 @@ jobs: vcpkgDirectory: '/usr/local/share/vcpkg' doNotUpdateVcpkg: false vcpkgGitCommitId: 2a42024b53ebb512fb5dd63c523338bf26c8489c - vcpkgArguments: 'liblzma libpng lzo' + vcpkgArguments: 'liblzma libpng lzo zstd' vcpkgTriplet: '${{ matrix.arch }}-osx' - name: Install OpenGFX @@ -232,7 +233,7 @@ jobs: with: vcpkgDirectory: 'c:/vcpkg' doNotUpdateVcpkg: true - vcpkgArguments: 'liblzma libpng lzo zlib' + vcpkgArguments: 'liblzma libpng lzo zlib zstd' vcpkgTriplet: '${{ matrix.arch }}-windows-static' - name: Install OpenGFX diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 8829c43483..844b3bb798 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -297,6 +297,7 @@ jobs: SDL2-devel \ wget \ xz-devel \ + libzstd-devel \ zlib-devel \ # EOF echo "::endgroup::" @@ -403,6 +404,7 @@ jobs: libfluidsynth-dev \ libicu-dev \ liblzma-dev \ + libzstd-dev \ liblzo2-dev \ libsdl2-dev \ lsb-release \ @@ -482,7 +484,7 @@ jobs: vcpkgDirectory: '/usr/local/share/vcpkg' doNotUpdateVcpkg: false vcpkgGitCommitId: 2a42024b53ebb512fb5dd63c523338bf26c8489c - vcpkgArguments: 'liblzma:x64-osx libpng:x64-osx lzo:x64-osx liblzma:arm64-osx libpng:arm64-osx lzo:arm64-osx' + vcpkgArguments: 'liblzma:x64-osx libpng:x64-osx lzo:x64-osx zstd:x64-osx liblzma:arm64-osx libpng:arm64-osx lzo:arm64-osx zstd:arm64-osx' - name: Build tools run: | @@ -667,7 +669,7 @@ jobs: with: vcpkgDirectory: 'c:/vcpkg' doNotUpdateVcpkg: true - vcpkgArguments: 'liblzma libpng lzo zlib' + vcpkgArguments: 'liblzma libpng lzo zlib zstd' vcpkgTriplet: '${{ matrix.arch }}-windows-static' - name: Build tools diff --git a/CMakeLists.txt b/CMakeLists.txt index 68e634476a..ab9963ec51 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -97,6 +97,7 @@ find_package(Threads REQUIRED) find_package(ZLIB) find_package(LibLZMA) find_package(LZO) +find_package(ZSTD) find_package(PNG) if(NOT WIN32) @@ -291,6 +292,7 @@ link_package(PNG TARGET PNG::PNG ENCOURAGED) link_package(ZLIB TARGET ZLIB::ZLIB ENCOURAGED) link_package(LIBLZMA TARGET LibLZMA::LibLZMA ENCOURAGED) link_package(LZO) +link_package(ZSTD TARGET ZSTD::ZSTD ENCOURAGED) if(NOT OPTION_DEDICATED) link_package(Fluidsynth) diff --git a/COMPILING.md b/COMPILING.md index 5e642dc874..9e63c3c9bc 100644 --- a/COMPILING.md +++ b/COMPILING.md @@ -8,6 +8,7 @@ The following libraries are used by OpenTTD for: heightmaps - liblzo2: (de)compressing of old (pre 0.3.0) savegames - liblzma: (de)compressing of savegames (1.1.0 and later) +- libzstd: (de)compressing of savegames (1.11.0 and later) - libpng: making screenshots and loading heightmaps - libfreetype: loading generic fonts and rendering them - libfontconfig: searching for fonts, resolving font names to actual fonts @@ -45,6 +46,7 @@ After this, you can install the dependencies OpenTTD needs. We advise to use the `static` versions, and OpenTTD currently needs the following dependencies: - liblzma +- libzstd - libpng - lzo - zlib @@ -52,8 +54,8 @@ the `static` versions, and OpenTTD currently needs the following dependencies: To install both the x64 (64bit) and x86 (32bit) variants (though only one is necessary), you can use: ```ps -.\vcpkg install liblzma:x64-windows-static libpng:x64-windows-static lzo:x64-windows-static zlib:x64-windows-static -.\vcpkg install liblzma:x86-windows-static libpng:x86-windows-static lzo:x86-windows-static zlib:x86-windows-static +.\vcpkg install liblzma:x64-windows-static zstd:x64-windows-static libpng:x64-windows-static lzo:x64-windows-static zlib:x64-windows-static +.\vcpkg install liblzma:x86-windows-static zstd:x86-windows-static libpng:x86-windows-static lzo:x86-windows-static zlib:x86-windows-static ``` You can open the folder (as a CMake project). CMake will be detected, and you can compile from there. diff --git a/Doxyfile.in b/Doxyfile.in index 8727594771..1068c9b7ba 100644 --- a/Doxyfile.in +++ b/Doxyfile.in @@ -290,6 +290,7 @@ INCLUDE_FILE_PATTERNS = PREDEFINED = WITH_ZLIB \ WITH_LZO \ WITH_LIBLZMA \ + WITH_ZSTD \ WITH_SDL \ WITH_PNG \ WITH_FONTCONFIG \ diff --git a/cmake/FindZSTD.cmake b/cmake/FindZSTD.cmake new file mode 100644 index 0000000000..2b730d508b --- /dev/null +++ b/cmake/FindZSTD.cmake @@ -0,0 +1,89 @@ +#[=======================================================================[.rst: +FindZSTD +------- + +Finds the ZSTD library. + +Result Variables +^^^^^^^^^^^^^^^^ + +This will define the following variables: + +``ZSTD_FOUND`` + True if the system has the ZSTD library. +``ZSTD_INCLUDE_DIRS`` + Include directories needed to use ZSTD. +``ZSTD_LIBRARIES`` + Libraries needed to link to ZSTD. +``ZSTD_VERSION`` + The version of the ZSTD library which was found. + +Cache Variables +^^^^^^^^^^^^^^^ + +The following cache variables may also be set: + +``ZSTD_INCLUDE_DIR`` + The directory containing ``zstd.h``. +``ZSTD_LIBRARY`` + The path to the ZSTD library. + +#]=======================================================================] + +find_package(PkgConfig QUIET) +pkg_check_modules(PC_ZSTD QUIET libzstd) + +find_path(ZSTD_INCLUDE_DIR + NAMES zstd.h + PATHS ${PC_ZSTD_INCLUDE_DIRS} +) + +find_library(ZSTD_LIBRARY + NAMES zstd + PATHS ${PC_ZSTD_LIBRARY_DIRS} +) + +# With vcpkg, the library path should contain both 'debug' and 'optimized' +# entries (see target_link_libraries() documentation for more information) +# +# NOTE: we only patch up when using vcpkg; the same issue might happen +# when not using vcpkg, but this is non-trivial to fix, as we have no idea +# what the paths are. With vcpkg we do. And we only official support vcpkg +# with Windows. +# +# NOTE: this is based on the assumption that the debug file has the same +# name as the optimized file. This is not always the case, but so far +# experiences has shown that in those case vcpkg CMake files do the right +# thing. +if(VCPKG_TOOLCHAIN AND ZSTD_LIBRARY) + if(ZSTD_LIBRARY MATCHES "/debug/") + set(ZSTD_LIBRARY_DEBUG ${ZSTD_LIBRARY}) + string(REPLACE "/debug/lib/" "/lib/" ZSTD_LIBRARY_RELEASE ${ZSTD_LIBRARY}) + else() + set(ZSTD_LIBRARY_RELEASE ${ZSTD_LIBRARY}) + string(REPLACE "/lib/" "/debug/lib/" ZSTD_LIBRARY_DEBUG ${ZSTD_LIBRARY}) + endif() + include(SelectLibraryConfigurations) + select_library_configurations(ZSTD) +endif() + +set(ZSTD_VERSION ${PC_ZSTD_VERSION}) + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(ZSTD + FOUND_VAR ZSTD_FOUND + REQUIRED_VARS + ZSTD_LIBRARY + ZSTD_INCLUDE_DIR + VERSION_VAR ZSTD_VERSION +) + +if(ZSTD_FOUND) + set(ZSTD_LIBRARIES ${ZSTD_LIBRARY}) + set(ZSTD_INCLUDE_DIRS ${ZSTD_INCLUDE_DIR}) +endif() + +mark_as_advanced( + ZSTD_INCLUDE_DIR + ZSTD_LIBRARY +) diff --git a/src/crashlog.cpp b/src/crashlog.cpp index 3b48aa045b..fc6e5d6a7a 100644 --- a/src/crashlog.cpp +++ b/src/crashlog.cpp @@ -60,6 +60,9 @@ #ifdef WITH_LIBLZMA # include #endif +#ifdef WITH_ZSTD +#include +#endif #ifdef WITH_LZO #include #endif @@ -292,6 +295,10 @@ char *CrashLog::LogLibraries(char *buffer, const char *last) const buffer += seprintf(buffer, last, " LZMA: %s\n", lzma_version_string()); #endif +#ifdef WITH_ZSTD + buffer += seprintf(buffer, last, " ZSTD: %s\n", ZSTD_versionString()); +#endif + #ifdef WITH_LZO buffer += seprintf(buffer, last, " LZO: %s\n", lzo_version_string()); #endif diff --git a/src/saveload/saveload.cpp b/src/saveload/saveload.cpp index 8fcba5a945..fe4720ece0 100644 --- a/src/saveload/saveload.cpp +++ b/src/saveload/saveload.cpp @@ -2761,6 +2761,119 @@ struct LZMASaveFilter : SaveFilter { #endif /* WITH_LIBLZMA */ +/******************************************** + ********** START OF ZSTD CODE ************** + ********************************************/ + +#if defined(WITH_ZSTD) +#include + + +/** Filter using ZSTD compression. */ +struct ZSTDLoadFilter : LoadFilter { + ZSTD_DCtx *zstd; ///< ZSTD decompression context + byte fread_buf[MEMORY_CHUNK_SIZE]; ///< Buffer for reading from the file + ZSTD_inBuffer input; ///< ZSTD input buffer for fread_buf + + /** + * Initialise this filter. + * @param chain The next filter in this chain. + */ + ZSTDLoadFilter(LoadFilter *chain) : LoadFilter(chain) + { + this->zstd = ZSTD_createDCtx(); + if (!this->zstd) SlError(STR_GAME_SAVELOAD_ERROR_BROKEN_INTERNAL_ERROR, "cannot initialize compressor"); + this->input = {this->fread_buf, 0, 0}; + } + + /** Clean everything up. */ + ~ZSTDLoadFilter() + { + ZSTD_freeDCtx(this->zstd); + } + + size_t Read(byte *buf, size_t size) override + { + ZSTD_outBuffer output{buf, size, 0}; + + do { + /* read more bytes from the file? */ + if (this->input.pos == this->input.size) { + this->input.size = this->chain->Read(this->fread_buf, sizeof(this->fread_buf)); + this->input.pos = 0; + } + + size_t ret = ZSTD_decompressStream(this->zstd, &output, &this->input); + if (ZSTD_isError(ret)) SlError(STR_GAME_SAVELOAD_ERROR_BROKEN_INTERNAL_ERROR, "libzstd returned error code"); + if (ret == 0) break; + } while (output.pos < output.size); + + return output.pos; + } +}; + +/** Filter using ZSTD compression. */ +struct ZSTDSaveFilter : SaveFilter { + ZSTD_CCtx *zstd; ///< ZSTD compression context + + /** + * Initialise this filter. + * @param chain The next filter in this chain. + * @param compression_level The requested level of compression. + */ + ZSTDSaveFilter(SaveFilter *chain, byte compression_level) : SaveFilter(chain) + { + this->zstd = ZSTD_createCCtx(); + if (!this->zstd) SlError(STR_GAME_SAVELOAD_ERROR_BROKEN_INTERNAL_ERROR, "cannot initialize compressor"); + if (ZSTD_isError(ZSTD_CCtx_setParameter(this->zstd, ZSTD_c_compressionLevel, (int)compression_level - 100))) { + ZSTD_freeCCtx(this->zstd); + SlError(STR_GAME_SAVELOAD_ERROR_BROKEN_INTERNAL_ERROR, "invalid compresison level"); + } + } + + /** Clean up what we allocated. */ + ~ZSTDSaveFilter() + { + ZSTD_freeCCtx(this->zstd); + } + + /** + * Helper loop for writing the data. + * @param p The bytes to write. + * @param len Amount of bytes to write. + * @param mode Mode for ZSTD_compressStream2. + */ + void WriteLoop(byte *p, size_t len, ZSTD_EndDirective mode) + { + byte buf[MEMORY_CHUNK_SIZE]; // output buffer + ZSTD_inBuffer input{p, len, 0}; + + bool finished; + do { + ZSTD_outBuffer output{buf, sizeof(buf), 0}; + size_t remaining = ZSTD_compressStream2(this->zstd, &output, &input, mode); + if (ZSTD_isError(remaining)) SlError(STR_GAME_SAVELOAD_ERROR_BROKEN_INTERNAL_ERROR, "libzstd returned error code"); + + if (output.pos != 0) this->chain->Write(buf, output.pos); + + finished = (mode == ZSTD_e_end ? (remaining == 0) : (input.pos == input.size)); + } while (!finished); + } + + void Write(byte *buf, size_t size) override + { + this->WriteLoop(buf, size, ZSTD_e_continue); + } + + void Finish() override + { + this->WriteLoop(nullptr, 0, ZSTD_e_end); + this->chain->Finish(); + } +}; + +#endif /* WITH_LIBZSTD */ + /******************************************* ************* END OF CODE ***************** *******************************************/ @@ -2797,6 +2910,17 @@ static const SaveLoadFormat _saveload_formats[] = { #else {"zlib", TO_BE32X('OTTZ'), nullptr, nullptr, 0, 0, 0, false}, #endif +#if defined(WITH_ZSTD) + /* Zstd provides a decent compression rate at a very high compression/decompression speed. Compared to lzma level 2 + * zstd saves are about 40% larger (on level 1) but it has about 30x faster compression and 5x decompression making it + * a good choice for multiplayer servers. And zstd level 1 seems to be the optimal one for client connection speed + * (compress + 10 MB/s download + decompress time), about 3x faster than lzma:2 and 1.5x than zlib:2 and lzo. + * As zstd has negative compression levels the values were increased by 100 moving zstd level range -100..22 into + * openttd 0..122. Also note that value 100 mathes zstd level 0 which is a special value for default level 3 (openttd 103) */ + {"zstd", TO_BE32X('OTTS'), CreateLoadFilter, CreateSaveFilter, 0, 101, 122, false}, +#else + {"zstd", TO_BE32X('OTTS'), nullptr, nullptr, 0, 0, 0, false}, +#endif #if defined(WITH_LIBLZMA) /* Level 2 compression is speed wise as fast as zlib level 6 compression (old default), but results in ~10% smaller saves. * Higher compression levels are possible, and might improve savegame size by up to 25%, but are also up to 10 times slower.