diff --git a/.gitmodules b/.gitmodules index 41e85ca7..5285d472 100644 --- a/.gitmodules +++ b/.gitmodules @@ -29,7 +29,3 @@ [submodule "deps/udis86"] path = deps/udis86 url = ../udis86.git -[submodule "deps/zstd"] - path = deps/zstd - url = https://github.com/facebook/zstd.git - branch = dev diff --git a/deps/mongoose b/deps/mongoose index 3e59e1c4..5a0b6fc3 160000 --- a/deps/mongoose +++ b/deps/mongoose @@ -1 +1 @@ -Subproject commit 3e59e1c465b39bebbb596df38be451307906109b +Subproject commit 5a0b6fc35abd21031ad749a1180a6f63cd70b52a diff --git a/deps/protobuf b/deps/protobuf index ecca6ea9..62d7fe56 160000 --- a/deps/protobuf +++ b/deps/protobuf @@ -1 +1 @@ -Subproject commit ecca6ea95d56a6f70ff7b223ec3f904758acc8b1 +Subproject commit 62d7fe569717b6da39401a94b22adf73466e1f2d diff --git a/deps/zstd b/deps/zstd deleted file mode 160000 index 1d395504..00000000 --- a/deps/zstd +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 1d39550471fa2f6bd994ce213539f1310ebadf88 diff --git a/premake/zstd.lua b/premake/zstd.lua deleted file mode 100644 index 83e3967c..00000000 --- a/premake/zstd.lua +++ /dev/null @@ -1,66 +0,0 @@ -zstd = { - settings = nil -} - -function zstd.setup(settings) - if not settings.source then error("Missing source.") end - - zstd.settings = settings - - if not zstd.settings.defines then zstd.settings.defines = {} end -end - -function zstd.import() - if not zstd.settings then error("You need to call zstd.setup first") end - - links { "zstd" } - zstd.includes() -end - -function zstd.includes() - if not zstd.settings then error("You need to call zstd.setup first") end - - includedirs - { - path.join(zstd.settings.source, "lib"), - path.join(zstd.settings.source, "lib/common"), - path.join(zstd.settings.source, "zlibWrapper") - } - defines(zstd.settings.defines) -end - -function zstd.project() - if not zstd.settings then error("You need to call zstd.setup first") end - - project "zstd" - language "C" - - zstd.includes() - files - { - path.join(zstd.settings.source, "lib/**.h"), - path.join(zstd.settings.source, "lib/**.c"), - path.join(zstd.settings.source, "zlibWrapper/zstd_zlibwrapper.h"), - path.join(zstd.settings.source, "zlibWrapper/zstd_zlibwrapper.c"), - } - removefiles - { - path.join(zstd.settings.source, "lib/legacy/**.*"), - --path.join(zstd.settings.source, "zlibWrapper/examples/**.*"), - } - defines - { - "zstd_DLL", - "_CRT_SECURE_NO_DEPRECATE", - } - - zlib.import() - - -- not our code, ignore POSIX usage warnings for now - warnings "Off" - - --configuration "*Static" - defines { "_LIB" } - removedefines { "_USRDLL", "_DLL", "zstd_DLL" } - kind "StaticLib" -end \ No newline at end of file diff --git a/premake5.lua b/premake5.lua index 03640e98..9b0be67f 100644 --- a/premake5.lua +++ b/premake5.lua @@ -180,7 +180,6 @@ require "premake/mongoose" require "premake/pdcurses" require "premake/protobuf" require "premake/zlib" -require "premake/zstd" require "premake/udis86" json11.setup @@ -222,10 +221,6 @@ zlib.setup }, source = path.join(depsBasePath, "zlib"), } -zstd.setup -{ - source = path.join(depsBasePath, "zstd"), -} udis86.setup { source = path.join(depsBasePath, "udis86"), @@ -316,7 +311,6 @@ workspace "iw4x" pdcurses.import() protobuf.import() zlib.import() - zstd.import() udis86.import() -- fix vpaths for protobuf sources @@ -429,7 +423,6 @@ workspace "iw4x" pdcurses.project() protobuf.project() zlib.project() - zstd.project() udis86.project() rule "ProtobufCompiler" diff --git a/src/Components/Modules/FastFiles.cpp b/src/Components/Modules/FastFiles.cpp index 9724d6ce..f97c3837 100644 --- a/src/Components/Modules/FastFiles.cpp +++ b/src/Components/Modules/FastFiles.cpp @@ -6,9 +6,6 @@ namespace Components symmetric_CTR FastFiles::CurrentCTR; std::vector FastFiles::ZonePaths; - bool FastFiles::UseZstd = false; - Utils::Compression::Deflate::Semaphore* FastFiles::ZlibLock = nullptr; - bool FastFiles::IsIW4xZone = false; bool FastFiles::StreamRead = false; @@ -336,28 +333,21 @@ namespace Components void FastFiles::ReadHeaderStub(unsigned int* header, int size) { - FastFiles::UseZstd = false; FastFiles::IsIW4xZone = false; FastFiles::LastByteRead = 0; Game::DB_ReadXFileUncompressed(header, size); if (header[0] == XFILE_HEADER_IW4X) { - FastFiles::UseZstd = true; FastFiles::IsIW4xZone = true; - static_assert((XFILE_VERSION_IW4X - 1) == 3, "FastFile backwards-compatibility not granted!"); - if (header[1] == XFILE_VERSION_IW4X - 1) - { - FastFiles::UseZstd = false; - } - else if (header[1] < XFILE_VERSION_IW4X) + if (header[1] < XFILE_VERSION_IW4X) { Logger::Error("The fastfile you are trying to load is outdated (%d, expected %d)", header[1], XFILE_VERSION_IW4X); } else if (header[1] > XFILE_VERSION_IW4X) { - Logger::Error("You are loading a fastfile that is too new (%d, expected %d), update your game", header[1], XFILE_VERSION_IW4X); + Logger::Error("You are loading a fastfile that is too new (%d, expected %d), update your game or rebuild the fastfile", header[1], XFILE_VERSION_IW4X); } *reinterpret_cast(header) = XFILE_MAGIC_UNSIGNED; @@ -406,9 +396,8 @@ namespace Components ctr_decrypt(strm->next_in, const_cast(strm->next_in), strm->avail_in, &FastFiles::CurrentCTR); } - //return Utils::Hook::Call(0x4D8090)(strm, version, stream_size); + return Utils::Hook::Call(0x4D8090)(strm, version, stream_size); //return inflateInit_(strm, version, stream_size); - return FastFiles::InflateInitStub(strm, version, stream_size); } void FastFiles::AuthLoadInflateDecryptBaseFunc(unsigned char* buffer) @@ -495,44 +484,6 @@ namespace Components } #endif - int FastFiles::InflateInitStub(z_streamp strm, const char *version, int stream_size) - { - if (FastFiles::UseZstd) - { - if (FastFiles::ZlibLock) delete FastFiles::ZlibLock; - FastFiles::ZlibLock = new Utils::Compression::Deflate::Semaphore(DEFLATE_ZSTD); - - return inflateInit_(strm, version, stream_size); - } - - return Utils::Hook::Call(0x4D8090)(strm, version, stream_size); - } - - int FastFiles::InflateStub(z_streamp strm, int flush) - { - if (FastFiles::UseZstd) - { - return inflate(strm, flush); - } - - return Utils::Hook::Call(0x49EA00)(strm, flush); - } - - int FastFiles::InflateEndStub(z_streamp strm) - { - if (FastFiles::UseZstd && FastFiles::ZlibLock) - { - int result = inflateEnd(strm); - - delete FastFiles::ZlibLock; - FastFiles::ZlibLock = nullptr; - - return result; - } - - return Utils::Hook::Call(0x453750)(strm); - } - FastFiles::FastFiles() { Dvar::Register("ui_zoneDebug", false, Game::dvar_flag::DVAR_FLAG_SAVED, "Display current loaded zone."); @@ -602,14 +553,6 @@ namespace Components // XFile header loading Utils::Hook(0x4159E2, FastFiles::ReadXFileHeader, HOOK_CALL).install()->quick(); - // Replace internal ZLib - //Utils::Hook(0x4D0306, FastFiles::InflateInitStub, HOOK_CALL).install()->quick(); - Utils::Hook(0x4D034B, FastFiles::InflateInitStub, HOOK_JUMP).install()->quick(); - Utils::Hook(0x480A1A, FastFiles::InflateStub, HOOK_JUMP).install()->quick(); - Utils::Hook(0x5B99DE, FastFiles::InflateStub, HOOK_CALL).install()->quick(); - Utils::Hook(0x449D8F, FastFiles::InflateEndStub, HOOK_CALL).install()->quick(); - Utils::Hook(0x449DA3, FastFiles::InflateEndStub, HOOK_CALL).install()->quick(); - // Add custom zone paths FastFiles::AddZonePath("zone\\patch\\"); FastFiles::AddZonePath("zone\\dlc\\"); @@ -670,12 +613,6 @@ namespace Components FastFiles::~FastFiles() { - if (FastFiles::ZlibLock) - { - delete FastFiles::ZlibLock; - FastFiles::ZlibLock = nullptr; - } - FastFiles::ZonePaths.clear(); } } diff --git a/src/Components/Modules/FastFiles.hpp b/src/Components/Modules/FastFiles.hpp index 6b37afa6..eb5569a1 100644 --- a/src/Components/Modules/FastFiles.hpp +++ b/src/Components/Modules/FastFiles.hpp @@ -34,9 +34,6 @@ namespace Components static unsigned int CurrentZone; static unsigned int MaxZones; - static bool UseZstd; - static Utils::Compression::Deflate::Semaphore* ZlibLock; - static bool IsIW4xZone; static bool StreamRead; @@ -66,10 +63,6 @@ namespace Components static void ReadXFile(void* buffer, int size); static void ReadXFileStub(char* buffer, int size); - static int InflateInitStub(z_streamp strm, const char *version, int stream_size); - static int InflateStub(z_streamp strm, int flush); - static int InflateEndStub(z_streamp strm); - #ifdef DEBUG static void LogStreamRead(int len); #endif diff --git a/src/Components/Modules/Node.cpp b/src/Components/Modules/Node.cpp index b7ea148a..9fd9e02c 100644 --- a/src/Components/Modules/Node.cpp +++ b/src/Components/Modules/Node.cpp @@ -68,12 +68,12 @@ namespace Components if (Monitor::IsEnabled()) { std::string nodes = Utils::IO::ReadFile("players/nodes_default.dat"); - if (nodes.empty() || !list.ParseFromString(Utils::Compression::Deflate::ZStd::Decompress(nodes))) return; + if (nodes.empty() || !list.ParseFromString(Utils::Compression::ZLib::Decompress(nodes))) return; } else { FileSystem::File defaultNodes("nodes_default.dat"); - if (!defaultNodes.exists() || !list.ParseFromString(Utils::Compression::Deflate::ZStd::Decompress(defaultNodes.getBuffer()))) return; + if (!defaultNodes.exists() || !list.ParseFromString(Utils::Compression::ZLib::Decompress(defaultNodes.getBuffer()))) return; } for (int i = 0; i < list.nodes_size(); ++i) @@ -91,7 +91,7 @@ namespace Components { Proto::Node::List list; std::string nodes = Utils::IO::ReadFile("players/nodes.dat"); - if (nodes.empty() || !list.ParseFromString(Utils::Compression::Deflate::ZStd::Decompress(nodes))) return; + if (nodes.empty() || !list.ParseFromString(Utils::Compression::ZLib::Decompress(nodes))) return; for (int i = 0; i < list.nodes_size(); ++i) { @@ -127,7 +127,7 @@ namespace Components } Node::Mutex.unlock(); - Utils::IO::WriteFile("players/nodes.dat", Utils::Compression::Deflate::ZStd::Compress(list.SerializeAsString())); + Utils::IO::WriteFile("players/nodes.dat", Utils::Compression::ZLib::Compress(list.SerializeAsString())); } void Node::Add(Network::Address address) diff --git a/src/Components/Modules/Playlist.cpp b/src/Components/Modules/Playlist.cpp index bdf5b236..3fb43aff 100644 --- a/src/Components/Modules/Playlist.cpp +++ b/src/Components/Modules/Playlist.cpp @@ -39,7 +39,7 @@ namespace Components DWORD Playlist::StorePlaylistStub(const char** buffer) { Playlist::MapRelocation.clear(); - Playlist::CurrentPlaylistBuffer = Utils::Compression::Deflate::ZStd::Compress(*buffer); + Playlist::CurrentPlaylistBuffer = Utils::Compression::ZLib::Compress(*buffer); return Utils::Hook::Call(0x4C0350)(buffer); } @@ -95,7 +95,7 @@ namespace Components } // Decompress buffer - Playlist::ReceivedPlaylistBuffer = Utils::Compression::Deflate::ZStd::Decompress(compressedData); + Playlist::ReceivedPlaylistBuffer = Utils::Compression::ZLib::Decompress(compressedData); // Load and continue connection Logger::Print("Received playlist, loading and continuing connection...\n"); diff --git a/src/Components/Modules/QuickPatch.cpp b/src/Components/Modules/QuickPatch.cpp index bdf74bba..04d19fe1 100644 --- a/src/Components/Modules/QuickPatch.cpp +++ b/src/Components/Modules/QuickPatch.cpp @@ -702,32 +702,8 @@ namespace Components for (int i = 0; i < 21; ++i) { - std::string compressed = Utils::Compression::Deflate::ZLib::Compress(test); - std::string decompressed = Utils::Compression::Deflate::ZLib::Decompress(compressed); - - if (test != decompressed) - { - printf("Error\n"); - printf("Compressing %d bytes and decompressing failed!\n", test.size()); - return false; - } - - auto size = test.size(); - for (unsigned int j = 0; j < size; ++j) - { - test.append(Utils::String::VA("%c", Utils::Cryptography::Rand::GenerateInt())); - } - } - - printf("Success\n"); - printf("Testing ZStd compression..."); - - test = Utils::String::VA("%c", Utils::Cryptography::Rand::GenerateInt()); - - for (int i = 0; i < 21; ++i) - { - std::string compressed = Utils::Compression::Deflate::ZStd::Compress(test); - std::string decompressed = Utils::Compression::Deflate::ZStd::Decompress(compressed); + std::string compressed = Utils::Compression::ZLib::Compress(test); + std::string decompressed = Utils::Compression::ZLib::Decompress(compressed); if (test != decompressed) { diff --git a/src/Components/Modules/ZoneBuilder.cpp b/src/Components/Modules/ZoneBuilder.cpp index 9e69e8a4..8f9d14bf 100644 --- a/src/Components/Modules/ZoneBuilder.cpp +++ b/src/Components/Modules/ZoneBuilder.cpp @@ -410,7 +410,7 @@ namespace Components #ifdef DEBUG zoneBuffer = Utils::Compression::Deflate::ZLib::Compress(zoneBuffer); #else - zoneBuffer = Utils::Compression::Deflate::ZStd::Compress(zoneBuffer); + zoneBuffer = Utils::Compression::ZLib::Compress(zoneBuffer); #endif outBuffer.append(zoneBuffer); diff --git a/src/Components/Modules/ZoneBuilder.hpp b/src/Components/Modules/ZoneBuilder.hpp index a5971492..da5022a8 100644 --- a/src/Components/Modules/ZoneBuilder.hpp +++ b/src/Components/Modules/ZoneBuilder.hpp @@ -4,7 +4,7 @@ #define XFILE_VERSION 276 #define XFILE_HEADER_IW4X 0x78345749 // 'IW4x' -#define XFILE_VERSION_IW4X 4 +#define XFILE_VERSION_IW4X 3 namespace Components { diff --git a/src/STDInclude.hpp b/src/STDInclude.hpp index 3b5ecb59..f6fcd70b 100644 --- a/src/STDInclude.hpp +++ b/src/STDInclude.hpp @@ -73,11 +73,7 @@ template class Sizer { }; #pragma warning(disable: 6386) #pragma warning(disable: 6387) -// #include -#include - -#define ZWRAP_USE_ZSTD 1 -#include +#include #include #include diff --git a/src/Utils/Compression.cpp b/src/Utils/Compression.cpp index 1302b448..19df17bf 100644 --- a/src/Utils/Compression.cpp +++ b/src/Utils/Compression.cpp @@ -4,9 +4,7 @@ namespace Utils { namespace Compression { - std::mutex Deflate::Mutex; - - std::string Deflate::Compress(std::string data) + std::string ZLib::Compress(std::string data) { Utils::Memory::Allocator allocator; unsigned long length = (data.size() * 2); @@ -16,9 +14,7 @@ namespace Utils if (length < 100) length *= 10; char* buffer = allocator.allocateArray(length); - - int level = (ZWRAP_isUsingZSTDcompression() ? ZSTD_maxCLevel() : Z_BEST_COMPRESSION); - if (compress2(reinterpret_cast(buffer), &length, reinterpret_cast(const_cast(data.data())), data.size(), level) != Z_OK) + if (compress2(reinterpret_cast(buffer), &length, reinterpret_cast(const_cast(data.data())), data.size(), Z_BEST_COMPRESSION) != Z_OK) { return ""; } @@ -29,7 +25,7 @@ namespace Utils return data; } - std::string Deflate::Decompress(std::string data) + std::string ZLib::Decompress(std::string data) { z_stream stream; ZeroMemory(&stream, sizeof(stream)); @@ -73,87 +69,5 @@ namespace Utils inflateEnd(&stream); return buffer; } - - std::string Deflate::ZLib::Compress(std::string data) - { - Deflate::Semaphore _(DEFLATE_ZLIB); - return Deflate::Compress(data); - } - - std::string Deflate::ZLib::Decompress(std::string data) - { - Deflate::Semaphore _(DEFLATE_ZLIB); - return Deflate::Decompress(data); - } - - std::string Deflate::ZStd::Compress(std::string data, bool safe) - { - if (safe) - { - Utils::Memory::Allocator allocator; - - size_t size = data.size() + 100; - size = ZSTD_compressBound(size); - char* buffer = allocator.allocateArray(size); - - size = ZSTD_compress(buffer, size, data.data(), data.size(), ZSTD_maxCLevel()); - if (size == 0 || ZSTD_isError(size)) return ""; - - return std::string(buffer, size); - } - else - { - Deflate::Semaphore _(DEFLATE_ZSTD); - return Deflate::Compress(data); - } - } - - std::string Deflate::ZStd::Decompress(std::string data, bool safe) - { - if (safe) - { - if (data.empty()) return ""; - - Utils::Memory::Allocator allocator; - - size_t size = data.size() + 100; - size = ZSTD_compressBound(size); - size_t maxSize = size; - char* buffer = allocator.allocateArray(size); - - while(true) - { - size = ZSTD_decompress(buffer, size, data.data(), data.size()); - if (!ZSTD_isError(size)) break; - - maxSize++; - maxSize *= 2; - size = maxSize; - - allocator.free(buffer); - buffer = allocator.allocateArray(size); - } - - return std::string(buffer, size); - } - else - { - Deflate::Semaphore _(DEFLATE_ZSTD); - return Deflate::Decompress(data); - } - } - - Deflate::Semaphore::Semaphore(bool zstd) - { - Deflate::Mutex.lock(); - this->state = ZWRAP_isUsingZSTDcompression(); - ZWRAP_useZSTDcompression(zstd); - } - - Deflate::Semaphore::~Semaphore() - { - ZWRAP_useZSTDcompression(this->state); - Deflate::Mutex.unlock(); - } - }; + } } diff --git a/src/Utils/Compression.hpp b/src/Utils/Compression.hpp index f106b88d..b79693ae 100644 --- a/src/Utils/Compression.hpp +++ b/src/Utils/Compression.hpp @@ -8,35 +8,9 @@ namespace Utils { namespace Compression { - class Deflate + class ZLib { public: - class ZLib - { - public: - static std::string Compress(std::string data); - static std::string Decompress(std::string data); - }; - - class ZStd - { - public: - static std::string Compress(std::string data, bool safe = true); - static std::string Decompress(std::string data, bool safe = true); - }; - - class Semaphore - { - public: - Semaphore(bool zstd); - ~Semaphore(); - - private: - int state; - }; - - private: - static std::mutex Mutex; static std::string Compress(std::string data); static std::string Decompress(std::string data); };