diff --git a/.gitmodules b/.gitmodules index 5285d472..41e85ca7 100644 --- a/.gitmodules +++ b/.gitmodules @@ -29,3 +29,7 @@ [submodule "deps/udis86"] path = deps/udis86 url = ../udis86.git +[submodule "deps/zstd"] + path = deps/zstd + url = https://github.com/facebook/zstd.git + branch = dev diff --git a/deps/zstd b/deps/zstd new file mode 160000 index 00000000..379f9d87 --- /dev/null +++ b/deps/zstd @@ -0,0 +1 @@ +Subproject commit 379f9d874d578111c611b2262211aa763171a94c diff --git a/premake/zstd.lua b/premake/zstd.lua new file mode 100644 index 00000000..83e3967c --- /dev/null +++ b/premake/zstd.lua @@ -0,0 +1,66 @@ +zstd = { + settings = nil +} + +function zstd.setup(settings) + if not settings.source then error("Missing source.") end + + zstd.settings = settings + + if not zstd.settings.defines then zstd.settings.defines = {} end +end + +function zstd.import() + if not zstd.settings then error("You need to call zstd.setup first") end + + links { "zstd" } + zstd.includes() +end + +function zstd.includes() + if not zstd.settings then error("You need to call zstd.setup first") end + + includedirs + { + path.join(zstd.settings.source, "lib"), + path.join(zstd.settings.source, "lib/common"), + path.join(zstd.settings.source, "zlibWrapper") + } + defines(zstd.settings.defines) +end + +function zstd.project() + if not zstd.settings then error("You need to call zstd.setup first") end + + project "zstd" + language "C" + + zstd.includes() + files + { + path.join(zstd.settings.source, "lib/**.h"), + path.join(zstd.settings.source, "lib/**.c"), + path.join(zstd.settings.source, "zlibWrapper/zstd_zlibwrapper.h"), + path.join(zstd.settings.source, "zlibWrapper/zstd_zlibwrapper.c"), + } + removefiles + { + path.join(zstd.settings.source, "lib/legacy/**.*"), + --path.join(zstd.settings.source, "zlibWrapper/examples/**.*"), + } + defines + { + "zstd_DLL", + "_CRT_SECURE_NO_DEPRECATE", + } + + zlib.import() + + -- not our code, ignore POSIX usage warnings for now + warnings "Off" + + --configuration "*Static" + defines { "_LIB" } + removedefines { "_USRDLL", "_DLL", "zstd_DLL" } + kind "StaticLib" +end \ No newline at end of file diff --git a/premake5.lua b/premake5.lua index 9b0be67f..03640e98 100644 --- a/premake5.lua +++ b/premake5.lua @@ -180,6 +180,7 @@ require "premake/mongoose" require "premake/pdcurses" require "premake/protobuf" require "premake/zlib" +require "premake/zstd" require "premake/udis86" json11.setup @@ -221,6 +222,10 @@ zlib.setup }, source = path.join(depsBasePath, "zlib"), } +zstd.setup +{ + source = path.join(depsBasePath, "zstd"), +} udis86.setup { source = path.join(depsBasePath, "udis86"), @@ -311,6 +316,7 @@ workspace "iw4x" pdcurses.import() protobuf.import() zlib.import() + zstd.import() udis86.import() -- fix vpaths for protobuf sources @@ -423,6 +429,7 @@ workspace "iw4x" pdcurses.project() protobuf.project() zlib.project() + zstd.project() udis86.project() rule "ProtobufCompiler" diff --git a/src/Components/Modules/FastFiles.cpp b/src/Components/Modules/FastFiles.cpp index 8d50114c..9724d6ce 100644 --- a/src/Components/Modules/FastFiles.cpp +++ b/src/Components/Modules/FastFiles.cpp @@ -6,6 +6,9 @@ namespace Components symmetric_CTR FastFiles::CurrentCTR; std::vector FastFiles::ZonePaths; + bool FastFiles::UseZstd = false; + Utils::Compression::Deflate::Semaphore* FastFiles::ZlibLock = nullptr; + bool FastFiles::IsIW4xZone = false; bool FastFiles::StreamRead = false; @@ -333,15 +336,22 @@ namespace Components void FastFiles::ReadHeaderStub(unsigned int* header, int size) { + FastFiles::UseZstd = false; FastFiles::IsIW4xZone = false; FastFiles::LastByteRead = 0; Game::DB_ReadXFileUncompressed(header, size); if (header[0] == XFILE_HEADER_IW4X) { + FastFiles::UseZstd = true; FastFiles::IsIW4xZone = true; - if (header[1] < XFILE_VERSION_IW4X) + static_assert((XFILE_VERSION_IW4X - 1) == 3, "FastFile backwards-compatibility not granted!"); + if (header[1] == XFILE_VERSION_IW4X - 1) + { + FastFiles::UseZstd = false; + } + else if (header[1] < XFILE_VERSION_IW4X) { Logger::Error("The fastfile you are trying to load is outdated (%d, expected %d)", header[1], XFILE_VERSION_IW4X); } @@ -396,8 +406,9 @@ namespace Components ctr_decrypt(strm->next_in, const_cast(strm->next_in), strm->avail_in, &FastFiles::CurrentCTR); } - return Utils::Hook::Call(0x4D8090)(strm, version, stream_size); + //return Utils::Hook::Call(0x4D8090)(strm, version, stream_size); //return inflateInit_(strm, version, stream_size); + return FastFiles::InflateInitStub(strm, version, stream_size); } void FastFiles::AuthLoadInflateDecryptBaseFunc(unsigned char* buffer) @@ -484,6 +495,44 @@ namespace Components } #endif + int FastFiles::InflateInitStub(z_streamp strm, const char *version, int stream_size) + { + if (FastFiles::UseZstd) + { + if (FastFiles::ZlibLock) delete FastFiles::ZlibLock; + FastFiles::ZlibLock = new Utils::Compression::Deflate::Semaphore(DEFLATE_ZSTD); + + return inflateInit_(strm, version, stream_size); + } + + return Utils::Hook::Call(0x4D8090)(strm, version, stream_size); + } + + int FastFiles::InflateStub(z_streamp strm, int flush) + { + if (FastFiles::UseZstd) + { + return inflate(strm, flush); + } + + return Utils::Hook::Call(0x49EA00)(strm, flush); + } + + int FastFiles::InflateEndStub(z_streamp strm) + { + if (FastFiles::UseZstd && FastFiles::ZlibLock) + { + int result = inflateEnd(strm); + + delete FastFiles::ZlibLock; + FastFiles::ZlibLock = nullptr; + + return result; + } + + return Utils::Hook::Call(0x453750)(strm); + } + FastFiles::FastFiles() { Dvar::Register("ui_zoneDebug", false, Game::dvar_flag::DVAR_FLAG_SAVED, "Display current loaded zone."); @@ -554,9 +603,12 @@ namespace Components Utils::Hook(0x4159E2, FastFiles::ReadXFileHeader, HOOK_CALL).install()->quick(); // Replace internal ZLib - //Utils::Hook(0x44B160, inflateInit2_, HOOK_JUMP).install()->quick(); - //Utils::Hook(0x453750, inflateEnd, HOOK_JUMP).install()->quick(); - //Utils::Hook(0x49EA00, inflate, HOOK_JUMP).install()->quick(); + //Utils::Hook(0x4D0306, FastFiles::InflateInitStub, HOOK_CALL).install()->quick(); + Utils::Hook(0x4D034B, FastFiles::InflateInitStub, HOOK_JUMP).install()->quick(); + Utils::Hook(0x480A1A, FastFiles::InflateStub, HOOK_JUMP).install()->quick(); + Utils::Hook(0x5B99DE, FastFiles::InflateStub, HOOK_CALL).install()->quick(); + Utils::Hook(0x449D8F, FastFiles::InflateEndStub, HOOK_CALL).install()->quick(); + Utils::Hook(0x449DA3, FastFiles::InflateEndStub, HOOK_CALL).install()->quick(); // Add custom zone paths FastFiles::AddZonePath("zone\\patch\\"); @@ -618,6 +670,12 @@ namespace Components FastFiles::~FastFiles() { + if (FastFiles::ZlibLock) + { + delete FastFiles::ZlibLock; + FastFiles::ZlibLock = nullptr; + } + FastFiles::ZonePaths.clear(); } } diff --git a/src/Components/Modules/FastFiles.hpp b/src/Components/Modules/FastFiles.hpp index eb5569a1..6b37afa6 100644 --- a/src/Components/Modules/FastFiles.hpp +++ b/src/Components/Modules/FastFiles.hpp @@ -34,6 +34,9 @@ namespace Components static unsigned int CurrentZone; static unsigned int MaxZones; + static bool UseZstd; + static Utils::Compression::Deflate::Semaphore* ZlibLock; + static bool IsIW4xZone; static bool StreamRead; @@ -63,6 +66,10 @@ namespace Components static void ReadXFile(void* buffer, int size); static void ReadXFileStub(char* buffer, int size); + static int InflateInitStub(z_streamp strm, const char *version, int stream_size); + static int InflateStub(z_streamp strm, int flush); + static int InflateEndStub(z_streamp strm); + #ifdef DEBUG static void LogStreamRead(int len); #endif diff --git a/src/Components/Modules/Node.cpp b/src/Components/Modules/Node.cpp index a596fbad..92306e47 100644 --- a/src/Components/Modules/Node.cpp +++ b/src/Components/Modules/Node.cpp @@ -68,12 +68,12 @@ namespace Components if (Monitor::IsEnabled()) { std::string nodes = Utils::IO::ReadFile("players/nodes_default.dat"); - if (nodes.empty() || !list.ParseFromString(Utils::Compression::ZLib::Decompress(nodes))) return; + if (nodes.empty() || !list.ParseFromString(Utils::Compression::Deflate::ZStd::Decompress(nodes))) return; } else { FileSystem::File defaultNodes("nodes_default.dat"); - if (!defaultNodes.exists() || !list.ParseFromString(Utils::Compression::ZLib::Decompress(defaultNodes.getBuffer()))) return; + if (!defaultNodes.exists() || !list.ParseFromString(Utils::Compression::Deflate::ZStd::Decompress(defaultNodes.getBuffer()))) return; } for (int i = 0; i < list.nodes_size(); ++i) @@ -91,7 +91,7 @@ namespace Components { Proto::Node::List list; std::string nodes = Utils::IO::ReadFile("players/nodes.dat"); - if (nodes.empty() || !list.ParseFromString(Utils::Compression::ZLib::Decompress(nodes))) return; + if (nodes.empty() || !list.ParseFromString(Utils::Compression::Deflate::ZStd::Decompress(nodes))) return; for (int i = 0; i < list.nodes_size(); ++i) { @@ -125,10 +125,9 @@ namespace Components } } - Utils::IO::WriteFile("players/nodes.dat", Utils::Compression::ZLib::Compress(list.SerializeAsString())); + Utils::IO::WriteFile("players/nodes.dat", Utils::Compression::Deflate::ZStd::Compress(list.SerializeAsString())); } - void Node::Add(Network::Address address) { #ifndef DEBUG diff --git a/src/Components/Modules/Playlist.cpp b/src/Components/Modules/Playlist.cpp index 82ea30ad..7f6db9c1 100644 --- a/src/Components/Modules/Playlist.cpp +++ b/src/Components/Modules/Playlist.cpp @@ -57,7 +57,7 @@ namespace Components Logger::Print("Received playlist request, sending currently stored buffer.\n"); - std::string compressedList = Utils::Compression::ZLib::Compress(Playlist::CurrentPlaylistBuffer); + std::string compressedList = Utils::Compression::Deflate::ZStd::Compress(Playlist::CurrentPlaylistBuffer); Proto::Party::Playlist list; list.set_hash(Utils::Cryptography::JenkinsOneAtATime::Compute(compressedList)); @@ -95,7 +95,7 @@ namespace Components } // Decompress buffer - Playlist::ReceivedPlaylistBuffer = Utils::Compression::ZLib::Decompress(compressedData); + Playlist::ReceivedPlaylistBuffer = Utils::Compression::Deflate::ZStd::Decompress(compressedData); // Load and continue connection Logger::Print("Received playlist, loading and continuing connection...\n"); diff --git a/src/Components/Modules/QuickPatch.cpp b/src/Components/Modules/QuickPatch.cpp index cc7f145e..341de498 100644 --- a/src/Components/Modules/QuickPatch.cpp +++ b/src/Components/Modules/QuickPatch.cpp @@ -718,8 +718,32 @@ namespace Components for (int i = 0; i < 21; ++i) { - std::string compressed = Utils::Compression::ZLib::Compress(test); - std::string decompressed = Utils::Compression::ZLib::Decompress(compressed); + std::string compressed = Utils::Compression::Deflate::ZLib::Compress(test); + std::string decompressed = Utils::Compression::Deflate::ZLib::Decompress(compressed); + + if (test != decompressed) + { + printf("Error\n"); + printf("Compressing %d bytes and decompressing failed!\n", test.size()); + return false; + } + + auto size = test.size(); + for (unsigned int j = 0; j < size; ++j) + { + test.append(Utils::String::VA("%c", Utils::Cryptography::Rand::GenerateInt())); + } + } + + printf("Success\n"); + printf("Testing ZStd compression..."); + + test = Utils::String::VA("%c", Utils::Cryptography::Rand::GenerateInt()); + + for (int i = 0; i < 21; ++i) + { + std::string compressed = Utils::Compression::Deflate::ZStd::Compress(test); + std::string decompressed = Utils::Compression::Deflate::ZStd::Decompress(compressed); if (test != decompressed) { diff --git a/src/Components/Modules/ZoneBuilder.cpp b/src/Components/Modules/ZoneBuilder.cpp index 86c909ae..9e69e8a4 100644 --- a/src/Components/Modules/ZoneBuilder.cpp +++ b/src/Components/Modules/ZoneBuilder.cpp @@ -407,7 +407,12 @@ namespace Components } #endif - zoneBuffer = Utils::Compression::ZLib::Compress(zoneBuffer); +#ifdef DEBUG + zoneBuffer = Utils::Compression::Deflate::ZLib::Compress(zoneBuffer); +#else + zoneBuffer = Utils::Compression::Deflate::ZStd::Compress(zoneBuffer); +#endif + outBuffer.append(zoneBuffer); std::string outFile = "zone/" + this->zoneName + ".ff"; diff --git a/src/Components/Modules/ZoneBuilder.hpp b/src/Components/Modules/ZoneBuilder.hpp index da5022a8..a5971492 100644 --- a/src/Components/Modules/ZoneBuilder.hpp +++ b/src/Components/Modules/ZoneBuilder.hpp @@ -4,7 +4,7 @@ #define XFILE_VERSION 276 #define XFILE_HEADER_IW4X 0x78345749 // 'IW4x' -#define XFILE_VERSION_IW4X 3 +#define XFILE_VERSION_IW4X 4 namespace Components { diff --git a/src/STDInclude.hpp b/src/STDInclude.hpp index a0c9b0cf..3b5ecb59 100644 --- a/src/STDInclude.hpp +++ b/src/STDInclude.hpp @@ -73,7 +73,12 @@ template class Sizer { }; #pragma warning(disable: 6386) #pragma warning(disable: 6387) -#include +// #include +#include + +#define ZWRAP_USE_ZSTD 1 +#include + #include #include #include diff --git a/src/Utils/Compression.cpp b/src/Utils/Compression.cpp index c35837b6..73e5d8c3 100644 --- a/src/Utils/Compression.cpp +++ b/src/Utils/Compression.cpp @@ -4,7 +4,9 @@ namespace Utils { namespace Compression { - std::string ZLib::Compress(std::string data) + std::mutex Deflate::Mutex; + + std::string Deflate::Compress(std::string data) { Utils::Memory::Allocator allocator; unsigned long length = (data.size() * 2); @@ -15,7 +17,8 @@ namespace Utils char* buffer = allocator.allocateArray(length); - if (compress2(reinterpret_cast(buffer), &length, reinterpret_cast(const_cast(data.data())), data.size(), Z_BEST_COMPRESSION) != Z_OK) + int level = (ZWRAP_isUsingZSTDcompression() ? ZSTD_maxCLevel() : Z_BEST_COMPRESSION); + if (compress2(reinterpret_cast(buffer), &length, reinterpret_cast(const_cast(data.data())), data.size(), level) != Z_OK) { return ""; } @@ -26,7 +29,7 @@ namespace Utils return data; } - std::string ZLib::Decompress(std::string data) + std::string Deflate::Decompress(std::string data) { z_stream stream; ZeroMemory(&stream, sizeof(stream)); @@ -70,5 +73,42 @@ namespace Utils inflateEnd(&stream); return buffer; } + + std::string Deflate::ZLib::Compress(std::string data) + { + Deflate::Semaphore _(DEFLATE_ZLIB); + return Deflate::Compress(data); + } + + std::string Deflate::ZLib::Decompress(std::string data) + { + Deflate::Semaphore _(DEFLATE_ZLIB); + return Deflate::Decompress(data); + } + + std::string Deflate::ZStd::Compress(std::string data) + { + Deflate::Semaphore _(DEFLATE_ZSTD); + return Deflate::Compress(data); + } + + std::string Deflate::ZStd::Decompress(std::string data) + { + Deflate::Semaphore _(DEFLATE_ZSTD); + return Deflate::Decompress(data); + } + + Deflate::Semaphore::Semaphore(bool zstd) + { + Deflate::Mutex.lock(); + this->state = ZWRAP_isUsingZSTDcompression(); + ZWRAP_useZSTDcompression(zstd); + } + + Deflate::Semaphore::~Semaphore() + { + ZWRAP_useZSTDcompression(this->state); + Deflate::Mutex.unlock(); + } }; } diff --git a/src/Utils/Compression.hpp b/src/Utils/Compression.hpp index 35ee73e9..1c48fb25 100644 --- a/src/Utils/Compression.hpp +++ b/src/Utils/Compression.hpp @@ -1,14 +1,42 @@ #pragma once #define CHUNK 16384 +#define DEFLATE_ZLIB false +#define DEFLATE_ZSTD true namespace Utils { namespace Compression { - class ZLib + class Deflate { public: + class ZLib + { + public: + static std::string Compress(std::string data); + static std::string Decompress(std::string data); + }; + + class ZStd + { + public: + static std::string Compress(std::string data); + static std::string Decompress(std::string data); + }; + + class Semaphore + { + public: + Semaphore(bool zstd); + ~Semaphore(); + + private: + int state; + }; + + private: + static std::mutex Mutex; static std::string Compress(std::string data); static std::string Decompress(std::string data); };