diff --git a/src/Components/Modules/FastFiles.cpp b/src/Components/Modules/FastFiles.cpp index 8d50114c..9724d6ce 100644 --- a/src/Components/Modules/FastFiles.cpp +++ b/src/Components/Modules/FastFiles.cpp @@ -6,6 +6,9 @@ namespace Components symmetric_CTR FastFiles::CurrentCTR; std::vector FastFiles::ZonePaths; + bool FastFiles::UseZstd = false; + Utils::Compression::Deflate::Semaphore* FastFiles::ZlibLock = nullptr; + bool FastFiles::IsIW4xZone = false; bool FastFiles::StreamRead = false; @@ -333,15 +336,22 @@ namespace Components void FastFiles::ReadHeaderStub(unsigned int* header, int size) { + FastFiles::UseZstd = false; FastFiles::IsIW4xZone = false; FastFiles::LastByteRead = 0; Game::DB_ReadXFileUncompressed(header, size); if (header[0] == XFILE_HEADER_IW4X) { + FastFiles::UseZstd = true; FastFiles::IsIW4xZone = true; - if (header[1] < XFILE_VERSION_IW4X) + static_assert((XFILE_VERSION_IW4X - 1) == 3, "FastFile backwards-compatibility not granted!"); + if (header[1] == XFILE_VERSION_IW4X - 1) + { + FastFiles::UseZstd = false; + } + else if (header[1] < XFILE_VERSION_IW4X) { Logger::Error("The fastfile you are trying to load is outdated (%d, expected %d)", header[1], XFILE_VERSION_IW4X); } @@ -396,8 +406,9 @@ namespace Components ctr_decrypt(strm->next_in, const_cast(strm->next_in), strm->avail_in, &FastFiles::CurrentCTR); } - return Utils::Hook::Call(0x4D8090)(strm, version, stream_size); + //return Utils::Hook::Call(0x4D8090)(strm, version, stream_size); //return inflateInit_(strm, version, stream_size); + return FastFiles::InflateInitStub(strm, version, stream_size); } void FastFiles::AuthLoadInflateDecryptBaseFunc(unsigned char* buffer) @@ -484,6 +495,44 @@ namespace Components } #endif + int FastFiles::InflateInitStub(z_streamp strm, const char *version, int stream_size) + { + if (FastFiles::UseZstd) + { + if (FastFiles::ZlibLock) delete FastFiles::ZlibLock; + FastFiles::ZlibLock = new Utils::Compression::Deflate::Semaphore(DEFLATE_ZSTD); + + return inflateInit_(strm, version, stream_size); + } + + return Utils::Hook::Call(0x4D8090)(strm, version, stream_size); + } + + int FastFiles::InflateStub(z_streamp strm, int flush) + { + if (FastFiles::UseZstd) + { + return inflate(strm, flush); + } + + return Utils::Hook::Call(0x49EA00)(strm, flush); + } + + int FastFiles::InflateEndStub(z_streamp strm) + { + if (FastFiles::UseZstd && FastFiles::ZlibLock) + { + int result = inflateEnd(strm); + + delete FastFiles::ZlibLock; + FastFiles::ZlibLock = nullptr; + + return result; + } + + return Utils::Hook::Call(0x453750)(strm); + } + FastFiles::FastFiles() { Dvar::Register("ui_zoneDebug", false, Game::dvar_flag::DVAR_FLAG_SAVED, "Display current loaded zone."); @@ -554,9 +603,12 @@ namespace Components Utils::Hook(0x4159E2, FastFiles::ReadXFileHeader, HOOK_CALL).install()->quick(); // Replace internal ZLib - //Utils::Hook(0x44B160, inflateInit2_, HOOK_JUMP).install()->quick(); - //Utils::Hook(0x453750, inflateEnd, HOOK_JUMP).install()->quick(); - //Utils::Hook(0x49EA00, inflate, HOOK_JUMP).install()->quick(); + //Utils::Hook(0x4D0306, FastFiles::InflateInitStub, HOOK_CALL).install()->quick(); + Utils::Hook(0x4D034B, FastFiles::InflateInitStub, HOOK_JUMP).install()->quick(); + Utils::Hook(0x480A1A, FastFiles::InflateStub, HOOK_JUMP).install()->quick(); + Utils::Hook(0x5B99DE, FastFiles::InflateStub, HOOK_CALL).install()->quick(); + Utils::Hook(0x449D8F, FastFiles::InflateEndStub, HOOK_CALL).install()->quick(); + Utils::Hook(0x449DA3, FastFiles::InflateEndStub, HOOK_CALL).install()->quick(); // Add custom zone paths FastFiles::AddZonePath("zone\\patch\\"); @@ -618,6 +670,12 @@ namespace Components FastFiles::~FastFiles() { + if (FastFiles::ZlibLock) + { + delete FastFiles::ZlibLock; + FastFiles::ZlibLock = nullptr; + } + FastFiles::ZonePaths.clear(); } } diff --git a/src/Components/Modules/FastFiles.hpp b/src/Components/Modules/FastFiles.hpp index eb5569a1..6b37afa6 100644 --- a/src/Components/Modules/FastFiles.hpp +++ b/src/Components/Modules/FastFiles.hpp @@ -34,6 +34,9 @@ namespace Components static unsigned int CurrentZone; static unsigned int MaxZones; + static bool UseZstd; + static Utils::Compression::Deflate::Semaphore* ZlibLock; + static bool IsIW4xZone; static bool StreamRead; @@ -63,6 +66,10 @@ namespace Components static void ReadXFile(void* buffer, int size); static void ReadXFileStub(char* buffer, int size); + static int InflateInitStub(z_streamp strm, const char *version, int stream_size); + static int InflateStub(z_streamp strm, int flush); + static int InflateEndStub(z_streamp strm); + #ifdef DEBUG static void LogStreamRead(int len); #endif diff --git a/src/Components/Modules/Node.cpp b/src/Components/Modules/Node.cpp index a596fbad..92306e47 100644 --- a/src/Components/Modules/Node.cpp +++ b/src/Components/Modules/Node.cpp @@ -68,12 +68,12 @@ namespace Components if (Monitor::IsEnabled()) { std::string nodes = Utils::IO::ReadFile("players/nodes_default.dat"); - if (nodes.empty() || !list.ParseFromString(Utils::Compression::ZLib::Decompress(nodes))) return; + if (nodes.empty() || !list.ParseFromString(Utils::Compression::Deflate::ZStd::Decompress(nodes))) return; } else { FileSystem::File defaultNodes("nodes_default.dat"); - if (!defaultNodes.exists() || !list.ParseFromString(Utils::Compression::ZLib::Decompress(defaultNodes.getBuffer()))) return; + if (!defaultNodes.exists() || !list.ParseFromString(Utils::Compression::Deflate::ZStd::Decompress(defaultNodes.getBuffer()))) return; } for (int i = 0; i < list.nodes_size(); ++i) @@ -91,7 +91,7 @@ namespace Components { Proto::Node::List list; std::string nodes = Utils::IO::ReadFile("players/nodes.dat"); - if (nodes.empty() || !list.ParseFromString(Utils::Compression::ZLib::Decompress(nodes))) return; + if (nodes.empty() || !list.ParseFromString(Utils::Compression::Deflate::ZStd::Decompress(nodes))) return; for (int i = 0; i < list.nodes_size(); ++i) { @@ -125,10 +125,9 @@ namespace Components } } - Utils::IO::WriteFile("players/nodes.dat", Utils::Compression::ZLib::Compress(list.SerializeAsString())); + Utils::IO::WriteFile("players/nodes.dat", Utils::Compression::Deflate::ZStd::Compress(list.SerializeAsString())); } - void Node::Add(Network::Address address) { #ifndef DEBUG diff --git a/src/Components/Modules/Playlist.cpp b/src/Components/Modules/Playlist.cpp index 82ea30ad..7f6db9c1 100644 --- a/src/Components/Modules/Playlist.cpp +++ b/src/Components/Modules/Playlist.cpp @@ -57,7 +57,7 @@ namespace Components Logger::Print("Received playlist request, sending currently stored buffer.\n"); - std::string compressedList = Utils::Compression::ZLib::Compress(Playlist::CurrentPlaylistBuffer); + std::string compressedList = Utils::Compression::Deflate::ZStd::Compress(Playlist::CurrentPlaylistBuffer); Proto::Party::Playlist list; list.set_hash(Utils::Cryptography::JenkinsOneAtATime::Compute(compressedList)); @@ -95,7 +95,7 @@ namespace Components } // Decompress buffer - Playlist::ReceivedPlaylistBuffer = Utils::Compression::ZLib::Decompress(compressedData); + Playlist::ReceivedPlaylistBuffer = Utils::Compression::Deflate::ZStd::Decompress(compressedData); // Load and continue connection Logger::Print("Received playlist, loading and continuing connection...\n"); diff --git a/src/Components/Modules/QuickPatch.cpp b/src/Components/Modules/QuickPatch.cpp index cc7f145e..341de498 100644 --- a/src/Components/Modules/QuickPatch.cpp +++ b/src/Components/Modules/QuickPatch.cpp @@ -718,8 +718,32 @@ namespace Components for (int i = 0; i < 21; ++i) { - std::string compressed = Utils::Compression::ZLib::Compress(test); - std::string decompressed = Utils::Compression::ZLib::Decompress(compressed); + std::string compressed = Utils::Compression::Deflate::ZLib::Compress(test); + std::string decompressed = Utils::Compression::Deflate::ZLib::Decompress(compressed); + + if (test != decompressed) + { + printf("Error\n"); + printf("Compressing %d bytes and decompressing failed!\n", test.size()); + return false; + } + + auto size = test.size(); + for (unsigned int j = 0; j < size; ++j) + { + test.append(Utils::String::VA("%c", Utils::Cryptography::Rand::GenerateInt())); + } + } + + printf("Success\n"); + printf("Testing ZStd compression..."); + + test = Utils::String::VA("%c", Utils::Cryptography::Rand::GenerateInt()); + + for (int i = 0; i < 21; ++i) + { + std::string compressed = Utils::Compression::Deflate::ZStd::Compress(test); + std::string decompressed = Utils::Compression::Deflate::ZStd::Decompress(compressed); if (test != decompressed) { diff --git a/src/Components/Modules/ZoneBuilder.cpp b/src/Components/Modules/ZoneBuilder.cpp index 80013bae..9e69e8a4 100644 --- a/src/Components/Modules/ZoneBuilder.cpp +++ b/src/Components/Modules/ZoneBuilder.cpp @@ -407,7 +407,12 @@ namespace Components } #endif - zoneBuffer = Utils::Compression::ZStd::Compress(zoneBuffer); +#ifdef DEBUG + zoneBuffer = Utils::Compression::Deflate::ZLib::Compress(zoneBuffer); +#else + zoneBuffer = Utils::Compression::Deflate::ZStd::Compress(zoneBuffer); +#endif + outBuffer.append(zoneBuffer); std::string outFile = "zone/" + this->zoneName + ".ff"; diff --git a/src/Components/Modules/ZoneBuilder.hpp b/src/Components/Modules/ZoneBuilder.hpp index da5022a8..a5971492 100644 --- a/src/Components/Modules/ZoneBuilder.hpp +++ b/src/Components/Modules/ZoneBuilder.hpp @@ -4,7 +4,7 @@ #define XFILE_VERSION 276 #define XFILE_HEADER_IW4X 0x78345749 // 'IW4x' -#define XFILE_VERSION_IW4X 3 +#define XFILE_VERSION_IW4X 4 namespace Components { diff --git a/src/STDInclude.hpp b/src/STDInclude.hpp index 31650b36..58e65b3e 100644 --- a/src/STDInclude.hpp +++ b/src/STDInclude.hpp @@ -73,11 +73,11 @@ template class Sizer { }; #pragma warning(disable: 6386) #pragma warning(disable: 6387) -#include -#include +// #include +// #include -//#define ZWRAP_USE_ZSTD 1 -//#include +#define ZWRAP_USE_ZSTD 1 +#include #include #include diff --git a/src/Utils/Compression.cpp b/src/Utils/Compression.cpp index de213981..6b772ef1 100644 --- a/src/Utils/Compression.cpp +++ b/src/Utils/Compression.cpp @@ -4,7 +4,9 @@ namespace Utils { namespace Compression { - std::string ZLib::Compress(std::string data) + std::mutex Deflate::Mutex; + + std::string Deflate::Compress(std::string data) { Utils::Memory::Allocator allocator; unsigned long length = (data.size() * 2); @@ -26,7 +28,7 @@ namespace Utils return data; } - std::string ZLib::Decompress(std::string data) + std::string Deflate::Decompress(std::string data) { z_stream stream; ZeroMemory(&stream, sizeof(stream)); @@ -71,71 +73,41 @@ namespace Utils return buffer; } - - - - std::string ZStd::Compress(std::string data) + std::string Deflate::ZLib::Compress(std::string data) { - Utils::Memory::Allocator allocator; - - size_t length = (ZSTD_compressBound(data.size() + 1) + 1) * 2; - char* buffer = allocator.allocateArray(length); - - length = ZSTD_compress(buffer, length, data.data(), data.size(), ZSTD_maxCLevel()); - if (length <= 0/* || ZSTD_isError()*/) - { - return ""; - } - - data.clear(); - data.append(buffer, length); - - return data; + Deflate::Semaphore _(DEFLATE_ZLIB); + return Deflate::Compress(data); } - std::string ZStd::Decompress(std::string data) + std::string Deflate::ZLib::Decompress(std::string data) { - z_stream stream; - ZeroMemory(&stream, sizeof(stream)); - std::string buffer; + Deflate::Semaphore _(DEFLATE_ZLIB); + return Deflate::Decompress(data); + } - if (inflateInit(&stream) != Z_OK) - { - return ""; - } + std::string Deflate::ZStd::Compress(std::string data) + { + Deflate::Semaphore _(DEFLATE_ZSTD); + return Deflate::Compress(data); + } - int ret; - Utils::Memory::Allocator allocator; + std::string Deflate::ZStd::Decompress(std::string data) + { + Deflate::Semaphore _(DEFLATE_ZSTD); + return Deflate::Decompress(data); + } - uint8_t* dest = allocator.allocateArray(CHUNK); - const char* dataPtr = data.data(); + Deflate::Semaphore::Semaphore(bool zstd) + { + Deflate::Mutex.lock(); + this->state = ZWRAP_isUsingZSTDcompression(); + ZWRAP_useZSTDcompression(zstd); + } - do - { - stream.avail_in = std::min(static_cast(CHUNK), data.size() - (dataPtr - data.data())); - stream.next_in = reinterpret_cast(dataPtr); - dataPtr += stream.avail_in; - - do - { - stream.avail_out = CHUNK; - stream.next_out = dest; - - ret = inflate(&stream, Z_NO_FLUSH); - if (ret != Z_OK && ret != Z_STREAM_END) - { - inflateEnd(&stream); - return ""; - } - - buffer.append(reinterpret_cast(dest), CHUNK - stream.avail_out); - - } while (stream.avail_out == 0); - - } while (ret != Z_STREAM_END); - - inflateEnd(&stream); - return buffer; + Deflate::Semaphore::~Semaphore() + { + ZWRAP_useZSTDcompression(this->state); + Deflate::Mutex.unlock(); } }; } diff --git a/src/Utils/Compression.hpp b/src/Utils/Compression.hpp index 2ab684ad..1c48fb25 100644 --- a/src/Utils/Compression.hpp +++ b/src/Utils/Compression.hpp @@ -1,21 +1,42 @@ #pragma once #define CHUNK 16384 +#define DEFLATE_ZLIB false +#define DEFLATE_ZSTD true namespace Utils { namespace Compression { - class ZLib + class Deflate { public: - static std::string Compress(std::string data); - static std::string Decompress(std::string data); - }; + class ZLib + { + public: + static std::string Compress(std::string data); + static std::string Decompress(std::string data); + }; - class ZStd - { - public: + class ZStd + { + public: + static std::string Compress(std::string data); + static std::string Decompress(std::string data); + }; + + class Semaphore + { + public: + Semaphore(bool zstd); + ~Semaphore(); + + private: + int state; + }; + + private: + static std::mutex Mutex; static std::string Compress(std::string data); static std::string Decompress(std::string data); };