[ZoneBuilder] Use ZSTD for fastfile compression

src/Components/Modules/FastFiles.cpp

@@ -6,6 +6,9 @@ namespace Components
 	symmetric_CTR FastFiles::CurrentCTR;
 	std::vector<std::string> FastFiles::ZonePaths;
 
+	bool FastFiles::UseZstd = false;
+	Utils::Compression::Deflate::Semaphore* FastFiles::ZlibLock = nullptr;
+
 	bool FastFiles::IsIW4xZone = false;
 	bool FastFiles::StreamRead = false;
 
@@ -333,15 +336,22 @@ namespace Components
 
 	void FastFiles::ReadHeaderStub(unsigned int* header, int size)
 	{
+		FastFiles::UseZstd = false;
 		FastFiles::IsIW4xZone = false;
 		FastFiles::LastByteRead = 0;
 		Game::DB_ReadXFileUncompressed(header, size);
 
 		if (header[0] == XFILE_HEADER_IW4X)
 		{
+			FastFiles::UseZstd = true;
 			FastFiles::IsIW4xZone = true;
 
-			if (header[1] < XFILE_VERSION_IW4X)
+			static_assert((XFILE_VERSION_IW4X - 1) == 3, "FastFile backwards-compatibility not granted!");
+			if (header[1] == XFILE_VERSION_IW4X - 1)
+			{
+				FastFiles::UseZstd = false;
+			}
+			else if (header[1] < XFILE_VERSION_IW4X)
 			{
 				Logger::Error("The fastfile you are trying to load is outdated (%d, expected %d)", header[1], XFILE_VERSION_IW4X);
 			}
@@ -396,8 +406,9 @@ namespace Components
 			ctr_decrypt(strm->next_in, const_cast<unsigned char*>(strm->next_in), strm->avail_in, &FastFiles::CurrentCTR);
 		}
 
-		return Utils::Hook::Call<int(z_streamp, const char*, int)>(0x4D8090)(strm, version, stream_size);
+		//return Utils::Hook::Call<int(z_streamp, const char*, int)>(0x4D8090)(strm, version, stream_size);
 		//return inflateInit_(strm, version, stream_size);
+		return FastFiles::InflateInitStub(strm, version, stream_size);
 	}
 
 	void FastFiles::AuthLoadInflateDecryptBaseFunc(unsigned char* buffer)
@@ -484,6 +495,44 @@ namespace Components
 	}
 #endif
 
+	int FastFiles::InflateInitStub(z_streamp strm, const char *version, int stream_size)
+	{
+		if (FastFiles::UseZstd)
+		{
+			if (FastFiles::ZlibLock) delete FastFiles::ZlibLock;
+			FastFiles::ZlibLock = new Utils::Compression::Deflate::Semaphore(DEFLATE_ZSTD);
+
+			return inflateInit_(strm, version, stream_size);
+		}
+
+		return Utils::Hook::Call<int(z_streamp, const char*, int)>(0x4D8090)(strm, version, stream_size);
+	}
+
+	int FastFiles::InflateStub(z_streamp strm, int flush)
+	{
+		if (FastFiles::UseZstd)
+		{
+			return inflate(strm, flush);
+		}
+
+		return Utils::Hook::Call<int(z_streamp, int)>(0x49EA00)(strm, flush);
+	}
+
+	int FastFiles::InflateEndStub(z_streamp strm)
+	{
+		if (FastFiles::UseZstd && FastFiles::ZlibLock)
+		{
+			int result = inflateEnd(strm);
+
+			delete FastFiles::ZlibLock;
+			FastFiles::ZlibLock = nullptr;
+
+			return result;
+		}
+
+		return Utils::Hook::Call<int(z_streamp)>(0x453750)(strm);
+	}
+
 	FastFiles::FastFiles()
 	{
 		Dvar::Register<bool>("ui_zoneDebug", false, Game::dvar_flag::DVAR_FLAG_SAVED, "Display current loaded zone.");
@@ -554,9 +603,12 @@ namespace Components
 		Utils::Hook(0x4159E2, FastFiles::ReadXFileHeader, HOOK_CALL).install()->quick();
 
 		// Replace internal ZLib
-		//Utils::Hook(0x44B160, inflateInit2_, HOOK_JUMP).install()->quick();
+		//Utils::Hook(0x4D0306, FastFiles::InflateInitStub, HOOK_CALL).install()->quick();
-		//Utils::Hook(0x453750, inflateEnd, HOOK_JUMP).install()->quick();
+		Utils::Hook(0x4D034B, FastFiles::InflateInitStub, HOOK_JUMP).install()->quick();
-		//Utils::Hook(0x49EA00, inflate, HOOK_JUMP).install()->quick();
+		Utils::Hook(0x480A1A, FastFiles::InflateStub, HOOK_JUMP).install()->quick();
+		Utils::Hook(0x5B99DE, FastFiles::InflateStub, HOOK_CALL).install()->quick();
+		Utils::Hook(0x449D8F, FastFiles::InflateEndStub, HOOK_CALL).install()->quick();
+		Utils::Hook(0x449DA3, FastFiles::InflateEndStub, HOOK_CALL).install()->quick();
 
 		// Add custom zone paths
 		FastFiles::AddZonePath("zone\\patch\\");
@@ -618,6 +670,12 @@ namespace Components
 
 	FastFiles::~FastFiles()
 	{
+		if (FastFiles::ZlibLock)
+		{
+			delete FastFiles::ZlibLock;
+			FastFiles::ZlibLock = nullptr;
+		}
+
 		FastFiles::ZonePaths.clear();
 	}
 }

src/Components/Modules/FastFiles.hpp

@@ -34,6 +34,9 @@ namespace Components
 		static unsigned int CurrentZone;
 		static unsigned int MaxZones;
 
+		static bool UseZstd;
+		static Utils::Compression::Deflate::Semaphore* ZlibLock;
+
 		static bool IsIW4xZone;
 		static bool StreamRead;
 
@@ -63,6 +66,10 @@ namespace Components
 		static void ReadXFile(void* buffer, int size);
 		static void ReadXFileStub(char* buffer, int size);
 
+		static int InflateInitStub(z_streamp strm, const char *version, int stream_size);
+		static int InflateStub(z_streamp strm, int flush);
+		static int InflateEndStub(z_streamp strm);
+
 #ifdef DEBUG
 		static void LogStreamRead(int len);
 #endif

src/Components/Modules/Node.cpp

@@ -68,12 +68,12 @@ namespace Components
 		if (Monitor::IsEnabled())
 		{
 			std::string nodes = Utils::IO::ReadFile("players/nodes_default.dat");
-			if (nodes.empty() || !list.ParseFromString(Utils::Compression::ZLib::Decompress(nodes))) return;
+			if (nodes.empty() || !list.ParseFromString(Utils::Compression::Deflate::ZStd::Decompress(nodes))) return;
 		}
 		else
 		{
 			FileSystem::File defaultNodes("nodes_default.dat");
-			if (!defaultNodes.exists() || !list.ParseFromString(Utils::Compression::ZLib::Decompress(defaultNodes.getBuffer()))) return;
+			if (!defaultNodes.exists() || !list.ParseFromString(Utils::Compression::Deflate::ZStd::Decompress(defaultNodes.getBuffer()))) return;
 		}
 
 		for (int i = 0; i < list.nodes_size(); ++i)
@@ -91,7 +91,7 @@ namespace Components
 	{
 		Proto::Node::List list;
 		std::string nodes = Utils::IO::ReadFile("players/nodes.dat");
-		if (nodes.empty() || !list.ParseFromString(Utils::Compression::ZLib::Decompress(nodes))) return;
+		if (nodes.empty() || !list.ParseFromString(Utils::Compression::Deflate::ZStd::Decompress(nodes))) return;
 
 		for (int i = 0; i < list.nodes_size(); ++i)
 		{
@@ -125,10 +125,9 @@ namespace Components
 			}
 		}
 
-		Utils::IO::WriteFile("players/nodes.dat", Utils::Compression::ZLib::Compress(list.SerializeAsString()));
+		Utils::IO::WriteFile("players/nodes.dat", Utils::Compression::Deflate::ZStd::Compress(list.SerializeAsString()));
 	}
 
-
 	void Node::Add(Network::Address address)
 	{
 #ifndef DEBUG

src/Components/Modules/Playlist.cpp

@@ -57,7 +57,7 @@ namespace Components
 
 		Logger::Print("Received playlist request, sending currently stored buffer.\n");
 
-		std::string compressedList = Utils::Compression::ZLib::Compress(Playlist::CurrentPlaylistBuffer);
+		std::string compressedList = Utils::Compression::Deflate::ZStd::Compress(Playlist::CurrentPlaylistBuffer);
 
 		Proto::Party::Playlist list;
 		list.set_hash(Utils::Cryptography::JenkinsOneAtATime::Compute(compressedList));
@@ -95,7 +95,7 @@ namespace Components
 					}
 
 					// Decompress buffer
-					Playlist::ReceivedPlaylistBuffer = Utils::Compression::ZLib::Decompress(compressedData);
+					Playlist::ReceivedPlaylistBuffer = Utils::Compression::Deflate::ZStd::Decompress(compressedData);
 
 					// Load and continue connection
 					Logger::Print("Received playlist, loading and continuing connection...\n");

src/Components/Modules/QuickPatch.cpp

@@ -718,8 +718,32 @@ namespace Components
 
 		for (int i = 0; i < 21; ++i)
 		{
-			std::string compressed = Utils::Compression::ZLib::Compress(test);
+			std::string compressed = Utils::Compression::Deflate::ZLib::Compress(test);
-			std::string decompressed = Utils::Compression::ZLib::Decompress(compressed);
+			std::string decompressed = Utils::Compression::Deflate::ZLib::Decompress(compressed);
+
+			if (test != decompressed)
+			{
+				printf("Error\n");
+				printf("Compressing %d bytes and decompressing failed!\n", test.size());
+				return false;
+			}
+
+			auto size = test.size();
+			for (unsigned int j = 0; j < size; ++j)
+			{
+				test.append(Utils::String::VA("%c", Utils::Cryptography::Rand::GenerateInt()));
+			}
+		}
+
+		printf("Success\n");
+		printf("Testing ZStd compression...");
+
+		test = Utils::String::VA("%c", Utils::Cryptography::Rand::GenerateInt());
+
+		for (int i = 0; i < 21; ++i)
+		{
+			std::string compressed = Utils::Compression::Deflate::ZStd::Compress(test);
+			std::string decompressed = Utils::Compression::Deflate::ZStd::Decompress(compressed);
 
 			if (test != decompressed)
 			{

src/Components/Modules/ZoneBuilder.cpp

@@ -407,7 +407,12 @@ namespace Components
 		}
 #endif
 
-		zoneBuffer = Utils::Compression::ZStd::Compress(zoneBuffer);
+#ifdef DEBUG
+		zoneBuffer = Utils::Compression::Deflate::ZLib::Compress(zoneBuffer);
+#else
+		zoneBuffer = Utils::Compression::Deflate::ZStd::Compress(zoneBuffer);
+#endif
+
 		outBuffer.append(zoneBuffer);
 
 		std::string outFile = "zone/" + this->zoneName + ".ff";

src/Components/Modules/ZoneBuilder.hpp

@@ -4,7 +4,7 @@
 #define XFILE_VERSION 276
 
 #define XFILE_HEADER_IW4X 0x78345749 // 'IW4x'
-#define XFILE_VERSION_IW4X 3
+#define XFILE_VERSION_IW4X 4
 
 namespace Components
 {

src/STDInclude.hpp

@@ -73,11 +73,11 @@ template <size_t S> class Sizer { };
 #pragma warning(disable: 6386)
 #pragma warning(disable: 6387)
 
-#include <zlib.h>
+// #include <zlib.h>
-#include <zstd.h>
+// #include <zstd.h>
 
-//#define ZWRAP_USE_ZSTD 1
+#define ZWRAP_USE_ZSTD 1
-//#include <zstd_zlibwrapper.h>
+#include <zstd_zlibwrapper.h>
 
 #include <curses.h>
 #include <mongoose.h>

src/Utils/Compression.cpp

@@ -4,7 +4,9 @@ namespace Utils
 {
 	namespace Compression
 	{
-		std::string ZLib::Compress(std::string data)
+		std::mutex Deflate::Mutex;
+
+		std::string Deflate::Compress(std::string data)
 		{
 			Utils::Memory::Allocator allocator;
 			unsigned long length = (data.size() * 2);
@@ -26,7 +28,7 @@ namespace Utils
 			return data;
 		}
 
-		std::string ZLib::Decompress(std::string data)
+		std::string Deflate::Decompress(std::string data)
 		{
 			z_stream stream;
 			ZeroMemory(&stream, sizeof(stream));
@@ -71,71 +73,41 @@ namespace Utils
 			return buffer;
 		}
 
-
+		std::string Deflate::ZLib::Compress(std::string data)
-
-
-		std::string ZStd::Compress(std::string data)
 		{
-			Utils::Memory::Allocator allocator;
+			Deflate::Semaphore _(DEFLATE_ZLIB);
-
+			return Deflate::Compress(data);
-			size_t length = (ZSTD_compressBound(data.size() + 1) + 1) * 2;
-			char* buffer = allocator.allocateArray<char>(length);
-
-			length = ZSTD_compress(buffer, length, data.data(), data.size(), ZSTD_maxCLevel());
-			if (length <= 0/* || ZSTD_isError()*/)
-			{
-				return "";
 		}
 
-			data.clear();
+		std::string Deflate::ZLib::Decompress(std::string data)
-			data.append(buffer, length);
+		{
-
+			Deflate::Semaphore _(DEFLATE_ZLIB);
-			return data;
+			return Deflate::Decompress(data);
 		}
 
-		std::string ZStd::Decompress(std::string data)
+		std::string Deflate::ZStd::Compress(std::string data)
 		{
-			z_stream stream;
+			Deflate::Semaphore _(DEFLATE_ZSTD);
-			ZeroMemory(&stream, sizeof(stream));
+			return Deflate::Compress(data);
-			std::string buffer;
-
-			if (inflateInit(&stream) != Z_OK)
-			{
-				return "";
 		}
 
-			int ret;
+		std::string Deflate::ZStd::Decompress(std::string data)
-			Utils::Memory::Allocator allocator;
-
-			uint8_t* dest = allocator.allocateArray<uint8_t>(CHUNK);
-			const char* dataPtr = data.data();
-
-			do
 		{
-				stream.avail_in = std::min(static_cast<size_t>(CHUNK), data.size() - (dataPtr - data.data()));
+			Deflate::Semaphore _(DEFLATE_ZSTD);
-				stream.next_in = reinterpret_cast<const uint8_t*>(dataPtr);
+			return Deflate::Decompress(data);
-				dataPtr += stream.avail_in;
-
-				do
-				{
-					stream.avail_out = CHUNK;
-					stream.next_out = dest;
-
-					ret = inflate(&stream, Z_NO_FLUSH);
-					if (ret != Z_OK && ret != Z_STREAM_END)
-					{
-						inflateEnd(&stream);
-						return "";
 		}
 
-					buffer.append(reinterpret_cast<const char*>(dest), CHUNK - stream.avail_out);
+		Deflate::Semaphore::Semaphore(bool zstd)
+		{
+			Deflate::Mutex.lock();
+			this->state = ZWRAP_isUsingZSTDcompression();
+			ZWRAP_useZSTDcompression(zstd);
+		}
 
-				} while (stream.avail_out == 0);
+		Deflate::Semaphore::~Semaphore()
-
+		{
-			} while (ret != Z_STREAM_END);
+			ZWRAP_useZSTDcompression(this->state);
-
+			Deflate::Mutex.unlock();
-			inflateEnd(&stream);
-			return buffer;
 		}
 	};
 }

src/Utils/Compression.hpp

@@ -1,11 +1,16 @@
 #pragma once
 
 #define CHUNK 16384
+#define DEFLATE_ZLIB false
+#define DEFLATE_ZSTD true
 
 namespace Utils
 {
 	namespace Compression
 	{
+		class Deflate
+		{
+		public:
 			class ZLib
 			{
 			public:
@@ -19,5 +24,21 @@ namespace Utils
 				static std::string Compress(std::string data);
 				static std::string Decompress(std::string data);
 			};
+
+			class Semaphore
+			{
+			public:
+				Semaphore(bool zstd);
+				~Semaphore();
+
+			private:
+				int state;
+			};
+
+		private:
+			static std::mutex Mutex;
+			static std::string Compress(std::string data);
+			static std::string Decompress(std::string data);
+		};
 	};
 }