Merge branch 'feature/zstd-zone' into 'develop'

[Merge] feature/zstd-zone -> develop

.gitmodules

@@ -29,3 +29,7 @@
 [submodule "deps/udis86"]
 	path = deps/udis86
 	url = ../udis86.git
+[submodule "deps/zstd"]
+	path = deps/zstd
+	url = https://github.com/facebook/zstd.git
+	branch = dev

deps/zstd

@@ -0,0 +1 @@
+Subproject commit 379f9d874d578111c611b2262211aa763171a94c

premake/zstd.lua

@@ -0,0 +1,66 @@
+zstd = {
+	settings = nil
+}
+
+function zstd.setup(settings)
+	if not settings.source then error("Missing source.") end
+
+	zstd.settings = settings
+
+	if not zstd.settings.defines then zstd.settings.defines = {} end
+end
+
+function zstd.import()
+	if not zstd.settings then error("You need to call zstd.setup first") end
+
+	links { "zstd" }
+	zstd.includes()
+end
+
+function zstd.includes()
+	if not zstd.settings then error("You need to call zstd.setup first") end
+
+	includedirs
+	{
+		path.join(zstd.settings.source, "lib"),
+		path.join(zstd.settings.source, "lib/common"),
+		path.join(zstd.settings.source, "zlibWrapper")
+	}
+	defines(zstd.settings.defines)
+end
+
+function zstd.project()
+	if not zstd.settings then error("You need to call zstd.setup first") end
+
+	project "zstd"
+		language "C"
+
+		zstd.includes()
+		files
+		{
+			path.join(zstd.settings.source, "lib/**.h"),
+			path.join(zstd.settings.source, "lib/**.c"),
+			path.join(zstd.settings.source, "zlibWrapper/zstd_zlibwrapper.h"),
+			path.join(zstd.settings.source, "zlibWrapper/zstd_zlibwrapper.c"),
+		}
+		removefiles
+		{
+			path.join(zstd.settings.source, "lib/legacy/**.*"),
+			--path.join(zstd.settings.source, "zlibWrapper/examples/**.*"),
+		}
+		defines
+		{
+			"zstd_DLL",
+			"_CRT_SECURE_NO_DEPRECATE",
+		}
+		
+		zlib.import()
+
+		-- not our code, ignore POSIX usage warnings for now
+		warnings "Off"
+		
+		--configuration "*Static"
+			defines { "_LIB" }
+			removedefines { "_USRDLL", "_DLL", "zstd_DLL" }
+			kind "StaticLib"
+end

premake5.lua

@@ -180,6 +180,7 @@ require "premake/mongoose"
 require "premake/pdcurses"
 require "premake/protobuf"
 require "premake/zlib"
+require "premake/zstd"
 require "premake/udis86"
 
 json11.setup
@@ -221,6 +222,10 @@ zlib.setup
 	},
 	source = path.join(depsBasePath, "zlib"),
 }
+zstd.setup
+{
+	source = path.join(depsBasePath, "zstd"),
+}
 udis86.setup
 {
 	source = path.join(depsBasePath, "udis86"),
@@ -311,6 +316,7 @@ workspace "iw4x"
 		pdcurses.import()
 		protobuf.import()
 		zlib.import()
+		zstd.import()
 		udis86.import()
 
 		-- fix vpaths for protobuf sources
@@ -423,6 +429,7 @@ workspace "iw4x"
 		pdcurses.project()
 		protobuf.project()
 		zlib.project()
+		zstd.project()
 		udis86.project()
 
 rule "ProtobufCompiler"

src/Components/Modules/FastFiles.cpp

@@ -6,6 +6,9 @@ namespace Components
 	symmetric_CTR FastFiles::CurrentCTR;
 	std::vector<std::string> FastFiles::ZonePaths;
 
+	bool FastFiles::UseZstd = false;
+	Utils::Compression::Deflate::Semaphore* FastFiles::ZlibLock = nullptr;
+
 	bool FastFiles::IsIW4xZone = false;
 	bool FastFiles::StreamRead = false;
 
@@ -333,15 +336,22 @@ namespace Components
 
 	void FastFiles::ReadHeaderStub(unsigned int* header, int size)
 	{
+		FastFiles::UseZstd = false;
 		FastFiles::IsIW4xZone = false;
 		FastFiles::LastByteRead = 0;
 		Game::DB_ReadXFileUncompressed(header, size);
 
 		if (header[0] == XFILE_HEADER_IW4X)
 		{
+			FastFiles::UseZstd = true;
 			FastFiles::IsIW4xZone = true;
 
-			if (header[1] < XFILE_VERSION_IW4X)
+			static_assert((XFILE_VERSION_IW4X - 1) == 3, "FastFile backwards-compatibility not granted!");
+			if (header[1] == XFILE_VERSION_IW4X - 1)
+			{
+				FastFiles::UseZstd = false;
+			}
+			else if (header[1] < XFILE_VERSION_IW4X)
 			{
 				Logger::Error("The fastfile you are trying to load is outdated (%d, expected %d)", header[1], XFILE_VERSION_IW4X);
 			}
@@ -396,8 +406,9 @@ namespace Components
 			ctr_decrypt(strm->next_in, const_cast<unsigned char*>(strm->next_in), strm->avail_in, &FastFiles::CurrentCTR);
 		}
 
-		return Utils::Hook::Call<int(z_streamp, const char*, int)>(0x4D8090)(strm, version, stream_size);
+		//return Utils::Hook::Call<int(z_streamp, const char*, int)>(0x4D8090)(strm, version, stream_size);
 		//return inflateInit_(strm, version, stream_size);
+		return FastFiles::InflateInitStub(strm, version, stream_size);
 	}
 
 	void FastFiles::AuthLoadInflateDecryptBaseFunc(unsigned char* buffer)
@@ -484,6 +495,44 @@ namespace Components
 	}
 #endif
 
+	int FastFiles::InflateInitStub(z_streamp strm, const char *version, int stream_size)
+	{
+		if (FastFiles::UseZstd)
+		{
+			if (FastFiles::ZlibLock) delete FastFiles::ZlibLock;
+			FastFiles::ZlibLock = new Utils::Compression::Deflate::Semaphore(DEFLATE_ZSTD);
+
+			return inflateInit_(strm, version, stream_size);
+		}
+
+		return Utils::Hook::Call<int(z_streamp, const char*, int)>(0x4D8090)(strm, version, stream_size);
+	}
+
+	int FastFiles::InflateStub(z_streamp strm, int flush)
+	{
+		if (FastFiles::UseZstd)
+		{
+			return inflate(strm, flush);
+		}
+
+		return Utils::Hook::Call<int(z_streamp, int)>(0x49EA00)(strm, flush);
+	}
+
+	int FastFiles::InflateEndStub(z_streamp strm)
+	{
+		if (FastFiles::UseZstd && FastFiles::ZlibLock)
+		{
+			int result = inflateEnd(strm);
+
+			delete FastFiles::ZlibLock;
+			FastFiles::ZlibLock = nullptr;
+
+			return result;
+		}
+
+		return Utils::Hook::Call<int(z_streamp)>(0x453750)(strm);
+	}
+
 	FastFiles::FastFiles()
 	{
 		Dvar::Register<bool>("ui_zoneDebug", false, Game::dvar_flag::DVAR_FLAG_SAVED, "Display current loaded zone.");
@@ -554,9 +603,12 @@ namespace Components
 		Utils::Hook(0x4159E2, FastFiles::ReadXFileHeader, HOOK_CALL).install()->quick();
 
 		// Replace internal ZLib
-		//Utils::Hook(0x44B160, inflateInit2_, HOOK_JUMP).install()->quick();
+		//Utils::Hook(0x4D0306, FastFiles::InflateInitStub, HOOK_CALL).install()->quick();
-		//Utils::Hook(0x453750, inflateEnd, HOOK_JUMP).install()->quick();
+		Utils::Hook(0x4D034B, FastFiles::InflateInitStub, HOOK_JUMP).install()->quick();
-		//Utils::Hook(0x49EA00, inflate, HOOK_JUMP).install()->quick();
+		Utils::Hook(0x480A1A, FastFiles::InflateStub, HOOK_JUMP).install()->quick();
+		Utils::Hook(0x5B99DE, FastFiles::InflateStub, HOOK_CALL).install()->quick();
+		Utils::Hook(0x449D8F, FastFiles::InflateEndStub, HOOK_CALL).install()->quick();
+		Utils::Hook(0x449DA3, FastFiles::InflateEndStub, HOOK_CALL).install()->quick();
 
 		// Add custom zone paths
 		FastFiles::AddZonePath("zone\\patch\\");
@@ -618,6 +670,12 @@ namespace Components
 
 	FastFiles::~FastFiles()
 	{
+		if (FastFiles::ZlibLock)
+		{
+			delete FastFiles::ZlibLock;
+			FastFiles::ZlibLock = nullptr;
+		}
+
 		FastFiles::ZonePaths.clear();
 	}
 }

src/Components/Modules/FastFiles.hpp

@@ -34,6 +34,9 @@ namespace Components
 		static unsigned int CurrentZone;
 		static unsigned int MaxZones;
 
+		static bool UseZstd;
+		static Utils::Compression::Deflate::Semaphore* ZlibLock;
+
 		static bool IsIW4xZone;
 		static bool StreamRead;
 
@@ -63,6 +66,10 @@ namespace Components
 		static void ReadXFile(void* buffer, int size);
 		static void ReadXFileStub(char* buffer, int size);
 
+		static int InflateInitStub(z_streamp strm, const char *version, int stream_size);
+		static int InflateStub(z_streamp strm, int flush);
+		static int InflateEndStub(z_streamp strm);
+
 #ifdef DEBUG
 		static void LogStreamRead(int len);
 #endif

src/Components/Modules/Node.cpp

@@ -68,12 +68,12 @@ namespace Components
 		if (Monitor::IsEnabled())
 		{
 			std::string nodes = Utils::IO::ReadFile("players/nodes_default.dat");
-			if (nodes.empty() || !list.ParseFromString(Utils::Compression::ZLib::Decompress(nodes))) return;
+			if (nodes.empty() || !list.ParseFromString(Utils::Compression::Deflate::ZStd::Decompress(nodes))) return;
 		}
 		else
 		{
 			FileSystem::File defaultNodes("nodes_default.dat");
-			if (!defaultNodes.exists() || !list.ParseFromString(Utils::Compression::ZLib::Decompress(defaultNodes.getBuffer()))) return;
+			if (!defaultNodes.exists() || !list.ParseFromString(Utils::Compression::Deflate::ZStd::Decompress(defaultNodes.getBuffer()))) return;
 		}
 
 		for (int i = 0; i < list.nodes_size(); ++i)
@@ -91,7 +91,7 @@ namespace Components
 	{
 		Proto::Node::List list;
 		std::string nodes = Utils::IO::ReadFile("players/nodes.dat");
-		if (nodes.empty() || !list.ParseFromString(Utils::Compression::ZLib::Decompress(nodes))) return;
+		if (nodes.empty() || !list.ParseFromString(Utils::Compression::Deflate::ZStd::Decompress(nodes))) return;
 
 		for (int i = 0; i < list.nodes_size(); ++i)
 		{
@@ -125,10 +125,9 @@ namespace Components
 			}
 		}
 
-		Utils::IO::WriteFile("players/nodes.dat", Utils::Compression::ZLib::Compress(list.SerializeAsString()));
+		Utils::IO::WriteFile("players/nodes.dat", Utils::Compression::Deflate::ZStd::Compress(list.SerializeAsString()));
 	}
 
-
 	void Node::Add(Network::Address address)
 	{
 #ifndef DEBUG

src/Components/Modules/Playlist.cpp

@@ -57,7 +57,7 @@ namespace Components
 
 		Logger::Print("Received playlist request, sending currently stored buffer.\n");
 
-		std::string compressedList = Utils::Compression::ZLib::Compress(Playlist::CurrentPlaylistBuffer);
+		std::string compressedList = Utils::Compression::Deflate::ZStd::Compress(Playlist::CurrentPlaylistBuffer);
 
 		Proto::Party::Playlist list;
 		list.set_hash(Utils::Cryptography::JenkinsOneAtATime::Compute(compressedList));
@@ -95,7 +95,7 @@ namespace Components
 					}
 
 					// Decompress buffer
-					Playlist::ReceivedPlaylistBuffer = Utils::Compression::ZLib::Decompress(compressedData);
+					Playlist::ReceivedPlaylistBuffer = Utils::Compression::Deflate::ZStd::Decompress(compressedData);
 
 					// Load and continue connection
 					Logger::Print("Received playlist, loading and continuing connection...\n");

src/Components/Modules/QuickPatch.cpp

@@ -718,8 +718,32 @@ namespace Components
 
 		for (int i = 0; i < 21; ++i)
 		{
-			std::string compressed = Utils::Compression::ZLib::Compress(test);
+			std::string compressed = Utils::Compression::Deflate::ZLib::Compress(test);
-			std::string decompressed = Utils::Compression::ZLib::Decompress(compressed);
+			std::string decompressed = Utils::Compression::Deflate::ZLib::Decompress(compressed);
+
+			if (test != decompressed)
+			{
+				printf("Error\n");
+				printf("Compressing %d bytes and decompressing failed!\n", test.size());
+				return false;
+			}
+
+			auto size = test.size();
+			for (unsigned int j = 0; j < size; ++j)
+			{
+				test.append(Utils::String::VA("%c", Utils::Cryptography::Rand::GenerateInt()));
+			}
+		}
+
+		printf("Success\n");
+		printf("Testing ZStd compression...");
+
+		test = Utils::String::VA("%c", Utils::Cryptography::Rand::GenerateInt());
+
+		for (int i = 0; i < 21; ++i)
+		{
+			std::string compressed = Utils::Compression::Deflate::ZStd::Compress(test);
+			std::string decompressed = Utils::Compression::Deflate::ZStd::Decompress(compressed);
 
 			if (test != decompressed)
 			{

src/Components/Modules/ZoneBuilder.cpp

@@ -407,7 +407,12 @@ namespace Components
 		}
 #endif
 
-		zoneBuffer = Utils::Compression::ZLib::Compress(zoneBuffer);
+#ifdef DEBUG
+		zoneBuffer = Utils::Compression::Deflate::ZLib::Compress(zoneBuffer);
+#else
+		zoneBuffer = Utils::Compression::Deflate::ZStd::Compress(zoneBuffer);
+#endif
+
 		outBuffer.append(zoneBuffer);
 
 		std::string outFile = "zone/" + this->zoneName + ".ff";

src/Components/Modules/ZoneBuilder.hpp

@@ -4,7 +4,7 @@
 #define XFILE_VERSION 276
 
 #define XFILE_HEADER_IW4X 0x78345749 // 'IW4x'
-#define XFILE_VERSION_IW4X 3
+#define XFILE_VERSION_IW4X 4
 
 namespace Components
 {

src/STDInclude.hpp

@@ -73,7 +73,12 @@ template <size_t S> class Sizer { };
 #pragma warning(disable: 6386)
 #pragma warning(disable: 6387)
 
-#include <zlib.h>
+// #include <zlib.h>
+#include <zstd.h>
+
+#define ZWRAP_USE_ZSTD 1
+#include <zstd_zlibwrapper.h>
+
 #include <curses.h>
 #include <mongoose.h>
 #include <json11.hpp>

src/Utils/Compression.cpp

@@ -4,7 +4,9 @@ namespace Utils
 {
 	namespace Compression
 	{
-		std::string ZLib::Compress(std::string data)
+		std::mutex Deflate::Mutex;
+
+		std::string Deflate::Compress(std::string data)
 		{
 			Utils::Memory::Allocator allocator;
 			unsigned long length = (data.size() * 2);
@@ -15,7 +17,8 @@ namespace Utils
 
 			char* buffer = allocator.allocateArray<char>(length);
 
-			if (compress2(reinterpret_cast<Bytef*>(buffer), &length, reinterpret_cast<Bytef*>(const_cast<char*>(data.data())), data.size(), Z_BEST_COMPRESSION) != Z_OK)
+			int level = (ZWRAP_isUsingZSTDcompression() ? ZSTD_maxCLevel() : Z_BEST_COMPRESSION);
+			if (compress2(reinterpret_cast<Bytef*>(buffer), &length, reinterpret_cast<Bytef*>(const_cast<char*>(data.data())), data.size(), level) != Z_OK)
 			{
 				return "";
 			}
@@ -26,7 +29,7 @@ namespace Utils
 			return data;
 		}
 
-		std::string ZLib::Decompress(std::string data)
+		std::string Deflate::Decompress(std::string data)
 		{
 			z_stream stream;
 			ZeroMemory(&stream, sizeof(stream));
@@ -70,5 +73,42 @@ namespace Utils
 			inflateEnd(&stream);
 			return buffer;
 		}
+
+		std::string Deflate::ZLib::Compress(std::string data)
+		{
+			Deflate::Semaphore _(DEFLATE_ZLIB);
+			return Deflate::Compress(data);
+		}
+
+		std::string Deflate::ZLib::Decompress(std::string data)
+		{
+			Deflate::Semaphore _(DEFLATE_ZLIB);
+			return Deflate::Decompress(data);
+		}
+
+		std::string Deflate::ZStd::Compress(std::string data)
+		{
+			Deflate::Semaphore _(DEFLATE_ZSTD);
+			return Deflate::Compress(data);
+		}
+
+		std::string Deflate::ZStd::Decompress(std::string data)
+		{
+			Deflate::Semaphore _(DEFLATE_ZSTD);
+			return Deflate::Decompress(data);
+		}
+
+		Deflate::Semaphore::Semaphore(bool zstd)
+		{
+			Deflate::Mutex.lock();
+			this->state = ZWRAP_isUsingZSTDcompression();
+			ZWRAP_useZSTDcompression(zstd);
+		}
+
+		Deflate::Semaphore::~Semaphore()
+		{
+			ZWRAP_useZSTDcompression(this->state);
+			Deflate::Mutex.unlock();
+		}
 	};
 }

src/Utils/Compression.hpp

@@ -1,14 +1,42 @@
 #pragma once
 
 #define CHUNK 16384
+#define DEFLATE_ZLIB false
+#define DEFLATE_ZSTD true
 
 namespace Utils
 {
 	namespace Compression
 	{
-		class ZLib
+		class Deflate
 		{
 		public:
+			class ZLib
+			{
+			public:
+				static std::string Compress(std::string data);
+				static std::string Decompress(std::string data);
+			};
+
+			class ZStd
+			{
+			public:
+				static std::string Compress(std::string data);
+				static std::string Decompress(std::string data);
+			};
+
+			class Semaphore
+			{
+			public:
+				Semaphore(bool zstd);
+				~Semaphore();
+
+			private:
+				int state;
+			};
+
+		private:
+			static std::mutex Mutex;
 			static std::string Compress(std::string data);
 			static std::string Decompress(std::string data);
 		};