[Submodules] Get rid of zstd

.gitmodules

@@ -29,7 +29,3 @@
 [submodule "deps/udis86"]
 	path = deps/udis86
 	url = ../udis86.git
-[submodule "deps/zstd"]
-	path = deps/zstd
-	url = https://github.com/facebook/zstd.git
-	branch = dev

deps/mongoose

@@ -1 +1 @@
-Subproject commit 3e59e1c465b39bebbb596df38be451307906109b
+Subproject commit 5a0b6fc35abd21031ad749a1180a6f63cd70b52a

deps/protobuf

@@ -1 +1 @@
-Subproject commit ecca6ea95d56a6f70ff7b223ec3f904758acc8b1
+Subproject commit 62d7fe569717b6da39401a94b22adf73466e1f2d

deps/zstd

@@ -1 +0,0 @@
-Subproject commit 1d39550471fa2f6bd994ce213539f1310ebadf88

premake/zstd.lua

@@ -1,66 +0,0 @@
-zstd = {
-	settings = nil
-}
-
-function zstd.setup(settings)
-	if not settings.source then error("Missing source.") end
-
-	zstd.settings = settings
-
-	if not zstd.settings.defines then zstd.settings.defines = {} end
-end
-
-function zstd.import()
-	if not zstd.settings then error("You need to call zstd.setup first") end
-
-	links { "zstd" }
-	zstd.includes()
-end
-
-function zstd.includes()
-	if not zstd.settings then error("You need to call zstd.setup first") end
-
-	includedirs
-	{
-		path.join(zstd.settings.source, "lib"),
-		path.join(zstd.settings.source, "lib/common"),
-		path.join(zstd.settings.source, "zlibWrapper")
-	}
-	defines(zstd.settings.defines)
-end
-
-function zstd.project()
-	if not zstd.settings then error("You need to call zstd.setup first") end
-
-	project "zstd"
-		language "C"
-
-		zstd.includes()
-		files
-		{
-			path.join(zstd.settings.source, "lib/**.h"),
-			path.join(zstd.settings.source, "lib/**.c"),
-			path.join(zstd.settings.source, "zlibWrapper/zstd_zlibwrapper.h"),
-			path.join(zstd.settings.source, "zlibWrapper/zstd_zlibwrapper.c"),
-		}
-		removefiles
-		{
-			path.join(zstd.settings.source, "lib/legacy/**.*"),
-			--path.join(zstd.settings.source, "zlibWrapper/examples/**.*"),
-		}
-		defines
-		{
-			"zstd_DLL",
-			"_CRT_SECURE_NO_DEPRECATE",
-		}
-		
-		zlib.import()
-
-		-- not our code, ignore POSIX usage warnings for now
-		warnings "Off"
-		
-		--configuration "*Static"
-			defines { "_LIB" }
-			removedefines { "_USRDLL", "_DLL", "zstd_DLL" }
-			kind "StaticLib"
-end

premake5.lua

@@ -180,7 +180,6 @@ require "premake/mongoose"
 require "premake/pdcurses"
 require "premake/protobuf"
 require "premake/zlib"
-require "premake/zstd"
 require "premake/udis86"
 
 json11.setup
@@ -222,10 +221,6 @@ zlib.setup
 	},
 	source = path.join(depsBasePath, "zlib"),
 }
-zstd.setup
-{
-	source = path.join(depsBasePath, "zstd"),
-}
 udis86.setup
 {
 	source = path.join(depsBasePath, "udis86"),
@@ -316,7 +311,6 @@ workspace "iw4x"
 		pdcurses.import()
 		protobuf.import()
 		zlib.import()
-		zstd.import()
 		udis86.import()
 
 		-- fix vpaths for protobuf sources
@@ -429,7 +423,6 @@ workspace "iw4x"
 		pdcurses.project()
 		protobuf.project()
 		zlib.project()
-		zstd.project()
 		udis86.project()
 
 rule "ProtobufCompiler"

src/Components/Modules/FastFiles.cpp

@@ -6,9 +6,6 @@ namespace Components
 	symmetric_CTR FastFiles::CurrentCTR;
 	std::vector<std::string> FastFiles::ZonePaths;
 
-	bool FastFiles::UseZstd = false;
-	Utils::Compression::Deflate::Semaphore* FastFiles::ZlibLock = nullptr;
-
 	bool FastFiles::IsIW4xZone = false;
 	bool FastFiles::StreamRead = false;
 
@@ -336,28 +333,21 @@ namespace Components
 
 	void FastFiles::ReadHeaderStub(unsigned int* header, int size)
 	{
-		FastFiles::UseZstd = false;
 		FastFiles::IsIW4xZone = false;
 		FastFiles::LastByteRead = 0;
 		Game::DB_ReadXFileUncompressed(header, size);
 
 		if (header[0] == XFILE_HEADER_IW4X)
 		{
-			FastFiles::UseZstd = true;
 			FastFiles::IsIW4xZone = true;
 
-			static_assert((XFILE_VERSION_IW4X - 1) == 3, "FastFile backwards-compatibility not granted!");
+			if (header[1] < XFILE_VERSION_IW4X)
-			if (header[1] == XFILE_VERSION_IW4X - 1)
-			{
-				FastFiles::UseZstd = false;
-			}
-			else if (header[1] < XFILE_VERSION_IW4X)
 			{
 				Logger::Error("The fastfile you are trying to load is outdated (%d, expected %d)", header[1], XFILE_VERSION_IW4X);
 			}
 			else if (header[1] > XFILE_VERSION_IW4X)
 			{
-				Logger::Error("You are loading a fastfile that is too new (%d, expected %d), update your game", header[1], XFILE_VERSION_IW4X);
+				Logger::Error("You are loading a fastfile that is too new (%d, expected %d), update your game or rebuild the fastfile", header[1], XFILE_VERSION_IW4X);
 			}
 
 			*reinterpret_cast<unsigned __int64*>(header) = XFILE_MAGIC_UNSIGNED;
@@ -406,9 +396,8 @@ namespace Components
 			ctr_decrypt(strm->next_in, const_cast<unsigned char*>(strm->next_in), strm->avail_in, &FastFiles::CurrentCTR);
 		}
 
-		//return Utils::Hook::Call<int(z_streamp, const char*, int)>(0x4D8090)(strm, version, stream_size);
+		return Utils::Hook::Call<int(z_streamp, const char*, int)>(0x4D8090)(strm, version, stream_size);
 		//return inflateInit_(strm, version, stream_size);
-		return FastFiles::InflateInitStub(strm, version, stream_size);
 	}
 
 	void FastFiles::AuthLoadInflateDecryptBaseFunc(unsigned char* buffer)
@@ -495,44 +484,6 @@ namespace Components
 	}
 #endif
 
-	int FastFiles::InflateInitStub(z_streamp strm, const char *version, int stream_size)
-	{
-		if (FastFiles::UseZstd)
-		{
-			if (FastFiles::ZlibLock) delete FastFiles::ZlibLock;
-			FastFiles::ZlibLock = new Utils::Compression::Deflate::Semaphore(DEFLATE_ZSTD);
-
-			return inflateInit_(strm, version, stream_size);
-		}
-
-		return Utils::Hook::Call<int(z_streamp, const char*, int)>(0x4D8090)(strm, version, stream_size);
-	}
-
-	int FastFiles::InflateStub(z_streamp strm, int flush)
-	{
-		if (FastFiles::UseZstd)
-		{
-			return inflate(strm, flush);
-		}
-
-		return Utils::Hook::Call<int(z_streamp, int)>(0x49EA00)(strm, flush);
-	}
-
-	int FastFiles::InflateEndStub(z_streamp strm)
-	{
-		if (FastFiles::UseZstd && FastFiles::ZlibLock)
-		{
-			int result = inflateEnd(strm);
-
-			delete FastFiles::ZlibLock;
-			FastFiles::ZlibLock = nullptr;
-
-			return result;
-		}
-
-		return Utils::Hook::Call<int(z_streamp)>(0x453750)(strm);
-	}
-
 	FastFiles::FastFiles()
 	{
 		Dvar::Register<bool>("ui_zoneDebug", false, Game::dvar_flag::DVAR_FLAG_SAVED, "Display current loaded zone.");
@@ -602,14 +553,6 @@ namespace Components
 		// XFile header loading
 		Utils::Hook(0x4159E2, FastFiles::ReadXFileHeader, HOOK_CALL).install()->quick();
 
-		// Replace internal ZLib
-		//Utils::Hook(0x4D0306, FastFiles::InflateInitStub, HOOK_CALL).install()->quick();
-		Utils::Hook(0x4D034B, FastFiles::InflateInitStub, HOOK_JUMP).install()->quick();
-		Utils::Hook(0x480A1A, FastFiles::InflateStub, HOOK_JUMP).install()->quick();
-		Utils::Hook(0x5B99DE, FastFiles::InflateStub, HOOK_CALL).install()->quick();
-		Utils::Hook(0x449D8F, FastFiles::InflateEndStub, HOOK_CALL).install()->quick();
-		Utils::Hook(0x449DA3, FastFiles::InflateEndStub, HOOK_CALL).install()->quick();
-
 		// Add custom zone paths
 		FastFiles::AddZonePath("zone\\patch\\");
 		FastFiles::AddZonePath("zone\\dlc\\");
@@ -670,12 +613,6 @@ namespace Components
 
 	FastFiles::~FastFiles()
 	{
-		if (FastFiles::ZlibLock)
-		{
-			delete FastFiles::ZlibLock;
-			FastFiles::ZlibLock = nullptr;
-		}
-
 		FastFiles::ZonePaths.clear();
 	}
 }

src/Components/Modules/FastFiles.hpp

@@ -34,9 +34,6 @@ namespace Components
 		static unsigned int CurrentZone;
 		static unsigned int MaxZones;
 
-		static bool UseZstd;
-		static Utils::Compression::Deflate::Semaphore* ZlibLock;
-
 		static bool IsIW4xZone;
 		static bool StreamRead;
 
@@ -66,10 +63,6 @@ namespace Components
 		static void ReadXFile(void* buffer, int size);
 		static void ReadXFileStub(char* buffer, int size);
 
-		static int InflateInitStub(z_streamp strm, const char *version, int stream_size);
-		static int InflateStub(z_streamp strm, int flush);
-		static int InflateEndStub(z_streamp strm);
-
 #ifdef DEBUG
 		static void LogStreamRead(int len);
 #endif

src/Components/Modules/Node.cpp

@@ -68,12 +68,12 @@ namespace Components
 		if (Monitor::IsEnabled())
 		{
 			std::string nodes = Utils::IO::ReadFile("players/nodes_default.dat");
-			if (nodes.empty() || !list.ParseFromString(Utils::Compression::Deflate::ZStd::Decompress(nodes))) return;
+			if (nodes.empty() || !list.ParseFromString(Utils::Compression::ZLib::Decompress(nodes))) return;
 		}
 		else
 		{
 			FileSystem::File defaultNodes("nodes_default.dat");
-			if (!defaultNodes.exists() || !list.ParseFromString(Utils::Compression::Deflate::ZStd::Decompress(defaultNodes.getBuffer()))) return;
+			if (!defaultNodes.exists() || !list.ParseFromString(Utils::Compression::ZLib::Decompress(defaultNodes.getBuffer()))) return;
 		}
 
 		for (int i = 0; i < list.nodes_size(); ++i)
@@ -91,7 +91,7 @@ namespace Components
 	{
 		Proto::Node::List list;
 		std::string nodes = Utils::IO::ReadFile("players/nodes.dat");
-		if (nodes.empty() || !list.ParseFromString(Utils::Compression::Deflate::ZStd::Decompress(nodes))) return;
+		if (nodes.empty() || !list.ParseFromString(Utils::Compression::ZLib::Decompress(nodes))) return;
 
 		for (int i = 0; i < list.nodes_size(); ++i)
 		{
@@ -127,7 +127,7 @@ namespace Components
 		}
 		Node::Mutex.unlock();
 
-		Utils::IO::WriteFile("players/nodes.dat", Utils::Compression::Deflate::ZStd::Compress(list.SerializeAsString()));
+		Utils::IO::WriteFile("players/nodes.dat", Utils::Compression::ZLib::Compress(list.SerializeAsString()));
 	}
 
 	void Node::Add(Network::Address address)

src/Components/Modules/Playlist.cpp

@@ -39,7 +39,7 @@ namespace Components
 	DWORD Playlist::StorePlaylistStub(const char** buffer)
 	{
 		Playlist::MapRelocation.clear();
-		Playlist::CurrentPlaylistBuffer = Utils::Compression::Deflate::ZStd::Compress(*buffer);
+		Playlist::CurrentPlaylistBuffer = Utils::Compression::ZLib::Compress(*buffer);
 		return Utils::Hook::Call<DWORD(const char**)>(0x4C0350)(buffer);
 	}
 
@@ -95,7 +95,7 @@ namespace Components
 					}
 
 					// Decompress buffer
-					Playlist::ReceivedPlaylistBuffer = Utils::Compression::Deflate::ZStd::Decompress(compressedData);
+					Playlist::ReceivedPlaylistBuffer = Utils::Compression::ZLib::Decompress(compressedData);
 
 					// Load and continue connection
 					Logger::Print("Received playlist, loading and continuing connection...\n");

src/Components/Modules/QuickPatch.cpp

@@ -702,32 +702,8 @@ namespace Components
 
 		for (int i = 0; i < 21; ++i)
 		{
-			std::string compressed = Utils::Compression::Deflate::ZLib::Compress(test);
+			std::string compressed = Utils::Compression::ZLib::Compress(test);
-			std::string decompressed = Utils::Compression::Deflate::ZLib::Decompress(compressed);
+			std::string decompressed = Utils::Compression::ZLib::Decompress(compressed);
-
-			if (test != decompressed)
-			{
-				printf("Error\n");
-				printf("Compressing %d bytes and decompressing failed!\n", test.size());
-				return false;
-			}
-
-			auto size = test.size();
-			for (unsigned int j = 0; j < size; ++j)
-			{
-				test.append(Utils::String::VA("%c", Utils::Cryptography::Rand::GenerateInt()));
-			}
-		}
-
-		printf("Success\n");
-		printf("Testing ZStd compression...");
-
-		test = Utils::String::VA("%c", Utils::Cryptography::Rand::GenerateInt());
-
-		for (int i = 0; i < 21; ++i)
-		{
-			std::string compressed = Utils::Compression::Deflate::ZStd::Compress(test);
-			std::string decompressed = Utils::Compression::Deflate::ZStd::Decompress(compressed);
 
 			if (test != decompressed)
 			{

src/Components/Modules/ZoneBuilder.cpp

@@ -410,7 +410,7 @@ namespace Components
 #ifdef DEBUG
 		zoneBuffer = Utils::Compression::Deflate::ZLib::Compress(zoneBuffer);
 #else
-		zoneBuffer = Utils::Compression::Deflate::ZStd::Compress(zoneBuffer);
+		zoneBuffer = Utils::Compression::ZLib::Compress(zoneBuffer);
 #endif
 
 		outBuffer.append(zoneBuffer);

src/Components/Modules/ZoneBuilder.hpp

@@ -4,7 +4,7 @@
 #define XFILE_VERSION 276
 
 #define XFILE_HEADER_IW4X 0x78345749 // 'IW4x'
-#define XFILE_VERSION_IW4X 4
+#define XFILE_VERSION_IW4X 3
 
 namespace Components
 {

src/STDInclude.hpp

@@ -73,11 +73,7 @@ template <size_t S> class Sizer { };
 #pragma warning(disable: 6386)
 #pragma warning(disable: 6387)
 
-// #include <zlib.h>
+#include <zlib.h>
-#include <zstd.h>
-
-#define ZWRAP_USE_ZSTD 1
-#include <zstd_zlibwrapper.h>
 
 #include <curses.h>
 #include <mongoose.h>

src/Utils/Compression.cpp

@@ -4,9 +4,7 @@ namespace Utils
 {
 	namespace Compression
 	{
-		std::mutex Deflate::Mutex;
+		std::string ZLib::Compress(std::string data)
-
-		std::string Deflate::Compress(std::string data)
 		{
 			Utils::Memory::Allocator allocator;
 			unsigned long length = (data.size() * 2);
@@ -16,9 +14,7 @@ namespace Utils
 			if (length < 100) length *= 10;
 
 			char* buffer = allocator.allocateArray<char>(length);
-
+			if (compress2(reinterpret_cast<Bytef*>(buffer), &length, reinterpret_cast<Bytef*>(const_cast<char*>(data.data())), data.size(), Z_BEST_COMPRESSION) != Z_OK)
-			int level = (ZWRAP_isUsingZSTDcompression() ? ZSTD_maxCLevel() : Z_BEST_COMPRESSION);
-			if (compress2(reinterpret_cast<Bytef*>(buffer), &length, reinterpret_cast<Bytef*>(const_cast<char*>(data.data())), data.size(), level) != Z_OK)
 			{
 				return "";
 			}
@@ -29,7 +25,7 @@ namespace Utils
 			return data;
 		}
 
-		std::string Deflate::Decompress(std::string data)
+		std::string ZLib::Decompress(std::string data)
 		{
 			z_stream stream;
 			ZeroMemory(&stream, sizeof(stream));
@@ -73,87 +69,5 @@ namespace Utils
 			inflateEnd(&stream);
 			return buffer;
 		}
-
+	}
-		std::string Deflate::ZLib::Compress(std::string data)
-		{
-			Deflate::Semaphore _(DEFLATE_ZLIB);
-			return Deflate::Compress(data);
-		}
-
-		std::string Deflate::ZLib::Decompress(std::string data)
-		{
-			Deflate::Semaphore _(DEFLATE_ZLIB);
-			return Deflate::Decompress(data);
-		}
-
-		std::string Deflate::ZStd::Compress(std::string data, bool safe)
-		{
-			if (safe)
-			{
-				Utils::Memory::Allocator allocator;
-
-				size_t size = data.size() + 100;
-				size = ZSTD_compressBound(size);
-				char* buffer = allocator.allocateArray<char>(size);
-
-				size = ZSTD_compress(buffer, size, data.data(), data.size(), ZSTD_maxCLevel());
-				if (size == 0 || ZSTD_isError(size)) return "";
-
-				return std::string(buffer, size);
-			}
-			else
-			{
-				Deflate::Semaphore _(DEFLATE_ZSTD);
-				return Deflate::Compress(data);
-			}
-		}
-
-		std::string Deflate::ZStd::Decompress(std::string data, bool safe)
-		{
-			if (safe)
-			{
-				if (data.empty()) return "";
-
-				Utils::Memory::Allocator allocator;
-
-				size_t size = data.size() + 100;
-				size = ZSTD_compressBound(size);
-				size_t maxSize = size;
-				char* buffer = allocator.allocateArray<char>(size);
-
-				while(true)
-				{
-					size = ZSTD_decompress(buffer, size, data.data(), data.size());
-					if (!ZSTD_isError(size)) break;
-
-					maxSize++;
-					maxSize *= 2;
-					size = maxSize;
-
-					allocator.free(buffer);
-					buffer = allocator.allocateArray<char>(size);
-				}
-
-				return std::string(buffer, size);
-			}
-			else
-			{
-				Deflate::Semaphore _(DEFLATE_ZSTD);
-				return Deflate::Decompress(data);
-			}
-		}
-
-		Deflate::Semaphore::Semaphore(bool zstd)
-		{
-			Deflate::Mutex.lock();
-			this->state = ZWRAP_isUsingZSTDcompression();
-			ZWRAP_useZSTDcompression(zstd);
-		}
-
-		Deflate::Semaphore::~Semaphore()
-		{
-			ZWRAP_useZSTDcompression(this->state);
-			Deflate::Mutex.unlock();
-		}
-	};
 }

src/Utils/Compression.hpp

@@ -8,35 +8,9 @@ namespace Utils
 {
 	namespace Compression
 	{
-		class Deflate
+		class ZLib
 		{
 		public:
-			class ZLib
-			{
-			public:
-				static std::string Compress(std::string data);
-				static std::string Decompress(std::string data);
-			};
-
-			class ZStd
-			{
-			public:
-				static std::string Compress(std::string data, bool safe = true);
-				static std::string Decompress(std::string data, bool safe = true);
-			};
-
-			class Semaphore
-			{
-			public:
-				Semaphore(bool zstd);
-				~Semaphore();
-
-			private:
-				int state;
-			};
-
-		private:
-			static std::mutex Mutex;
 			static std::string Compress(std::string data);
 			static std::string Decompress(std::string data);
 		};