Rim/iw4x-client
Rim/iw4x-client
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 2 Wiki Activity

[Submodules] Get rid of zstd

Browse Source
This commit is contained in:
momo5502 2017-07-03 16:32:10 +02:00
parent 41a85262f5
commit 4aba6fa06d
16 changed files with 21 additions and 309 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
.gitmodules vendored
View File

@ -29,7 +29,3 @@
[submodule "deps/udis86"]
path = deps/udis86
url = ../udis86.git
[submodule "deps/zstd"]
path = deps/zstd
url = https://github.com/facebook/zstd.git
branch = dev

2
deps/mongoose vendored

@ -1 +1 @@
Subproject commit 3e59e1c465b39bebbb596df38be451307906109b
Subproject commit 5a0b6fc35abd21031ad749a1180a6f63cd70b52a

2
deps/protobuf vendored

@ -1 +1 @@
Subproject commit ecca6ea95d56a6f70ff7b223ec3f904758acc8b1
Subproject commit 62d7fe569717b6da39401a94b22adf73466e1f2d

1
deps/zstd vendored

@ -1 +0,0 @@
Subproject commit 1d39550471fa2f6bd994ce213539f1310ebadf88

66
premake/zstd.lua
View File

@ -1,66 +0,0 @@
zstd = {
settings = nil
}
function zstd.setup(settings)
if not settings.source then error("Missing source.") end
zstd.settings = settings
if not zstd.settings.defines then zstd.settings.defines = {} end
end
function zstd.import()
if not zstd.settings then error("You need to call zstd.setup first") end
links { "zstd" }
zstd.includes()
end
function zstd.includes()
if not zstd.settings then error("You need to call zstd.setup first") end
includedirs
{
path.join(zstd.settings.source, "lib"),
path.join(zstd.settings.source, "lib/common"),
path.join(zstd.settings.source, "zlibWrapper")
}
defines(zstd.settings.defines)
end
function zstd.project()
if not zstd.settings then error("You need to call zstd.setup first") end
project "zstd"
language "C"
zstd.includes()
files
{
path.join(zstd.settings.source, "lib/**.h"),
path.join(zstd.settings.source, "lib/**.c"),
path.join(zstd.settings.source, "zlibWrapper/zstd_zlibwrapper.h"),
path.join(zstd.settings.source, "zlibWrapper/zstd_zlibwrapper.c"),
}
removefiles
{
path.join(zstd.settings.source, "lib/legacy/**.*"),
--path.join(zstd.settings.source, "zlibWrapper/examples/**.*"),
}
defines
{
"zstd_DLL",
"_CRT_SECURE_NO_DEPRECATE",
}
zlib.import()
-- not our code, ignore POSIX usage warnings for now
warnings "Off"
--configuration "*Static"
defines { "_LIB" }
removedefines { "_USRDLL", "_DLL", "zstd_DLL" }
kind "StaticLib"
end

7
premake5.lua
View File

@ -180,7 +180,6 @@ require "premake/mongoose"
require "premake/pdcurses"
require "premake/protobuf"
require "premake/zlib"
require "premake/zstd"
require "premake/udis86"
json11.setup
@ -222,10 +221,6 @@ zlib.setup
},
source = path.join(depsBasePath, "zlib"),
}
zstd.setup
{
source = path.join(depsBasePath, "zstd"),
}
udis86.setup
{
source = path.join(depsBasePath, "udis86"),
@ -316,7 +311,6 @@ workspace "iw4x"
pdcurses.import()
protobuf.import()
zlib.import()
zstd.import()
udis86.import()
-- fix vpaths for protobuf sources
@ -429,7 +423,6 @@ workspace "iw4x"
pdcurses.project()
protobuf.project()
zlib.project()
zstd.project()
udis86.project()
rule "ProtobufCompiler"

69
src/Components/Modules/FastFiles.cpp
View File

@ -6,9 +6,6 @@ namespace Components
symmetric_CTR FastFiles::CurrentCTR;
std::vector<std::string> FastFiles::ZonePaths;
bool FastFiles::UseZstd = false;
Utils::Compression::Deflate::Semaphore* FastFiles::ZlibLock = nullptr;
bool FastFiles::IsIW4xZone = false;
bool FastFiles::StreamRead = false;
@ -336,28 +333,21 @@ namespace Components
void FastFiles::ReadHeaderStub(unsigned int* header, int size)
{
FastFiles::UseZstd = false;
FastFiles::IsIW4xZone = false;
FastFiles::LastByteRead = 0;
Game::DB_ReadXFileUncompressed(header, size);
if (header[0] == XFILE_HEADER_IW4X)
{
FastFiles::UseZstd = true;
FastFiles::IsIW4xZone = true;
static_assert((XFILE_VERSION_IW4X - 1) == 3, "FastFile backwards-compatibility not granted!");
if (header[1] == XFILE_VERSION_IW4X - 1)
{
FastFiles::UseZstd = false;
}
else if (header[1] < XFILE_VERSION_IW4X)
if (header[1] < XFILE_VERSION_IW4X)
{
Logger::Error("The fastfile you are trying to load is outdated (%d, expected %d)", header[1], XFILE_VERSION_IW4X);
}
else if (header[1] > XFILE_VERSION_IW4X)
{
Logger::Error("You are loading a fastfile that is too new (%d, expected %d), update your game", header[1], XFILE_VERSION_IW4X);
Logger::Error("You are loading a fastfile that is too new (%d, expected %d), update your game or rebuild the fastfile", header[1], XFILE_VERSION_IW4X);
}
*reinterpret_cast<unsigned __int64*>(header) = XFILE_MAGIC_UNSIGNED;
@ -406,9 +396,8 @@ namespace Components
ctr_decrypt(strm->next_in, const_cast<unsigned char*>(strm->next_in), strm->avail_in, &FastFiles::CurrentCTR);
}
//return Utils::Hook::Call<int(z_streamp, const char*, int)>(0x4D8090)(strm, version, stream_size);
return Utils::Hook::Call<int(z_streamp, const char*, int)>(0x4D8090)(strm, version, stream_size);
//return inflateInit_(strm, version, stream_size);
return FastFiles::InflateInitStub(strm, version, stream_size);
}
void FastFiles::AuthLoadInflateDecryptBaseFunc(unsigned char* buffer)
@ -495,44 +484,6 @@ namespace Components
}
#endif
int FastFiles::InflateInitStub(z_streamp strm, const char *version, int stream_size)
{
if (FastFiles::UseZstd)
{
if (FastFiles::ZlibLock) delete FastFiles::ZlibLock;
FastFiles::ZlibLock = new Utils::Compression::Deflate::Semaphore(DEFLATE_ZSTD);
return inflateInit_(strm, version, stream_size);
}
return Utils::Hook::Call<int(z_streamp, const char*, int)>(0x4D8090)(strm, version, stream_size);
}
int FastFiles::InflateStub(z_streamp strm, int flush)
{
if (FastFiles::UseZstd)
{
return inflate(strm, flush);
}
return Utils::Hook::Call<int(z_streamp, int)>(0x49EA00)(strm, flush);
}
int FastFiles::InflateEndStub(z_streamp strm)
{
if (FastFiles::UseZstd && FastFiles::ZlibLock)
{
int result = inflateEnd(strm);
delete FastFiles::ZlibLock;
FastFiles::ZlibLock = nullptr;
return result;
}
return Utils::Hook::Call<int(z_streamp)>(0x453750)(strm);
}
FastFiles::FastFiles()
{
Dvar::Register<bool>("ui_zoneDebug", false, Game::dvar_flag::DVAR_FLAG_SAVED, "Display current loaded zone.");
@ -602,14 +553,6 @@ namespace Components
// XFile header loading
Utils::Hook(0x4159E2, FastFiles::ReadXFileHeader, HOOK_CALL).install()->quick();
// Replace internal ZLib
//Utils::Hook(0x4D0306, FastFiles::InflateInitStub, HOOK_CALL).install()->quick();
Utils::Hook(0x4D034B, FastFiles::InflateInitStub, HOOK_JUMP).install()->quick();
Utils::Hook(0x480A1A, FastFiles::InflateStub, HOOK_JUMP).install()->quick();
Utils::Hook(0x5B99DE, FastFiles::InflateStub, HOOK_CALL).install()->quick();
Utils::Hook(0x449D8F, FastFiles::InflateEndStub, HOOK_CALL).install()->quick();
Utils::Hook(0x449DA3, FastFiles::InflateEndStub, HOOK_CALL).install()->quick();
// Add custom zone paths
FastFiles::AddZonePath("zone\\patch\\");
FastFiles::AddZonePath("zone\\dlc\\");
@ -670,12 +613,6 @@ namespace Components
FastFiles::~FastFiles()
{
if (FastFiles::ZlibLock)
{
delete FastFiles::ZlibLock;
FastFiles::ZlibLock = nullptr;
}
FastFiles::ZonePaths.clear();
}
}

7
src/Components/Modules/FastFiles.hpp
View File

@ -34,9 +34,6 @@ namespace Components
static unsigned int CurrentZone;
static unsigned int MaxZones;
static bool UseZstd;
static Utils::Compression::Deflate::Semaphore* ZlibLock;
static bool IsIW4xZone;
static bool StreamRead;
@ -66,10 +63,6 @@ namespace Components
static void ReadXFile(void* buffer, int size);
static void ReadXFileStub(char* buffer, int size);
static int InflateInitStub(z_streamp strm, const char *version, int stream_size);
static int InflateStub(z_streamp strm, int flush);
static int InflateEndStub(z_streamp strm);
#ifdef DEBUG
static void LogStreamRead(int len);
#endif

8
src/Components/Modules/Node.cpp
View File

@ -68,12 +68,12 @@ namespace Components
if (Monitor::IsEnabled())
{
std::string nodes = Utils::IO::ReadFile("players/nodes_default.dat");
if (nodes.empty() || !list.ParseFromString(Utils::Compression::Deflate::ZStd::Decompress(nodes))) return;
if (nodes.empty() || !list.ParseFromString(Utils::Compression::ZLib::Decompress(nodes))) return;
}
else
{
FileSystem::File defaultNodes("nodes_default.dat");
if (!defaultNodes.exists() || !list.ParseFromString(Utils::Compression::Deflate::ZStd::Decompress(defaultNodes.getBuffer()))) return;
if (!defaultNodes.exists() || !list.ParseFromString(Utils::Compression::ZLib::Decompress(defaultNodes.getBuffer()))) return;
}
for (int i = 0; i < list.nodes_size(); ++i)
@ -91,7 +91,7 @@ namespace Components
{
Proto::Node::List list;
std::string nodes = Utils::IO::ReadFile("players/nodes.dat");
if (nodes.empty() || !list.ParseFromString(Utils::Compression::Deflate::ZStd::Decompress(nodes))) return;
if (nodes.empty() || !list.ParseFromString(Utils::Compression::ZLib::Decompress(nodes))) return;
for (int i = 0; i < list.nodes_size(); ++i)
{
@ -127,7 +127,7 @@ namespace Components
}
Node::Mutex.unlock();
Utils::IO::WriteFile("players/nodes.dat", Utils::Compression::Deflate::ZStd::Compress(list.SerializeAsString()));
Utils::IO::WriteFile("players/nodes.dat", Utils::Compression::ZLib::Compress(list.SerializeAsString()));
}
void Node::Add(Network::Address address)

4
src/Components/Modules/Playlist.cpp
View File

@ -39,7 +39,7 @@ namespace Components
DWORD Playlist::StorePlaylistStub(const char** buffer)
{
Playlist::MapRelocation.clear();
Playlist::CurrentPlaylistBuffer = Utils::Compression::Deflate::ZStd::Compress(*buffer);
Playlist::CurrentPlaylistBuffer = Utils::Compression::ZLib::Compress(*buffer);
return Utils::Hook::Call<DWORD(const char**)>(0x4C0350)(buffer);
}
@ -95,7 +95,7 @@ namespace Components
}
// Decompress buffer
Playlist::ReceivedPlaylistBuffer = Utils::Compression::Deflate::ZStd::Decompress(compressedData);
Playlist::ReceivedPlaylistBuffer = Utils::Compression::ZLib::Decompress(compressedData);
// Load and continue connection
Logger::Print("Received playlist, loading and continuing connection...\n");

28
src/Components/Modules/QuickPatch.cpp
View File

@ -702,32 +702,8 @@ namespace Components
for (int i = 0; i < 21; ++i)
{
std::string compressed = Utils::Compression::Deflate::ZLib::Compress(test);
std::string decompressed = Utils::Compression::Deflate::ZLib::Decompress(compressed);
if (test != decompressed)
{
printf("Error\n");
printf("Compressing %d bytes and decompressing failed!\n", test.size());
return false;
}
auto size = test.size();
for (unsigned int j = 0; j < size; ++j)
{
test.append(Utils::String::VA("%c", Utils::Cryptography::Rand::GenerateInt()));
}
}
printf("Success\n");
printf("Testing ZStd compression...");
test = Utils::String::VA("%c", Utils::Cryptography::Rand::GenerateInt());
for (int i = 0; i < 21; ++i)
{
std::string compressed = Utils::Compression::Deflate::ZStd::Compress(test);
std::string decompressed = Utils::Compression::Deflate::ZStd::Decompress(compressed);
std::string compressed = Utils::Compression::ZLib::Compress(test);
std::string decompressed = Utils::Compression::ZLib::Decompress(compressed);
if (test != decompressed)
{

2
src/Components/Modules/ZoneBuilder.cpp
View File

@ -410,7 +410,7 @@ namespace Components
#ifdef DEBUG
zoneBuffer = Utils::Compression::Deflate::ZLib::Compress(zoneBuffer);
#else
zoneBuffer = Utils::Compression::Deflate::ZStd::Compress(zoneBuffer);
zoneBuffer = Utils::Compression::ZLib::Compress(zoneBuffer);
#endif
outBuffer.append(zoneBuffer);

2
src/Components/Modules/ZoneBuilder.hpp
View File

@ -4,7 +4,7 @@
#define XFILE_VERSION 276
#define XFILE_HEADER_IW4X 0x78345749 // 'IW4x'
#define XFILE_VERSION_IW4X 4
#define XFILE_VERSION_IW4X 3
namespace Components
{

6
src/STDInclude.hpp
View File

@ -73,11 +73,7 @@ template <size_t S> class Sizer { };
#pragma warning(disable: 6386)
#pragma warning(disable: 6387)
// #include <zlib.h>
#include <zstd.h>
#define ZWRAP_USE_ZSTD 1
#include <zstd_zlibwrapper.h>
#include <zlib.h>
#include <curses.h>
#include <mongoose.h>

94
src/Utils/Compression.cpp
View File

@ -4,9 +4,7 @@ namespace Utils
{
namespace Compression
{
std::mutex Deflate::Mutex;
std::string Deflate::Compress(std::string data)
std::string ZLib::Compress(std::string data)
{
Utils::Memory::Allocator allocator;
unsigned long length = (data.size() * 2);
@ -16,9 +14,7 @@ namespace Utils
if (length < 100) length *= 10;
char* buffer = allocator.allocateArray<char>(length);
int level = (ZWRAP_isUsingZSTDcompression() ? ZSTD_maxCLevel() : Z_BEST_COMPRESSION);
if (compress2(reinterpret_cast<Bytef*>(buffer), &length, reinterpret_cast<Bytef*>(const_cast<char*>(data.data())), data.size(), level) != Z_OK)
if (compress2(reinterpret_cast<Bytef*>(buffer), &length, reinterpret_cast<Bytef*>(const_cast<char*>(data.data())), data.size(), Z_BEST_COMPRESSION) != Z_OK)
{
return "";
}
@ -29,7 +25,7 @@ namespace Utils
return data;
}
std::string Deflate::Decompress(std::string data)
std::string ZLib::Decompress(std::string data)
{
z_stream stream;
ZeroMemory(&stream, sizeof(stream));
@ -73,87 +69,5 @@ namespace Utils
inflateEnd(&stream);
return buffer;
}
std::string Deflate::ZLib::Compress(std::string data)
{
Deflate::Semaphore _(DEFLATE_ZLIB);
return Deflate::Compress(data);
}
std::string Deflate::ZLib::Decompress(std::string data)
{
Deflate::Semaphore _(DEFLATE_ZLIB);
return Deflate::Decompress(data);
}
std::string Deflate::ZStd::Compress(std::string data, bool safe)
{
if (safe)
{
Utils::Memory::Allocator allocator;
size_t size = data.size() + 100;
size = ZSTD_compressBound(size);
char* buffer = allocator.allocateArray<char>(size);
size = ZSTD_compress(buffer, size, data.data(), data.size(), ZSTD_maxCLevel());
if (size == 0 || ZSTD_isError(size)) return "";
return std::string(buffer, size);
}
else
{
Deflate::Semaphore _(DEFLATE_ZSTD);
return Deflate::Compress(data);
}
}
std::string Deflate::ZStd::Decompress(std::string data, bool safe)
{
if (safe)
{
if (data.empty()) return "";
Utils::Memory::Allocator allocator;
size_t size = data.size() + 100;
size = ZSTD_compressBound(size);
size_t maxSize = size;
char* buffer = allocator.allocateArray<char>(size);
while(true)
{
size = ZSTD_decompress(buffer, size, data.data(), data.size());
if (!ZSTD_isError(size)) break;
maxSize++;
maxSize *= 2;
size = maxSize;
allocator.free(buffer);
buffer = allocator.allocateArray<char>(size);
}
return std::string(buffer, size);
}
else
{
Deflate::Semaphore _(DEFLATE_ZSTD);
return Deflate::Decompress(data);
}
}
Deflate::Semaphore::Semaphore(bool zstd)
{
Deflate::Mutex.lock();
this->state = ZWRAP_isUsingZSTDcompression();
ZWRAP_useZSTDcompression(zstd);
}
Deflate::Semaphore::~Semaphore()
{
ZWRAP_useZSTDcompression(this->state);
Deflate::Mutex.unlock();
}
};
}
}

28
src/Utils/Compression.hpp
View File

@ -8,35 +8,9 @@ namespace Utils
{
namespace Compression
{
class Deflate
class ZLib
{
public:
class ZLib
{
public:
static std::string Compress(std::string data);
static std::string Decompress(std::string data);
};
class ZStd
{
public:
static std::string Compress(std::string data, bool safe = true);
static std::string Decompress(std::string data, bool safe = true);
};
class Semaphore
{
public:
Semaphore(bool zstd);
~Semaphore();
private:
int state;
};
private:
static std::mutex Mutex;
static std::string Compress(std::string data);
static std::string Decompress(std::string data);
};