Rim/iw4x-client
Rim/iw4x-client
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 2 Wiki Activity

Merge branch 'feature/zstd-zone' into 'develop'

Browse Source 
[Merge] feature/zstd-zone -> develop
This commit is contained in:
momo5502 2017-06-26 20:48:51 +02:00
commit 3a9d4231bb
14 changed files with 265 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
.gitmodules vendored
View File

@ -29,3 +29,7 @@
[submodule "deps/udis86"]
path = deps/udis86
url = ../udis86.git
[submodule "deps/zstd"]
path = deps/zstd
url = https://github.com/facebook/zstd.git
branch = dev

1
deps/zstd vendored Submodule

@ -0,0 +1 @@
Subproject commit 379f9d874d578111c611b2262211aa763171a94c

66
premake/zstd.lua Normal file
View File

@ -0,0 +1,66 @@
zstd = {
settings = nil
}
function zstd.setup(settings)
if not settings.source then error("Missing source.") end
zstd.settings = settings
if not zstd.settings.defines then zstd.settings.defines = {} end
end
function zstd.import()
if not zstd.settings then error("You need to call zstd.setup first") end
links { "zstd" }
zstd.includes()
end
function zstd.includes()
if not zstd.settings then error("You need to call zstd.setup first") end
includedirs
{
path.join(zstd.settings.source, "lib"),
path.join(zstd.settings.source, "lib/common"),
path.join(zstd.settings.source, "zlibWrapper")
}
defines(zstd.settings.defines)
end
function zstd.project()
if not zstd.settings then error("You need to call zstd.setup first") end
project "zstd"
language "C"
zstd.includes()
files
{
path.join(zstd.settings.source, "lib/**.h"),
path.join(zstd.settings.source, "lib/**.c"),
path.join(zstd.settings.source, "zlibWrapper/zstd_zlibwrapper.h"),
path.join(zstd.settings.source, "zlibWrapper/zstd_zlibwrapper.c"),
}
removefiles
{
path.join(zstd.settings.source, "lib/legacy/**.*"),
--path.join(zstd.settings.source, "zlibWrapper/examples/**.*"),
}
defines
{
"zstd_DLL",
"_CRT_SECURE_NO_DEPRECATE",
}
zlib.import()
-- not our code, ignore POSIX usage warnings for now
warnings "Off"
--configuration "*Static"
defines { "_LIB" }
removedefines { "_USRDLL", "_DLL", "zstd_DLL" }
kind "StaticLib"
end

7
premake5.lua
View File

@ -180,6 +180,7 @@ require "premake/mongoose"
require "premake/pdcurses"
require "premake/protobuf"
require "premake/zlib"
require "premake/zstd"
require "premake/udis86"
json11.setup
@ -221,6 +222,10 @@ zlib.setup
},
source = path.join(depsBasePath, "zlib"),
}
zstd.setup
{
source = path.join(depsBasePath, "zstd"),
}
udis86.setup
{
source = path.join(depsBasePath, "udis86"),
@ -311,6 +316,7 @@ workspace "iw4x"
pdcurses.import()
protobuf.import()
zlib.import()
zstd.import()
udis86.import()
-- fix vpaths for protobuf sources
@ -423,6 +429,7 @@ workspace "iw4x"
pdcurses.project()
protobuf.project()
zlib.project()
zstd.project()
udis86.project()
rule "ProtobufCompiler"

68
src/Components/Modules/FastFiles.cpp
View File

@ -6,6 +6,9 @@ namespace Components
symmetric_CTR FastFiles::CurrentCTR;
std::vector<std::string> FastFiles::ZonePaths;
bool FastFiles::UseZstd = false;
Utils::Compression::Deflate::Semaphore* FastFiles::ZlibLock = nullptr;
bool FastFiles::IsIW4xZone = false;
bool FastFiles::StreamRead = false;
@ -333,15 +336,22 @@ namespace Components
void FastFiles::ReadHeaderStub(unsigned int* header, int size)
{
FastFiles::UseZstd = false;
FastFiles::IsIW4xZone = false;
FastFiles::LastByteRead = 0;
Game::DB_ReadXFileUncompressed(header, size);
if (header[0] == XFILE_HEADER_IW4X)
{
FastFiles::UseZstd = true;
FastFiles::IsIW4xZone = true;
if (header[1] < XFILE_VERSION_IW4X)
static_assert((XFILE_VERSION_IW4X - 1) == 3, "FastFile backwards-compatibility not granted!");
if (header[1] == XFILE_VERSION_IW4X - 1)
{
FastFiles::UseZstd = false;
}
else if (header[1] < XFILE_VERSION_IW4X)
{
Logger::Error("The fastfile you are trying to load is outdated (%d, expected %d)", header[1], XFILE_VERSION_IW4X);
}
@ -396,8 +406,9 @@ namespace Components
ctr_decrypt(strm->next_in, const_cast<unsigned char*>(strm->next_in), strm->avail_in, &FastFiles::CurrentCTR);
}
return Utils::Hook::Call<int(z_streamp, const char*, int)>(0x4D8090)(strm, version, stream_size);
//return Utils::Hook::Call<int(z_streamp, const char*, int)>(0x4D8090)(strm, version, stream_size);
//return inflateInit_(strm, version, stream_size);
return FastFiles::InflateInitStub(strm, version, stream_size);
}
void FastFiles::AuthLoadInflateDecryptBaseFunc(unsigned char* buffer)
@ -484,6 +495,44 @@ namespace Components
}
#endif
int FastFiles::InflateInitStub(z_streamp strm, const char *version, int stream_size)
{
if (FastFiles::UseZstd)
{
if (FastFiles::ZlibLock) delete FastFiles::ZlibLock;
FastFiles::ZlibLock = new Utils::Compression::Deflate::Semaphore(DEFLATE_ZSTD);
return inflateInit_(strm, version, stream_size);
}
return Utils::Hook::Call<int(z_streamp, const char*, int)>(0x4D8090)(strm, version, stream_size);
}
int FastFiles::InflateStub(z_streamp strm, int flush)
{
if (FastFiles::UseZstd)
{
return inflate(strm, flush);
}
return Utils::Hook::Call<int(z_streamp, int)>(0x49EA00)(strm, flush);
}
int FastFiles::InflateEndStub(z_streamp strm)
{
if (FastFiles::UseZstd && FastFiles::ZlibLock)
{
int result = inflateEnd(strm);
delete FastFiles::ZlibLock;
FastFiles::ZlibLock = nullptr;
return result;
}
return Utils::Hook::Call<int(z_streamp)>(0x453750)(strm);
}
FastFiles::FastFiles()
{
Dvar::Register<bool>("ui_zoneDebug", false, Game::dvar_flag::DVAR_FLAG_SAVED, "Display current loaded zone.");
@ -554,9 +603,12 @@ namespace Components
Utils::Hook(0x4159E2, FastFiles::ReadXFileHeader, HOOK_CALL).install()->quick();
// Replace internal ZLib
//Utils::Hook(0x44B160, inflateInit2_, HOOK_JUMP).install()->quick();
//Utils::Hook(0x453750, inflateEnd, HOOK_JUMP).install()->quick();
//Utils::Hook(0x49EA00, inflate, HOOK_JUMP).install()->quick();
//Utils::Hook(0x4D0306, FastFiles::InflateInitStub, HOOK_CALL).install()->quick();
Utils::Hook(0x4D034B, FastFiles::InflateInitStub, HOOK_JUMP).install()->quick();
Utils::Hook(0x480A1A, FastFiles::InflateStub, HOOK_JUMP).install()->quick();
Utils::Hook(0x5B99DE, FastFiles::InflateStub, HOOK_CALL).install()->quick();
Utils::Hook(0x449D8F, FastFiles::InflateEndStub, HOOK_CALL).install()->quick();
Utils::Hook(0x449DA3, FastFiles::InflateEndStub, HOOK_CALL).install()->quick();
// Add custom zone paths
FastFiles::AddZonePath("zone\\patch\\");
@ -618,6 +670,12 @@ namespace Components
FastFiles::~FastFiles()
{
if (FastFiles::ZlibLock)
{
delete FastFiles::ZlibLock;
FastFiles::ZlibLock = nullptr;
}
FastFiles::ZonePaths.clear();
}
}

7
src/Components/Modules/FastFiles.hpp
View File

@ -34,6 +34,9 @@ namespace Components
static unsigned int CurrentZone;
static unsigned int MaxZones;
static bool UseZstd;
static Utils::Compression::Deflate::Semaphore* ZlibLock;
static bool IsIW4xZone;
static bool StreamRead;
@ -63,6 +66,10 @@ namespace Components
static void ReadXFile(void* buffer, int size);
static void ReadXFileStub(char* buffer, int size);
static int InflateInitStub(z_streamp strm, const char *version, int stream_size);
static int InflateStub(z_streamp strm, int flush);
static int InflateEndStub(z_streamp strm);
#ifdef DEBUG
static void LogStreamRead(int len);
#endif

9
src/Components/Modules/Node.cpp
View File

@ -68,12 +68,12 @@ namespace Components
if (Monitor::IsEnabled())
{
std::string nodes = Utils::IO::ReadFile("players/nodes_default.dat");
if (nodes.empty() || !list.ParseFromString(Utils::Compression::ZLib::Decompress(nodes))) return;
if (nodes.empty() || !list.ParseFromString(Utils::Compression::Deflate::ZStd::Decompress(nodes))) return;
}
else
{
FileSystem::File defaultNodes("nodes_default.dat");
if (!defaultNodes.exists() || !list.ParseFromString(Utils::Compression::ZLib::Decompress(defaultNodes.getBuffer()))) return;
if (!defaultNodes.exists() || !list.ParseFromString(Utils::Compression::Deflate::ZStd::Decompress(defaultNodes.getBuffer()))) return;
}
for (int i = 0; i < list.nodes_size(); ++i)
@ -91,7 +91,7 @@ namespace Components
{
Proto::Node::List list;
std::string nodes = Utils::IO::ReadFile("players/nodes.dat");
if (nodes.empty() || !list.ParseFromString(Utils::Compression::ZLib::Decompress(nodes))) return;
if (nodes.empty() || !list.ParseFromString(Utils::Compression::Deflate::ZStd::Decompress(nodes))) return;
for (int i = 0; i < list.nodes_size(); ++i)
{
@ -125,10 +125,9 @@ namespace Components
}
}
Utils::IO::WriteFile("players/nodes.dat", Utils::Compression::ZLib::Compress(list.SerializeAsString()));
Utils::IO::WriteFile("players/nodes.dat", Utils::Compression::Deflate::ZStd::Compress(list.SerializeAsString()));
}
void Node::Add(Network::Address address)
{
#ifndef DEBUG

4
src/Components/Modules/Playlist.cpp
View File

@ -57,7 +57,7 @@ namespace Components
Logger::Print("Received playlist request, sending currently stored buffer.\n");
std::string compressedList = Utils::Compression::ZLib::Compress(Playlist::CurrentPlaylistBuffer);
std::string compressedList = Utils::Compression::Deflate::ZStd::Compress(Playlist::CurrentPlaylistBuffer);
Proto::Party::Playlist list;
list.set_hash(Utils::Cryptography::JenkinsOneAtATime::Compute(compressedList));
@ -95,7 +95,7 @@ namespace Components
}
// Decompress buffer
Playlist::ReceivedPlaylistBuffer = Utils::Compression::ZLib::Decompress(compressedData);
Playlist::ReceivedPlaylistBuffer = Utils::Compression::Deflate::ZStd::Decompress(compressedData);
// Load and continue connection
Logger::Print("Received playlist, loading and continuing connection...\n");

28
src/Components/Modules/QuickPatch.cpp
View File

@ -718,8 +718,32 @@ namespace Components
for (int i = 0; i < 21; ++i)
{
std::string compressed = Utils::Compression::ZLib::Compress(test);
std::string decompressed = Utils::Compression::ZLib::Decompress(compressed);
std::string compressed = Utils::Compression::Deflate::ZLib::Compress(test);
std::string decompressed = Utils::Compression::Deflate::ZLib::Decompress(compressed);
if (test != decompressed)
{
printf("Error\n");
printf("Compressing %d bytes and decompressing failed!\n", test.size());
return false;
}
auto size = test.size();
for (unsigned int j = 0; j < size; ++j)
{
test.append(Utils::String::VA("%c", Utils::Cryptography::Rand::GenerateInt()));
}
}
printf("Success\n");
printf("Testing ZStd compression...");
test = Utils::String::VA("%c", Utils::Cryptography::Rand::GenerateInt());
for (int i = 0; i < 21; ++i)
{
std::string compressed = Utils::Compression::Deflate::ZStd::Compress(test);
std::string decompressed = Utils::Compression::Deflate::ZStd::Decompress(compressed);
if (test != decompressed)
{

7
src/Components/Modules/ZoneBuilder.cpp
View File

@ -407,7 +407,12 @@ namespace Components
}
#endif
zoneBuffer = Utils::Compression::ZLib::Compress(zoneBuffer);
#ifdef DEBUG
zoneBuffer = Utils::Compression::Deflate::ZLib::Compress(zoneBuffer);
#else
zoneBuffer = Utils::Compression::Deflate::ZStd::Compress(zoneBuffer);
#endif
outBuffer.append(zoneBuffer);
std::string outFile = "zone/" + this->zoneName + ".ff";

2
src/Components/Modules/ZoneBuilder.hpp
View File

@ -4,7 +4,7 @@
#define XFILE_VERSION 276
#define XFILE_HEADER_IW4X 0x78345749 // 'IW4x'
#define XFILE_VERSION_IW4X 3
#define XFILE_VERSION_IW4X 4
namespace Components
{

7
src/STDInclude.hpp
View File

@ -73,7 +73,12 @@ template <size_t S> class Sizer { };
#pragma warning(disable: 6386)
#pragma warning(disable: 6387)
#include <zlib.h>
// #include <zlib.h>
#include <zstd.h>
#define ZWRAP_USE_ZSTD 1
#include <zstd_zlibwrapper.h>
#include <curses.h>
#include <mongoose.h>
#include <json11.hpp>

46
src/Utils/Compression.cpp
View File

@ -4,7 +4,9 @@ namespace Utils
{
namespace Compression
{
std::string ZLib::Compress(std::string data)
std::mutex Deflate::Mutex;
std::string Deflate::Compress(std::string data)
{
Utils::Memory::Allocator allocator;
unsigned long length = (data.size() * 2);
@ -15,7 +17,8 @@ namespace Utils
char* buffer = allocator.allocateArray<char>(length);
if (compress2(reinterpret_cast<Bytef*>(buffer), &length, reinterpret_cast<Bytef*>(const_cast<char*>(data.data())), data.size(), Z_BEST_COMPRESSION) != Z_OK)
int level = (ZWRAP_isUsingZSTDcompression() ? ZSTD_maxCLevel() : Z_BEST_COMPRESSION);
if (compress2(reinterpret_cast<Bytef*>(buffer), &length, reinterpret_cast<Bytef*>(const_cast<char*>(data.data())), data.size(), level) != Z_OK)
{
return "";
}
@ -26,7 +29,7 @@ namespace Utils
return data;
}
std::string ZLib::Decompress(std::string data)
std::string Deflate::Decompress(std::string data)
{
z_stream stream;
ZeroMemory(&stream, sizeof(stream));
@ -70,5 +73,42 @@ namespace Utils
inflateEnd(&stream);
return buffer;
}
std::string Deflate::ZLib::Compress(std::string data)
{
Deflate::Semaphore _(DEFLATE_ZLIB);
return Deflate::Compress(data);
}
std::string Deflate::ZLib::Decompress(std::string data)
{
Deflate::Semaphore _(DEFLATE_ZLIB);
return Deflate::Decompress(data);
}
std::string Deflate::ZStd::Compress(std::string data)
{
Deflate::Semaphore _(DEFLATE_ZSTD);
return Deflate::Compress(data);
}
std::string Deflate::ZStd::Decompress(std::string data)
{
Deflate::Semaphore _(DEFLATE_ZSTD);
return Deflate::Decompress(data);
}
Deflate::Semaphore::Semaphore(bool zstd)
{
Deflate::Mutex.lock();
this->state = ZWRAP_isUsingZSTDcompression();
ZWRAP_useZSTDcompression(zstd);
}
Deflate::Semaphore::~Semaphore()
{
ZWRAP_useZSTDcompression(this->state);
Deflate::Mutex.unlock();
}
};
}

30
src/Utils/Compression.hpp
View File

@ -1,14 +1,42 @@
#pragma once
#define CHUNK 16384
#define DEFLATE_ZLIB false
#define DEFLATE_ZSTD true
namespace Utils
{
namespace Compression
{
class ZLib
class Deflate
{
public:
class ZLib
{
public:
static std::string Compress(std::string data);
static std::string Decompress(std::string data);
};
class ZStd
{
public:
static std::string Compress(std::string data);
static std::string Decompress(std::string data);
};
class Semaphore
{
public:
Semaphore(bool zstd);
~Semaphore();
private:
int state;
};
private:
static std::mutex Mutex;
static std::string Compress(std::string data);
static std::string Decompress(std::string data);
};