Rim/iw4x-client
Rim/iw4x-client
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 2 Wiki Activity

[ZoneBuilder] Use ZSTD for fastfile compression

Browse Source
This commit is contained in:
momo5502 2017-06-26 20:30:29 +02:00
parent 19f49a0273
commit a011022da5
10 changed files with 172 additions and 86 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

68
src/Components/Modules/FastFiles.cpp
View File

@ -6,6 +6,9 @@ namespace Components
symmetric_CTR FastFiles::CurrentCTR;
std::vector<std::string> FastFiles::ZonePaths;
bool FastFiles::UseZstd = false;
Utils::Compression::Deflate::Semaphore* FastFiles::ZlibLock = nullptr;
bool FastFiles::IsIW4xZone = false;
bool FastFiles::StreamRead = false;
@ -333,15 +336,22 @@ namespace Components
void FastFiles::ReadHeaderStub(unsigned int* header, int size)
{
FastFiles::UseZstd = false;
FastFiles::IsIW4xZone = false;
FastFiles::LastByteRead = 0;
Game::DB_ReadXFileUncompressed(header, size);
if (header[0] == XFILE_HEADER_IW4X)
{
FastFiles::UseZstd = true;
FastFiles::IsIW4xZone = true;
if (header[1] < XFILE_VERSION_IW4X)
static_assert((XFILE_VERSION_IW4X - 1) == 3, "FastFile backwards-compatibility not granted!");
if (header[1] == XFILE_VERSION_IW4X - 1)
{
FastFiles::UseZstd = false;
}
else if (header[1] < XFILE_VERSION_IW4X)
{
Logger::Error("The fastfile you are trying to load is outdated (%d, expected %d)", header[1], XFILE_VERSION_IW4X);
}
@ -396,8 +406,9 @@ namespace Components
ctr_decrypt(strm->next_in, const_cast<unsigned char*>(strm->next_in), strm->avail_in, &FastFiles::CurrentCTR);
}
return Utils::Hook::Call<int(z_streamp, const char*, int)>(0x4D8090)(strm, version, stream_size);
//return Utils::Hook::Call<int(z_streamp, const char*, int)>(0x4D8090)(strm, version, stream_size);
//return inflateInit_(strm, version, stream_size);
return FastFiles::InflateInitStub(strm, version, stream_size);
}
void FastFiles::AuthLoadInflateDecryptBaseFunc(unsigned char* buffer)
@ -484,6 +495,44 @@ namespace Components
}
#endif
int FastFiles::InflateInitStub(z_streamp strm, const char *version, int stream_size)
{
if (FastFiles::UseZstd)
{
if (FastFiles::ZlibLock) delete FastFiles::ZlibLock;
FastFiles::ZlibLock = new Utils::Compression::Deflate::Semaphore(DEFLATE_ZSTD);
return inflateInit_(strm, version, stream_size);
}
return Utils::Hook::Call<int(z_streamp, const char*, int)>(0x4D8090)(strm, version, stream_size);
}
int FastFiles::InflateStub(z_streamp strm, int flush)
{
if (FastFiles::UseZstd)
{
return inflate(strm, flush);
}
return Utils::Hook::Call<int(z_streamp, int)>(0x49EA00)(strm, flush);
}
int FastFiles::InflateEndStub(z_streamp strm)
{
if (FastFiles::UseZstd && FastFiles::ZlibLock)
{
int result = inflateEnd(strm);
delete FastFiles::ZlibLock;
FastFiles::ZlibLock = nullptr;
return result;
}
return Utils::Hook::Call<int(z_streamp)>(0x453750)(strm);
}
FastFiles::FastFiles()
{
Dvar::Register<bool>("ui_zoneDebug", false, Game::dvar_flag::DVAR_FLAG_SAVED, "Display current loaded zone.");
@ -554,9 +603,12 @@ namespace Components
Utils::Hook(0x4159E2, FastFiles::ReadXFileHeader, HOOK_CALL).install()->quick();
// Replace internal ZLib
//Utils::Hook(0x44B160, inflateInit2_, HOOK_JUMP).install()->quick();
//Utils::Hook(0x453750, inflateEnd, HOOK_JUMP).install()->quick();
//Utils::Hook(0x49EA00, inflate, HOOK_JUMP).install()->quick();
//Utils::Hook(0x4D0306, FastFiles::InflateInitStub, HOOK_CALL).install()->quick();
Utils::Hook(0x4D034B, FastFiles::InflateInitStub, HOOK_JUMP).install()->quick();
Utils::Hook(0x480A1A, FastFiles::InflateStub, HOOK_JUMP).install()->quick();
Utils::Hook(0x5B99DE, FastFiles::InflateStub, HOOK_CALL).install()->quick();
Utils::Hook(0x449D8F, FastFiles::InflateEndStub, HOOK_CALL).install()->quick();
Utils::Hook(0x449DA3, FastFiles::InflateEndStub, HOOK_CALL).install()->quick();
// Add custom zone paths
FastFiles::AddZonePath("zone\\patch\\");
@ -618,6 +670,12 @@ namespace Components
FastFiles::~FastFiles()
{
if (FastFiles::ZlibLock)
{
delete FastFiles::ZlibLock;
FastFiles::ZlibLock = nullptr;
}
FastFiles::ZonePaths.clear();
}
}

7
src/Components/Modules/FastFiles.hpp
View File

@ -34,6 +34,9 @@ namespace Components
static unsigned int CurrentZone;
static unsigned int MaxZones;
static bool UseZstd;
static Utils::Compression::Deflate::Semaphore* ZlibLock;
static bool IsIW4xZone;
static bool StreamRead;
@ -63,6 +66,10 @@ namespace Components
static void ReadXFile(void* buffer, int size);
static void ReadXFileStub(char* buffer, int size);
static int InflateInitStub(z_streamp strm, const char *version, int stream_size);
static int InflateStub(z_streamp strm, int flush);
static int InflateEndStub(z_streamp strm);
#ifdef DEBUG
static void LogStreamRead(int len);
#endif

9
src/Components/Modules/Node.cpp
View File

@ -68,12 +68,12 @@ namespace Components
if (Monitor::IsEnabled())
{
std::string nodes = Utils::IO::ReadFile("players/nodes_default.dat");
if (nodes.empty() || !list.ParseFromString(Utils::Compression::ZLib::Decompress(nodes))) return;
if (nodes.empty() || !list.ParseFromString(Utils::Compression::Deflate::ZStd::Decompress(nodes))) return;
}
else
{
FileSystem::File defaultNodes("nodes_default.dat");
if (!defaultNodes.exists() || !list.ParseFromString(Utils::Compression::ZLib::Decompress(defaultNodes.getBuffer()))) return;
if (!defaultNodes.exists() || !list.ParseFromString(Utils::Compression::Deflate::ZStd::Decompress(defaultNodes.getBuffer()))) return;
}
for (int i = 0; i < list.nodes_size(); ++i)
@ -91,7 +91,7 @@ namespace Components
{
Proto::Node::List list;
std::string nodes = Utils::IO::ReadFile("players/nodes.dat");
if (nodes.empty() || !list.ParseFromString(Utils::Compression::ZLib::Decompress(nodes))) return;
if (nodes.empty() || !list.ParseFromString(Utils::Compression::Deflate::ZStd::Decompress(nodes))) return;
for (int i = 0; i < list.nodes_size(); ++i)
{
@ -125,10 +125,9 @@ namespace Components
}
}
Utils::IO::WriteFile("players/nodes.dat", Utils::Compression::ZLib::Compress(list.SerializeAsString()));
Utils::IO::WriteFile("players/nodes.dat", Utils::Compression::Deflate::ZStd::Compress(list.SerializeAsString()));
}
void Node::Add(Network::Address address)
{
#ifndef DEBUG

4
src/Components/Modules/Playlist.cpp
View File

@ -57,7 +57,7 @@ namespace Components
Logger::Print("Received playlist request, sending currently stored buffer.\n");
std::string compressedList = Utils::Compression::ZLib::Compress(Playlist::CurrentPlaylistBuffer);
std::string compressedList = Utils::Compression::Deflate::ZStd::Compress(Playlist::CurrentPlaylistBuffer);
Proto::Party::Playlist list;
list.set_hash(Utils::Cryptography::JenkinsOneAtATime::Compute(compressedList));
@ -95,7 +95,7 @@ namespace Components
}
// Decompress buffer
Playlist::ReceivedPlaylistBuffer = Utils::Compression::ZLib::Decompress(compressedData);
Playlist::ReceivedPlaylistBuffer = Utils::Compression::Deflate::ZStd::Decompress(compressedData);
// Load and continue connection
Logger::Print("Received playlist, loading and continuing connection...\n");

28
src/Components/Modules/QuickPatch.cpp
View File

@ -718,8 +718,32 @@ namespace Components
for (int i = 0; i < 21; ++i)
{
std::string compressed = Utils::Compression::ZLib::Compress(test);
std::string decompressed = Utils::Compression::ZLib::Decompress(compressed);
std::string compressed = Utils::Compression::Deflate::ZLib::Compress(test);
std::string decompressed = Utils::Compression::Deflate::ZLib::Decompress(compressed);
if (test != decompressed)
{
printf("Error\n");
printf("Compressing %d bytes and decompressing failed!\n", test.size());
return false;
}
auto size = test.size();
for (unsigned int j = 0; j < size; ++j)
{
test.append(Utils::String::VA("%c", Utils::Cryptography::Rand::GenerateInt()));
}
}
printf("Success\n");
printf("Testing ZStd compression...");
test = Utils::String::VA("%c", Utils::Cryptography::Rand::GenerateInt());
for (int i = 0; i < 21; ++i)
{
std::string compressed = Utils::Compression::Deflate::ZStd::Compress(test);
std::string decompressed = Utils::Compression::Deflate::ZStd::Decompress(compressed);
if (test != decompressed)
{

7
src/Components/Modules/ZoneBuilder.cpp
View File

@ -407,7 +407,12 @@ namespace Components
}
#endif
zoneBuffer = Utils::Compression::ZStd::Compress(zoneBuffer);
#ifdef DEBUG
zoneBuffer = Utils::Compression::Deflate::ZLib::Compress(zoneBuffer);
#else
zoneBuffer = Utils::Compression::Deflate::ZStd::Compress(zoneBuffer);
#endif
outBuffer.append(zoneBuffer);
std::string outFile = "zone/" + this->zoneName + ".ff";

2
src/Components/Modules/ZoneBuilder.hpp
View File

@ -4,7 +4,7 @@
#define XFILE_VERSION 276
#define XFILE_HEADER_IW4X 0x78345749 // 'IW4x'
#define XFILE_VERSION_IW4X 3
#define XFILE_VERSION_IW4X 4
namespace Components
{

8
src/STDInclude.hpp
View File

@ -73,11 +73,11 @@ template <size_t S> class Sizer { };
#pragma warning(disable: 6386)
#pragma warning(disable: 6387)
#include <zlib.h>
#include <zstd.h>
// #include <zlib.h>
// #include <zstd.h>
//#define ZWRAP_USE_ZSTD 1
//#include <zstd_zlibwrapper.h>
#define ZWRAP_USE_ZSTD 1
#include <zstd_zlibwrapper.h>
#include <curses.h>
#include <mongoose.h>

90
src/Utils/Compression.cpp
View File

@ -4,7 +4,9 @@ namespace Utils
{
namespace Compression
{
std::string ZLib::Compress(std::string data)
std::mutex Deflate::Mutex;
std::string Deflate::Compress(std::string data)
{
Utils::Memory::Allocator allocator;
unsigned long length = (data.size() * 2);
@ -26,7 +28,7 @@ namespace Utils
return data;
}
std::string ZLib::Decompress(std::string data)
std::string Deflate::Decompress(std::string data)
{
z_stream stream;
ZeroMemory(&stream, sizeof(stream));
@ -71,71 +73,41 @@ namespace Utils
return buffer;
}
std::string ZStd::Compress(std::string data)
std::string Deflate::ZLib::Compress(std::string data)
{
Utils::Memory::Allocator allocator;
size_t length = (ZSTD_compressBound(data.size() + 1) + 1) * 2;
char* buffer = allocator.allocateArray<char>(length);
length = ZSTD_compress(buffer, length, data.data(), data.size(), ZSTD_maxCLevel());
if (length <= 0/* || ZSTD_isError()*/)
{
return "";
}
data.clear();
data.append(buffer, length);
return data;
Deflate::Semaphore _(DEFLATE_ZLIB);
return Deflate::Compress(data);
}
std::string ZStd::Decompress(std::string data)
std::string Deflate::ZLib::Decompress(std::string data)
{
z_stream stream;
ZeroMemory(&stream, sizeof(stream));
std::string buffer;
Deflate::Semaphore _(DEFLATE_ZLIB);
return Deflate::Decompress(data);
}
if (inflateInit(&stream) != Z_OK)
{
return "";
}
std::string Deflate::ZStd::Compress(std::string data)
{
Deflate::Semaphore _(DEFLATE_ZSTD);
return Deflate::Compress(data);
}
int ret;
Utils::Memory::Allocator allocator;
std::string Deflate::ZStd::Decompress(std::string data)
{
Deflate::Semaphore _(DEFLATE_ZSTD);
return Deflate::Decompress(data);
}
uint8_t* dest = allocator.allocateArray<uint8_t>(CHUNK);
const char* dataPtr = data.data();
Deflate::Semaphore::Semaphore(bool zstd)
{
Deflate::Mutex.lock();
this->state = ZWRAP_isUsingZSTDcompression();
ZWRAP_useZSTDcompression(zstd);
}
do
{
stream.avail_in = std::min(static_cast<size_t>(CHUNK), data.size() - (dataPtr - data.data()));
stream.next_in = reinterpret_cast<const uint8_t*>(dataPtr);
dataPtr += stream.avail_in;
do
{
stream.avail_out = CHUNK;
stream.next_out = dest;
ret = inflate(&stream, Z_NO_FLUSH);
if (ret != Z_OK && ret != Z_STREAM_END)
{
inflateEnd(&stream);
return "";
}
buffer.append(reinterpret_cast<const char*>(dest), CHUNK - stream.avail_out);
} while (stream.avail_out == 0);
} while (ret != Z_STREAM_END);
inflateEnd(&stream);
return buffer;
Deflate::Semaphore::~Semaphore()
{
ZWRAP_useZSTDcompression(this->state);
Deflate::Mutex.unlock();
}
};
}

35
src/Utils/Compression.hpp
View File

@ -1,21 +1,42 @@
#pragma once
#define CHUNK 16384
#define DEFLATE_ZLIB false
#define DEFLATE_ZSTD true
namespace Utils
{
namespace Compression
{
class ZLib
class Deflate
{
public:
static std::string Compress(std::string data);
static std::string Decompress(std::string data);
};
class ZLib
{
public:
static std::string Compress(std::string data);
static std::string Decompress(std::string data);
};
class ZStd
{
public:
class ZStd
{
public:
static std::string Compress(std::string data);
static std::string Decompress(std::string data);
};
class Semaphore
{
public:
Semaphore(bool zstd);
~Semaphore();
private:
int state;
};
private:
static std::mutex Mutex;
static std::string Compress(std::string data);
static std::string Decompress(std::string data);
};