Better file hash system

This commit is contained in:
fed 2023-12-12 17:17:47 +01:00
parent 80597dd07e
commit 0ccde7640a
No known key found for this signature in database
GPG Key ID: 1D2C630F04722996
4 changed files with 153 additions and 65 deletions

View File

@ -17,6 +17,8 @@
#include "steam/steam.hpp"
#include "utils/hash.hpp"
#include <utils/properties.hpp>
#include <utils/string.hpp>
#include <utils/info_string.hpp>
@ -50,17 +52,17 @@ namespace party
// snake case these names before release
std::vector<usermap_file> usermap_files =
{
{".ff", "usermaphash", false},
{"_load.ff", "usermaploadhash", true},
{".arena", "usermaparenahash", true},
{".pak", "usermappakhash", true},
{".ff", "usermap_hash", false},
{"_load.ff", "usermap_load_hash", true},
{".arena", "usermap_arena_hash", true},
{".pak", "usermap_pak_hash", true},
};
std::vector<usermap_file> mod_files =
{
{".ff", "modHash", false},
{"_pre_gfx.ff", "modpregfxhash", true},
{".pak", "modpakhash", true},
{".ff", "mod_hash", false},
{"_pre_gfx.ff", "mod_pre_gfx_hash", true},
{".pak", "mod_pak_hash", true},
};
struct
@ -199,71 +201,23 @@ namespace party
}
std::unordered_map<std::string, std::string> hash_cache;
std::string get_file_hash(const std::string& file, bool use_threads = false)
std::string get_file_hash(const std::string& file)
{
const auto iter = hash_cache.find(file);
if (iter != hash_cache.end())
{
const auto iter = hash_cache.find(file);
if (iter != hash_cache.end())
{
return iter->second;
}
return iter->second;
}
constexpr std::streamsize max_block_size = 1 * 1024 * 1024; // 1MB
std::ifstream file_stream(file, std::ios::binary);
if (!file_stream.good())
{
return {};
}
console::debug("[HASH::%s]: computing hash...\n", file.data());
file_stream.seekg(0, std::ios::end);
std::streampos file_size = file_stream.tellg();
file_stream.seekg(0, std::ios::beg);
uLong crc = crc32(0L, Z_NULL, 0);
if (file_size <= max_block_size)
{
std::string buffer(file_size, '\0');
file_stream.read(&buffer[0], file_size);
crc = crc32(crc, reinterpret_cast<const Bytef*>(buffer.data()), static_cast<uInt>(file_size));
}
else
{
constexpr std::streamsize buffer_size = max_block_size;
std::vector<char> buffer(buffer_size);
std::streamsize remaining_size = file_size;
while (remaining_size > 0)
{
std::streamsize block_size = std::min(remaining_size, buffer_size);
file_stream.read(buffer.data(), block_size);
crc = crc32(crc, reinterpret_cast<const Bytef*>(buffer.data()), static_cast<uInt>(block_size));
remaining_size -= block_size;
}
}
file_stream.close();
// Convert the hash to a string
std::string hash = std::to_string(crc);
{
hash_cache[file] = hash;
}
console::debug("[HASH::%s]: done. (%s)\n", file.data(), hash.data());
const auto hash = utils::hash::get_file_hash(file);
hash_cache.insert(std::make_pair(file, hash));
return hash;
}
std::string get_usermap_file_path(const std::string& mapname, const std::string& extension)
{
return utils::string::va("usermaps\\%s\\%s%s", mapname.data(), mapname.data(), extension.data());
return std::format("usermaps\\{}\\{}{}", mapname, mapname, extension);
}
// generate hashes so they are cached
@ -282,7 +236,8 @@ namespace party
{
for (const auto& file : mod_files)
{
get_file_hash(utils::string::va("%s/mod%s", fs_game.data(), file.extension.data()));
const auto path = std::format("{}\\mod{}", fs_game, file.extension);
get_file_hash(path);
}
}
}
@ -1074,7 +1029,7 @@ namespace party
return;
}
auto hash = get_file_hash(params.get(1), std::atoi(params.get(2)));
const auto hash = get_file_hash(params.get(1));
console::info("hash output: %s\n", hash.data());
});

View File

@ -2401,6 +2401,25 @@ namespace game
assert_offsetof(GfxWorld, mdaoVolumes, 2792);
assert_offsetof(GfxWorld, buildInfo, 2832);
struct DB_AuthSignature
{
unsigned char bytes[256];
};
struct DB_AuthHash
{
unsigned char bytes[32];
};
struct XPakHeader
{
char header[8];
std::int32_t version;
unsigned char unknown[16];
DB_AuthHash hash;
DB_AuthSignature signature;
};
namespace hks
{
struct lua_State;

108
src/client/utils/hash.cpp Normal file
View File

@ -0,0 +1,108 @@
#include <std_include.hpp>
#include "game/game.hpp"
#include "hash.hpp"
#include <zlib.h>
#include <utils/cryptography.hpp>
#include <utils/string.hpp>
namespace utils::hash
{
namespace
{
constexpr auto read_buffer_size = 1ull * 1024ull * 1024ull; // 1MB
std::string get_file_hash_pakfile(std::ifstream& file_stream, const std::size_t file_size,
const std::string& filename)
{
if (file_size < sizeof(game::XPakHeader))
{
return {};
}
game::XPakHeader header{};
file_stream.read(reinterpret_cast<char*>(&header), sizeof(game::XPakHeader));
constexpr auto hash_size = sizeof(game::DB_AuthHash);
game::DB_AuthHash empty_hash{};
if (!std::memcmp(header.hash.bytes, empty_hash.bytes, hash_size))
{
hash_state state;
sha256_init(&state);
std::string buffer;
buffer.resize(read_buffer_size);
auto bytes_to_read = file_size - sizeof(game::XPakHeader);
while (bytes_to_read > 0)
{
const auto read_size = std::min(read_buffer_size, bytes_to_read);
file_stream.read(buffer.data(), read_size);
sha256_process(&state, reinterpret_cast<std::uint8_t*>(buffer.data()), static_cast<std::uint32_t>(read_size));
bytes_to_read -= read_size;
}
file_stream.close();
if (sha256_done(&state, header.hash.bytes) != CRYPT_OK)
{
return {};
}
std::fstream out_stream;
out_stream.open(filename, std::ios_base::binary | std::ios_base::out | std::ios_base::in);
out_stream.write(reinterpret_cast<char*>(&header), sizeof(game::XPakHeader));
}
const auto hash_str = std::string{header.hash.bytes, header.hash.bytes + hash_size};
return utils::string::dump_hex(hash_str, "");
}
std::string get_file_hash_generic(std::ifstream& file_stream, const std::size_t file_size)
{
auto crc_value = crc32(0L, Z_NULL, 0);
auto bytes_to_read = file_size;
std::string buffer;
buffer.resize(read_buffer_size);
while (bytes_to_read > 0)
{
const auto read_size = std::min(bytes_to_read, read_buffer_size);
file_stream.read(buffer.data(), read_size);
crc_value = crc32(crc_value, reinterpret_cast<std::uint8_t*>(buffer.data()), static_cast<std::uint32_t>(read_size));
bytes_to_read -= read_size;
}
std::string hash;
hash.append(reinterpret_cast<char*>(&crc_value), sizeof(crc_value));
return utils::string::dump_hex(hash, "");
}
}
std::string get_file_hash(const std::string& file)
{
std::ifstream file_stream(file, std::ios::binary);
if (!file_stream.is_open())
{
return {};
}
file_stream.seekg(0, std::ios::end);
const auto file_size = static_cast<std::size_t>(file_stream.tellg());
file_stream.seekg(0, std::ios::beg);
if (file.ends_with(".pak"))
{
return get_file_hash_pakfile(file_stream, file_size, file);
}
else
{
return get_file_hash_generic(file_stream, file_size);
}
}
}

View File

@ -0,0 +1,6 @@
#pragma once
namespace utils::hash
{
std::string get_file_hash(const std::string& file);
}