Better file hash system
This commit is contained in:
parent
80597dd07e
commit
0ccde7640a
@ -17,6 +17,8 @@
|
||||
|
||||
#include "steam/steam.hpp"
|
||||
|
||||
#include "utils/hash.hpp"
|
||||
|
||||
#include <utils/properties.hpp>
|
||||
#include <utils/string.hpp>
|
||||
#include <utils/info_string.hpp>
|
||||
@ -50,17 +52,17 @@ namespace party
|
||||
// snake case these names before release
|
||||
std::vector<usermap_file> usermap_files =
|
||||
{
|
||||
{".ff", "usermaphash", false},
|
||||
{"_load.ff", "usermaploadhash", true},
|
||||
{".arena", "usermaparenahash", true},
|
||||
{".pak", "usermappakhash", true},
|
||||
{".ff", "usermap_hash", false},
|
||||
{"_load.ff", "usermap_load_hash", true},
|
||||
{".arena", "usermap_arena_hash", true},
|
||||
{".pak", "usermap_pak_hash", true},
|
||||
};
|
||||
|
||||
std::vector<usermap_file> mod_files =
|
||||
{
|
||||
{".ff", "modHash", false},
|
||||
{"_pre_gfx.ff", "modpregfxhash", true},
|
||||
{".pak", "modpakhash", true},
|
||||
{".ff", "mod_hash", false},
|
||||
{"_pre_gfx.ff", "mod_pre_gfx_hash", true},
|
||||
{".pak", "mod_pak_hash", true},
|
||||
};
|
||||
|
||||
struct
|
||||
@ -199,71 +201,23 @@ namespace party
|
||||
}
|
||||
|
||||
std::unordered_map<std::string, std::string> hash_cache;
|
||||
std::string get_file_hash(const std::string& file, bool use_threads = false)
|
||||
|
||||
std::string get_file_hash(const std::string& file)
|
||||
{
|
||||
const auto iter = hash_cache.find(file);
|
||||
if (iter != hash_cache.end())
|
||||
{
|
||||
const auto iter = hash_cache.find(file);
|
||||
if (iter != hash_cache.end())
|
||||
{
|
||||
return iter->second;
|
||||
}
|
||||
return iter->second;
|
||||
}
|
||||
|
||||
constexpr std::streamsize max_block_size = 1 * 1024 * 1024; // 1MB
|
||||
|
||||
std::ifstream file_stream(file, std::ios::binary);
|
||||
|
||||
if (!file_stream.good())
|
||||
{
|
||||
return {};
|
||||
}
|
||||
|
||||
console::debug("[HASH::%s]: computing hash...\n", file.data());
|
||||
|
||||
file_stream.seekg(0, std::ios::end);
|
||||
std::streampos file_size = file_stream.tellg();
|
||||
file_stream.seekg(0, std::ios::beg);
|
||||
|
||||
uLong crc = crc32(0L, Z_NULL, 0);
|
||||
|
||||
if (file_size <= max_block_size)
|
||||
{
|
||||
std::string buffer(file_size, '\0');
|
||||
file_stream.read(&buffer[0], file_size);
|
||||
crc = crc32(crc, reinterpret_cast<const Bytef*>(buffer.data()), static_cast<uInt>(file_size));
|
||||
}
|
||||
else
|
||||
{
|
||||
constexpr std::streamsize buffer_size = max_block_size;
|
||||
std::vector<char> buffer(buffer_size);
|
||||
|
||||
std::streamsize remaining_size = file_size;
|
||||
while (remaining_size > 0)
|
||||
{
|
||||
std::streamsize block_size = std::min(remaining_size, buffer_size);
|
||||
file_stream.read(buffer.data(), block_size);
|
||||
crc = crc32(crc, reinterpret_cast<const Bytef*>(buffer.data()), static_cast<uInt>(block_size));
|
||||
remaining_size -= block_size;
|
||||
}
|
||||
}
|
||||
|
||||
file_stream.close();
|
||||
|
||||
// Convert the hash to a string
|
||||
std::string hash = std::to_string(crc);
|
||||
|
||||
{
|
||||
hash_cache[file] = hash;
|
||||
}
|
||||
|
||||
console::debug("[HASH::%s]: done. (%s)\n", file.data(), hash.data());
|
||||
|
||||
const auto hash = utils::hash::get_file_hash(file);
|
||||
hash_cache.insert(std::make_pair(file, hash));
|
||||
return hash;
|
||||
}
|
||||
|
||||
std::string get_usermap_file_path(const std::string& mapname, const std::string& extension)
|
||||
{
|
||||
return utils::string::va("usermaps\\%s\\%s%s", mapname.data(), mapname.data(), extension.data());
|
||||
return std::format("usermaps\\{}\\{}{}", mapname, mapname, extension);
|
||||
}
|
||||
|
||||
// generate hashes so they are cached
|
||||
@ -282,7 +236,8 @@ namespace party
|
||||
{
|
||||
for (const auto& file : mod_files)
|
||||
{
|
||||
get_file_hash(utils::string::va("%s/mod%s", fs_game.data(), file.extension.data()));
|
||||
const auto path = std::format("{}\\mod{}", fs_game, file.extension);
|
||||
get_file_hash(path);
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1074,7 +1029,7 @@ namespace party
|
||||
return;
|
||||
}
|
||||
|
||||
auto hash = get_file_hash(params.get(1), std::atoi(params.get(2)));
|
||||
const auto hash = get_file_hash(params.get(1));
|
||||
console::info("hash output: %s\n", hash.data());
|
||||
});
|
||||
|
||||
|
@ -2401,6 +2401,25 @@ namespace game
|
||||
assert_offsetof(GfxWorld, mdaoVolumes, 2792);
|
||||
assert_offsetof(GfxWorld, buildInfo, 2832);
|
||||
|
||||
struct DB_AuthSignature
|
||||
{
|
||||
unsigned char bytes[256];
|
||||
};
|
||||
|
||||
struct DB_AuthHash
|
||||
{
|
||||
unsigned char bytes[32];
|
||||
};
|
||||
|
||||
struct XPakHeader
|
||||
{
|
||||
char header[8];
|
||||
std::int32_t version;
|
||||
unsigned char unknown[16];
|
||||
DB_AuthHash hash;
|
||||
DB_AuthSignature signature;
|
||||
};
|
||||
|
||||
namespace hks
|
||||
{
|
||||
struct lua_State;
|
||||
|
108
src/client/utils/hash.cpp
Normal file
108
src/client/utils/hash.cpp
Normal file
@ -0,0 +1,108 @@
|
||||
#include <std_include.hpp>
|
||||
|
||||
#include "game/game.hpp"
|
||||
|
||||
#include "hash.hpp"
|
||||
|
||||
#include <zlib.h>
|
||||
#include <utils/cryptography.hpp>
|
||||
#include <utils/string.hpp>
|
||||
|
||||
namespace utils::hash
|
||||
{
|
||||
namespace
|
||||
{
|
||||
constexpr auto read_buffer_size = 1ull * 1024ull * 1024ull; // 1MB
|
||||
|
||||
std::string get_file_hash_pakfile(std::ifstream& file_stream, const std::size_t file_size,
|
||||
const std::string& filename)
|
||||
{
|
||||
if (file_size < sizeof(game::XPakHeader))
|
||||
{
|
||||
return {};
|
||||
}
|
||||
|
||||
game::XPakHeader header{};
|
||||
file_stream.read(reinterpret_cast<char*>(&header), sizeof(game::XPakHeader));
|
||||
|
||||
constexpr auto hash_size = sizeof(game::DB_AuthHash);
|
||||
|
||||
game::DB_AuthHash empty_hash{};
|
||||
if (!std::memcmp(header.hash.bytes, empty_hash.bytes, hash_size))
|
||||
{
|
||||
hash_state state;
|
||||
sha256_init(&state);
|
||||
|
||||
std::string buffer;
|
||||
buffer.resize(read_buffer_size);
|
||||
|
||||
auto bytes_to_read = file_size - sizeof(game::XPakHeader);
|
||||
while (bytes_to_read > 0)
|
||||
{
|
||||
const auto read_size = std::min(read_buffer_size, bytes_to_read);
|
||||
file_stream.read(buffer.data(), read_size);
|
||||
sha256_process(&state, reinterpret_cast<std::uint8_t*>(buffer.data()), static_cast<std::uint32_t>(read_size));
|
||||
bytes_to_read -= read_size;
|
||||
}
|
||||
|
||||
file_stream.close();
|
||||
|
||||
if (sha256_done(&state, header.hash.bytes) != CRYPT_OK)
|
||||
{
|
||||
return {};
|
||||
}
|
||||
|
||||
std::fstream out_stream;
|
||||
out_stream.open(filename, std::ios_base::binary | std::ios_base::out | std::ios_base::in);
|
||||
out_stream.write(reinterpret_cast<char*>(&header), sizeof(game::XPakHeader));
|
||||
}
|
||||
|
||||
const auto hash_str = std::string{header.hash.bytes, header.hash.bytes + hash_size};
|
||||
|
||||
return utils::string::dump_hex(hash_str, "");
|
||||
}
|
||||
|
||||
std::string get_file_hash_generic(std::ifstream& file_stream, const std::size_t file_size)
|
||||
{
|
||||
auto crc_value = crc32(0L, Z_NULL, 0);
|
||||
auto bytes_to_read = file_size;
|
||||
|
||||
std::string buffer;
|
||||
buffer.resize(read_buffer_size);
|
||||
|
||||
while (bytes_to_read > 0)
|
||||
{
|
||||
const auto read_size = std::min(bytes_to_read, read_buffer_size);
|
||||
file_stream.read(buffer.data(), read_size);
|
||||
crc_value = crc32(crc_value, reinterpret_cast<std::uint8_t*>(buffer.data()), static_cast<std::uint32_t>(read_size));
|
||||
bytes_to_read -= read_size;
|
||||
}
|
||||
|
||||
std::string hash;
|
||||
hash.append(reinterpret_cast<char*>(&crc_value), sizeof(crc_value));
|
||||
return utils::string::dump_hex(hash, "");
|
||||
}
|
||||
}
|
||||
|
||||
std::string get_file_hash(const std::string& file)
|
||||
{
|
||||
std::ifstream file_stream(file, std::ios::binary);
|
||||
if (!file_stream.is_open())
|
||||
{
|
||||
return {};
|
||||
}
|
||||
|
||||
file_stream.seekg(0, std::ios::end);
|
||||
const auto file_size = static_cast<std::size_t>(file_stream.tellg());
|
||||
file_stream.seekg(0, std::ios::beg);
|
||||
|
||||
if (file.ends_with(".pak"))
|
||||
{
|
||||
return get_file_hash_pakfile(file_stream, file_size, file);
|
||||
}
|
||||
else
|
||||
{
|
||||
return get_file_hash_generic(file_stream, file_size);
|
||||
}
|
||||
}
|
||||
}
|
6
src/client/utils/hash.hpp
Normal file
6
src/client/utils/hash.hpp
Normal file
@ -0,0 +1,6 @@
|
||||
#pragma once
|
||||
|
||||
namespace utils::hash
|
||||
{
|
||||
std::string get_file_hash(const std::string& file);
|
||||
}
|
Loading…
Reference in New Issue
Block a user