Better file hash system

src/client/component/party.cpp

@@ -17,6 +17,8 @@
 
 #include "steam/steam.hpp"
 
+#include "utils/hash.hpp"
+
 #include <utils/properties.hpp>
 #include <utils/string.hpp>
 #include <utils/info_string.hpp>
@@ -50,17 +52,17 @@ namespace party
 		// snake case these names before release
 		std::vector<usermap_file> usermap_files =
 		{
-			{".ff", "usermaphash", false},
+			{".ff", "usermap_hash", false},
-			{"_load.ff", "usermaploadhash", true},
+			{"_load.ff", "usermap_load_hash", true},
-			{".arena", "usermaparenahash", true},
+			{".arena", "usermap_arena_hash", true},
-			{".pak", "usermappakhash", true},
+			{".pak", "usermap_pak_hash", true},
 		};
 
 		std::vector<usermap_file> mod_files =
 		{
-			{".ff", "modHash", false},
+			{".ff", "mod_hash", false},
-			{"_pre_gfx.ff", "modpregfxhash", true},
+			{"_pre_gfx.ff", "mod_pre_gfx_hash", true},
-			{".pak", "modpakhash", true},
+			{".pak", "mod_pak_hash", true},
 		};
 
 		struct
@@ -199,71 +201,23 @@ namespace party
 		}
 
 		std::unordered_map<std::string, std::string> hash_cache;
-		std::string get_file_hash(const std::string& file, bool use_threads = false)
+
+		std::string get_file_hash(const std::string& file)
 		{
+			const auto iter = hash_cache.find(file);
+			if (iter != hash_cache.end())
 			{
-				const auto iter = hash_cache.find(file);
+				return iter->second;
-				if (iter != hash_cache.end())
-				{
-					return iter->second;
-				}
 			}
 
-			constexpr std::streamsize max_block_size = 1 * 1024 * 1024; // 1MB
+			const auto hash = utils::hash::get_file_hash(file);
-
+			hash_cache.insert(std::make_pair(file, hash));
-			std::ifstream file_stream(file, std::ios::binary);
-
-			if (!file_stream.good())
-			{
-				return {};
-			}
-
-			console::debug("[HASH::%s]: computing hash...\n", file.data());
-
-			file_stream.seekg(0, std::ios::end);
-			std::streampos file_size = file_stream.tellg();
-			file_stream.seekg(0, std::ios::beg);
-
-			uLong crc = crc32(0L, Z_NULL, 0);
-
-			if (file_size <= max_block_size)
-			{
-				std::string buffer(file_size, '\0');
-				file_stream.read(&buffer[0], file_size);
-				crc = crc32(crc, reinterpret_cast<const Bytef*>(buffer.data()), static_cast<uInt>(file_size));
-			}
-			else
-			{
-				constexpr std::streamsize buffer_size = max_block_size;
-				std::vector<char> buffer(buffer_size);
-
-				std::streamsize remaining_size = file_size;
-				while (remaining_size > 0)
-				{
-					std::streamsize block_size = std::min(remaining_size, buffer_size);
-					file_stream.read(buffer.data(), block_size);
-					crc = crc32(crc, reinterpret_cast<const Bytef*>(buffer.data()), static_cast<uInt>(block_size));
-					remaining_size -= block_size;
-				}
-			}
-
-			file_stream.close();
-
-			// Convert the hash to a string
-			std::string hash = std::to_string(crc);
-
-			{
-				hash_cache[file] = hash;
-			}
-
-			console::debug("[HASH::%s]: done. (%s)\n", file.data(), hash.data());
-
 			return hash;
 		}
 
 		std::string get_usermap_file_path(const std::string& mapname, const std::string& extension)
 		{
-			return utils::string::va("usermaps\\%s\\%s%s", mapname.data(), mapname.data(), extension.data());
+			return std::format("usermaps\\{}\\{}{}", mapname, mapname, extension);
 		}
 
 		// generate hashes so they are cached
@@ -282,7 +236,8 @@ namespace party
 			{
 				for (const auto& file : mod_files)
 				{
-					get_file_hash(utils::string::va("%s/mod%s", fs_game.data(), file.extension.data()));
+					const auto path = std::format("{}\\mod{}", fs_game, file.extension);
+					get_file_hash(path);
 				}
 			}
 		}
@@ -1074,7 +1029,7 @@ namespace party
 					return;
 				}
 
-				auto hash = get_file_hash(params.get(1), std::atoi(params.get(2)));
+				const auto hash = get_file_hash(params.get(1));
 				console::info("hash output: %s\n", hash.data());
 			});
 

src/client/game/structs.hpp

@@ -2401,6 +2401,25 @@ namespace game
 	assert_offsetof(GfxWorld, mdaoVolumes, 2792);
 	assert_offsetof(GfxWorld, buildInfo, 2832);
 
+	struct DB_AuthSignature
+	{
+		unsigned char bytes[256];
+	};
+
+	struct DB_AuthHash
+	{
+		unsigned char bytes[32];
+	};
+
+	struct XPakHeader
+	{
+		char header[8];
+		std::int32_t version;
+		unsigned char unknown[16];
+		DB_AuthHash hash;
+		DB_AuthSignature signature;
+	};
+
 	namespace hks
 	{
 		struct lua_State;

src/client/utils/hash.cpp

@@ -0,0 +1,108 @@
+#include <std_include.hpp>
+
+#include "game/game.hpp"
+
+#include "hash.hpp"
+
+#include <zlib.h>
+#include <utils/cryptography.hpp>
+#include <utils/string.hpp>
+
+namespace utils::hash
+{
+	namespace
+	{
+		constexpr auto read_buffer_size = 1ull * 1024ull * 1024ull; // 1MB
+
+		std::string get_file_hash_pakfile(std::ifstream& file_stream, const std::size_t file_size,
+			const std::string& filename)
+		{
+			if (file_size < sizeof(game::XPakHeader))
+			{
+				return {};
+			}
+
+			game::XPakHeader header{};
+			file_stream.read(reinterpret_cast<char*>(&header), sizeof(game::XPakHeader));
+
+			constexpr auto hash_size = sizeof(game::DB_AuthHash);
+
+			game::DB_AuthHash empty_hash{};
+			if (!std::memcmp(header.hash.bytes, empty_hash.bytes, hash_size))
+			{
+				hash_state state;
+				sha256_init(&state);
+
+				std::string buffer;
+				buffer.resize(read_buffer_size);
+
+				auto bytes_to_read = file_size - sizeof(game::XPakHeader);
+				while (bytes_to_read > 0)
+				{
+					const auto read_size = std::min(read_buffer_size, bytes_to_read);
+					file_stream.read(buffer.data(), read_size);
+					sha256_process(&state, reinterpret_cast<std::uint8_t*>(buffer.data()), static_cast<std::uint32_t>(read_size));
+					bytes_to_read -= read_size;
+				}
+
+				file_stream.close();
+
+				if (sha256_done(&state, header.hash.bytes) != CRYPT_OK)
+				{
+					return {};
+				}
+
+				std::fstream out_stream;
+				out_stream.open(filename, std::ios_base::binary | std::ios_base::out | std::ios_base::in);
+				out_stream.write(reinterpret_cast<char*>(&header), sizeof(game::XPakHeader));
+			}
+
+			const auto hash_str = std::string{header.hash.bytes, header.hash.bytes + hash_size};
+
+			return utils::string::dump_hex(hash_str, "");
+		}
+
+		std::string get_file_hash_generic(std::ifstream& file_stream, const std::size_t file_size)
+		{
+			auto crc_value = crc32(0L, Z_NULL, 0);
+			auto bytes_to_read = file_size;
+
+			std::string buffer;
+			buffer.resize(read_buffer_size);
+
+			while (bytes_to_read > 0)
+			{
+				const auto read_size = std::min(bytes_to_read, read_buffer_size);
+				file_stream.read(buffer.data(), read_size);
+				crc_value = crc32(crc_value, reinterpret_cast<std::uint8_t*>(buffer.data()), static_cast<std::uint32_t>(read_size));
+				bytes_to_read -= read_size;
+			}
+
+			std::string hash;
+			hash.append(reinterpret_cast<char*>(&crc_value), sizeof(crc_value));
+			return utils::string::dump_hex(hash, "");
+		}
+	}
+
+	std::string get_file_hash(const std::string& file)
+	{
+		std::ifstream file_stream(file, std::ios::binary);
+		if (!file_stream.is_open())
+		{
+			return {};
+		}
+
+		file_stream.seekg(0, std::ios::end);
+		const auto file_size = static_cast<std::size_t>(file_stream.tellg());
+		file_stream.seekg(0, std::ios::beg);
+
+		if (file.ends_with(".pak"))
+		{
+			return get_file_hash_pakfile(file_stream, file_size, file);
+		}
+		else
+		{
+			return get_file_hash_generic(file_stream, file_size);
+		}
+	}
+}

src/client/utils/hash.hpp

@@ -0,0 +1,6 @@
+#pragma once
+
+namespace utils::hash
+{
+	std::string get_file_hash(const std::string& file);
+}