From 1e71472b46ad506dc0122550cb19a498172f6488 Mon Sep 17 00:00:00 2001 From: Federico Cecchetto Date: Fri, 22 Jul 2022 19:51:26 +0200 Subject: [PATCH] Allow bnet filesystem to read file from disk + soundfile stuff --- src/client/component/database.cpp | 506 ++++++++++++++++++++++++++++ src/client/component/filesystem.cpp | 15 + src/client/component/filesystem.hpp | 1 + src/client/game/structs.hpp | 44 ++- 4 files changed, 563 insertions(+), 3 deletions(-) diff --git a/src/client/component/database.cpp b/src/client/component/database.cpp index e04c2119..1bf4c79c 100644 --- a/src/client/component/database.cpp +++ b/src/client/component/database.cpp @@ -1,16 +1,45 @@ #include #include "loader/component_loader.hpp" +#include "filesystem.hpp" +#include "console.hpp" +#include "command.hpp" + #include "game/dvars.hpp" #include "game/game.hpp" #include #include +#include namespace database { namespace { + struct bnet_file_handle_t + { + std::unique_ptr stream; + uint64_t offset{}; + }; + + std::unordered_map bnet_file_handles{}; + utils::hook::detour bnet_fs_open_file_hook; + utils::hook::detour bnet_fs_read_hook; + utils::hook::detour bnet_fs_tell_hook; + utils::hook::detour bnet_fs_size_hook; + utils::hook::detour bnet_fs_close_hook; + utils::hook::detour bnet_fs_exists_hook; + utils::hook::detour bink_io_read_hook; + utils::hook::detour bink_io_seek_hook; + + utils::hook::detour open_sound_handle_hook; + + utils::memory::allocator handle_allocator; + + using sound_file_t = std::unordered_map; + std::unordered_map sound_files = {}; + std::unordered_map sound_sizes = {}; + game::dvar_t* db_filesysImpl = nullptr; utils::hook::detour db_fs_initialize_hook; @@ -26,6 +55,414 @@ namespace database return nullptr; // this should not happen } } + + std::string get_sound_file_name(unsigned short file_index, uint64_t packed_file_offset) + { + if (sound_files.find(file_index) != sound_files.end() && + sound_files[file_index].find(packed_file_offset) != sound_files[file_index].end()) + { + return sound_files[file_index][packed_file_offset]; + } + + return utils::string::va("soundfile_%i_%llX.flac", file_index, packed_file_offset); + } + + std::string get_sound_file_name(game::StreamedSound* sound) + { + const auto file_index = sound->filename.fileIndex; + const auto packed_file_offset = sound->filename.info.packed.offset; + return get_sound_file_name(file_index, packed_file_offset); + } + + void* disk_fs_sound_file_open_stub(game::StreamFile* file, game::StreamedSound* sound) + { + const auto name = "sound/" + get_sound_file_name(sound); + std::string path{}; + + if (!filesystem::find_file(name, &path)) + { + return open_sound_handle_hook.invoke(file, sound); + } + + const auto disk_fs = reinterpret_cast(0x140BEFDC0); + const auto handle = disk_fs->vftbl->OpenFile(disk_fs, game::SF_ZONE_REGION, path.data()); + if (handle) + { + const auto size = disk_fs->vftbl->Size(disk_fs, handle); + file->isPacked = 0; + file->length = size; + file->handle = handle; + } + else + { + file->handle = nullptr; + file->length = 0; + } + + file->startOffset = 0; + return 0; + } + + thread_local game::StreamedSound current_sound{}; + + void* sound_file_open_stub(game::StreamFile* file, game::StreamedSound* sound) + { + if (db_filesysImpl->current.integer == 1) + { + return disk_fs_sound_file_open_stub(file, sound); + } + else + { + current_sound = *sound; + return open_sound_handle_hook.invoke(file, sound); + } + } + + game::DB_IFileSysFile* bnet_fs_open_file_stub(game::DB_FileSysInterface* this_, int folder, const char* file) + { + const auto _0 = gsl::finally([] + { + current_sound = {}; + }); + + std::string name = file; + + bool is_sound = false; + if (name.starts_with("soundfile") && name.ends_with(".pak")) + { + is_sound = true; + name = "sound/" + get_sound_file_name(¤t_sound); + } + + if (name.ends_with(".bik")) + { + name = "videos/" + name; + } + + std::string path{}; + if (!filesystem::find_file(name, &path)) + { + return bnet_fs_open_file_hook.invoke(this_, folder, file); + } + + const auto handle = handle_allocator.allocate(); + std::memset(handle, 0, sizeof(handle)); + + try + { +#ifdef DEBUG + console::info("[Database] Opening file %s\n", path.data()); +#endif + + auto stream = std::make_unique(); + stream->open(path, std::ios::binary); + + bnet_file_handle_t bnet_handle{}; + bnet_handle.stream = std::move(stream); + bnet_handle.offset = current_sound.filename.info.packed.offset; + + bnet_file_handles[handle] = std::move(bnet_handle); + return handle; + } + catch (const std::exception& e) + { + console::error("[Database] Error opening file %s: %s\n", path.data(), e.what()); + } + + return handle; + } + + game::FileSysResult bnet_fs_read_stub(game::DB_FileSysInterface* this_, game::DB_IFileSysFile* handle, + unsigned __int64 offset, unsigned __int64 size, void* dest) + { + if (bnet_file_handles.find(handle) == bnet_file_handles.end()) + { + return bnet_fs_read_hook.invoke(this_, handle, offset, size, dest); + } + else + { + auto& handle_ = bnet_file_handles[handle]; + if (!handle_.stream->is_open()) + { + return game::FILESYSRESULT_ERROR; + } + + try + { + const auto start_pos = offset - handle_.offset; + handle_.stream->seekg(0, std::ios::end); + const auto end_pos = static_cast(handle_.stream->tellg()); + handle_.stream->seekg(start_pos); + + const auto len = end_pos - start_pos; + const auto bytes_to_read = len <= size ? len : size; + + const auto& res = handle_.stream->read(reinterpret_cast(dest), bytes_to_read); + if (res.bad()) + { + return game::FILESYSRESULT_ERROR; + } + + const auto bytes_read = static_cast(res.gcount()); + handle->bytes_read += bytes_read; + handle->last_read = bytes_read; + + return game::FILESYSRESULT_SUCCESS; + } + catch (const std::exception& e) + { + console::error("[Database] bnet_fs_read_stub: %s\n", e.what()); + return game::FILESYSRESULT_ERROR; + } + } + } + + game::FileSysResult bnet_fs_tell_stub(game::DB_FileSysInterface* this_, game::DB_IFileSysFile* handle, uint64_t* bytes_read) + { + if (bnet_file_handles.find(handle) == bnet_file_handles.end()) + { + return bnet_fs_tell_hook.invoke(this_, handle, bytes_read); + } + else + { + auto& handle_ = bnet_file_handles[handle]; + if (!handle_.stream->is_open()) + { + return game::FILESYSRESULT_ERROR; + } + + try + { + *bytes_read = handle->last_read; + return game::FILESYSRESULT_SUCCESS; + } + catch (const std::exception& e) + { + console::error("[Database] bnet_fs_tell_stub: %s\n", e.what()); + return game::FILESYSRESULT_ERROR; + } + } + } + + uint64_t bnet_fs_size_stub(game::DB_FileSysInterface* this_, game::DB_IFileSysFile* handle) + { + if (bnet_file_handles.find(handle) == bnet_file_handles.end()) + { + return bnet_fs_size_hook.invoke(this_, handle); + } + else + { + auto& handle_ = bnet_file_handles[handle]; + try + { + handle_.stream->seekg(0, std::ios::end); + const std::streamsize size = handle_.stream->tellg(); + handle_.stream->seekg(0, std::ios::beg); + return static_cast(size); + } + catch (const std::exception& e) + { + console::error("[Database] bnet_fs_size_stub: %s\n", e.what()); + return 0; + } + } + } + + void bnet_fs_close_stub(game::DB_FileSysInterface* this_, game::DB_IFileSysFile* handle) + { + if (bnet_file_handles.find(handle) == bnet_file_handles.end()) + { + bnet_fs_close_hook.invoke(this_, handle); + } + else + { + handle_allocator.free(handle); + bnet_file_handles.erase(handle); + } + } + + bool bnet_fs_exists_stub(game::DB_FileSysInterface* this_, game::DB_IFileSysFile* handle, const char* filename) + { + std::string path{}; + return filesystem::find_file(filename, &path) || bnet_fs_exists_hook.invoke(this_, handle, filename); + } + + uint64_t bink_io_read_stub(game::DB_IFileSysFile** handle, void* dest, uint64_t bytes) + { + const auto handle_ptr = *handle; + if (bnet_file_handles.find(handle_ptr) == bnet_file_handles.end()) + { + return bink_io_read_hook.invoke(handle, dest, bytes); + } + else + { + auto& handle_ = bnet_file_handles[handle_ptr]; + if (!handle_.stream->is_open()) + { + return 0; + } + + try + { + const auto& res = handle_.stream->read(reinterpret_cast(dest), bytes); + return static_cast(res.gcount()); + } + catch (const std::exception& e) + { + console::error("[Database] bink_io_read_stub: %s\n", e.what()); + return 0; + } + } + } + + bool bink_io_seek_stub(game::DB_IFileSysFile** handle, uint64_t pos) + { + const auto handle_ptr = *handle; + if (bnet_file_handles.find(handle_ptr) == bnet_file_handles.end()) + { + return bink_io_seek_hook.invoke(handle, pos); + } + else + { + auto& handle_ = bnet_file_handles[handle_ptr]; + if (!handle_.stream->is_open()) + { + return false; + } + + try + { + const auto& res = handle_.stream->seekg(pos); + return !(res.fail() || res.bad()); + } + catch (const std::exception& e) + { + console::error("[Database] bink_io_seek_stub: %s\n", e.what()); + return 0; + } + } + } + + utils::hook::detour db_link_xasset_entry1_hook; + game::XAssetEntry* db_link_xasset_entry1_stub(game::XAssetType type, game::XAssetHeader* header) + { + const auto result = db_link_xasset_entry1_hook.invoke(type, header); + if (result->asset.type == game::ASSET_TYPE_SOUND) + { + const auto sound = result->asset.header.sound; + for (auto i = 0; i < sound->count; i++) + { + const auto alias = &sound->head[i]; + if (alias->soundFile != nullptr && alias->soundFile->type == 2) + { + const auto file_index = alias->soundFile->u.streamSnd.filename.fileIndex; + const auto length = alias->soundFile->u.streamSnd.filename.info.packed.length; + const auto offset = alias->soundFile->u.streamSnd.filename.info.packed.offset; + const auto name = utils::string::va("%s.flac", alias->aliasName); + + sound_files[file_index][offset] = name; + sound_sizes[name] = length; + } + } + } + + return result; + } + + void dump_flac_sound(const std::string& sound_name, unsigned short file_index, uint64_t start_offset, uint64_t size) + { + const auto name = utils::string::va("soundfile%i.pak", file_index); + const auto fs_interface = db_fs_initialize_stub(); + + const auto handle = fs_interface->vftbl->OpenFile(fs_interface, game::Sys_Folder::SF_PAKFILE, name); + if (handle == nullptr) + { + console::error("Sound file %s not found\n", name); + return; + } + + const auto buffer = utils::memory::get_allocator()->allocate_array(size); + const auto _0 = gsl::finally([&]() + { + utils::memory::get_allocator()->free(buffer); + }); + + const auto result = fs_interface->vftbl->Read(fs_interface, handle, start_offset, size, buffer); + if (result != game::FILESYSRESULT_SUCCESS) + { + console::error("Error reading file %s\n", name); + } + + const auto path = utils::string::va("dumps/sound/%s", sound_name.data()); + utils::io::write_file(path, std::string(buffer, size), false); + console::info("Sound dumped to %s\n", path); + } + + void dump_sound_file(unsigned short file_index) + { + const auto name = utils::string::va("soundfile%i.pak", file_index); + const auto fs_interface = db_fs_initialize_stub(); + + const auto handle = fs_interface->vftbl->OpenFile(fs_interface, game::Sys_Folder::SF_PAKFILE, name); + if (handle == nullptr) + { + console::error("Sound file %s not found\n", name); + return; + } + + const auto size = fs_interface->vftbl->Size(fs_interface, handle); + const auto buffer = utils::memory::get_allocator()->allocate_array(size); + const auto _0 = gsl::finally([&]() + { + utils::memory::get_allocator()->free(buffer); + }); + + const auto result = fs_interface->vftbl->Read(fs_interface, handle, 0, size, buffer); + if (result != game::FILESYSRESULT_SUCCESS) + { + console::error("Error reading file %s\n", name); + } + + std::vector signature = {0x66, 0x4C, 0x61, 0x43, 0x00, 0x00, 0x00, 0x22}; + + const auto check_signature = [&](char* start) + { + for (auto i = 0; i < signature.size(); i++) + { + if (start[i] != signature[i]) + { + return false; + } + } + + return true; + }; + + const auto end = buffer + size - signature.size() - 1; + for (auto pos = buffer; pos < end;) + { + const auto start_pos = pos; + if (check_signature(start_pos)) + { + ++pos; + + while (pos < end && !check_signature(pos)) + { + ++pos; + } + + const auto flac_size = static_cast(pos - start_pos); + std::string data{start_pos, flac_size}; + + const auto progress = static_cast(100 * (static_cast(start_pos - buffer) / static_cast(end - buffer))); + const auto sound_name = get_sound_file_name(file_index, static_cast(start_pos - buffer)); + const auto path = utils::string::va("dumps/sound/%s", sound_name.data()); + + utils::io::write_file(path, data, false); + console::info("Sound dumped: %s (%i%%)\n", path, progress); + } + } + } } class component final : public component_interface @@ -53,6 +490,75 @@ namespace database } db_fs_initialize_hook.create(game::DB_FSInitialize, db_fs_initialize_stub); + + open_sound_handle_hook.create(0x1406233B0, sound_file_open_stub); + db_link_xasset_entry1_hook.create(0x140414900, db_link_xasset_entry1_stub); + + // Allow bnet filesystem to also load files from disk + if (db_filesysImpl->current.integer == 0) + { + const auto bnet_interface = reinterpret_cast(0x140BE82F8); + + bnet_fs_open_file_hook.create(bnet_interface->vftbl->OpenFile, bnet_fs_open_file_stub); + bnet_fs_read_hook.create(bnet_interface->vftbl->Read, bnet_fs_read_stub); + bnet_fs_tell_hook.create(bnet_interface->vftbl->Tell, bnet_fs_tell_stub); + bnet_fs_size_hook.create(bnet_interface->vftbl->Size, bnet_fs_size_stub); + bnet_fs_close_hook.create(bnet_interface->vftbl->Close, bnet_fs_close_stub); + bnet_fs_exists_hook.create(bnet_interface->vftbl->Exists, bnet_fs_exists_stub); + + bink_io_read_hook.create(0x1407191B0, bink_io_read_stub); + bink_io_seek_hook.create(0x140719200, bink_io_seek_stub); + } + + command::add("listSoundFiles", []() + { + for (const auto& packed : sound_files) + { + for (const auto& sound : packed.second) + { + console::info("soundfile%i.pak %s %llX", packed.first, sound.second.data(), sound.first); + } + } + }); + + command::add("dumpSoundFile", [](const command::params& params) + { + if (params.size() < 2) + { + console::info("Usage: dumpSoundFile \n"); + return; + } + + const auto index = static_cast(atoi(params.get(1))); + dump_sound_file(index); + }); + + command::add("dumpSound", [](const command::params& params) + { + if (params.size() < 2) + { + console::info("Usage: dumpSound \n"); + return; + } + + const auto name = params.get(1); + for (const auto& packed : sound_files) + { + for (auto i = packed.second.begin(); i != packed.second.end();) + { + if (i->second != name) + { + ++i; + continue; + } + + const auto& sound_name = i->second; + const auto start_offset = i->first; + dump_flac_sound(sound_name, packed.first, start_offset, sound_sizes[sound_name]); + break; + } + } + }); } }; } diff --git a/src/client/component/filesystem.cpp b/src/client/component/filesystem.cpp index 954e9e29..038b231a 100644 --- a/src/client/component/filesystem.cpp +++ b/src/client/component/filesystem.cpp @@ -108,6 +108,21 @@ namespace filesystem return false; } + bool find_file(const std::string& path, std::string* real_path) + { + for (const auto& search_path : get_search_paths_internal()) + { + const auto path_ = search_path / path; + if (utils::io::file_exists(path_.generic_string())) + { + *real_path = path_.generic_string(); + return true; + } + } + + return false; + } + void register_path(const std::filesystem::path& path) { if (!initialized) diff --git a/src/client/component/filesystem.hpp b/src/client/component/filesystem.hpp index cfdf855e..cc07b7d6 100644 --- a/src/client/component/filesystem.hpp +++ b/src/client/component/filesystem.hpp @@ -6,6 +6,7 @@ namespace filesystem { std::string read_file(const std::string& path); bool read_file(const std::string& path, std::string* data, std::string* real_path = nullptr); + bool find_file(const std::string& path, std::string* real_path); void register_path(const std::filesystem::path& path); void unregister_path(const std::filesystem::path& path); diff --git a/src/client/game/structs.hpp b/src/client/game/structs.hpp index 2a0907b7..e0fe5ab0 100644 --- a/src/client/game/structs.hpp +++ b/src/client/game/structs.hpp @@ -692,8 +692,37 @@ namespace game unsigned int totalMsec; }; + struct StreamFile + { + void* handle; + __int64 length; + __int64 startOffset; + bool isPacked; + }; + + struct LoadedSoundInfo + { + char* data; + unsigned int sampleRate; + unsigned int dataByteCount; + unsigned int numSamples; + char channels; + char numBits; + char blockAlign; + short format; + int loadedSize; + }; static_assert(sizeof(LoadedSoundInfo) == 0x20); + + struct LoadedSound + { + const char* name; + StreamFileName filename; + LoadedSoundInfo info; + }; static_assert(sizeof(LoadedSound) == 0x40); + union SoundFileRef { + LoadedSound* loadSnd; StreamedSound streamSnd; }; @@ -709,10 +738,11 @@ namespace game const char* aliasName; char __pad0[24]; SoundFile* soundFile; - char __pad1[198]; - // not gonna map this out... + char __pad1[216]; }; + static_assert(sizeof(snd_alias_t) == 256); + struct snd_alias_list_t { const char* aliasName; @@ -841,6 +871,7 @@ namespace game MapEnts* mapents; AddonMapEnts* addon_mapents; LocalizeEntry* localize; + snd_alias_list_t* sound; }; struct XAsset @@ -1165,7 +1196,14 @@ namespace game FILESYSRESULT_ERROR = 0x2, }; - struct DB_IFileSysFile; + struct DB_IFileSysFile + { + void* file; + uint64_t last_read; + uint64_t bytes_read; + }; + + static_assert(sizeof(DB_IFileSysFile) == 24); struct DB_FileSysInterface;