[RawFiles] Load info strings from raw (#557)

src/Components/Modules/AssetHandler.hpp

@@ -8,12 +8,12 @@ namespace Components
 		class IAsset
 		{
 		public:
-			virtual ~IAsset() {};
+			virtual ~IAsset() {}
-			virtual Game::XAssetType getType() { return Game::XAssetType::ASSET_TYPE_INVALID; };
+			virtual Game::XAssetType getType() { return Game::XAssetType::ASSET_TYPE_INVALID; }
-			virtual void mark(Game::XAssetHeader /*header*/, ZoneBuilder::Zone* /*builder*/) { /*ErrorTypeNotSupported(this);*/ };
+			virtual void mark(Game::XAssetHeader /*header*/, ZoneBuilder::Zone* /*builder*/) {}
-			virtual void save(Game::XAssetHeader /*header*/, ZoneBuilder::Zone* /*builder*/) { /*ErrorTypeNotSupported(this);*/ };
+			virtual void save(Game::XAssetHeader /*header*/, ZoneBuilder::Zone* /*builder*/) {}
-			virtual void dump(Game::XAssetHeader /*header*/) { /*ErrorTypeNotSupported(this);*/ };
+			virtual void dump(Game::XAssetHeader /*header*/) {}
-			virtual void load(Game::XAssetHeader* /*header*/, const std::string& /*name*/, ZoneBuilder::Zone* /*builder*/) { /*ErrorTypeNotSupported(this);*/ };
+			virtual void load(Game::XAssetHeader* /*header*/, const std::string& /*name*/, ZoneBuilder::Zone* /*builder*/) {}
 		};
 
 		typedef Game::XAssetHeader(Callback)(Game::XAssetType type, const std::string& name);

src/Components/Modules/AssetInterfaces/ILocalizeEntry.cpp

@@ -3,13 +3,37 @@
 
 namespace Assets
 {
+	void ILocalizeEntry::load(Game::XAssetHeader* header, const std::string& name, Components::ZoneBuilder::Zone* builder)
+	{
+		const auto path = "localizedstrings/" + name;
+
+		Components::FileSystem::File rawFile(path);
+		if (!rawFile.exists())
+		{
+			return;
+		}
+
+		Components::Logger::Debug("Parsing localized string \"{}\"...", path);
+
+		auto* asset = builder->getAllocator()->allocate<Game::LocalizeEntry>();
+		if (!asset)
+		{
+			return;
+		}
+
+		asset->name = builder->getAllocator()->duplicateString(name);
+		asset->value = builder->getAllocator()->duplicateString(rawFile.getBuffer());
+
+		header->localize = asset;
+	}
+
 	void ILocalizeEntry::save(Game::XAssetHeader header, Components::ZoneBuilder::Zone* builder)
 	{
 		AssertSize(Game::LocalizeEntry, 8);
 
-		Utils::Stream* buffer = builder->getBuffer();
+		auto* buffer = builder->getBuffer();
-		Game::LocalizeEntry* asset = header.localize;
+		auto* asset = header.localize;
-		Game::LocalizeEntry* dest = buffer->dest<Game::LocalizeEntry>();
+		auto* dest = buffer->dest<Game::LocalizeEntry>();
 		buffer->save(asset);
 
 		buffer->pushBlock(Game::XFILE_BLOCK_VIRTUAL);

src/Components/Modules/AssetInterfaces/ILocalizeEntry.hpp

@@ -7,6 +7,7 @@ namespace Assets
 	public:
 		Game::XAssetType getType() override { return Game::XAssetType::ASSET_TYPE_LOCALIZE_ENTRY; };
 
+		void load(Game::XAssetHeader* header, const std::string& name, Components::ZoneBuilder::Zone* builder) override;
 		void save(Game::XAssetHeader header, Components::ZoneBuilder::Zone* builder) override;
 	};
 }

src/Components/Modules/AssetInterfaces/IRawFile.cpp

@@ -7,31 +7,44 @@ namespace Assets
 	{
 		Components::FileSystem::File rawFile(name);
 
-		if (rawFile.exists())
+		if (!rawFile.exists())
 		{
-			Game::RawFile* asset = builder->getAllocator()->allocate<Game::RawFile>();
+			return;
-
-			if (asset)
-			{
-				//std::string data = Utils::Compression::ZLib::Compress(rawFile.getBuffer());
-
-				asset->name = builder->getAllocator()->duplicateString(name);
-				asset->buffer = builder->getAllocator()->duplicateString(rawFile.getBuffer());
-				asset->compressedLen = 0;//data.size();
-				asset->len = rawFile.getBuffer().size();
-
-				header->rawfile = asset;
-			}
 		}
+
+		auto* asset = builder->getAllocator()->allocate<Game::RawFile>();
+		if (!asset)
+		{
+			return;
+		}
+
+		const auto data = Utils::Compression::ZLib::Compress(rawFile.getBuffer());
+
+		asset->name = builder->getAllocator()->duplicateString(name);
+
+		if (data.size() < rawFile.getBuffer().size())
+		{
+			asset->buffer = builder->getAllocator()->duplicateString(data);
+			asset->compressedLen = static_cast<int>(data.size());
+		}
+		else
+		{
+			asset->buffer = builder->getAllocator()->duplicateString(rawFile.getBuffer());
+			asset->compressedLen = 0;
+		}
+
+		asset->len = static_cast<int>(rawFile.getBuffer().size());
+
+		header->rawfile = asset;
 	}
 
 	void IRawFile::save(Game::XAssetHeader header, Components::ZoneBuilder::Zone* builder)
 	{
 		AssertSize(Game::RawFile, 16);
 
-		Utils::Stream* buffer = builder->getBuffer();
+		auto* buffer = builder->getBuffer();
-		Game::RawFile* asset = header.rawfile;
+		auto* asset = header.rawfile;
-		Game::RawFile* dest = buffer->dest<Game::RawFile>();
+		auto* dest = buffer->dest<Game::RawFile>();
 		buffer->save(asset);
 
 		buffer->pushBlock(Game::XFILE_BLOCK_VIRTUAL);

src/Components/Modules/AssetInterfaces/IStringTable.hpp

@@ -10,6 +10,6 @@ namespace Assets
 		void save(Game::XAssetHeader header, Components::ZoneBuilder::Zone* builder) override;
 
 	private:
-		void saveStringTableCellArray(Components::ZoneBuilder::Zone* builder, Game::StringTableCell* values, int count);
+		static void saveStringTableCellArray(Components::ZoneBuilder::Zone* builder, Game::StringTableCell* values, int count);
 	};
 }

src/Components/Modules/AssetInterfaces/IStructuredDataDefSet.hpp

@@ -9,7 +9,7 @@ namespace Assets
 
 		void save(Game::XAssetHeader header, Components::ZoneBuilder::Zone* builder) override;
 
-		void saveStructuredDataEnumArray(Game::StructuredDataEnum* enums, int numEnums, Components::ZoneBuilder::Zone* builder);
+		static void saveStructuredDataEnumArray(Game::StructuredDataEnum* enums, int numEnums, Components::ZoneBuilder::Zone* builder);
-		void saveStructuredDataStructArray(Game::StructuredDataStruct* structs, int numStructs, Components::ZoneBuilder::Zone* builder);
+		static void saveStructuredDataStructArray(Game::StructuredDataStruct* structs, int numStructs, Components::ZoneBuilder::Zone* builder);
 	};
 }

src/Components/Modules/Localization.cpp

@@ -5,7 +5,6 @@ namespace Components
 	std::recursive_mutex Localization::LocalizeMutex;
 	Dvar::Var Localization::UseLocalization;
 	std::unordered_map<std::string, Game::LocalizeEntry*> Localization::LocalizeMap;
-	std::unordered_map<std::string, Game::LocalizeEntry*> Localization::TempLocalizeMap;
 
 	void Localization::Set(const std::string& key, const std::string& value)
 	{
@@ -49,14 +48,11 @@ namespace Components
 		if (!Localization::UseLocalization.get<bool>()) return key;
 
 		Game::LocalizeEntry* entry = nullptr;
+
 		{
 			std::lock_guard _(Localization::LocalizeMutex);
 
-			if (Localization::TempLocalizeMap.contains(key))
+			if (Localization::LocalizeMap.contains(key))
-			{
-				entry = Localization::TempLocalizeMap[key];
-			}
-			else if (Localization::LocalizeMap.contains(key))
 			{
 				entry = Localization::LocalizeMap[key];
 			}
@@ -75,59 +71,6 @@ namespace Components
 		return key;
 	}
 
-	void Localization::SetTemp(const std::string& key, const std::string& value)
-	{
-		std::lock_guard _(Localization::LocalizeMutex);
-		Utils::Memory::Allocator* allocator = Utils::Memory::GetAllocator();
-
-		if (Localization::TempLocalizeMap.contains(key))
-		{
-			Game::LocalizeEntry* entry = Localization::TempLocalizeMap[key];
-			if (entry->value) allocator->free(entry->value);
-			entry->value = allocator->duplicateString(value);
-		}
-		else
-		{
-			Game::LocalizeEntry* entry = allocator->allocate<Game::LocalizeEntry>();
-			if (!entry) return;
-
-			entry->name = allocator->duplicateString(key);
-			if (!entry->name)
-			{
-				allocator->free(entry);
-				return;
-			}
-
-			entry->value = allocator->duplicateString(value);
-			if (!entry->value)
-			{
-				allocator->free(entry->name);
-				allocator->free(entry);
-				return;
-			}
-
-			Localization::TempLocalizeMap[key] = entry;
-		}
-	}
-
-	void Localization::ClearTemp()
-	{
-		std::lock_guard _(Localization::LocalizeMutex);
-		Utils::Memory::Allocator* allocator = Utils::Memory::GetAllocator();
-
-		for (auto i = Localization::TempLocalizeMap.begin(); i != Localization::TempLocalizeMap.end(); ++i)
-		{
-			if (i->second)
-			{
-				if (i->second->name)  allocator->free(i->second->name);
-				if (i->second->value) allocator->free(i->second->value);
-				allocator->free(i->second);
-			}
-		}
-
-		Localization::TempLocalizeMap.clear();
-	}
-
 	void __stdcall Localization::SetStringStub(const char* key, const char* value, bool /*isEnglish*/)
 	{
 		Localization::Set(key, value);
@@ -393,11 +336,7 @@ namespace Components
 			Game::XAssetHeader header = { nullptr };
 			std::lock_guard _(Localization::LocalizeMutex);
 
-			if (Localization::TempLocalizeMap.contains(filename))
+			if (Localization::LocalizeMap.contains(filename))
-			{
-				header.localize = Localization::TempLocalizeMap[filename];
-			}
-			else if (Localization::LocalizeMap.contains(filename))
 			{
 				header.localize = Localization::LocalizeMap[filename];
 			}
@@ -441,7 +380,6 @@ namespace Components
 
 	Localization::~Localization()
 	{
-		Localization::ClearTemp();
 		Localization::LocalizeMap.clear();
 	}
 }

src/Components/Modules/Localization.hpp

@@ -11,13 +11,9 @@ namespace Components
 		static void Set(const std::string& key, const std::string& value);
 		static const char* Get(const char* key);
 
-		static void SetTemp(const std::string& key, const std::string& value);
-		static void ClearTemp();
-
 	private:
 		static std::recursive_mutex LocalizeMutex;
 		static std::unordered_map<std::string, Game::LocalizeEntry*> LocalizeMap;
-		static std::unordered_map<std::string, Game::LocalizeEntry*> TempLocalizeMap;
 		static Dvar::Var UseLocalization;
 
 		static void __stdcall SetStringStub(const char* key, const char* value, bool isEnglish);

src/Components/Modules/Party.cpp

@@ -65,7 +65,6 @@ namespace Components
 
 	void Party::ConnectError(const std::string& message)
 	{
-		Localization::ClearTemp();
 		Command::Execute("closemenu popup_reconnectingtoparty");
 		Dvar::Var("partyend_reason").set(message);
 		Command::Execute("openmenu menu_xboxlive_partyended");

src/Components/Modules/RawFiles.cpp

@@ -12,7 +12,7 @@ namespace Components
 			if ((fileSize + 1) <= size)
 			{
 				Game::FS_Read(buf, fileSize, fileHandle);
-				buf[fileSize] = 0;
+				buf[fileSize] = '\0';
 				Game::FS_FCloseFile(fileHandle);
 				return buf;
 			}
@@ -62,6 +62,59 @@ namespace Components
 		return buffer;
 	}
 
+	char* RawFiles::Com_LoadInfoString_LoadObj(const char* fileName, const char* fileDesc, const char* ident, char* loadBuffer)
+	{
+		auto fileHandle = 0;
+
+		const auto fileLen = Game::FS_FOpenFileByMode(fileName, &fileHandle, Game::FS_READ);
+		if (fileLen < 0)
+		{
+			Logger::Debug("Could not load {} [{}] as rawfile", fileDesc, fileName);
+			return nullptr;
+		}
+
+		const auto identLen = static_cast<int>(std::strlen(ident));
+		Game::FS_Read(loadBuffer, identLen, fileHandle);
+		loadBuffer[identLen] = '\0';
+
+		if (std::strncmp(loadBuffer, ident, identLen) != 0)
+		{
+			Game::Com_Error(Game::ERR_DROP, "\x15" "File [%s] is not a %s\n", fileName, fileDesc);
+			return nullptr;
+		}
+
+		if ((fileLen - identLen) >= 0x4000)
+		{
+			Game::Com_Error(Game::ERR_DROP, "\x15" "File [%s] is too long of a %s to parse\n", fileName, fileDesc);
+			return nullptr;
+		}
+
+		Game::FS_Read(loadBuffer, fileLen - identLen, fileHandle);
+		loadBuffer[fileLen - identLen] = '\0';
+		Game::FS_FCloseFile(fileHandle);
+
+		return loadBuffer;
+	}
+
+	const char* RawFiles::Com_LoadInfoString_Hk(const char* fileName, const char* fileDesc, const char* ident, char* loadBuffer)
+	{
+		const char* buffer;
+
+		buffer = Com_LoadInfoString_LoadObj(fileName, fileDesc, ident, loadBuffer);
+		if (!buffer)
+		{
+			buffer = Game::Com_LoadInfoString_FastFile(fileName, fileDesc, ident, loadBuffer);
+		}
+
+		if (!Game::Info_Validate(buffer))
+		{
+			Game::Com_Error(Game::ERR_DROP, "\x15" "File [%s] is not a valid %s\n", fileName, fileDesc);
+			return nullptr;
+		}
+
+		return buffer;
+	}
+
 	RawFiles::RawFiles()
 	{
 		// remove fs_game check for moddable rawfiles - allows non-fs_game to modify rawfiles
@@ -69,6 +122,7 @@ namespace Components
 
 		Utils::Hook(0x4DA0D0, ReadRawFile, HOOK_JUMP).install()->quick();
 		Utils::Hook(0x631640, GetMenuBuffer, HOOK_JUMP).install()->quick();
+		Utils::Hook(0x463500, Com_LoadInfoString_Hk, HOOK_JUMP).install()->quick();
 
 		Command::Add("dumpraw", [](Command::Params* params)
 		{

src/Components/Modules/RawFiles.hpp

@@ -11,5 +11,7 @@ namespace Components
 
 	private:
 		static char* GetMenuBuffer(const char* filename);
+		static char* Com_LoadInfoString_LoadObj(const char* fileName, const char* fileDesc, const char* ident, char* loadBuffer);
+		static const char* Com_LoadInfoString_Hk(const char* fileName, const char* fileDesc, const char* ident, char* loadBuffer);
 	};
 }

src/Components/Modules/ZoneBuilder.cpp

@@ -76,7 +76,6 @@ namespace Components
 		Game::DB_LoadXAssets(&info, 1, true);
 
 		AssetHandler::ClearTemporaryAssets();
-		Localization::ClearTemp();
 	}
 
 	Utils::Stream* ZoneBuilder::Zone::getBuffer()
@@ -151,29 +150,18 @@ namespace Components
 			{
 				if (this->dataMap.getColumns(i) > 2)
 				{
-					if (this->dataMap.getElementAt(i, 0) == "localize")
+					std::string oldName = this->dataMap.getElementAt(i, 1);
-					{
+					std::string newName = this->dataMap.getElementAt(i, 2);
-						std::string stringOverride = this->dataMap.getElementAt(i, 2);
+					std::string typeName = this->dataMap.getElementAt(i, 0);
-						Utils::String::Replace(stringOverride, "\\n", "\n");
+					Game::XAssetType type = Game::DB_GetXAssetNameType(typeName.data());
 
-						Localization::SetTemp(this->dataMap.getElementAt(i, 1), stringOverride);
+					if (type < Game::XAssetType::ASSET_TYPE_COUNT && type >= 0)
+					{
+							this->renameAsset(type, oldName, newName);
 					}
 					else
 					{
-						std::string oldName = this->dataMap.getElementAt(i, 1);
+						Logger::Error(Game::ERR_FATAL, "Unable to rename '{}' to '{}' as the asset type '{}' is invalid!", oldName, newName, typeName);
-						std::string newName = this->dataMap.getElementAt(i, 2);
-						std::string typeName = this->dataMap.getElementAt(i, 0).data();
-						Game::XAssetType type = Game::DB_GetXAssetNameType(typeName.data());
-
-						if (type < Game::XAssetType::ASSET_TYPE_COUNT && type >= 0)
-						{
-							this->renameAsset(type, oldName, newName);
-						}
-						else
-						{
-							Logger::Error(Game::ERR_FATAL, "Unable to rename '{}' to '{}' as the asset type '{}' is invalid!",
-								oldName, newName, typeName);
-						}
 					}
 				}
 
@@ -914,7 +902,6 @@ namespace Components
 			frames++;
 		}
 
-		// ReSharper disable once CppUnreachableCode
 		return 0;
 	}
 
@@ -923,7 +910,7 @@ namespace Components
 		char buffer[4096] = {0};
 		va_list args;
 		va_start(args, fmt);
-		_vsnprintf_s(buffer, _TRUNCATE, fmt, args);
+		vsnprintf_s(buffer, _TRUNCATE, fmt, args);
 		va_end(args);
 
 		if (!Flags::HasFlag("stdout"))

src/Components/Modules/Zones.cpp

@@ -3627,7 +3627,7 @@ namespace Components
 		Utils::Hook(0x4B4EA1, Zones::ParseShellShock_Stub, HOOK_CALL).install()->quick();
 		Utils::Hook(0x4B4F0C, Zones::ParseShellShock_Stub, HOOK_CALL).install()->quick();
 		
-		Utils::Hook(0x4F4D3B, [] ()
+		Utils::Hook(0x4F4D3B, []
 		{
 			if (Zones::ZoneVersion >= VERSION_ALPHA3)
 			{

src/Game/Common.cpp

@@ -61,4 +61,28 @@ namespace Game
 			g_parse[1056 * *(reinterpret_cast<DWORD*>(g_parse) + 4224) + 1032] = parse != 0;
 		}
 	}
+
+	const char* Com_LoadInfoString_FastFile(const char* fileName, const char* fileDesc, const char* ident, char* loadBuffer)
+	{
+		static DWORD Com_LoadInfoString_FastFile_t = 0x609B60;
+
+		const char* result{};
+
+		__asm
+		{
+			pushad
+
+			mov edi, fileName
+			mov ebx, loadBuffer
+			push ident
+			push fileDesc
+			call Com_LoadInfoString_FastFile_t
+			add esp, 0x8
+			mov result, eax
+
+			popad
+		}
+
+		return result;
+	}
 }

src/Game/Common.hpp

@@ -70,4 +70,6 @@ namespace Game
 
 	extern char* Com_GetParseThreadInfo();
 	extern void Com_SetParseNegativeNumbers(int parse);
+
+	extern const char* Com_LoadInfoString_FastFile(const char* fileName, const char* fileDesc, const char* ident, char* loadBuffer);
 }

src/Game/FileSystem.cpp

@@ -13,6 +13,7 @@ namespace Game
 	FS_FOpenFileRead_t FS_FOpenFileRead = FS_FOpenFileRead_t(0x46CBF0);
 	FS_FOpenFileReadDatabase_t FS_FOpenFileReadDatabase = FS_FOpenFileReadDatabase_t(0x42ECA0);
 	FS_FOpenFileReadForThread_t FS_FOpenFileReadForThread = FS_FOpenFileReadForThread_t(0x643270);
+	FS_FOpenFileByMode_t FS_FOpenFileByMode = FS_FOpenFileByMode_t(0x4C0700);
 	FS_FCloseFile_t FS_FCloseFile = FS_FCloseFile_t(0x462000);
 	FS_WriteFile_t FS_WriteFile = FS_WriteFile_t(0x426450);
 	FS_WriteToDemo_t  FS_WriteToDemo = FS_WriteToDemo_t(0x4C06E0);

src/Game/FileSystem.hpp

@@ -32,6 +32,9 @@ namespace Game
 	typedef int(*FS_FOpenFileReadForThread_t)(const char* filename, int* file, int thread);
 	extern FS_FOpenFileReadForThread_t FS_FOpenFileReadForThread;
 
+	typedef int(*FS_FOpenFileByMode_t)(const char* qpath, int* f, fsMode_t mode);
+	extern FS_FOpenFileByMode_t FS_FOpenFileByMode;
+
 	typedef int(*FS_FCloseFile_t)(int stream);
 	extern FS_FCloseFile_t FS_FCloseFile;
 
@@ -50,7 +53,7 @@ namespace Game
 	typedef int(*FS_Printf_t)(int file, const char* fmt, ...);
 	extern FS_Printf_t FS_Printf;
 
-	typedef int(*FS_Read_t)(void* buffer, size_t size, int file);
+	typedef int(*FS_Read_t)(void* buffer, int len, int h);
 	extern FS_Read_t FS_Read;
 
 	typedef int(*FS_Seek_t)(int fileHandle, int seekPosition, int seekOrigin);

src/Game/Functions.cpp

@@ -48,6 +48,7 @@ namespace Game
 	Image_Release_t Image_Release = Image_Release_t(0x51F010);
 
 	Info_ValueForKey_t Info_ValueForKey = Info_ValueForKey_t(0x47C820);
+	Info_Validate_t Info_Validate = Info_Validate_t(0x436AE0);
 
 	Key_SetCatcher_t Key_SetCatcher = Key_SetCatcher_t(0x43BD00);
 	Key_RemoveCatcher_t Key_RemoveCatcher = Key_RemoveCatcher_t(0x408260);

src/Game/Functions.hpp

@@ -95,6 +95,9 @@ namespace Game
 	typedef char*(*Info_ValueForKey_t)(const char* s, const char* key);
 	extern Info_ValueForKey_t Info_ValueForKey;
 
+	typedef int(*Info_Validate_t)(const char* s);
+	extern Info_Validate_t Info_Validate;
+
 	typedef void(*Key_SetCatcher_t)(int localClientNum, int catcher);
 	extern Key_SetCatcher_t Key_SetCatcher;
 

src/Game/Structs.hpp

@@ -5332,6 +5332,14 @@ namespace Game
 		char uiName[32];
 	};
 
+	enum fsMode_t
+	{
+		FS_READ = 0x0,
+		FS_WRITE = 0x1,
+		FS_APPEND = 0x2,
+		FS_APPEND_SYNC = 0x3,
+	};
+
 	struct fileInIwd_s
 	{
 		unsigned int pos;

src/Utils/Compression.cpp

@@ -6,7 +6,7 @@ namespace Utils
 	{
 		std::string ZLib::Compress(const std::string& data)
 		{
-			Utils::Memory::Allocator allocator;
+			Memory::Allocator allocator;
 			unsigned long length = (data.size() * 2);
 			if (!length) length = 2;
 
@@ -34,7 +34,7 @@ namespace Utils
 			}
 
 			int ret;
-			Utils::Memory::Allocator allocator;
+			Memory::Allocator allocator;
 
 			uint8_t* dest = allocator.allocateArray<uint8_t>(CHUNK);
 			const char* dataPtr = data.data();