Filter CSV while building zonebuilder zone

src/Components/Modules/ZoneBuilder.cpp

@@ -14,7 +14,7 @@
 namespace Components
 {
 	std::string ZoneBuilder::TraceZone;
-	std::vector<std::pair<Game::XAssetType, std::string>> ZoneBuilder::TraceAssets;
+	std::vector<ZoneBuilder::NamedAsset> ZoneBuilder::TraceAssets;
 
 	bool ZoneBuilder::MainThreadInterrupted;
 	DWORD ZoneBuilder::InterruptingThreadId;
@@ -733,11 +733,11 @@ namespace Components
 		ZoneBuilder::TraceZone = zone;
 	}
 
-	std::vector<std::pair<Game::XAssetType, std::string>> ZoneBuilder::EndAssetTrace()
+	std::vector<ZoneBuilder::NamedAsset> ZoneBuilder::EndAssetTrace()
 	{
 		ZoneBuilder::TraceZone.clear();
 
-		std::vector<std::pair<Game::XAssetType, std::string>> AssetTrace;
+		std::vector<NamedAsset> AssetTrace;
 		Utils::Merge(&AssetTrace, ZoneBuilder::TraceAssets);
 
 		ZoneBuilder::TraceAssets.clear();
@@ -1164,20 +1164,15 @@ namespace Components
 		ZoneBuilder::DumpingZone = zone;
 		ZoneBuilder::RefreshExporterWorkDirectory();
 
-		std::vector<std::pair<Game::XAssetType, std::string>> assets{};
+		std::vector<NamedAsset> assets{};
 		const auto unload = ZoneBuilder::LoadZoneWithTrace(zone, assets);
 
 		Logger::Print("Dumping zone '{}'...\n", zone);
 
 		{
 			Utils::IO::CreateDir(GetDumpingZonePath());
-			std::ofstream csv(std::filesystem::path(GetDumpingZonePath()) / std::format("{}.csv", zone));
-			csv
-				<< std::format("### Zone '{}' dumped with Zonebuilder {}", zone, Components::Branding::GetVersionString())
-				<< "\n\n";
 
 			// Order the CSV around
-			// TODO: Trim asset list using IW4OF dependencies
 			constexpr Game::XAssetType typeOrder[] = {
 				Game::XAssetType::ASSET_TYPE_GAMEWORLD_MP,
 				Game::XAssetType::ASSET_TYPE_GAMEWORLD_SP,
@@ -1197,11 +1192,13 @@ namespace Components
 				Game::XAssetType::ASSET_TYPE_PIXELSHADER,
 				Game::XAssetType::ASSET_TYPE_VERTEXSHADER,
 				Game::XAssetType::ASSET_TYPE_VERTEXDECL,
+				Game::XAssetType::ASSET_TYPE_LIGHT_DEF,
 				Game::XAssetType::ASSET_TYPE_IMAGE,
 				Game::XAssetType::ASSET_TYPE_SOUND,
 				Game::XAssetType::ASSET_TYPE_LOADED_SOUND,
 				Game::XAssetType::ASSET_TYPE_SOUND_CURVE,
 				Game::XAssetType::ASSET_TYPE_PHYSPRESET,
+				Game::XAssetType::ASSET_TYPE_LOCALIZE_ENTRY,
 			};
 
 			std::unordered_map<Game::XAssetType, int> typePriority{};
@@ -1211,25 +1208,35 @@ namespace Components
 				typePriority.emplace(type, 1 + ARRAYSIZE(typeOrder) - i);
 			}
 
-			std::map<std::string, std::vector<std::string>> invalidAssets{};
+			enum AssetCategory
+			{
+				EXPLICIT,
+				IMPLICIT,
+				NOT_SUPPORTED,
+				DISAPPEARED,
+
+				COUNT
+			};
+
+			std::vector<std::string> assetsToList[AssetCategory::COUNT]{};
 
 			std::sort(assets.begin(), assets.end(), [&](
-				const std::pair<Game::XAssetType, std::string>& a,
+				const NamedAsset& a,
-				const std::pair<Game::XAssetType, std::string>& b
+				const NamedAsset& b
 				) {
-					if (a.first == b.first)
+					if (a.type == b.type)
 					{
 
-						return a.second.compare(b.second) < 0;
+						return a.name.compare(b.name) < 0;
 					}
 					else
 					{
-						const auto priorityA = typePriority[a.first];
+						const auto priorityA = typePriority[a.type];
-						const auto priorityB = typePriority[b.first];
+						const auto priorityB = typePriority[b.type];
 
 						if (priorityA == priorityB)
 						{
-							return a.second.compare(b.second) < 0;
+							return a.name.compare(b.name) < 0;
 						}
 						else
 						{
@@ -1238,47 +1245,112 @@ namespace Components
 					}
 				});
 
-			// Used to format the CSV
+			std::unordered_set<NamedAsset, NamedAsset::Hash> assetsToSkip{};
-			Game::XAssetType lastTypeEncountered{};
+			const std::function<void(std::unordered_set<iw4of::iw4_native_asset, iw4of::iw4_native_asset::hash>)> recursivelyAddChildren =
+				[&](std::unordered_set<iw4of::iw4_native_asset, iw4of::iw4_native_asset::hash> children)
+				{
+					for (const auto& k : children)
+					{
+						const auto type = static_cast<Game::XAssetType>(k.type);
+						Game::XAsset asset = { type, {k.data} };
+						const auto insertionInfo = assetsToSkip.insert({ type, Game::DB_GetXAssetName(&asset) });
 
+						if (insertionInfo.second)
+						{
+							// True means it was inserted, so we might need to check children aswell
+							if (ExporterAPI.is_type_supported(k.type))
+							{
+								const auto deps = ExporterAPI.get_children(k.type, k.data);
+								recursivelyAddChildren(deps);
+							}
+						}
+					}
+				};
+
+			Logger::Print("Filtering asset list for '{}.csv'...\n", zone);
+			std::vector<Game::XAsset> explicitAssetsToFilter{};
+
+			// Filter implicit and assets that don't need dumping
 			for (const auto& asset : assets)
 			{
-				const auto type = asset.first;
+				const auto type = asset.type;
-				const auto name = asset.second;
+				const auto& name = asset.name;
+
+				if (assetsToSkip.contains({ type, name.data() }))
+				{
+					assetsToList[AssetCategory::IMPLICIT].push_back(std::format("{},{}", Game::DB_GetXAssetTypeName(type), name));
+					continue;
+				}
+
 				if (ExporterAPI.is_type_supported(type) && name[0] != ',')
 				{
 					const auto assetHeader = Game::DB_FindXAssetHeader(type, name.data());
 					if (assetHeader.data)
 					{
-						ExporterAPI.write(type, assetHeader.data);
+						const auto children = ExporterAPI.get_children(type, assetHeader.data);
-						const auto typeName = Game::DB_GetXAssetTypeName(type);
 
-						if (type != lastTypeEncountered)
+						recursivelyAddChildren(children);
-						{
+						explicitAssetsToFilter.push_back({ type, assetHeader });
-							csv << "\n### " << typeName << "\n";
-							lastTypeEncountered = type;
-						}
-
-						csv << typeName << "," << name << "\n";
 					}
 					else
 					{
 						Logger::Warning(Game::conChannel_t::CON_CHANNEL_ERROR, "Asset {} has disappeared while dumping!\n", name);
-						invalidAssets["The following assets disappeared while dumping"].push_back(std::format("{},{}", Game::DB_GetXAssetTypeName(type), name));
+						assetsToList[AssetCategory::DISAPPEARED].push_back(std::format("{},{}", Game::DB_GetXAssetTypeName(type), name));
 					}
 				}
 				else
 				{
-					invalidAssets["The following assets are unsupported or not dumped as individual assets, but still present in the zone"].push_back(std::format("{},{}", Game::DB_GetXAssetTypeName(type), name));
+					assetsToList[AssetCategory::NOT_SUPPORTED].push_back(std::format("{},{}", Game::DB_GetXAssetTypeName(type), name));
 				}
 			}
 
-			for (const auto& kv : invalidAssets)
+			// Write explicit assets
+			Logger::Print("Dumping remaining assets...\n");
+			for (auto& asset : explicitAssetsToFilter)
 			{
-				csv << "\n### " << kv.first << "\n";
+				const auto& name = Game::DB_GetXAssetName(&asset);
-				for (const auto& line : kv.second)
+				if (!assetsToSkip.contains({ asset.type, name }))
 				{
-					csv << "#" << line << "\n";
+					ExporterAPI.write(asset.type, asset.header.data);
+					Logger::Print(".");
+					assetsToList[AssetCategory::EXPLICIT].push_back(std::format("{},{}", Game::DB_GetXAssetTypeName(asset.type), name));
+				}
+			}
+
+			Logger::Print("\n");
+
+			// Write zone source
+			Logger::Print("Writing zone source...\n");
+			std::ofstream csv(std::filesystem::path(GetDumpingZonePath()) / std::format("{}.csv", zone));
+			csv
+				<< std::format("### Zone '{}' dumped with Zonebuilder {}", zone, Components::Branding::GetVersionString())
+				<< "\n\n";
+
+			for (size_t i = 0; i < AssetCategory::COUNT; i++)
+			{
+				if (assetsToList[i].size() == 0)
+				{
+					continue;
+				}
+
+				switch (static_cast<AssetCategory>(i))
+				{
+					case AssetCategory::EXPLICIT:
+						break;
+					case AssetCategory::IMPLICIT:
+						csv << "\n### The following assets are implicitly built with previous assets:\n";
+						break;
+					case AssetCategory::NOT_SUPPORTED:
+						csv << "\n### The following assets are not supported for dumping:\n";
+						break;
+					case AssetCategory::DISAPPEARED:
+						csv << "\n### The following assets disappeared while dumping but are mentioned in the zone:\n";
+						break;
+				}
+
+				for (const auto& asset : assetsToList[i])
+				{
+					csv << (i == AssetCategory::EXPLICIT ? "" : "#") << asset << "\n";
 				}
 			}
 
@@ -1292,7 +1364,7 @@ namespace Components
 	}
 
 
-	std::function<void()> ZoneBuilder::LoadZoneWithTrace(const std::string& zone, OUT std::vector<std::pair<Game::XAssetType, std::string>>& assets)
+	std::function<void()> ZoneBuilder::LoadZoneWithTrace(const std::string& zone, OUT std::vector<NamedAsset>& assets)
 	{
 		ZoneBuilder::BeginAssetTrace(zone);
 
@@ -1433,7 +1505,7 @@ namespace Components
 				{
 					if (!ZoneBuilder::TraceZone.empty() && ZoneBuilder::TraceZone == FastFiles::Current())
 					{
-						ZoneBuilder::TraceAssets.emplace_back(std::make_pair(type, name));
+						ZoneBuilder::TraceAssets.push_back({ type, name });
 					}
 				});
 
@@ -1451,13 +1523,13 @@ namespace Components
 					if (params->size() < 2) return;
 
 					std::string zone = params->get(1);
-					std::vector<std::pair<Game::XAssetType, std::string>> assets{};
+					std::vector<NamedAsset> assets{};
 					const auto unload = ZoneBuilder::LoadZoneWithTrace(zone, assets);
 
 					int count = 0;
 					for (auto i = assets.begin(); i != assets.end(); ++i, ++count)
 					{
-						Logger::Print(" {}: {}: {}\n", count, Game::DB_GetXAssetTypeName(i->first), i->second);
+						Logger::Print(" {}: {}: {}\n", count, Game::DB_GetXAssetTypeName(i->type), i->name);
 					}
 
 					Logger::Print("\n");
@@ -1547,8 +1619,8 @@ namespace Components
 #endif
 							}
 						}
-					}
+				}
-				});
+		});
 
 			Command::Add("listassets", [](const Command::Params* params)
 				{
@@ -1564,8 +1636,8 @@ namespace Components
 							}, &type, false);
 					}
 				});
-		}
 	}
+}
 
 	ZoneBuilder::~ZoneBuilder()
 	{

src/Components/Modules/ZoneBuilder.hpp

@@ -32,7 +32,7 @@ namespace Components
 			private:
 				Zone* builder;
 			};
-			
+
 			Zone(const std::string& zoneName, const std::string& sourceName, const std::string& destination);
 			Zone(const std::string& zoneName);
 			~Zone();
@@ -123,6 +123,22 @@ namespace Components
 			size_t assetDepth;
 		};
 
+		struct NamedAsset
+		{
+			Game::XAssetType type;
+			std::string name;
+
+			bool operator==(const NamedAsset& other) const {
+				return type == other.type && name == other.name;
+			};
+
+			struct Hash {
+				size_t operator()(const NamedAsset& k) const {
+					return static_cast<size_t>(k.type) ^ std::hash<std::string>{}(k.name);
+				}
+			};
+		};
+
 		ZoneBuilder();
 		~ZoneBuilder();
 
@@ -130,10 +146,10 @@ namespace Components
 		static bool IsDumpingZone() { return DumpingZone.length() > 0; };
 
 		static std::string TraceZone;
-		static std::vector<std::pair<Game::XAssetType, std::string>> TraceAssets;
+		static std::vector<NamedAsset> TraceAssets;
 
 		static void BeginAssetTrace(const std::string& zone);
-		static std::vector<std::pair<Game::XAssetType, std::string>> EndAssetTrace();
+		static std::vector<NamedAsset> EndAssetTrace();
 
 		static Dvar::Var zb_sp_to_mp;
 
@@ -164,7 +180,7 @@ namespace Components
 
 		static void DumpZone(const std::string& zone);
 
-		static std::function<void()> LoadZoneWithTrace(const std::string& zone, OUT std::vector<std::pair<Game::XAssetType, std::string>> &assets);
+		static std::function<void()> LoadZoneWithTrace(const std::string& zone, OUT std::vector<NamedAsset>& assets);
 
 		static void Com_Quitf_t();