Rim/iw4x-client
Rim/iw4x-client
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 2 Wiki Activity

Filter CSV while building zonebuilder zone

Browse Source
This commit is contained in:
Louvenarde 2024-02-03 12:33:53 +01:00
parent 67bd07949d
commit a1b024ba67
2 changed files with 135 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

154
src/Components/Modules/ZoneBuilder.cpp
View File

@ -14,7 +14,7 @@
namespace Components
{
std::string ZoneBuilder::TraceZone;
std::vector<std::pair<Game::XAssetType, std::string>> ZoneBuilder::TraceAssets;
std::vector<ZoneBuilder::NamedAsset> ZoneBuilder::TraceAssets;
bool ZoneBuilder::MainThreadInterrupted;
DWORD ZoneBuilder::InterruptingThreadId;
@ -733,11 +733,11 @@ namespace Components
ZoneBuilder::TraceZone = zone;
}
std::vector<std::pair<Game::XAssetType, std::string>> ZoneBuilder::EndAssetTrace()
std::vector<ZoneBuilder::NamedAsset> ZoneBuilder::EndAssetTrace()
{
ZoneBuilder::TraceZone.clear();
std::vector<std::pair<Game::XAssetType, std::string>> AssetTrace;
std::vector<NamedAsset> AssetTrace;
Utils::Merge(&AssetTrace, ZoneBuilder::TraceAssets);
ZoneBuilder::TraceAssets.clear();
@ -1164,20 +1164,15 @@ namespace Components
ZoneBuilder::DumpingZone = zone;
ZoneBuilder::RefreshExporterWorkDirectory();
std::vector<std::pair<Game::XAssetType, std::string>> assets{};
std::vector<NamedAsset> assets{};
const auto unload = ZoneBuilder::LoadZoneWithTrace(zone, assets);
Logger::Print("Dumping zone '{}'...\n", zone);
{
Utils::IO::CreateDir(GetDumpingZonePath());
std::ofstream csv(std::filesystem::path(GetDumpingZonePath()) / std::format("{}.csv", zone));
csv
<< std::format("### Zone '{}' dumped with Zonebuilder {}", zone, Components::Branding::GetVersionString())
<< "\n\n";
// Order the CSV around
// TODO: Trim asset list using IW4OF dependencies
constexpr Game::XAssetType typeOrder[] = {
Game::XAssetType::ASSET_TYPE_GAMEWORLD_MP,
Game::XAssetType::ASSET_TYPE_GAMEWORLD_SP,
@ -1197,11 +1192,13 @@ namespace Components
Game::XAssetType::ASSET_TYPE_PIXELSHADER,
Game::XAssetType::ASSET_TYPE_VERTEXSHADER,
Game::XAssetType::ASSET_TYPE_VERTEXDECL,
Game::XAssetType::ASSET_TYPE_LIGHT_DEF,
Game::XAssetType::ASSET_TYPE_IMAGE,
Game::XAssetType::ASSET_TYPE_SOUND,
Game::XAssetType::ASSET_TYPE_LOADED_SOUND,
Game::XAssetType::ASSET_TYPE_SOUND_CURVE,
Game::XAssetType::ASSET_TYPE_PHYSPRESET,
Game::XAssetType::ASSET_TYPE_LOCALIZE_ENTRY,
};
std::unordered_map<Game::XAssetType, int> typePriority{};
@ -1211,25 +1208,35 @@ namespace Components
typePriority.emplace(type, 1 + ARRAYSIZE(typeOrder) - i);
}
std::map<std::string, std::vector<std::string>> invalidAssets{};
enum AssetCategory
{
EXPLICIT,
IMPLICIT,
NOT_SUPPORTED,
DISAPPEARED,
COUNT
};
std::vector<std::string> assetsToList[AssetCategory::COUNT]{};
std::sort(assets.begin(), assets.end(), [&](
const std::pair<Game::XAssetType, std::string>& a,
const std::pair<Game::XAssetType, std::string>& b
const NamedAsset& a,
const NamedAsset& b
) {
if (a.first == b.first)
if (a.type == b.type)
{
return a.second.compare(b.second) < 0;
return a.name.compare(b.name) < 0;
}
else
{
const auto priorityA = typePriority[a.first];
const auto priorityB = typePriority[b.first];
const auto priorityA = typePriority[a.type];
const auto priorityB = typePriority[b.type];
if (priorityA == priorityB)
{
return a.second.compare(b.second) < 0;
return a.name.compare(b.name) < 0;
}
else
{
@ -1238,47 +1245,112 @@ namespace Components
}
});
// Used to format the CSV
Game::XAssetType lastTypeEncountered{};
std::unordered_set<NamedAsset, NamedAsset::Hash> assetsToSkip{};
const std::function<void(std::unordered_set<iw4of::iw4_native_asset, iw4of::iw4_native_asset::hash>)> recursivelyAddChildren =
[&](std::unordered_set<iw4of::iw4_native_asset, iw4of::iw4_native_asset::hash> children)
{
for (const auto& k : children)
{
const auto type = static_cast<Game::XAssetType>(k.type);
Game::XAsset asset = { type, {k.data} };
const auto insertionInfo = assetsToSkip.insert({ type, Game::DB_GetXAssetName(&asset) });
if (insertionInfo.second)
{
// True means it was inserted, so we might need to check children aswell
if (ExporterAPI.is_type_supported(k.type))
{
const auto deps = ExporterAPI.get_children(k.type, k.data);
recursivelyAddChildren(deps);
}
}
}
};
Logger::Print("Filtering asset list for '{}.csv'...\n", zone);
std::vector<Game::XAsset> explicitAssetsToFilter{};
// Filter implicit and assets that don't need dumping
for (const auto& asset : assets)
{
const auto type = asset.first;
const auto name = asset.second;
const auto type = asset.type;
const auto& name = asset.name;
if (assetsToSkip.contains({ type, name.data() }))
{
assetsToList[AssetCategory::IMPLICIT].push_back(std::format("{},{}", Game::DB_GetXAssetTypeName(type), name));
continue;
}
if (ExporterAPI.is_type_supported(type) && name[0] != ',')
{
const auto assetHeader = Game::DB_FindXAssetHeader(type, name.data());
if (assetHeader.data)
{
ExporterAPI.write(type, assetHeader.data);
const auto typeName = Game::DB_GetXAssetTypeName(type);
const auto children = ExporterAPI.get_children(type, assetHeader.data);
if (type != lastTypeEncountered)
{
csv << "\n### " << typeName << "\n";
lastTypeEncountered = type;
}
csv << typeName << "," << name << "\n";
recursivelyAddChildren(children);
explicitAssetsToFilter.push_back({ type, assetHeader });
}
else
{
Logger::Warning(Game::conChannel_t::CON_CHANNEL_ERROR, "Asset {} has disappeared while dumping!\n", name);
invalidAssets["The following assets disappeared while dumping"].push_back(std::format("{},{}", Game::DB_GetXAssetTypeName(type), name));
assetsToList[AssetCategory::DISAPPEARED].push_back(std::format("{},{}", Game::DB_GetXAssetTypeName(type), name));
}
}
else
{
invalidAssets["The following assets are unsupported or not dumped as individual assets, but still present in the zone"].push_back(std::format("{},{}", Game::DB_GetXAssetTypeName(type), name));
assetsToList[AssetCategory::NOT_SUPPORTED].push_back(std::format("{},{}", Game::DB_GetXAssetTypeName(type), name));
}
}
for (const auto& kv : invalidAssets)
// Write explicit assets
Logger::Print("Dumping remaining assets...\n");
for (auto& asset : explicitAssetsToFilter)
{
csv << "\n### " << kv.first << "\n";
for (const auto& line : kv.second)
const auto& name = Game::DB_GetXAssetName(&asset);
if (!assetsToSkip.contains({ asset.type, name }))
{
csv << "#" << line << "\n";
ExporterAPI.write(asset.type, asset.header.data);
Logger::Print(".");
assetsToList[AssetCategory::EXPLICIT].push_back(std::format("{},{}", Game::DB_GetXAssetTypeName(asset.type), name));
}
}
Logger::Print("\n");
// Write zone source
Logger::Print("Writing zone source...\n");
std::ofstream csv(std::filesystem::path(GetDumpingZonePath()) / std::format("{}.csv", zone));
csv
<< std::format("### Zone '{}' dumped with Zonebuilder {}", zone, Components::Branding::GetVersionString())
<< "\n\n";
for (size_t i = 0; i < AssetCategory::COUNT; i++)
{
if (assetsToList[i].size() == 0)
{
continue;
}
switch (static_cast<AssetCategory>(i))
{
case AssetCategory::EXPLICIT:
break;
case AssetCategory::IMPLICIT:
csv << "\n### The following assets are implicitly built with previous assets:\n";
break;
case AssetCategory::NOT_SUPPORTED:
csv << "\n### The following assets are not supported for dumping:\n";
break;
case AssetCategory::DISAPPEARED:
csv << "\n### The following assets disappeared while dumping but are mentioned in the zone:\n";
break;
}
for (const auto& asset : assetsToList[i])
{
csv << (i == AssetCategory::EXPLICIT ? "" : "#") << asset << "\n";
}
}
@ -1292,7 +1364,7 @@ namespace Components
}
std::function<void()> ZoneBuilder::LoadZoneWithTrace(const std::string& zone, OUT std::vector<std::pair<Game::XAssetType, std::string>>& assets)
std::function<void()> ZoneBuilder::LoadZoneWithTrace(const std::string& zone, OUT std::vector<NamedAsset>& assets)
{
ZoneBuilder::BeginAssetTrace(zone);
@ -1433,7 +1505,7 @@ namespace Components
{
if (!ZoneBuilder::TraceZone.empty() && ZoneBuilder::TraceZone == FastFiles::Current())
{
ZoneBuilder::TraceAssets.emplace_back(std::make_pair(type, name));
ZoneBuilder::TraceAssets.push_back({ type, name });
}
});
@ -1451,13 +1523,13 @@ namespace Components
if (params->size() < 2) return;
std::string zone = params->get(1);
std::vector<std::pair<Game::XAssetType, std::string>> assets{};
std::vector<NamedAsset> assets{};
const auto unload = ZoneBuilder::LoadZoneWithTrace(zone, assets);
int count = 0;
for (auto i = assets.begin(); i != assets.end(); ++i, ++count)
{
Logger::Print(" {}: {}: {}\n", count, Game::DB_GetXAssetTypeName(i->first), i->second);
Logger::Print(" {}: {}: {}\n", count, Game::DB_GetXAssetTypeName(i->type), i->name);
}
Logger::Print("\n");
@ -1565,7 +1637,7 @@ namespace Components
}
});
}
}
}
ZoneBuilder::~ZoneBuilder()
{

22
src/Components/Modules/ZoneBuilder.hpp
View File

@ -123,6 +123,22 @@ namespace Components
size_t assetDepth;
};
struct NamedAsset
{
Game::XAssetType type;
std::string name;
bool operator==(const NamedAsset& other) const {
return type == other.type && name == other.name;
};
struct Hash {
size_t operator()(const NamedAsset& k) const {
return static_cast<size_t>(k.type) ^ std::hash<std::string>{}(k.name);
}
};
};
ZoneBuilder();
~ZoneBuilder();
@ -130,10 +146,10 @@ namespace Components
static bool IsDumpingZone() { return DumpingZone.length() > 0; };
static std::string TraceZone;
static std::vector<std::pair<Game::XAssetType, std::string>> TraceAssets;
static std::vector<NamedAsset> TraceAssets;
static void BeginAssetTrace(const std::string& zone);
static std::vector<std::pair<Game::XAssetType, std::string>> EndAssetTrace();
static std::vector<NamedAsset> EndAssetTrace();
static Dvar::Var zb_sp_to_mp;
@ -164,7 +180,7 @@ namespace Components
static void DumpZone(const std::string& zone);
static std::function<void()> LoadZoneWithTrace(const std::string& zone, OUT std::vector<std::pair<Game::XAssetType, std::string>> &assets);
static std::function<void()> LoadZoneWithTrace(const std::string& zone, OUT std::vector<NamedAsset>& assets);
static void Com_Quitf_t();