From 3896ec0e4eb3793fd5fce762f4e83bbd4a8630fd Mon Sep 17 00:00:00 2001 From: INeedGames Date: Sun, 7 Jan 2024 06:07:34 -0600 Subject: [PATCH] feat(compiler): produce a developer source map (#167) Co-authored-by: xensik --- include/xsk/arc/assembler.hpp | 4 ++- include/xsk/arc/common/assembly.hpp | 7 ++++++ include/xsk/arc/common/location.hpp | 2 +- include/xsk/arc/compiler.hpp | 1 + include/xsk/gsc/assembler.hpp | 4 ++- include/xsk/gsc/common/assembly.hpp | 7 ++++++ include/xsk/gsc/common/location.hpp | 2 +- include/xsk/gsc/compiler.hpp | 1 + src/arc/assembler.cpp | 18 ++++++++++--- src/arc/compiler.cpp | 8 ++++++ src/gsc/assembler.cpp | 16 ++++++++++-- src/gsc/compiler.cpp | 8 ++++++ src/tool/main.cpp | 39 +++++++++++++++++------------ 13 files changed, 92 insertions(+), 25 deletions(-) diff --git a/include/xsk/arc/assembler.hpp b/include/xsk/arc/assembler.hpp index 792f502d..58ecfd03 100644 --- a/include/xsk/arc/assembler.hpp +++ b/include/xsk/arc/assembler.hpp @@ -17,15 +17,17 @@ class assembler function const* func_; assembly const* assembly_; utils::writer script_; + utils::writer devmap_; std::unordered_map strpool_; std::vector exports_; std::vector imports_; std::vector strings_; std::vector anims_; + u32 devmap_count_; public: assembler(context const* ctx); - auto assemble(assembly const& data, std::string const& name = {}) -> buffer; + auto assemble(assembly const& data, std::string const& name = {}) -> std::pair; private: auto assemble_function(function& func) -> void; diff --git a/include/xsk/arc/common/assembly.hpp b/include/xsk/arc/common/assembly.hpp index a8c1b3a6..d9abbb1c 100644 --- a/include/xsk/arc/common/assembly.hpp +++ b/include/xsk/arc/common/assembly.hpp @@ -158,12 +158,19 @@ enum class opcode : u8 OP_Count, }; +struct sourcepos +{ + u16 line; + u16 column; +}; + struct instruction { using ptr = std::unique_ptr; u32 index; u32 size; + sourcepos pos; opcode opcode; std::vector data; diff --git a/include/xsk/arc/common/location.hpp b/include/xsk/arc/common/location.hpp index 3a93247f..7e7b608a 100644 --- a/include/xsk/arc/common/location.hpp +++ b/include/xsk/arc/common/location.hpp @@ -12,7 +12,7 @@ class position { public: typedef const std::string filename_type; - typedef int counter_type; + typedef u16 counter_type; filename_type *filename; counter_type line; diff --git a/include/xsk/arc/compiler.hpp b/include/xsk/arc/compiler.hpp index af62a995..180ec95a 100644 --- a/include/xsk/arc/compiler.hpp +++ b/include/xsk/arc/compiler.hpp @@ -20,6 +20,7 @@ class compiler std::vector scopes_; std::unordered_map constants_; std::string animtree_; + sourcepos debug_pos_; u32 index_; u32 label_idx_; bool can_break_; diff --git a/include/xsk/gsc/assembler.hpp b/include/xsk/gsc/assembler.hpp index 98fa1613..1b1f100f 100644 --- a/include/xsk/gsc/assembler.hpp +++ b/include/xsk/gsc/assembler.hpp @@ -18,10 +18,12 @@ class assembler assembly const* assembly_; utils::writer script_; utils::writer stack_; + utils::writer devmap_; + u32 devmap_count_; public: assembler(context const* ctx); - auto assemble(assembly const& data) -> std::pair; + auto assemble(assembly const& data) -> std::tuple; private: auto assemble_function(function const& func) -> void; diff --git a/include/xsk/gsc/common/assembly.hpp b/include/xsk/gsc/common/assembly.hpp index e4dd6df1..c82a928a 100644 --- a/include/xsk/gsc/common/assembly.hpp +++ b/include/xsk/gsc/common/assembly.hpp @@ -220,12 +220,19 @@ enum class opcode : u8 OP_count, }; +struct sourcepos +{ + u16 line; + u16 column; +}; + struct instruction { using ptr = std::unique_ptr; u32 index; u32 size; + sourcepos pos; opcode opcode; std::vector data; diff --git a/include/xsk/gsc/common/location.hpp b/include/xsk/gsc/common/location.hpp index 16629284..62637d60 100644 --- a/include/xsk/gsc/common/location.hpp +++ b/include/xsk/gsc/common/location.hpp @@ -12,7 +12,7 @@ class position { public: typedef const std::string filename_type; - typedef int counter_type; + typedef u16 counter_type; filename_type *filename; counter_type line; diff --git a/include/xsk/gsc/compiler.hpp b/include/xsk/gsc/compiler.hpp index 19eb0726..0518506e 100644 --- a/include/xsk/gsc/compiler.hpp +++ b/include/xsk/gsc/compiler.hpp @@ -22,6 +22,7 @@ class compiler std::vector break_blks_; std::vector continue_blks_; std::string animname_; + sourcepos debug_pos_; u32 index_; u32 label_idx_; bool can_break_; diff --git a/src/arc/assembler.cpp b/src/arc/assembler.cpp index c7f53f2e..2e258c02 100644 --- a/src/arc/assembler.cpp +++ b/src/arc/assembler.cpp @@ -14,17 +14,20 @@ assembler::assembler(context const* ctx) : ctx_{ ctx }, script_{ ctx->endian() = { } -auto assembler::assemble(assembly const& data, std::string const& name) -> buffer +auto assembler::assemble(assembly const& data, std::string const& name) -> std::pair { assembly_ = &data; script_.clear(); + devmap_.clear(); strpool_.clear(); exports_.clear(); imports_.clear(); strings_.clear(); anims_.clear(); + devmap_count_ = 0; auto head = header{}; + devmap_.pos(sizeof(u32)); script_.pos((ctx_->props() & props::headerxx) ? 0 : (ctx_->props() & props::header72) ? 72 : 64); process_string(name); @@ -185,7 +188,7 @@ auto assembler::assemble(assembly const& data, std::string const& name) -> buffe head.flags = 0; head.name = resolve_string(name); - auto endpos = script_.pos(); + auto const endpos = script_.pos(); script_.pos(0); script_.write(ctx_->magic()); @@ -223,7 +226,12 @@ auto assembler::assemble(assembly const& data, std::string const& name) -> buffe script_.write(head.flags); script_.pos(endpos); - return buffer{ script_.data(), script_.pos() }; + auto const dev_endpos = devmap_.pos(); + devmap_.pos(0); + devmap_.write(devmap_count_); + devmap_.pos(dev_endpos); + + return { buffer{ script_.data(), script_.pos() }, buffer{ devmap_.data(), devmap_.pos() } }; } auto assembler::assemble_function(function& func) -> void @@ -270,6 +278,10 @@ auto assembler::assemble_function(function& func) -> void auto assembler::assemble_instruction(instruction const& inst) -> void { + devmap_count_++; + devmap_.write(script_.pos()); + devmap_.write(inst.pos.line); + devmap_.write(inst.pos.column); script_.write(static_cast(ctx_->opcode_id(inst.opcode))); switch (inst.opcode) diff --git a/src/arc/compiler.cpp b/src/arc/compiler.cpp index 875eded3..aa59c68b 100644 --- a/src/arc/compiler.cpp +++ b/src/arc/compiler.cpp @@ -33,6 +33,7 @@ auto compiler::emit_program(program const& prog) -> void developer_thread_ = false; animtree_ = {}; index_ = 0; + debug_pos_ = { 0, 0 }; for (auto const& include : prog.includes) { @@ -138,6 +139,8 @@ auto compiler::emit_decl_function(decl_function const& func) -> void auto compiler::emit_stmt(stmt const& stm) -> void { + debug_pos_ = { stm.loc().begin.line, stm.loc().begin.column }; + switch (stm.kind()) { case node::stmt_list: @@ -695,6 +698,8 @@ auto compiler::emit_stmt_prof_end(stmt_prof_end const&) -> void auto compiler::emit_expr(expr const& exp) -> void { + debug_pos_ = { exp.loc().begin.line, exp.loc().begin.column }; + switch (exp.kind()) { case node::expr_paren: @@ -1875,6 +1880,7 @@ auto compiler::emit_opcode(opcode op) -> void inst->opcode = op; inst->size = ctx_->opcode_size(op); inst->index = index_; + inst->pos = debug_pos_; index_ += inst->size; } @@ -1888,6 +1894,7 @@ auto compiler::emit_opcode(opcode op, std::string const& data) -> void inst->size = ctx_->opcode_size(op); inst->index = index_; inst->data.push_back(data); + inst->pos = debug_pos_; index_ += inst->size; } @@ -1901,6 +1908,7 @@ auto compiler::emit_opcode(opcode op, std::vector const& data) -> v inst->size = ctx_->opcode_size(op); inst->index = index_; inst->data = data; + inst->pos = debug_pos_; index_ += inst->size; } diff --git a/src/gsc/assembler.cpp b/src/gsc/assembler.cpp index b7113747..27f652d0 100644 --- a/src/gsc/assembler.cpp +++ b/src/gsc/assembler.cpp @@ -14,12 +14,15 @@ assembler::assembler(context const* ctx) : ctx_{ ctx }, script_{ ctx->endian() = { } -auto assembler::assemble(assembly const& data) -> std::pair +auto assembler::assemble(assembly const& data) -> std::tuple { assembly_ = &data; script_.clear(); stack_.clear(); + devmap_.clear(); + devmap_count_ = 0; + devmap_.pos(sizeof(u32)); script_.write(ctx_->opcode_id(opcode::OP_End)); for (auto const& func : data.functions) @@ -27,7 +30,12 @@ auto assembler::assemble(assembly const& data) -> std::pair assemble_function(*func); } - return { buffer{ script_.data(), script_.pos() }, buffer{ stack_.data(), stack_.pos() } }; + auto const dev_endpos = devmap_.pos(); + devmap_.pos(0); + devmap_.write(devmap_count_); + devmap_.pos(dev_endpos); + + return { buffer{ script_.data(), script_.pos() }, buffer{ stack_.data(), stack_.pos() }, buffer{ devmap_.data(), devmap_.pos() } }; } auto assembler::assemble_function(function const& func) -> void @@ -61,6 +69,10 @@ auto assembler::assemble_function(function const& func) -> void auto assembler::assemble_instruction(instruction const& inst) -> void { + devmap_count_++; + devmap_.write(script_.pos()); + devmap_.write(inst.pos.line); + devmap_.write(inst.pos.column); script_.write(ctx_->opcode_id(inst.opcode)); switch (inst.opcode) diff --git a/src/gsc/compiler.cpp b/src/gsc/compiler.cpp index 4b37b6df..080a9d5b 100644 --- a/src/gsc/compiler.cpp +++ b/src/gsc/compiler.cpp @@ -35,6 +35,7 @@ auto compiler::emit_program(program const& prog) -> void animload_ = false; animname_ = {}; index_ = 1; + debug_pos_ = { 0, 0 }; ctx_->init_includes(); @@ -147,6 +148,8 @@ auto compiler::emit_decl_function(decl_function const& func) -> void auto compiler::emit_stmt(stmt const& stm, scope& scp, bool last) -> void { + debug_pos_ = { stm.loc().begin.line, stm.loc().begin.column }; + switch (stm.kind()) { case node::stmt_list: @@ -925,6 +928,8 @@ auto compiler::emit_stmt_assertmsg(stmt_assertmsg const&, scope&) -> void auto compiler::emit_expr(expr const& exp, scope& scp) -> void { + debug_pos_ = { exp.loc().begin.line, exp.loc().begin.column }; + switch (exp.kind()) { case node::expr_paren: @@ -2197,6 +2202,7 @@ auto compiler::emit_opcode(opcode op) -> void inst->opcode = op; inst->size = ctx_->opcode_size(op); inst->index = index_; + inst->pos = debug_pos_; index_ += inst->size; } @@ -2210,6 +2216,7 @@ auto compiler::emit_opcode(opcode op, std::string const& data) -> void inst->size = ctx_->opcode_size(op); inst->index = index_; inst->data.push_back(data); + inst->pos = debug_pos_; index_ += inst->size; } @@ -2223,6 +2230,7 @@ auto compiler::emit_opcode(opcode op, std::vector const& data) -> v inst->size = ctx_->opcode_size(op); inst->index = index_; inst->data = data; + inst->pos = debug_pos_; index_ += inst->size; } diff --git a/src/tool/main.cpp b/src/tool/main.cpp index 5f9d6725..c48ee48a 100644 --- a/src/tool/main.cpp +++ b/src/tool/main.cpp @@ -176,8 +176,8 @@ auto assemble_file(game game, mach mach, fs::path file, fs::path rel) -> result if (zonetool) { auto path = fs::path{ "assembled" } / rel; - utils::file::save(path, outbin.first.data, outbin.first.size); - utils::file::save(path.replace_extension(".cgsc.stack"), outbin.second.data, outbin.second.size); + utils::file::save(path, std::get<0>(outbin).data, std::get<0>(outbin).size); + utils::file::save(path.replace_extension(".cgsc.stack"), std::get<1>(outbin).data, std::get<1>(outbin).size); std::cout << fmt::format("assembled {}\n", rel.generic_string()); } else @@ -185,14 +185,14 @@ auto assemble_file(game game, mach mach, fs::path file, fs::path rel) -> result asset script; script.name = "GSC"s; - script.bytecode.resize(outbin.first.size); - std::memcpy(script.bytecode.data(), outbin.first.data, script.bytecode.size()); + script.bytecode.resize(std::get<0>(outbin).size); + std::memcpy(script.bytecode.data(), std::get<0>(outbin).data, script.bytecode.size()); - script.buffer.resize(outbin.second.size); - std::memcpy(script.buffer.data(), outbin.second.data, script.buffer.size()); + script.buffer.resize(std::get<1>(outbin).size); + std::memcpy(script.buffer.data(), std::get<1>(outbin).data, script.buffer.size()); script.buffer = utils::zlib::compress(script.buffer); - script.len = static_cast(outbin.second.size); + script.len = static_cast(std::get<1>(outbin).size); script.compressedLen = static_cast(script.buffer.size()); script.bytecodeLen = static_cast(script.bytecode.size()); @@ -271,8 +271,8 @@ auto compile_file(game game, mach mach, fs::path file, fs::path rel) -> result if (zonetool) { auto path = fs::path{ "compiled" } / rel; - utils::file::save(path, outbin.first.data, outbin.first.size); - utils::file::save(path.replace_extension(".cgsc.stack"), outbin.second.data, outbin.second.size); + utils::file::save(path, std::get<0>(outbin).data, std::get<0>(outbin).size); + utils::file::save(path.replace_extension(".cgsc.stack"), std::get<1>(outbin).data, std::get<1>(outbin).size); std::cout << fmt::format("compiled {}\n", rel.generic_string()); } else @@ -280,20 +280,23 @@ auto compile_file(game game, mach mach, fs::path file, fs::path rel) -> result asset script; script.name = "GSC"s; - script.bytecode.resize(outbin.first.size); - std::memcpy(script.bytecode.data(), outbin.first.data, script.bytecode.size()); + script.bytecode.resize(std::get<0>(outbin).size); + std::memcpy(script.bytecode.data(), std::get<0>(outbin).data, script.bytecode.size()); - script.buffer.resize(outbin.second.size); - std::memcpy(script.buffer.data(), outbin.second.data, script.buffer.size()); + script.buffer.resize(std::get<1>(outbin).size); + std::memcpy(script.buffer.data(), std::get<1>(outbin).data, script.buffer.size()); script.buffer = utils::zlib::compress(script.buffer); - script.len = static_cast(outbin.second.size); + script.len = static_cast(std::get<1>(outbin).size); script.compressedLen = static_cast(script.buffer.size()); script.bytecodeLen = static_cast(script.bytecode.size()); auto result = script.serialize(); utils::file::save(fs::path{ "compiled" } / rel, result); std::cout << fmt::format("compiled {}\n", rel.generic_string()); + + utils::file::save(fs::path{ "compiled" } / fs::path{ "developer_maps" } / rel.replace_extension(".gscmap"), std::get<2>(outbin).data, std::get<2>(outbin).size); + std::cout << fmt::format("saved developer map {}\n", rel.generic_string()); } } @@ -741,7 +744,7 @@ auto assemble_file(game game, mach mach, fs::path const& file, fs::path rel) -> auto outasm = contexts[game][mach]->source().parse_assembly(data); auto outbin = contexts[game][mach]->assembler().assemble(*outasm); - utils::file::save(fs::path{ "assembled" } / rel, outbin.data, outbin.size); + utils::file::save(fs::path{ "assembled" } / rel, outbin.first.data, outbin.first.size); std::cout << fmt::format("assembled {}\n", rel.generic_string()); return result::success; } @@ -793,8 +796,12 @@ auto compile_file(game game, mach mach, fs::path const& file, fs::path rel) -> r auto outasm = contexts[game][mach]->compiler().compile(file.string(), data); auto outbin = contexts[game][mach]->assembler().assemble(*outasm); - utils::file::save(fs::path{ "compiled" } / rel, outbin.data, outbin.size); + utils::file::save(fs::path{ "compiled" } / rel, outbin.first.data, outbin.first.size); std::cout << fmt::format("compiled {}\n", rel.generic_string()); + + utils::file::save(fs::path{ "compiled" } / fs::path{ "developer_maps" } / rel.replace_extension(".gscmap"), outbin.second.data, outbin.second.size); + std::cout << fmt::format("saved developer map {}\n", rel.generic_string()); + return result::success; } catch (std::exception const& e)