feat(compiler): produce a developer source map (#167)

Co-authored-by: xensik <xensik@pm.me>

include/xsk/arc/assembler.hpp

@@ -17,15 +17,17 @@ class assembler
     function const* func_;
     assembly const* assembly_;
     utils::writer script_;
+    utils::writer devmap_;
     std::unordered_map<std::string, u16> strpool_;
     std::vector<export_ref> exports_;
     std::vector<import_ref> imports_;
     std::vector<string_ref> strings_;
     std::vector<animtree_ref> anims_;
+    u32 devmap_count_;
 
 public:
     assembler(context const* ctx);
-    auto assemble(assembly const& data, std::string const& name = {}) -> buffer;
+    auto assemble(assembly const& data, std::string const& name = {}) -> std::pair<buffer, buffer>;
 
 private:
     auto assemble_function(function& func) -> void;

include/xsk/arc/common/assembly.hpp

@@ -158,12 +158,19 @@ enum class opcode : u8
     OP_Count,
 };
 
+struct sourcepos
+{
+    u16 line;
+    u16 column;
+};
+
 struct instruction
 {
     using ptr = std::unique_ptr<instruction>;
 
     u32 index;
     u32 size;
+    sourcepos pos;
     opcode opcode;
     std::vector<std::string> data;
 

include/xsk/arc/common/location.hpp

@@ -12,7 +12,7 @@ class position
 {
 public:
     typedef const std::string filename_type;
-    typedef int counter_type;
+    typedef u16 counter_type;
 
     filename_type *filename;
     counter_type line;

include/xsk/arc/compiler.hpp

@@ -20,6 +20,7 @@ class compiler
     std::vector<scope> scopes_;
     std::unordered_map<std::string, expr const*> constants_;
     std::string animtree_;
+    sourcepos debug_pos_;
     u32 index_;
     u32 label_idx_;
     bool can_break_;

include/xsk/gsc/assembler.hpp

@@ -18,10 +18,12 @@ class assembler
     assembly const* assembly_;
     utils::writer script_;
     utils::writer stack_;
+    utils::writer devmap_;
+    u32 devmap_count_;
 
 public:
     assembler(context const* ctx);
-    auto assemble(assembly const& data) -> std::pair<buffer, buffer>;
+    auto assemble(assembly const& data) -> std::tuple<buffer, buffer, buffer>;
 
 private:
     auto assemble_function(function const& func) -> void;

include/xsk/gsc/common/assembly.hpp

@@ -220,12 +220,19 @@ enum class opcode : u8
     OP_count,
 };
 
+struct sourcepos
+{
+    u16 line;
+    u16 column;
+};
+
 struct instruction
 {
     using ptr = std::unique_ptr<instruction>;
 
     u32 index;
     u32 size;
+    sourcepos pos;
     opcode opcode;
     std::vector<std::string> data;
 

include/xsk/gsc/common/location.hpp

@@ -12,7 +12,7 @@ class position
 {
 public:
     typedef const std::string filename_type;
-    typedef int counter_type;
+    typedef u16 counter_type;
 
     filename_type *filename;
     counter_type line;

include/xsk/gsc/compiler.hpp

@@ -22,6 +22,7 @@ class compiler
     std::vector<scope*> break_blks_;
     std::vector<scope*> continue_blks_;
     std::string animname_;
+    sourcepos debug_pos_;
     u32 index_;
     u32 label_idx_;
     bool can_break_;

src/arc/assembler.cpp

@@ -14,17 +14,20 @@ assembler::assembler(context const* ctx) : ctx_{ ctx }, script_{ ctx->endian() =
 {
 }
 
-auto assembler::assemble(assembly const& data, std::string const& name) -> buffer
+auto assembler::assemble(assembly const& data, std::string const& name) -> std::pair<buffer, buffer>
 {
     assembly_ = &data;
     script_.clear();
+    devmap_.clear();
     strpool_.clear();
     exports_.clear();
     imports_.clear();
     strings_.clear();
     anims_.clear();
+    devmap_count_ = 0;
     auto head = header{};
 
+    devmap_.pos(sizeof(u32));
     script_.pos((ctx_->props() & props::headerxx) ? 0 : (ctx_->props() & props::header72) ? 72 : 64);
     process_string(name);
 
@@ -185,7 +188,7 @@ auto assembler::assemble(assembly const& data, std::string const& name) -> buffe
     head.flags = 0;
     head.name = resolve_string(name);
 
-    auto endpos = script_.pos();
+    auto const endpos = script_.pos();
 
     script_.pos(0);
     script_.write<u64>(ctx_->magic());
@@ -223,7 +226,12 @@ auto assembler::assemble(assembly const& data, std::string const& name) -> buffe
     script_.write<u8>(head.flags);
     script_.pos(endpos);
 
-    return buffer{ script_.data(), script_.pos() };
+    auto const dev_endpos = devmap_.pos();
+    devmap_.pos(0);
+    devmap_.write<u32>(devmap_count_);
+    devmap_.pos(dev_endpos);
+
+    return { buffer{ script_.data(), script_.pos() }, buffer{ devmap_.data(), devmap_.pos() } };
 }
 
 auto assembler::assemble_function(function& func) -> void
@@ -270,6 +278,10 @@ auto assembler::assemble_function(function& func) -> void
 
 auto assembler::assemble_instruction(instruction const& inst) -> void
 {
+    devmap_count_++;
+    devmap_.write<u32>(script_.pos());
+    devmap_.write<u16>(inst.pos.line);
+    devmap_.write<u16>(inst.pos.column);
     script_.write<u8>(static_cast<u8>(ctx_->opcode_id(inst.opcode)));
 
     switch (inst.opcode)

src/arc/compiler.cpp

@@ -33,6 +33,7 @@ auto compiler::emit_program(program const& prog) -> void
     developer_thread_ = false;
     animtree_ = {};
     index_ = 0;
+    debug_pos_ = { 0, 0 };
 
     for (auto const& include : prog.includes)
     {
@@ -138,6 +139,8 @@ auto compiler::emit_decl_function(decl_function const& func) -> void
 
 auto compiler::emit_stmt(stmt const& stm) -> void
 {
+    debug_pos_ = { stm.loc().begin.line, stm.loc().begin.column };
+
     switch (stm.kind())
     {
         case node::stmt_list:
@@ -695,6 +698,8 @@ auto compiler::emit_stmt_prof_end(stmt_prof_end const&) -> void
 
 auto compiler::emit_expr(expr const& exp) -> void
 {
+    debug_pos_ = { exp.loc().begin.line, exp.loc().begin.column };
+
     switch (exp.kind())
     {
         case node::expr_paren:
@@ -1875,6 +1880,7 @@ auto compiler::emit_opcode(opcode op) -> void
     inst->opcode = op;
     inst->size = ctx_->opcode_size(op);
     inst->index = index_;
+    inst->pos = debug_pos_;
 
     index_ += inst->size;
 }
@@ -1888,6 +1894,7 @@ auto compiler::emit_opcode(opcode op, std::string const& data) -> void
     inst->size = ctx_->opcode_size(op);
     inst->index = index_;
     inst->data.push_back(data);
+    inst->pos = debug_pos_;
 
     index_ += inst->size;
 }
@@ -1901,6 +1908,7 @@ auto compiler::emit_opcode(opcode op, std::vector<std::string> const& data) -> v
     inst->size = ctx_->opcode_size(op);
     inst->index = index_;
     inst->data = data;
+    inst->pos = debug_pos_;
 
     index_ += inst->size;
 }

src/gsc/assembler.cpp

@@ -14,12 +14,15 @@ assembler::assembler(context const* ctx) : ctx_{ ctx }, script_{ ctx->endian() =
 {
 }
 
-auto assembler::assemble(assembly const& data) -> std::pair<buffer, buffer>
+auto assembler::assemble(assembly const& data) -> std::tuple<buffer, buffer, buffer>
 {
     assembly_ = &data;
     script_.clear();
     stack_.clear();
+    devmap_.clear();
+    devmap_count_ = 0;
 
+    devmap_.pos(sizeof(u32));
     script_.write<u8>(ctx_->opcode_id(opcode::OP_End));
 
     for (auto const& func : data.functions)
@@ -27,7 +30,12 @@ auto assembler::assemble(assembly const& data) -> std::pair<buffer, buffer>
         assemble_function(*func);
     }
 
-    return { buffer{ script_.data(), script_.pos() }, buffer{ stack_.data(), stack_.pos() } };
+    auto const dev_endpos = devmap_.pos();
+    devmap_.pos(0);
+    devmap_.write<u32>(devmap_count_);
+    devmap_.pos(dev_endpos);
+
+    return { buffer{ script_.data(), script_.pos() }, buffer{ stack_.data(), stack_.pos() }, buffer{ devmap_.data(), devmap_.pos() } };
 }
 
 auto assembler::assemble_function(function const& func) -> void
@@ -61,6 +69,10 @@ auto assembler::assemble_function(function const& func) -> void
 
 auto assembler::assemble_instruction(instruction const& inst) -> void
 {
+    devmap_count_++;
+    devmap_.write<u32>(script_.pos());
+    devmap_.write<u16>(inst.pos.line);
+    devmap_.write<u16>(inst.pos.column);
     script_.write<u8>(ctx_->opcode_id(inst.opcode));
 
     switch (inst.opcode)

src/gsc/compiler.cpp

@@ -35,6 +35,7 @@ auto compiler::emit_program(program const& prog) -> void
     animload_ = false;
     animname_ = {};
     index_ = 1;
+    debug_pos_ = { 0, 0 };
 
     ctx_->init_includes();
 
@@ -147,6 +148,8 @@ auto compiler::emit_decl_function(decl_function const& func) -> void
 
 auto compiler::emit_stmt(stmt const& stm, scope& scp, bool last) -> void
 {
+    debug_pos_ = { stm.loc().begin.line, stm.loc().begin.column };
+
     switch (stm.kind())
     {
         case node::stmt_list:
@@ -925,6 +928,8 @@ auto compiler::emit_stmt_assertmsg(stmt_assertmsg const&, scope&) -> void
 
 auto compiler::emit_expr(expr const& exp, scope& scp) -> void
 {
+    debug_pos_ = { exp.loc().begin.line, exp.loc().begin.column };
+
     switch (exp.kind())
     {
         case node::expr_paren:
@@ -2197,6 +2202,7 @@ auto compiler::emit_opcode(opcode op) -> void
     inst->opcode = op;
     inst->size = ctx_->opcode_size(op);
     inst->index = index_;
+    inst->pos = debug_pos_;
 
     index_ += inst->size;
 }
@@ -2210,6 +2216,7 @@ auto compiler::emit_opcode(opcode op, std::string const& data) -> void
     inst->size = ctx_->opcode_size(op);
     inst->index = index_;
     inst->data.push_back(data);
+    inst->pos = debug_pos_;
 
     index_ += inst->size;
 }
@@ -2223,6 +2230,7 @@ auto compiler::emit_opcode(opcode op, std::vector<std::string> const& data) -> v
     inst->size = ctx_->opcode_size(op);
     inst->index = index_;
     inst->data = data;
+    inst->pos = debug_pos_;
 
     index_ += inst->size;
 }

src/tool/main.cpp

@@ -176,8 +176,8 @@ auto assemble_file(game game, mach mach, fs::path file, fs::path rel) -> result
             if (zonetool)
             {
                 auto path = fs::path{ "assembled" } / rel;
-                utils::file::save(path, outbin.first.data, outbin.first.size);
+                utils::file::save(path, std::get<0>(outbin).data, std::get<0>(outbin).size);
-                utils::file::save(path.replace_extension(".cgsc.stack"), outbin.second.data, outbin.second.size);
+                utils::file::save(path.replace_extension(".cgsc.stack"), std::get<1>(outbin).data, std::get<1>(outbin).size);
                 std::cout << fmt::format("assembled {}\n", rel.generic_string());
             }
             else
@@ -185,14 +185,14 @@ auto assemble_file(game game, mach mach, fs::path file, fs::path rel) -> result
                 asset script;
                 script.name = "GSC"s;
                 
-                script.bytecode.resize(outbin.first.size);
+                script.bytecode.resize(std::get<0>(outbin).size);
-                std::memcpy(script.bytecode.data(), outbin.first.data, script.bytecode.size());
+                std::memcpy(script.bytecode.data(), std::get<0>(outbin).data, script.bytecode.size());
 
-                script.buffer.resize(outbin.second.size);
+                script.buffer.resize(std::get<1>(outbin).size);
-                std::memcpy(script.buffer.data(), outbin.second.data, script.buffer.size());
+                std::memcpy(script.buffer.data(), std::get<1>(outbin).data, script.buffer.size());
                 script.buffer = utils::zlib::compress(script.buffer);
 
-                script.len = static_cast<u32>(outbin.second.size);
+                script.len = static_cast<u32>(std::get<1>(outbin).size);
                 script.compressedLen = static_cast<u32>(script.buffer.size());
                 script.bytecodeLen = static_cast<u32>(script.bytecode.size());
 
@@ -271,8 +271,8 @@ auto compile_file(game game, mach mach, fs::path file, fs::path rel) -> result
             if (zonetool)
             {
                 auto path = fs::path{ "compiled" } / rel;
-                utils::file::save(path, outbin.first.data, outbin.first.size);
+                utils::file::save(path, std::get<0>(outbin).data, std::get<0>(outbin).size);
-                utils::file::save(path.replace_extension(".cgsc.stack"), outbin.second.data, outbin.second.size);
+                utils::file::save(path.replace_extension(".cgsc.stack"), std::get<1>(outbin).data, std::get<1>(outbin).size);
                 std::cout << fmt::format("compiled {}\n", rel.generic_string());
             }
             else
@@ -280,20 +280,23 @@ auto compile_file(game game, mach mach, fs::path file, fs::path rel) -> result
                 asset script;
                 script.name = "GSC"s;
                 
-                script.bytecode.resize(outbin.first.size);
+                script.bytecode.resize(std::get<0>(outbin).size);
-                std::memcpy(script.bytecode.data(), outbin.first.data, script.bytecode.size());
+                std::memcpy(script.bytecode.data(), std::get<0>(outbin).data, script.bytecode.size());
 
-                script.buffer.resize(outbin.second.size);
+                script.buffer.resize(std::get<1>(outbin).size);
-                std::memcpy(script.buffer.data(), outbin.second.data, script.buffer.size());
+                std::memcpy(script.buffer.data(), std::get<1>(outbin).data, script.buffer.size());
                 script.buffer = utils::zlib::compress(script.buffer);
 
-                script.len = static_cast<std::uint32_t>(outbin.second.size);
+                script.len = static_cast<std::uint32_t>(std::get<1>(outbin).size);
                 script.compressedLen = static_cast<std::uint32_t>(script.buffer.size());
                 script.bytecodeLen = static_cast<std::uint32_t>(script.bytecode.size());
 
                 auto result = script.serialize();
                 utils::file::save(fs::path{ "compiled" } / rel, result);
                 std::cout << fmt::format("compiled {}\n", rel.generic_string());
+
+                utils::file::save(fs::path{ "compiled" } / fs::path{ "developer_maps" } / rel.replace_extension(".gscmap"), std::get<2>(outbin).data, std::get<2>(outbin).size);
+                std::cout << fmt::format("saved developer map {}\n", rel.generic_string());
             }
         }
 
@@ -741,7 +744,7 @@ auto assemble_file(game game, mach mach, fs::path const& file, fs::path rel) ->
         auto outasm = contexts[game][mach]->source().parse_assembly(data);
         auto outbin = contexts[game][mach]->assembler().assemble(*outasm);
 
-        utils::file::save(fs::path{ "assembled" } / rel, outbin.data, outbin.size);
+        utils::file::save(fs::path{ "assembled" } / rel, outbin.first.data, outbin.first.size);
         std::cout << fmt::format("assembled {}\n", rel.generic_string());
         return result::success;
     }
@@ -793,8 +796,12 @@ auto compile_file(game game, mach mach, fs::path const& file, fs::path rel) -> r
         auto outasm = contexts[game][mach]->compiler().compile(file.string(), data);
         auto outbin = contexts[game][mach]->assembler().assemble(*outasm);
 
-        utils::file::save(fs::path{ "compiled" } / rel, outbin.data, outbin.size);
+        utils::file::save(fs::path{ "compiled" } / rel, outbin.first.data, outbin.first.size);
         std::cout << fmt::format("compiled {}\n", rel.generic_string());
+
+        utils::file::save(fs::path{ "compiled" } / fs::path{ "developer_maps" } / rel.replace_extension(".gscmap"), outbin.second.data, outbin.second.size);
+        std::cout << fmt::format("saved developer map {}\n", rel.generic_string());
+
         return result::success;
     }
     catch (std::exception const& e)