Add devblock support in IW5

gen/iw5/Makefile

@@ -9,3 +9,4 @@ clean:
 IW5: lexer.lpp parser.ypp
 	flex lexer.lpp
 	bison parser.ypp -Wcounterexamples
+	mv lexer.hpp lexer.cpp parser.hpp parser.cpp ../../src/iw5/xsk/

gen/iw5/lexer.lpp

@@ -35,7 +35,12 @@ RGX_INT_DEC     [0-9]+
 RGX_DEFAULT     (.|\n)
 
 %x COMMENT_BLOCK_STATE
-%x DEVELOPER_BLOCK_STATE
+/*
+    this is an inclusive start condition, so rules not explicitly specifying "DEVBLOCK_ON_STATE" will also match
+    http://dinosaur.compilertools.net/flex/flex_11.html
+*/
+%s DEVBLOCK_ON_STATE
+%x DEVBLOCK_OFF_STATE
 
 %%
 
@@ -54,10 +59,16 @@ RGX_DEFAULT     (.|\n)
 <COMMENT_BLOCK_STATE>\n     { loc.lines(yyleng); loc.step(); }
 <COMMENT_BLOCK_STATE>"*/"   { BEGIN(INITIAL); }
 
-"/#"                        { BEGIN(DEVELOPER_BLOCK_STATE); }
+"/#"                        { BEGIN(devblock_mode == xsk::gsc::dev_blocks::on ? DEVBLOCK_ON_STATE : DEVBLOCK_OFF_STATE); }
-<DEVELOPER_BLOCK_STATE>.
+ /* ignore everything if we're in devblock-off state */
-<DEVELOPER_BLOCK_STATE>\n   { loc.lines(yyleng); loc.step(); }
+<DEVBLOCK_OFF_STATE>.
-<DEVELOPER_BLOCK_STATE>"#/" { BEGIN(INITIAL); }    
+<DEVBLOCK_OFF_STATE>\n      { loc.lines(yyleng); loc.step(); }
+ /* always exit devblock state when seeing #/ */
+<DEVBLOCK_OFF_STATE,DEVBLOCK_ON_STATE>"#/" { BEGIN(INITIAL); }
+
+ /* for better errors */
+<INITIAL>"*/"               { throw iw5::parser::syntax_error(loc, "unmatched multiline comment end ('*/')"); }
+<INITIAL>"#/"               { throw iw5::parser::syntax_error(loc, "unmatched devblock end ('#/')"); }
 
 "breakpoint"        { return iw5::parser::make_BREAKPOINT(loc); }
 "prof_begin"        { return iw5::parser::make_PROFBEGIN(loc); }

gen/iw5/parser.ypp

@@ -24,16 +24,18 @@
 
 %lex-param   { yyscan_t yyscanner }
 %lex-param   { xsk::gsc::location& loc }
+%lex-param   { xsk::gsc::dev_blocks devblock_mode }
 
 %parse-param { yyscan_t yyscanner }
 %parse-param { xsk::gsc::location& loc }
 %parse-param { xsk::gsc::program_ptr& ast }
+%parse-param { xsk::gsc::dev_blocks devblock_mode }
 
 %code requires
 {
 #include "iw5.hpp"
 typedef void *yyscan_t;
-#define YY_DECL xsk::gsc::iw5::parser::symbol_type IW5lex(yyscan_t yyscanner, xsk::gsc::location& loc)
+#define YY_DECL xsk::gsc::iw5::parser::symbol_type IW5lex(yyscan_t yyscanner, xsk::gsc::location& loc, xsk::gsc::dev_blocks devblock_mode)
 }
 
 %code top
@@ -42,7 +44,7 @@ typedef void *yyscan_t;
 #include "parser.hpp"
 #include "lexer.hpp"
 using namespace xsk::gsc;
-xsk::gsc::iw5::parser::symbol_type IW5lex(yyscan_t yyscanner, xsk::gsc::location& loc);
+xsk::gsc::iw5::parser::symbol_type IW5lex(yyscan_t yyscanner, xsk::gsc::location& loc, xsk::gsc::dev_blocks devblock_mode);
 }
 
 %token BREAKPOINT "breakpoint"

src/iw5/xsk/compiler.cpp

@@ -46,7 +46,7 @@ auto compiler::parse_buffer(const std::string& file, std::vector<std::uint8_t>&
 
     YY_BUFFER_STATE yybuffer = iw5__scan_buffer(reinterpret_cast<char*>(data.data()), data.size(), scanner);
 
-    parser parser(scanner, loc, result);
+    parser parser(scanner, loc, result, m_devblocks);
     
     if(parser.parse() || result == nullptr)
     {

src/iw5/xsk/compiler.hpp

@@ -30,6 +30,8 @@ class compiler : public gsc::compiler
     bool can_continue_;
 
 public:
+    compiler(gsc::dev_blocks devblocks);
+
     auto output() -> std::vector<gsc::function_ptr>;
     void compile(const std::string& file, std::vector<std::uint8_t>& data);
     void set_readf_callback(std::function<std::vector<std::uint8_t>(const std::string&)> func);
@@ -154,6 +156,8 @@ private:
     void print_function(const gsc::function_ptr& func);
     void print_instruction(const gsc::instruction_ptr& inst);
     void print_label(const std::string& label);
+
+    gsc::dev_blocks m_devblocks;
 };
 
 } // namespace xsk::gsc::iw5

src/iw5/xsk/lexer.hpp

@@ -2,13 +2,13 @@
 #define iw5_HEADER_H 1
 #define iw5_IN_HEADER 1
 
-#line 5 "lexer.hpp"
+#line 6 "lexer.hpp"
 #include "stdafx.hpp"
 #include "iw5.hpp"
 #include "parser.hpp"
 using namespace xsk::gsc;
 
-#line 11 "lexer.hpp"
+#line 12 "lexer.hpp"
 
 #define  YY_INT_ALIGNED short int
 
@@ -438,7 +438,8 @@ void yyfree ( void * , yyscan_t yyscanner );
 #ifdef YY_HEADER_EXPORT_START_CONDITIONS
 #define INITIAL 0
 #define COMMENT_BLOCK_STATE 1
-#define DEVELOPER_BLOCK_STATE 2
+#define DEVBLOCK_ON_STATE 2
+#define DEVBLOCK_OFF_STATE 3
 
 #endif
 
@@ -700,9 +701,9 @@ extern int yylex (yyscan_t yyscanner);
 #undef yyTABLES_NAME
 #endif
 
-#line 157 "lexer.lpp"
+#line 168 "lexer.lpp"
 
 
-#line 706 "lexer.hpp"
+#line 708 "lexer.hpp"
 #undef iw5_IN_HEADER
 #endif /* iw5_HEADER_H */

src/iw5/xsk/parser.hpp

@@ -1,4 +1,4 @@
-// A Bison parser, made by GNU Bison 3.7.5.
+// A Bison parser, made by GNU Bison 3.8.2.
 
 // Skeleton interface for Bison LALR(1) parsers in C++
 
@@ -15,7 +15,7 @@
 // GNU General Public License for more details.
 
 // You should have received a copy of the GNU General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
 // As a special exception, you may create a larger work that contains
 // part or all of the Bison parser skeleton and distribute that work
@@ -45,11 +45,11 @@
 #ifndef YY_IW5_PARSER_HPP_INCLUDED
 # define YY_IW5_PARSER_HPP_INCLUDED
 // "%code requires" blocks.
-#line 33 "parser.ypp"
+#line 35 "parser.ypp"
 
 #include "iw5.hpp"
 typedef void *yyscan_t;
-#define YY_DECL xsk::gsc::iw5::parser::symbol_type IW5lex(yyscan_t yyscanner, xsk::gsc::location& loc)
+#define YY_DECL xsk::gsc::iw5::parser::symbol_type IW5lex(yyscan_t yyscanner, xsk::gsc::location& loc, xsk::gsc::dev_blocks devblock_mode)
 
 #line 55 "parser.hpp"
 
@@ -127,12 +127,18 @@ typedef void *yyscan_t;
 # define YY_USE(E) /* empty */
 #endif
 
-#if defined __GNUC__ && ! defined __ICC && 407 <= __GNUC__ * 100 + __GNUC_MINOR__
 /* Suppress an incorrect diagnostic about yylval being uninitialized.  */
+#if defined __GNUC__ && ! defined __ICC && 406 <= __GNUC__ * 100 + __GNUC_MINOR__
+# if __GNUC__ * 100 + __GNUC_MINOR__ < 407
+#  define YY_IGNORE_MAYBE_UNINITIALIZED_BEGIN                           \
+    _Pragma ("GCC diagnostic push")                                     \
+    _Pragma ("GCC diagnostic ignored \"-Wuninitialized\"")
+# else
 #  define YY_IGNORE_MAYBE_UNINITIALIZED_BEGIN                           \
     _Pragma ("GCC diagnostic push")                                     \
     _Pragma ("GCC diagnostic ignored \"-Wuninitialized\"")              \
     _Pragma ("GCC diagnostic ignored \"-Wmaybe-uninitialized\"")
+# endif
 # define YY_IGNORE_MAYBE_UNINITIALIZED_END      \
     _Pragma ("GCC diagnostic pop")
 #else
@@ -194,7 +200,7 @@ typedef void *yyscan_t;
 
 #line 13 "parser.ypp"
 namespace xsk { namespace gsc { namespace iw5 {
-#line 198 "parser.hpp"
+#line 204 "parser.hpp"
 
 
 
@@ -203,27 +209,32 @@ namespace xsk { namespace gsc { namespace iw5 {
   class parser
   {
   public:
-#ifndef IW5STYPE
+#ifdef IW5STYPE
+# ifdef __GNUC__
+#  pragma GCC message "bison: do not #define IW5STYPE in C++, use %define api.value.type"
+# endif
+    typedef IW5STYPE value_type;
+#else
   /// A buffer to store and retrieve objects.
   ///
   /// Sort of a variant, but does not keep track of the nature
   /// of the stored data, since that knowledge is available
   /// via the current parser state.
-  class semantic_type
+  class value_type
   {
   public:
     /// Type of *this.
-    typedef semantic_type self_type;
+    typedef value_type self_type;
 
     /// Empty construction.
-    semantic_type () YY_NOEXCEPT
+    value_type () YY_NOEXCEPT
-      : yybuffer_ ()
+      : yyraw_ ()
       , yytypeid_ (YY_NULLPTR)
     {}
 
     /// Construct and fill.
     template <typename T>
-    semantic_type (YY_RVREF (T) t)
+    value_type (YY_RVREF (T) t)
       : yytypeid_ (&typeid (T))
     {
       IW5_ASSERT (sizeof (T) <= size);
@@ -232,13 +243,13 @@ namespace xsk { namespace gsc { namespace iw5 {
 
 #if 201103L <= YY_CPLUSPLUS
     /// Non copyable.
-    semantic_type (const self_type&) = delete;
+    value_type (const self_type&) = delete;
     /// Non copyable.
     self_type& operator= (const self_type&) = delete;
 #endif
 
     /// Destruction, allowed only if empty.
-    ~semantic_type () YY_NOEXCEPT
+    ~value_type () YY_NOEXCEPT
     {
       IW5_ASSERT (!yytypeid_);
     }
@@ -382,7 +393,7 @@ namespace xsk { namespace gsc { namespace iw5 {
   private:
 #if YY_CPLUSPLUS < 201103L
     /// Non copyable.
-    semantic_type (const self_type&);
+    value_type (const self_type&);
     /// Non copyable.
     self_type& operator= (const self_type&);
 #endif
@@ -392,7 +403,7 @@ namespace xsk { namespace gsc { namespace iw5 {
     T*
     yyas_ () YY_NOEXCEPT
     {
-      void *yyp = yybuffer_.yyraw;
+      void *yyp = yyraw_;
       return static_cast<T*> (yyp);
      }
 
@@ -401,7 +412,7 @@ namespace xsk { namespace gsc { namespace iw5 {
     const T*
     yyas_ () const YY_NOEXCEPT
     {
-      const void *yyp = yybuffer_.yyraw;
+      const void *yyp = yyraw_;
       return static_cast<const T*> (yyp);
      }
 
@@ -613,18 +624,19 @@ namespace xsk { namespace gsc { namespace iw5 {
     union
     {
       /// Strongest alignment constraints.
-      long double yyalign_me;
+      long double yyalign_me_;
       /// A buffer large enough to store any of the semantic values.
-      char yyraw[size];
+      char yyraw_[size];
-    } yybuffer_;
+    };
 
     /// Whether the content is built: if defined, the name of the stored type.
     const std::type_info *yytypeid_;
   };
 
-#else
-    typedef IW5STYPE semantic_type;
 #endif
+    /// Backward compatibility (Bison 3.8).
+    typedef value_type semantic_type;
+
     /// Symbol locations.
     typedef xsk::gsc::location location_type;
 
@@ -761,7 +773,7 @@ namespace xsk { namespace gsc { namespace iw5 {
     };
 
     /// Token kind, as returned by yylex.
-    typedef token::yytokentype token_kind_type;
+    typedef token::token_kind_type token_kind_type;
 
     /// Backward compatibility alias (Bison 3.6).
     typedef token_kind_type token_type;
@@ -973,7 +985,7 @@ namespace xsk { namespace gsc { namespace iw5 {
       typedef Base super_type;
 
       /// Default constructor.
-      basic_symbol ()
+      basic_symbol () YY_NOEXCEPT
         : value ()
         , location ()
       {}
@@ -2068,6 +2080,8 @@ namespace xsk { namespace gsc { namespace iw5 {
         clear ();
       }
 
+
+
       /// Destroy contents, and record that is empty.
       void clear () YY_NOEXCEPT
       {
@@ -2361,7 +2375,7 @@ switch (yykind)
       void move (basic_symbol& s);
 
       /// The semantic value.
-      semantic_type value;
+      value_type value;
 
       /// The location.
       location_type location;
@@ -2376,22 +2390,24 @@ switch (yykind)
     /// Type access provider for token (enum) based symbols.
     struct by_kind
     {
-      /// Default constructor.
-      by_kind ();
-
-#if 201103L <= YY_CPLUSPLUS
-      /// Move constructor.
-      by_kind (by_kind&& that);
-#endif
-
-      /// Copy constructor.
-      by_kind (const by_kind& that);
-
       /// The symbol kind as needed by the constructor.
       typedef token_kind_type kind_type;
 
+      /// Default constructor.
+      by_kind () YY_NOEXCEPT;
+
+#if 201103L <= YY_CPLUSPLUS
+      /// Move constructor.
+      by_kind (by_kind&& that) YY_NOEXCEPT;
+#endif
+
+      /// Copy constructor.
+      by_kind (const by_kind& that) YY_NOEXCEPT;
+
       /// Constructor from (external) token numbers.
-      by_kind (kind_type t);
+      by_kind (kind_type t) YY_NOEXCEPT;
+
+
 
       /// Record that this symbol is empty.
       void clear () YY_NOEXCEPT;
@@ -2421,35 +2437,39 @@ switch (yykind)
       typedef basic_symbol<by_kind> super_type;
 
       /// Empty symbol.
-      symbol_type () {}
+      symbol_type () YY_NOEXCEPT {}
 
       /// Constructor for valueless symbols, and symbols from each type.
 #if 201103L <= YY_CPLUSPLUS
       symbol_type (int tok, location_type l)
-        : super_type(token_type (tok), std::move (l))
+        : super_type (token_kind_type (tok), std::move (l))
 #else
       symbol_type (int tok, const location_type& l)
-        : super_type(token_type (tok), l)
+        : super_type (token_kind_type (tok), l)
 #endif
       {
+#if !defined _MSC_VER || defined __clang__
         IW5_ASSERT (tok == token::IW5EOF
                    || (token::IW5error <= tok && tok <= token::MOD)
                    || (token::ADD_ARRAY <= tok && tok <= token::POSTDEC));
+#endif
       }
 #if 201103L <= YY_CPLUSPLUS
       symbol_type (int tok, std::string v, location_type l)
-        : super_type(token_type (tok), std::move (v), std::move (l))
+        : super_type (token_kind_type (tok), std::move (v), std::move (l))
 #else
       symbol_type (int tok, const std::string& v, const location_type& l)
-        : super_type(token_type (tok), v, l)
+        : super_type (token_kind_type (tok), v, l)
 #endif
       {
+#if !defined _MSC_VER || defined __clang__
         IW5_ASSERT ((token::FILE <= tok && tok <= token::INT_HEX));
+#endif
       }
     };
 
     /// Build a parser object.
-    parser (yyscan_t yyscanner_yyarg, xsk::gsc::location& loc_yyarg, xsk::gsc::program_ptr& ast_yyarg);
+    parser (yyscan_t yyscanner_yyarg, xsk::gsc::location& loc_yyarg, xsk::gsc::program_ptr& ast_yyarg, xsk::gsc::dev_blocks devblock_mode_yyarg);
     virtual ~parser ();
 
 #if 201103L <= YY_CPLUSPLUS
@@ -2493,7 +2513,7 @@ switch (yykind)
     /// YYSYMBOL.  No bounds checking.
     static const char *symbol_name (symbol_kind_type yysymbol);
 
-    // Implementation of make_symbol for each symbol type.
+    // Implementation of make_symbol for each token kind.
 #if 201103L <= YY_CPLUSPLUS
       static
       symbol_type
@@ -4095,19 +4115,19 @@ switch (yykind)
 
     /// Whether the given \c yypact_ value indicates a defaulted state.
     /// \param yyvalue   the value to check
-    static bool yy_pact_value_is_default_ (int yyvalue);
+    static bool yy_pact_value_is_default_ (int yyvalue) YY_NOEXCEPT;
 
     /// Whether the given \c yytable_ value indicates a syntax error.
     /// \param yyvalue   the value to check
-    static bool yy_table_value_is_error_ (int yyvalue);
+    static bool yy_table_value_is_error_ (int yyvalue) YY_NOEXCEPT;
 
     static const short yypact_ninf_;
     static const short yytable_ninf_;
 
     /// Convert a scanner token kind \a t to a symbol kind.
     /// In theory \a t should be a token_kind_type, but character literals
-    /// are valid, yet not members of the token_type enum.
+    /// are valid, yet not members of the token_kind_type enum.
-    static symbol_kind_type yytranslate_ (int t);
+    static symbol_kind_type yytranslate_ (int t) YY_NOEXCEPT;
 
 
 
@@ -4134,14 +4154,14 @@ switch (yykind)
 
     static const short yycheck_[];
 
-    // YYSTOS[STATE-NUM] -- The (internal number of the) accessing
+    // YYSTOS[STATE-NUM] -- The symbol kind of the accessing symbol of
-    // symbol of state STATE-NUM.
+    // state STATE-NUM.
     static const unsigned char yystos_[];
 
-    // YYR1[YYN] -- Symbol number of symbol that rule YYN derives.
+    // YYR1[RULE-NUM] -- Symbol kind of the left-hand side of rule RULE-NUM.
     static const unsigned char yyr1_[];
 
-    // YYR2[YYN] -- Number of symbols on the right hand side of rule YYN.
+    // YYR2[RULE-NUM] -- Number of symbols on the right-hand side of rule RULE-NUM.
     static const signed char yyr2_[];
 
 
@@ -4240,7 +4260,7 @@ switch (yykind)
       typedef typename S::size_type size_type;
       typedef typename std::ptrdiff_t index_type;
 
-      stack (size_type n = 200)
+      stack (size_type n = 200) YY_NOEXCEPT
         : seq_ (n)
       {}
 
@@ -4319,7 +4339,7 @@ switch (yykind)
       class slice
       {
       public:
-        slice (const stack& stack, index_type range)
+        slice (const stack& stack, index_type range) YY_NOEXCEPT
           : stack_ (stack)
           , range_ (range)
         {}
@@ -4378,7 +4398,7 @@ switch (yykind)
     void yypush_ (const char* m, state_type s, YY_MOVE_REF (symbol_type) sym);
 
     /// Pop \a n symbols from the stack.
-    void yypop_ (int n = 1);
+    void yypop_ (int n = 1) YY_NOEXCEPT;
 
     /// Constants.
     enum
@@ -4393,12 +4413,13 @@ switch (yykind)
     yyscan_t yyscanner;
     xsk::gsc::location& loc;
     xsk::gsc::program_ptr& ast;
+    xsk::gsc::dev_blocks devblock_mode;
 
   };
 
   inline
   parser::symbol_kind_type
-  parser::yytranslate_ (int t)
+  parser::yytranslate_ (int t) YY_NOEXCEPT
   {
     return static_cast<symbol_kind_type> (t);
   }
@@ -4674,6 +4695,7 @@ switch (yykind)
 
 
 
+
   template <typename Base>
   parser::symbol_kind_type
   parser::basic_symbol<Base>::type_get () const YY_NOEXCEPT
@@ -4681,6 +4703,7 @@ switch (yykind)
     return this->kind ();
   }
 
+
   template <typename Base>
   bool
   parser::basic_symbol<Base>::empty () const YY_NOEXCEPT
@@ -4958,13 +4981,13 @@ switch (yykind)
 
   // by_kind.
   inline
-  parser::by_kind::by_kind ()
+  parser::by_kind::by_kind () YY_NOEXCEPT
     : kind_ (symbol_kind::S_YYEMPTY)
   {}
 
 #if 201103L <= YY_CPLUSPLUS
   inline
-  parser::by_kind::by_kind (by_kind&& that)
+  parser::by_kind::by_kind (by_kind&& that) YY_NOEXCEPT
     : kind_ (that.kind_)
   {
     that.clear ();
@@ -4972,15 +4995,17 @@ switch (yykind)
 #endif
 
   inline
-  parser::by_kind::by_kind (const by_kind& that)
+  parser::by_kind::by_kind (const by_kind& that) YY_NOEXCEPT
     : kind_ (that.kind_)
   {}
 
   inline
-  parser::by_kind::by_kind (token_kind_type t)
+  parser::by_kind::by_kind (token_kind_type t) YY_NOEXCEPT
     : kind_ (yytranslate_ (t))
   {}
 
+
+
   inline
   void
   parser::by_kind::clear () YY_NOEXCEPT
@@ -5003,6 +5028,7 @@ switch (yykind)
     return kind_;
   }
 
+
   inline
   parser::symbol_kind_type
   parser::by_kind::type_get () const YY_NOEXCEPT
@@ -5010,9 +5036,10 @@ switch (yykind)
     return this->kind ();
   }
 
+
 #line 13 "parser.ypp"
 } } } // xsk::gsc::iw5
-#line 5016 "parser.hpp"
+#line 5043 "parser.hpp"
 
 
 

src/tool/xsk/main.cpp

@@ -507,7 +507,7 @@ std::uint32_t main(std::uint32_t argc, char** argv)
         if (game == game::IW5)
         {
             gsc::iw5::assembler assembler;
-            gsc::iw5::compiler compiler;
+            gsc::iw5::compiler compiler(gsc::dev_blocks::off);
             compile_file(assembler, compiler, file ,zonetool);
         }
         else if (game == game::IW6)

src/utils/xsk/gsc/devblocks.hpp

@@ -0,0 +1,10 @@
+#pragma once
+
+namespace xsk::gsc
+{
+	enum class dev_blocks
+	{
+		on,
+		off
+	};
+}

src/utils/xsk/utils.hpp

@@ -12,6 +12,7 @@
 #include "compression.hpp"
 
 // GSC Types
+#include "gsc/devblocks.hpp"
 #include "gsc/pair.hpp"
 #include "gsc/asset.hpp"
 #include "gsc/assembly.hpp"