Rim/gsc-tool
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 3 Wiki Activity

lexer & grammar fixes

Browse Source
This commit is contained in:
xensik 2022-02-20 12:10:21 +01:00
parent e491993f93
commit 6980506e88
54 changed files with 10312 additions and 10375 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
gen/h1/parser.ypp
View File

@ -85,7 +85,7 @@ xsk::gsc::h1::parser::symbol_type H1lex(xsk::gsc::h1::lexer& lexer);
%token TRUE "true" %token TRUE "true"
%token FALSE "false" %token FALSE "false"
%token UNDEFINED "undefined" %token UNDEFINED "undefined"
%token SIZE ".size" %token SIZE "size"
%token GAME "game" %token GAME "game"
%token SELF "self" %token SELF "self"
%token ANIM "anim" %token ANIM "anim"
@ -135,7 +135,6 @@ xsk::gsc::h1::parser::symbol_type H1lex(xsk::gsc::h1::lexer& lexer);
%token MUL "*" %token MUL "*"
%token DIV "/" %token DIV "/"
%token MOD "%" %token MOD "%"
%token <std::string> FIELD "field"
%token <std::string> PATH "path" %token <std::string> PATH "path"
%token <std::string> IDENTIFIER "identifier" %token <std::string> IDENTIFIER "identifier"
%token <std::string> STRING "string literal" %token <std::string> STRING "string literal"
@ -213,6 +212,7 @@ xsk::gsc::h1::parser::symbol_type H1lex(xsk::gsc::h1::lexer& lexer);
%type <ast::expr_level::ptr> expr_level %type <ast::expr_level::ptr> expr_level
%type <ast::expr_animation::ptr> expr_animation %type <ast::expr_animation::ptr> expr_animation
%type <ast::expr_animtree::ptr> expr_animtree %type <ast::expr_animtree::ptr> expr_animtree
%type <ast::expr_identifier::ptr> expr_identifier_nosize
%type <ast::expr_identifier::ptr> expr_identifier %type <ast::expr_identifier::ptr> expr_identifier
%type <ast::expr_path::ptr> expr_path %type <ast::expr_path::ptr> expr_path
%type <ast::expr_istring::ptr> expr_istring %type <ast::expr_istring::ptr> expr_istring
@ -223,6 +223,7 @@ xsk::gsc::h1::parser::symbol_type H1lex(xsk::gsc::h1::lexer& lexer);
%type <ast::expr_false::ptr> expr_false %type <ast::expr_false::ptr> expr_false
%type <ast::expr_true::ptr> expr_true %type <ast::expr_true::ptr> expr_true
%nonassoc SIZEOF
%nonassoc ADD_ARRAY %nonassoc ADD_ARRAY
%nonassoc RBRACKET %nonassoc RBRACKET
%nonassoc THEN %nonassoc THEN
@ -722,14 +723,12 @@ expr_array
; ;
expr_field expr_field
: expr_object DOT expr_identifier : expr_object DOT expr_identifier_nosize
{ $$ = std::make_unique<ast::expr_field>(@$, std::move($1), std::move($3)); } { $$ = std::make_unique<ast::expr_field>(@$, std::move($1), std::move($3)); }
| expr_object FIELD
{ $$ = std::make_unique<ast::expr_field>(@$, std::move($1), std::make_unique<ast::expr_identifier>(@$, $2)); }
; ;
expr_size expr_size
: expr_object SIZE : expr_object DOT SIZE %prec SIZEOF
{ $$ = std::make_unique<ast::expr_size>(@$, std::move($1)); } { $$ = std::make_unique<ast::expr_size>(@$, std::move($1)); }
; ;
@ -795,16 +794,23 @@ expr_animtree
{ $$ = std::make_unique<ast::expr_animtree>(@$); }; { $$ = std::make_unique<ast::expr_animtree>(@$); };
; ;
expr_identifier expr_identifier_nosize
: IDENTIFIER : IDENTIFIER
{ $$ = std::make_unique<ast::expr_identifier>(@$, $1); }; { $$ = std::make_unique<ast::expr_identifier>(@$, $1); };
; ;
expr_identifier
: IDENTIFIER
{ $$ = std::make_unique<ast::expr_identifier>(@$, $1); };
| SIZE
{ $$ = std::make_unique<ast::expr_identifier>(@$, "size"); };
;
expr_path expr_path
: PATH : IDENTIFIER
{ $$ = std::make_unique<ast::expr_path>(@$, $1); };
| PATH
{ $$ = std::make_unique<ast::expr_path>(@$, $1); }; { $$ = std::make_unique<ast::expr_path>(@$, $1); };
| expr_identifier
{ $$ = std::make_unique<ast::expr_path>(@$, $1->value); };
; ;
expr_istring expr_istring

26
gen/h2/parser.ypp
View File

@ -83,7 +83,7 @@ xsk::gsc::h2::parser::symbol_type H2lex(xsk::gsc::h2::lexer& lexer);
%token TRUE "true" %token TRUE "true"
%token FALSE "false" %token FALSE "false"
%token UNDEFINED "undefined" %token UNDEFINED "undefined"
%token SIZE ".size" %token SIZE "size"
%token GAME "game" %token GAME "game"
%token SELF "self" %token SELF "self"
%token ANIM "anim" %token ANIM "anim"
@ -133,7 +133,6 @@ xsk::gsc::h2::parser::symbol_type H2lex(xsk::gsc::h2::lexer& lexer);
%token MUL "*" %token MUL "*"
%token DIV "/" %token DIV "/"
%token MOD "%" %token MOD "%"
%token <std::string> FIELD "field"
%token <std::string> PATH "path" %token <std::string> PATH "path"
%token <std::string> IDENTIFIER "identifier" %token <std::string> IDENTIFIER "identifier"
%token <std::string> STRING "string literal" %token <std::string> STRING "string literal"
@ -211,6 +210,7 @@ xsk::gsc::h2::parser::symbol_type H2lex(xsk::gsc::h2::lexer& lexer);
%type <ast::expr_level::ptr> expr_level %type <ast::expr_level::ptr> expr_level
%type <ast::expr_animation::ptr> expr_animation %type <ast::expr_animation::ptr> expr_animation
%type <ast::expr_animtree::ptr> expr_animtree %type <ast::expr_animtree::ptr> expr_animtree
%type <ast::expr_identifier::ptr> expr_identifier_nosize
%type <ast::expr_identifier::ptr> expr_identifier %type <ast::expr_identifier::ptr> expr_identifier
%type <ast::expr_path::ptr> expr_path %type <ast::expr_path::ptr> expr_path
%type <ast::expr_istring::ptr> expr_istring %type <ast::expr_istring::ptr> expr_istring
@ -221,6 +221,7 @@ xsk::gsc::h2::parser::symbol_type H2lex(xsk::gsc::h2::lexer& lexer);
%type <ast::expr_false::ptr> expr_false %type <ast::expr_false::ptr> expr_false
%type <ast::expr_true::ptr> expr_true %type <ast::expr_true::ptr> expr_true
%nonassoc SIZEOF
%nonassoc ADD_ARRAY %nonassoc ADD_ARRAY
%nonassoc RBRACKET %nonassoc RBRACKET
%nonassoc THEN %nonassoc THEN
@ -720,14 +721,12 @@ expr_array
; ;
expr_field expr_field
: expr_object DOT expr_identifier : expr_object DOT expr_identifier_nosize
{ $$ = std::make_unique<ast::expr_field>(@$, std::move($1), std::move($3)); } { $$ = std::make_unique<ast::expr_field>(@$, std::move($1), std::move($3)); }
| expr_object FIELD
{ $$ = std::make_unique<ast::expr_field>(@$, std::move($1), std::make_unique<ast::expr_identifier>(@$, $2)); }
; ;
expr_size expr_size
: expr_object SIZE : expr_object DOT SIZE %prec SIZEOF
{ $$ = std::make_unique<ast::expr_size>(@$, std::move($1)); } { $$ = std::make_unique<ast::expr_size>(@$, std::move($1)); }
; ;
@ -793,16 +792,23 @@ expr_animtree
{ $$ = std::make_unique<ast::expr_animtree>(@$); }; { $$ = std::make_unique<ast::expr_animtree>(@$); };
; ;
expr_identifier expr_identifier_nosize
: IDENTIFIER : IDENTIFIER
{ $$ = std::make_unique<ast::expr_identifier>(@$, $1); }; { $$ = std::make_unique<ast::expr_identifier>(@$, $1); };
; ;
expr_identifier
: IDENTIFIER
{ $$ = std::make_unique<ast::expr_identifier>(@$, $1); };
| SIZE
{ $$ = std::make_unique<ast::expr_identifier>(@$, "size"); };
;
expr_path expr_path
: PATH : IDENTIFIER
{ $$ = std::make_unique<ast::expr_path>(@$, $1); };
| PATH
{ $$ = std::make_unique<ast::expr_path>(@$, $1); }; { $$ = std::make_unique<ast::expr_path>(@$, $1); };
| expr_identifier
{ $$ = std::make_unique<ast::expr_path>(@$, $1->value); };
; ;
expr_istring expr_istring

26
gen/iw5/parser.ypp
View File

@ -82,7 +82,7 @@ xsk::gsc::iw5::parser::symbol_type IW5lex(xsk::gsc::iw5::lexer& lexer);
%token TRUE "true" %token TRUE "true"
%token FALSE "false" %token FALSE "false"
%token UNDEFINED "undefined" %token UNDEFINED "undefined"
%token SIZE ".size" %token SIZE "size"
%token GAME "game" %token GAME "game"
%token SELF "self" %token SELF "self"
%token ANIM "anim" %token ANIM "anim"
@ -132,7 +132,6 @@ xsk::gsc::iw5::parser::symbol_type IW5lex(xsk::gsc::iw5::lexer& lexer);
%token MUL "*" %token MUL "*"
%token DIV "/" %token DIV "/"
%token MOD "%" %token MOD "%"
%token <std::string> FIELD "field"
%token <std::string> PATH "path" %token <std::string> PATH "path"
%token <std::string> IDENTIFIER "identifier" %token <std::string> IDENTIFIER "identifier"
%token <std::string> STRING "string literal" %token <std::string> STRING "string literal"
@ -209,6 +208,7 @@ xsk::gsc::iw5::parser::symbol_type IW5lex(xsk::gsc::iw5::lexer& lexer);
%type <ast::expr_level::ptr> expr_level %type <ast::expr_level::ptr> expr_level
%type <ast::expr_animation::ptr> expr_animation %type <ast::expr_animation::ptr> expr_animation
%type <ast::expr_animtree::ptr> expr_animtree %type <ast::expr_animtree::ptr> expr_animtree
%type <ast::expr_identifier::ptr> expr_identifier_nosize
%type <ast::expr_identifier::ptr> expr_identifier %type <ast::expr_identifier::ptr> expr_identifier
%type <ast::expr_path::ptr> expr_path %type <ast::expr_path::ptr> expr_path
%type <ast::expr_istring::ptr> expr_istring %type <ast::expr_istring::ptr> expr_istring
@ -219,6 +219,7 @@ xsk::gsc::iw5::parser::symbol_type IW5lex(xsk::gsc::iw5::lexer& lexer);
%type <ast::expr_false::ptr> expr_false %type <ast::expr_false::ptr> expr_false
%type <ast::expr_true::ptr> expr_true %type <ast::expr_true::ptr> expr_true
%nonassoc SIZEOF
%nonassoc ADD_ARRAY %nonassoc ADD_ARRAY
%nonassoc RBRACKET %nonassoc RBRACKET
%nonassoc THEN %nonassoc THEN
@ -710,14 +711,12 @@ expr_array
; ;
expr_field expr_field
: expr_object DOT expr_identifier : expr_object DOT expr_identifier_nosize
{ $$ = std::make_unique<ast::expr_field>(@$, std::move($1), std::move($3)); } { $$ = std::make_unique<ast::expr_field>(@$, std::move($1), std::move($3)); }
| expr_object FIELD
{ $$ = std::make_unique<ast::expr_field>(@$, std::move($1), std::make_unique<ast::expr_identifier>(@$, $2)); }
; ;
expr_size expr_size
: expr_object SIZE : expr_object DOT SIZE %prec SIZEOF
{ $$ = std::make_unique<ast::expr_size>(@$, std::move($1)); } { $$ = std::make_unique<ast::expr_size>(@$, std::move($1)); }
; ;
@ -783,16 +782,23 @@ expr_animtree
{ $$ = std::make_unique<ast::expr_animtree>(@$); }; { $$ = std::make_unique<ast::expr_animtree>(@$); };
; ;
expr_identifier expr_identifier_nosize
: IDENTIFIER : IDENTIFIER
{ $$ = std::make_unique<ast::expr_identifier>(@$, $1); }; { $$ = std::make_unique<ast::expr_identifier>(@$, $1); };
; ;
expr_identifier
: IDENTIFIER
{ $$ = std::make_unique<ast::expr_identifier>(@$, $1); };
| SIZE
{ $$ = std::make_unique<ast::expr_identifier>(@$, "size"); };
;
expr_path expr_path
: PATH : IDENTIFIER
{ $$ = std::make_unique<ast::expr_path>(@$, $1); };
| PATH
{ $$ = std::make_unique<ast::expr_path>(@$, $1); }; { $$ = std::make_unique<ast::expr_path>(@$, $1); };
| expr_identifier
{ $$ = std::make_unique<ast::expr_path>(@$, $1->value); };
; ;
expr_istring expr_istring

26
gen/iw6/parser.ypp
View File

@ -82,7 +82,7 @@ xsk::gsc::iw6::parser::symbol_type IW6lex(xsk::gsc::iw6::lexer& lexer);
%token TRUE "true" %token TRUE "true"
%token FALSE "false" %token FALSE "false"
%token UNDEFINED "undefined" %token UNDEFINED "undefined"
%token SIZE ".size" %token SIZE "size"
%token GAME "game" %token GAME "game"
%token SELF "self" %token SELF "self"
%token ANIM "anim" %token ANIM "anim"
@ -132,7 +132,6 @@ xsk::gsc::iw6::parser::symbol_type IW6lex(xsk::gsc::iw6::lexer& lexer);
%token MUL "*" %token MUL "*"
%token DIV "/" %token DIV "/"
%token MOD "%" %token MOD "%"
%token <std::string> FIELD "field"
%token <std::string> PATH "path" %token <std::string> PATH "path"
%token <std::string> IDENTIFIER "identifier" %token <std::string> IDENTIFIER "identifier"
%token <std::string> STRING "string literal" %token <std::string> STRING "string literal"
@ -209,6 +208,7 @@ xsk::gsc::iw6::parser::symbol_type IW6lex(xsk::gsc::iw6::lexer& lexer);
%type <ast::expr_level::ptr> expr_level %type <ast::expr_level::ptr> expr_level
%type <ast::expr_animation::ptr> expr_animation %type <ast::expr_animation::ptr> expr_animation
%type <ast::expr_animtree::ptr> expr_animtree %type <ast::expr_animtree::ptr> expr_animtree
%type <ast::expr_identifier::ptr> expr_identifier_nosize
%type <ast::expr_identifier::ptr> expr_identifier %type <ast::expr_identifier::ptr> expr_identifier
%type <ast::expr_path::ptr> expr_path %type <ast::expr_path::ptr> expr_path
%type <ast::expr_istring::ptr> expr_istring %type <ast::expr_istring::ptr> expr_istring
@ -219,6 +219,7 @@ xsk::gsc::iw6::parser::symbol_type IW6lex(xsk::gsc::iw6::lexer& lexer);
%type <ast::expr_false::ptr> expr_false %type <ast::expr_false::ptr> expr_false
%type <ast::expr_true::ptr> expr_true %type <ast::expr_true::ptr> expr_true
%nonassoc SIZEOF
%nonassoc ADD_ARRAY %nonassoc ADD_ARRAY
%nonassoc RBRACKET %nonassoc RBRACKET
%nonassoc THEN %nonassoc THEN
@ -710,14 +711,12 @@ expr_array
; ;
expr_field expr_field
: expr_object DOT expr_identifier : expr_object DOT expr_identifier_nosize
{ $$ = std::make_unique<ast::expr_field>(@$, std::move($1), std::move($3)); } { $$ = std::make_unique<ast::expr_field>(@$, std::move($1), std::move($3)); }
| expr_object FIELD
{ $$ = std::make_unique<ast::expr_field>(@$, std::move($1), std::make_unique<ast::expr_identifier>(@$, $2)); }
; ;
expr_size expr_size
: expr_object SIZE : expr_object DOT SIZE %prec SIZEOF
{ $$ = std::make_unique<ast::expr_size>(@$, std::move($1)); } { $$ = std::make_unique<ast::expr_size>(@$, std::move($1)); }
; ;
@ -783,16 +782,23 @@ expr_animtree
{ $$ = std::make_unique<ast::expr_animtree>(@$); }; { $$ = std::make_unique<ast::expr_animtree>(@$); };
; ;
expr_identifier expr_identifier_nosize
: IDENTIFIER : IDENTIFIER
{ $$ = std::make_unique<ast::expr_identifier>(@$, $1); }; { $$ = std::make_unique<ast::expr_identifier>(@$, $1); };
; ;
expr_identifier
: IDENTIFIER
{ $$ = std::make_unique<ast::expr_identifier>(@$, $1); };
| SIZE
{ $$ = std::make_unique<ast::expr_identifier>(@$, "size"); };
;
expr_path expr_path
: PATH : IDENTIFIER
{ $$ = std::make_unique<ast::expr_path>(@$, $1); };
| PATH
{ $$ = std::make_unique<ast::expr_path>(@$, $1); }; { $$ = std::make_unique<ast::expr_path>(@$, $1); };
| expr_identifier
{ $$ = std::make_unique<ast::expr_path>(@$, $1->value); };
; ;
expr_istring expr_istring

26
gen/iw7/parser.ypp
View File

@ -82,7 +82,7 @@ xsk::gsc::iw7::parser::symbol_type IW7lex(xsk::gsc::iw7::lexer& lexer);
%token TRUE "true" %token TRUE "true"
%token FALSE "false" %token FALSE "false"
%token UNDEFINED "undefined" %token UNDEFINED "undefined"
%token SIZE ".size" %token SIZE "size"
%token GAME "game" %token GAME "game"
%token SELF "self" %token SELF "self"
%token ANIM "anim" %token ANIM "anim"
@ -132,7 +132,6 @@ xsk::gsc::iw7::parser::symbol_type IW7lex(xsk::gsc::iw7::lexer& lexer);
%token MUL "*" %token MUL "*"
%token DIV "/" %token DIV "/"
%token MOD "%" %token MOD "%"
%token <std::string> FIELD "field"
%token <std::string> PATH "path" %token <std::string> PATH "path"
%token <std::string> IDENTIFIER "identifier" %token <std::string> IDENTIFIER "identifier"
%token <std::string> STRING "string literal" %token <std::string> STRING "string literal"
@ -209,6 +208,7 @@ xsk::gsc::iw7::parser::symbol_type IW7lex(xsk::gsc::iw7::lexer& lexer);
%type <ast::expr_level::ptr> expr_level %type <ast::expr_level::ptr> expr_level
%type <ast::expr_animation::ptr> expr_animation %type <ast::expr_animation::ptr> expr_animation
%type <ast::expr_animtree::ptr> expr_animtree %type <ast::expr_animtree::ptr> expr_animtree
%type <ast::expr_identifier::ptr> expr_identifier_nosize
%type <ast::expr_identifier::ptr> expr_identifier %type <ast::expr_identifier::ptr> expr_identifier
%type <ast::expr_path::ptr> expr_path %type <ast::expr_path::ptr> expr_path
%type <ast::expr_istring::ptr> expr_istring %type <ast::expr_istring::ptr> expr_istring
@ -219,6 +219,7 @@ xsk::gsc::iw7::parser::symbol_type IW7lex(xsk::gsc::iw7::lexer& lexer);
%type <ast::expr_false::ptr> expr_false %type <ast::expr_false::ptr> expr_false
%type <ast::expr_true::ptr> expr_true %type <ast::expr_true::ptr> expr_true
%nonassoc SIZEOF
%nonassoc ADD_ARRAY %nonassoc ADD_ARRAY
%nonassoc RBRACKET %nonassoc RBRACKET
%nonassoc THEN %nonassoc THEN
@ -710,14 +711,12 @@ expr_array
; ;
expr_field expr_field
: expr_object DOT expr_identifier : expr_object DOT expr_identifier_nosize
{ $$ = std::make_unique<ast::expr_field>(@$, std::move($1), std::move($3)); } { $$ = std::make_unique<ast::expr_field>(@$, std::move($1), std::move($3)); }
| expr_object FIELD
{ $$ = std::make_unique<ast::expr_field>(@$, std::move($1), std::make_unique<ast::expr_identifier>(@$, $2)); }
; ;
expr_size expr_size
: expr_object SIZE : expr_object DOT SIZE %prec SIZEOF
{ $$ = std::make_unique<ast::expr_size>(@$, std::move($1)); } { $$ = std::make_unique<ast::expr_size>(@$, std::move($1)); }
; ;
@ -783,16 +782,23 @@ expr_animtree
{ $$ = std::make_unique<ast::expr_animtree>(@$); }; { $$ = std::make_unique<ast::expr_animtree>(@$); };
; ;
expr_identifier expr_identifier_nosize
: IDENTIFIER : IDENTIFIER
{ $$ = std::make_unique<ast::expr_identifier>(@$, $1); }; { $$ = std::make_unique<ast::expr_identifier>(@$, $1); };
; ;
expr_identifier
: IDENTIFIER
{ $$ = std::make_unique<ast::expr_identifier>(@$, $1); };
| SIZE
{ $$ = std::make_unique<ast::expr_identifier>(@$, "size"); };
;
expr_path expr_path
: PATH : IDENTIFIER
{ $$ = std::make_unique<ast::expr_path>(@$, $1); };
| PATH
{ $$ = std::make_unique<ast::expr_path>(@$, $1); }; { $$ = std::make_unique<ast::expr_path>(@$, $1); };
| expr_identifier
{ $$ = std::make_unique<ast::expr_path>(@$, $1->value); };
; ;
expr_istring expr_istring

26
gen/iw8/parser.ypp
View File

@ -83,7 +83,7 @@ xsk::gsc::iw8::parser::symbol_type IW8lex(xsk::gsc::iw8::lexer& lexer);
%token TRUE "true" %token TRUE "true"
%token FALSE "false" %token FALSE "false"
%token UNDEFINED "undefined" %token UNDEFINED "undefined"
%token SIZE ".size" %token SIZE "size"
%token GAME "game" %token GAME "game"
%token SELF "self" %token SELF "self"
%token ANIM "anim" %token ANIM "anim"
@ -135,7 +135,6 @@ xsk::gsc::iw8::parser::symbol_type IW8lex(xsk::gsc::iw8::lexer& lexer);
%token MUL "*" %token MUL "*"
%token DIV "/" %token DIV "/"
%token MOD "%" %token MOD "%"
%token <std::string> FIELD "field"
%token <std::string> PATH "path" %token <std::string> PATH "path"
%token <std::string> IDENTIFIER "identifier" %token <std::string> IDENTIFIER "identifier"
%token <std::string> STRING "string literal" %token <std::string> STRING "string literal"
@ -215,6 +214,7 @@ xsk::gsc::iw8::parser::symbol_type IW8lex(xsk::gsc::iw8::lexer& lexer);
%type <ast::expr_level::ptr> expr_level %type <ast::expr_level::ptr> expr_level
%type <ast::expr_animation::ptr> expr_animation %type <ast::expr_animation::ptr> expr_animation
%type <ast::expr_animtree::ptr> expr_animtree %type <ast::expr_animtree::ptr> expr_animtree
%type <ast::expr_identifier::ptr> expr_identifier_nosize
%type <ast::expr_identifier::ptr> expr_identifier %type <ast::expr_identifier::ptr> expr_identifier
%type <ast::expr_path::ptr> expr_path %type <ast::expr_path::ptr> expr_path
%type <ast::expr_istring::ptr> expr_istring %type <ast::expr_istring::ptr> expr_istring
@ -225,6 +225,7 @@ xsk::gsc::iw8::parser::symbol_type IW8lex(xsk::gsc::iw8::lexer& lexer);
%type <ast::expr_false::ptr> expr_false %type <ast::expr_false::ptr> expr_false
%type <ast::expr_true::ptr> expr_true %type <ast::expr_true::ptr> expr_true
%nonassoc SIZEOF
%nonassoc ADD_ARRAY %nonassoc ADD_ARRAY
%nonassoc RBRACKET %nonassoc RBRACKET
%nonassoc THEN %nonassoc THEN
@ -736,14 +737,12 @@ expr_array
; ;
expr_field expr_field
: expr_object DOT expr_identifier : expr_object DOT expr_identifier_nosize
{ $$ = std::make_unique<ast::expr_field>(@$, std::move($1), std::move($3)); } { $$ = std::make_unique<ast::expr_field>(@$, std::move($1), std::move($3)); }
| expr_object FIELD
{ $$ = std::make_unique<ast::expr_field>(@$, std::move($1), std::make_unique<ast::expr_identifier>(@$, $2)); }
; ;
expr_size expr_size
: expr_object SIZE : expr_object DOT SIZE %prec SIZEOF
{ $$ = std::make_unique<ast::expr_size>(@$, std::move($1)); } { $$ = std::make_unique<ast::expr_size>(@$, std::move($1)); }
; ;
@ -809,16 +808,23 @@ expr_animtree
{ $$ = std::make_unique<ast::expr_animtree>(@$); }; { $$ = std::make_unique<ast::expr_animtree>(@$); };
; ;
expr_identifier expr_identifier_nosize
: IDENTIFIER : IDENTIFIER
{ $$ = std::make_unique<ast::expr_identifier>(@$, $1); }; { $$ = std::make_unique<ast::expr_identifier>(@$, $1); };
; ;
expr_identifier
: IDENTIFIER
{ $$ = std::make_unique<ast::expr_identifier>(@$, $1); };
| SIZE
{ $$ = std::make_unique<ast::expr_identifier>(@$, "size"); };
;
expr_path expr_path
: PATH : IDENTIFIER
{ $$ = std::make_unique<ast::expr_path>(@$, $1); };
| PATH
{ $$ = std::make_unique<ast::expr_path>(@$, $1); }; { $$ = std::make_unique<ast::expr_path>(@$, $1); };
| expr_identifier
{ $$ = std::make_unique<ast::expr_path>(@$, $1->value); };
; ;
expr_istring expr_istring

26
gen/s1/parser.ypp
View File

@ -83,7 +83,7 @@ xsk::gsc::s1::parser::symbol_type S1lex(xsk::gsc::s1::lexer& lexer);
%token TRUE "true" %token TRUE "true"
%token FALSE "false" %token FALSE "false"
%token UNDEFINED "undefined" %token UNDEFINED "undefined"
%token SIZE ".size" %token SIZE "size"
%token GAME "game" %token GAME "game"
%token SELF "self" %token SELF "self"
%token ANIM "anim" %token ANIM "anim"
@ -133,7 +133,6 @@ xsk::gsc::s1::parser::symbol_type S1lex(xsk::gsc::s1::lexer& lexer);
%token MUL "*" %token MUL "*"
%token DIV "/" %token DIV "/"
%token MOD "%" %token MOD "%"
%token <std::string> FIELD "field"
%token <std::string> PATH "path" %token <std::string> PATH "path"
%token <std::string> IDENTIFIER "identifier" %token <std::string> IDENTIFIER "identifier"
%token <std::string> STRING "string literal" %token <std::string> STRING "string literal"
@ -211,6 +210,7 @@ xsk::gsc::s1::parser::symbol_type S1lex(xsk::gsc::s1::lexer& lexer);
%type <ast::expr_level::ptr> expr_level %type <ast::expr_level::ptr> expr_level
%type <ast::expr_animation::ptr> expr_animation %type <ast::expr_animation::ptr> expr_animation
%type <ast::expr_animtree::ptr> expr_animtree %type <ast::expr_animtree::ptr> expr_animtree
%type <ast::expr_identifier::ptr> expr_identifier_nosize
%type <ast::expr_identifier::ptr> expr_identifier %type <ast::expr_identifier::ptr> expr_identifier
%type <ast::expr_path::ptr> expr_path %type <ast::expr_path::ptr> expr_path
%type <ast::expr_istring::ptr> expr_istring %type <ast::expr_istring::ptr> expr_istring
@ -221,6 +221,7 @@ xsk::gsc::s1::parser::symbol_type S1lex(xsk::gsc::s1::lexer& lexer);
%type <ast::expr_false::ptr> expr_false %type <ast::expr_false::ptr> expr_false
%type <ast::expr_true::ptr> expr_true %type <ast::expr_true::ptr> expr_true
%nonassoc SIZEOF
%nonassoc ADD_ARRAY %nonassoc ADD_ARRAY
%nonassoc RBRACKET %nonassoc RBRACKET
%nonassoc THEN %nonassoc THEN
@ -720,14 +721,12 @@ expr_array
; ;
expr_field expr_field
: expr_object DOT expr_identifier : expr_object DOT expr_identifier_nosize
{ $$ = std::make_unique<ast::expr_field>(@$, std::move($1), std::move($3)); } { $$ = std::make_unique<ast::expr_field>(@$, std::move($1), std::move($3)); }
| expr_object FIELD
{ $$ = std::make_unique<ast::expr_field>(@$, std::move($1), std::make_unique<ast::expr_identifier>(@$, $2)); }
; ;
expr_size expr_size
: expr_object SIZE : expr_object DOT SIZE %prec SIZEOF
{ $$ = std::make_unique<ast::expr_size>(@$, std::move($1)); } { $$ = std::make_unique<ast::expr_size>(@$, std::move($1)); }
; ;
@ -793,16 +792,23 @@ expr_animtree
{ $$ = std::make_unique<ast::expr_animtree>(@$); }; { $$ = std::make_unique<ast::expr_animtree>(@$); };
; ;
expr_identifier expr_identifier_nosize
: IDENTIFIER : IDENTIFIER
{ $$ = std::make_unique<ast::expr_identifier>(@$, $1); }; { $$ = std::make_unique<ast::expr_identifier>(@$, $1); };
; ;
expr_identifier
: IDENTIFIER
{ $$ = std::make_unique<ast::expr_identifier>(@$, $1); };
| SIZE
{ $$ = std::make_unique<ast::expr_identifier>(@$, "size"); };
;
expr_path expr_path
: PATH : IDENTIFIER
{ $$ = std::make_unique<ast::expr_path>(@$, $1); };
| PATH
{ $$ = std::make_unique<ast::expr_path>(@$, $1); }; { $$ = std::make_unique<ast::expr_path>(@$, $1); };
| expr_identifier
{ $$ = std::make_unique<ast::expr_path>(@$, $1->value); };
; ;
expr_istring expr_istring

26
gen/s2/parser.ypp
View File

@ -83,7 +83,7 @@ xsk::gsc::s2::parser::symbol_type S2lex(xsk::gsc::s2::lexer& lexer);
%token TRUE "true" %token TRUE "true"
%token FALSE "false" %token FALSE "false"
%token UNDEFINED "undefined" %token UNDEFINED "undefined"
%token SIZE ".size" %token SIZE "size"
%token GAME "game" %token GAME "game"
%token SELF "self" %token SELF "self"
%token ANIM "anim" %token ANIM "anim"
@ -133,7 +133,6 @@ xsk::gsc::s2::parser::symbol_type S2lex(xsk::gsc::s2::lexer& lexer);
%token MUL "*" %token MUL "*"
%token DIV "/" %token DIV "/"
%token MOD "%" %token MOD "%"
%token <std::string> FIELD "field"
%token <std::string> PATH "path" %token <std::string> PATH "path"
%token <std::string> IDENTIFIER "identifier" %token <std::string> IDENTIFIER "identifier"
%token <std::string> STRING "string literal" %token <std::string> STRING "string literal"
@ -211,6 +210,7 @@ xsk::gsc::s2::parser::symbol_type S2lex(xsk::gsc::s2::lexer& lexer);
%type <ast::expr_level::ptr> expr_level %type <ast::expr_level::ptr> expr_level
%type <ast::expr_animation::ptr> expr_animation %type <ast::expr_animation::ptr> expr_animation
%type <ast::expr_animtree::ptr> expr_animtree %type <ast::expr_animtree::ptr> expr_animtree
%type <ast::expr_identifier::ptr> expr_identifier_nosize
%type <ast::expr_identifier::ptr> expr_identifier %type <ast::expr_identifier::ptr> expr_identifier
%type <ast::expr_path::ptr> expr_path %type <ast::expr_path::ptr> expr_path
%type <ast::expr_istring::ptr> expr_istring %type <ast::expr_istring::ptr> expr_istring
@ -221,6 +221,7 @@ xsk::gsc::s2::parser::symbol_type S2lex(xsk::gsc::s2::lexer& lexer);
%type <ast::expr_false::ptr> expr_false %type <ast::expr_false::ptr> expr_false
%type <ast::expr_true::ptr> expr_true %type <ast::expr_true::ptr> expr_true
%nonassoc SIZEOF
%nonassoc ADD_ARRAY %nonassoc ADD_ARRAY
%nonassoc RBRACKET %nonassoc RBRACKET
%nonassoc THEN %nonassoc THEN
@ -720,14 +721,12 @@ expr_array
; ;
expr_field expr_field
: expr_object DOT expr_identifier : expr_object DOT expr_identifier_nosize
{ $$ = std::make_unique<ast::expr_field>(@$, std::move($1), std::move($3)); } { $$ = std::make_unique<ast::expr_field>(@$, std::move($1), std::move($3)); }
| expr_object FIELD
{ $$ = std::make_unique<ast::expr_field>(@$, std::move($1), std::make_unique<ast::expr_identifier>(@$, $2)); }
; ;
expr_size expr_size
: expr_object SIZE : expr_object DOT SIZE %prec SIZEOF
{ $$ = std::make_unique<ast::expr_size>(@$, std::move($1)); } { $$ = std::make_unique<ast::expr_size>(@$, std::move($1)); }
; ;
@ -793,16 +792,23 @@ expr_animtree
{ $$ = std::make_unique<ast::expr_animtree>(@$); }; { $$ = std::make_unique<ast::expr_animtree>(@$); };
; ;
expr_identifier expr_identifier_nosize
: IDENTIFIER : IDENTIFIER
{ $$ = std::make_unique<ast::expr_identifier>(@$, $1); }; { $$ = std::make_unique<ast::expr_identifier>(@$, $1); };
; ;
expr_identifier
: IDENTIFIER
{ $$ = std::make_unique<ast::expr_identifier>(@$, $1); };
| SIZE
{ $$ = std::make_unique<ast::expr_identifier>(@$, "size"); };
;
expr_path expr_path
: PATH : IDENTIFIER
{ $$ = std::make_unique<ast::expr_path>(@$, $1); };
| PATH
{ $$ = std::make_unique<ast::expr_path>(@$, $1); }; { $$ = std::make_unique<ast::expr_path>(@$, $1); };
| expr_identifier
{ $$ = std::make_unique<ast::expr_path>(@$, $1->value); };
; ;
expr_istring expr_istring

26
gen/s4/parser.ypp
View File

@ -83,7 +83,7 @@ xsk::gsc::s4::parser::symbol_type S4lex(xsk::gsc::s4::lexer& lexer);
%token TRUE "true" %token TRUE "true"
%token FALSE "false" %token FALSE "false"
%token UNDEFINED "undefined" %token UNDEFINED "undefined"
%token SIZE ".size" %token SIZE "size"
%token GAME "game" %token GAME "game"
%token SELF "self" %token SELF "self"
%token ANIM "anim" %token ANIM "anim"
@ -135,7 +135,6 @@ xsk::gsc::s4::parser::symbol_type S4lex(xsk::gsc::s4::lexer& lexer);
%token MUL "*" %token MUL "*"
%token DIV "/" %token DIV "/"
%token MOD "%" %token MOD "%"
%token <std::string> FIELD "field"
%token <std::string> PATH "path" %token <std::string> PATH "path"
%token <std::string> IDENTIFIER "identifier" %token <std::string> IDENTIFIER "identifier"
%token <std::string> STRING "string literal" %token <std::string> STRING "string literal"
@ -215,6 +214,7 @@ xsk::gsc::s4::parser::symbol_type S4lex(xsk::gsc::s4::lexer& lexer);
%type <ast::expr_level::ptr> expr_level %type <ast::expr_level::ptr> expr_level
%type <ast::expr_animation::ptr> expr_animation %type <ast::expr_animation::ptr> expr_animation
%type <ast::expr_animtree::ptr> expr_animtree %type <ast::expr_animtree::ptr> expr_animtree
%type <ast::expr_identifier::ptr> expr_identifier_nosize
%type <ast::expr_identifier::ptr> expr_identifier %type <ast::expr_identifier::ptr> expr_identifier
%type <ast::expr_path::ptr> expr_path %type <ast::expr_path::ptr> expr_path
%type <ast::expr_istring::ptr> expr_istring %type <ast::expr_istring::ptr> expr_istring
@ -225,6 +225,7 @@ xsk::gsc::s4::parser::symbol_type S4lex(xsk::gsc::s4::lexer& lexer);
%type <ast::expr_false::ptr> expr_false %type <ast::expr_false::ptr> expr_false
%type <ast::expr_true::ptr> expr_true %type <ast::expr_true::ptr> expr_true
%nonassoc SIZEOF
%nonassoc ADD_ARRAY %nonassoc ADD_ARRAY
%nonassoc RBRACKET %nonassoc RBRACKET
%nonassoc THEN %nonassoc THEN
@ -736,14 +737,12 @@ expr_array
; ;
expr_field expr_field
: expr_object DOT expr_identifier : expr_object DOT expr_identifier_nosize
{ $$ = std::make_unique<ast::expr_field>(@$, std::move($1), std::move($3)); } { $$ = std::make_unique<ast::expr_field>(@$, std::move($1), std::move($3)); }
| expr_object FIELD
{ $$ = std::make_unique<ast::expr_field>(@$, std::move($1), std::make_unique<ast::expr_identifier>(@$, $2)); }
; ;
expr_size expr_size
: expr_object SIZE : expr_object DOT SIZE %prec SIZEOF
{ $$ = std::make_unique<ast::expr_size>(@$, std::move($1)); } { $$ = std::make_unique<ast::expr_size>(@$, std::move($1)); }
; ;
@ -809,16 +808,23 @@ expr_animtree
{ $$ = std::make_unique<ast::expr_animtree>(@$); }; { $$ = std::make_unique<ast::expr_animtree>(@$); };
; ;
expr_identifier expr_identifier_nosize
: IDENTIFIER : IDENTIFIER
{ $$ = std::make_unique<ast::expr_identifier>(@$, $1); }; { $$ = std::make_unique<ast::expr_identifier>(@$, $1); };
; ;
expr_identifier
: IDENTIFIER
{ $$ = std::make_unique<ast::expr_identifier>(@$, $1); };
| SIZE
{ $$ = std::make_unique<ast::expr_identifier>(@$, "size"); };
;
expr_path expr_path
: PATH : IDENTIFIER
{ $$ = std::make_unique<ast::expr_path>(@$, $1); };
| PATH
{ $$ = std::make_unique<ast::expr_path>(@$, $1); }; { $$ = std::make_unique<ast::expr_path>(@$, $1); };
| expr_identifier
{ $$ = std::make_unique<ast::expr_path>(@$, $1->value); };
; ;
expr_istring expr_istring

26
gen/t6/parser.ypp
View File

@ -78,7 +78,7 @@ xsk::arc::t6::parser::symbol_type T6lex(xsk::arc::t6::lexer& lexer);
%token TRUE "true" %token TRUE "true"
%token FALSE "false" %token FALSE "false"
%token UNDEFINED "undefined" %token UNDEFINED "undefined"
%token SIZE ".size" %token SIZE "size"
%token GAME "game" %token GAME "game"
%token SELF "self" %token SELF "self"
%token ANIM "anim" %token ANIM "anim"
@ -147,7 +147,6 @@ xsk::arc::t6::parser::symbol_type T6lex(xsk::arc::t6::lexer& lexer);
%token MUL "*" %token MUL "*"
%token DIV "/" %token DIV "/"
%token MOD "%" %token MOD "%"
%token <std::string> FIELD "field"
%token <std::string> PATH "path" %token <std::string> PATH "path"
%token <std::string> IDENTIFIER "identifier" %token <std::string> IDENTIFIER "identifier"
%token <std::string> STRING "string literal" %token <std::string> STRING "string literal"
@ -241,6 +240,7 @@ xsk::arc::t6::parser::symbol_type T6lex(xsk::arc::t6::lexer& lexer);
%type <ast::expr_anim::ptr> expr_anim %type <ast::expr_anim::ptr> expr_anim
%type <ast::expr_level::ptr> expr_level %type <ast::expr_level::ptr> expr_level
%type <ast::expr_animation::ptr> expr_animation %type <ast::expr_animation::ptr> expr_animation
%type <ast::expr_identifier::ptr> expr_identifier_nosize
%type <ast::expr_identifier::ptr> expr_identifier %type <ast::expr_identifier::ptr> expr_identifier
%type <ast::expr_path::ptr> expr_path %type <ast::expr_path::ptr> expr_path
%type <ast::expr_istring::ptr> expr_istring %type <ast::expr_istring::ptr> expr_istring
@ -252,6 +252,7 @@ xsk::arc::t6::parser::symbol_type T6lex(xsk::arc::t6::lexer& lexer);
%type <ast::expr_false::ptr> expr_false %type <ast::expr_false::ptr> expr_false
%type <ast::expr_true::ptr> expr_true %type <ast::expr_true::ptr> expr_true
%nonassoc SIZEOF
%nonassoc RBRACKET %nonassoc RBRACKET
%nonassoc THEN %nonassoc THEN
%nonassoc ELSE %nonassoc ELSE
@ -841,14 +842,12 @@ expr_array
; ;
expr_field expr_field
: expr_object DOT expr_identifier : expr_object DOT expr_identifier_nosize
{ $$ = std::make_unique<ast::expr_field>(@$, std::move($1), std::move($3)); } { $$ = std::make_unique<ast::expr_field>(@$, std::move($1), std::move($3)); }
| expr_object FIELD
{ $$ = std::make_unique<ast::expr_field>(@$, std::move($1), std::make_unique<ast::expr_identifier>(@$, $2)); }
; ;
expr_size expr_size
: expr_object SIZE : expr_object DOT SIZE %prec SIZEOF
{ $$ = std::make_unique<ast::expr_size>(@$, std::move($1)); } { $$ = std::make_unique<ast::expr_size>(@$, std::move($1)); }
; ;
@ -904,16 +903,23 @@ expr_animation
{ $$ = std::make_unique<ast::expr_animation>(@$, $2); }; { $$ = std::make_unique<ast::expr_animation>(@$, $2); };
; ;
expr_identifier expr_identifier_nosize
: IDENTIFIER : IDENTIFIER
{ $$ = std::make_unique<ast::expr_identifier>(@$, $1); }; { $$ = std::make_unique<ast::expr_identifier>(@$, $1); };
; ;
expr_identifier
: IDENTIFIER
{ $$ = std::make_unique<ast::expr_identifier>(@$, $1); };
| SIZE
{ $$ = std::make_unique<ast::expr_identifier>(@$, "size"); };
;
expr_path expr_path
: PATH : IDENTIFIER
{ $$ = std::make_unique<ast::expr_path>(@$, $1); };
| PATH
{ $$ = std::make_unique<ast::expr_path>(@$, $1); }; { $$ = std::make_unique<ast::expr_path>(@$, $1); };
| expr_identifier
{ $$ = std::make_unique<ast::expr_path>(@$, $1->value); };
; ;
expr_istring expr_istring

120
src/h1/xsk/lexer.cpp
View File

@ -60,6 +60,7 @@ const std::unordered_map<std::string_view, parser::token::token_kind_type> keywo
{ "true", parser::token::TRUE }, { "true", parser::token::TRUE },
{ "false", parser::token::FALSE }, { "false", parser::token::FALSE },
{ "undefined", parser::token::UNDEFINED }, { "undefined", parser::token::UNDEFINED },
{ "size", parser::token::SIZE },
{ "game", parser::token::GAME }, { "game", parser::token::GAME },
{ "self", parser::token::SELF }, { "self", parser::token::SELF },
{ "anim", parser::token::ANIM }, { "anim", parser::token::ANIM },
@ -85,8 +86,8 @@ bool buffer::push(char c)
return true; return true;
} }
reader::reader() : state(reader::end), buffer_pos(0), bytes_remaining(0), reader::reader() : state(reader::end), buffer_pos(0),
last_byte(0), current_byte(0) { } bytes_remaining(0), last_byte(0), current_byte(0) {}
void reader::init(const char* data, size_t size) void reader::init(const char* data, size_t size)
{ {
@ -174,7 +175,6 @@ auto lexer::lex() -> parser::symbol_type
{ {
buffer_.length = 0; buffer_.length = 0;
state_ = state::start; state_ = state::start;
loc_.step();
while (true) while (true)
{ {
@ -182,6 +182,7 @@ auto lexer::lex() -> parser::symbol_type
auto& last = reader_.last_byte; auto& last = reader_.last_byte;
auto& curr = reader_.current_byte; auto& curr = reader_.current_byte;
auto path = false; auto path = false;
loc_.step();
if (state == reader::end) if (state == reader::end)
{ {
@ -214,7 +215,7 @@ auto lexer::lex() -> parser::symbol_type
case '\\': case '\\':
throw comp_error(loc_, "invalid token ('\\')"); throw comp_error(loc_, "invalid token ('\\')");
case '/': case '/':
if (curr != '/' && curr != '*' && curr != '#' && curr != '=') if (curr != '=' && curr != '#' && curr != '@' && curr != '*' && curr != '/')
return parser::make_DIV(loc_); return parser::make_DIV(loc_);
advance(); advance();
@ -246,14 +247,35 @@ auto lexer::lex() -> parser::symbol_type
} }
else if (last == '#' && curr == '/') else if (last == '#' && curr == '/')
{ {
reader_.advance(); advance();
break; break;
} }
reader_.advance(); advance();
} }
} }
} }
else if (last == '@')
{
while (true)
{
if (state == reader::end)
throw comp_error(loc_, "unmatched script doc comment start ('/@')");
if (curr == '\n')
{
loc_.lines();
loc_.step();
}
else if (last == '@' && curr == '/')
{
advance();
break;
}
advance();
}
}
else if (last == '*') else if (last == '*')
{ {
while (true) while (true)
@ -268,11 +290,11 @@ auto lexer::lex() -> parser::symbol_type
} }
else if (last == '*' && curr == '/') else if (last == '*' && curr == '/')
{ {
reader_.advance(); advance();
break; break;
} }
reader_.advance(); advance();
} }
} }
else if (last == '/') else if (last == '/')
@ -282,30 +304,10 @@ auto lexer::lex() -> parser::symbol_type
if (state == reader::end) if (state == reader::end)
break; break;
if (last == '\\' && (curr == '\r' || curr == '\n'))
{
reader_.advance();
if (state == reader::end)
break;
if (last == '\r')
{
if (curr != '\n')
throw comp_error(loc_, "invalid token ('\')");
reader_.advance();
}
loc_.lines();
loc_.step();
continue;
}
if (curr == '\n') if (curr == '\n')
break; break;
reader_.advance(); advance();
} }
} }
continue; continue;
@ -315,8 +317,8 @@ auto lexer::lex() -> parser::symbol_type
if (!indev_) if (!indev_)
throw comp_error(loc_, "unmatched devblock end ('#/')"); throw comp_error(loc_, "unmatched devblock end ('#/')");
indev_ = false;
advance(); advance();
indev_ = false;
return parser::make_DEVEND(loc_); return parser::make_DEVEND(loc_);
} }
@ -337,7 +339,7 @@ auto lexer::lex() -> parser::symbol_type
state_ = state::preprocessor; state_ = state::preprocessor;
goto lex_name; goto lex_name;
case '*': case '*':
if (curr != '/' && curr != '=') if (curr != '=' && curr != '/')
return parser::make_MUL(loc_); return parser::make_MUL(loc_);
advance(); advance();
@ -350,13 +352,9 @@ auto lexer::lex() -> parser::symbol_type
state_ = state::string; state_ = state::string;
goto lex_string; goto lex_string;
case '.': case '.':
advance(); if (curr < '0' || curr > '9')
return parser::make_DOT(loc_);
if (state == reader::end) goto lex_number;
throw comp_error(loc_, "unterminated field ('.')");
state_ = state::field;
goto lex_name_or_number;
case '(': case '(':
return parser::make_LPAREN(loc_); return parser::make_LPAREN(loc_);
case ')': case ')':
@ -480,7 +478,6 @@ auto lexer::lex() -> parser::symbol_type
advance(); advance();
return parser::make_ASSIGN_RSHIFT(loc_); return parser::make_ASSIGN_RSHIFT(loc_);
default: default:
lex_name_or_number:
if (last >= '0' && last <= '9') if (last >= '0' && last <= '9')
goto lex_number; goto lex_number;
else if (last == '_' || last >= 'A' && last <= 'Z' || last >= 'a' && last <= 'z') else if (last == '_' || last >= 'A' && last <= 'Z' || last >= 'a' && last <= 'z')
@ -562,23 +559,8 @@ lex_name:
advance(); advance();
} }
if (state_ == state::field) if (state_ == state::preprocessor)
{ {
if (path)
throw comp_error(loc_, "invalid field token '\\'");
if (std::string_view(buffer_.data, buffer_.length) == "size")
{
return parser::make_SIZE(loc_);
}
return parser::make_FIELD(std::string(buffer_.data, buffer_.length), loc_);
}
else if (state_ == state::preprocessor)
{
if (path)
throw comp_error(loc_, "invalid preprocessor directive");
auto token = parser::token::H1UNDEF; auto token = parser::token::H1UNDEF;
if (buffer_.length < 16) if (buffer_.length < 16)
@ -594,7 +576,8 @@ lex_name:
} }
} }
preprocessor(token); preprocessor_run(token);
state_ = state::start; state_ = state::start;
continue; continue;
} }
@ -620,14 +603,11 @@ lex_name:
} }
lex_number: lex_number:
if (state_ == state::field) if (last == '.' || last != '0' || (last == '0' && (curr != 'o' && curr != 'b' && curr != 'x')))
buffer_.push('.');
if (state_ == state::field || last == '.' || last != '0' || (last == '0' && (curr != 'o' && curr != 'b' && curr != 'x')))
{ {
buffer_.push(last); buffer_.push(last);
auto dot = 0; auto dot = last == '.' ? 1 : 0;
auto flt = 0; auto flt = 0;
while (true) while (true)
@ -663,10 +643,10 @@ lex_number:
if (last == '\'') if (last == '\'')
throw comp_error(loc_, "invalid number literal"); throw comp_error(loc_, "invalid number literal");
if (state_ == state::field && dot || dot > 1 || flt > 1 || flt && buffer_.data[buffer_.length - 1] != 'f') if (dot > 1 || flt > 1 || flt && buffer_.data[buffer_.length - 1] != 'f')
throw comp_error(loc_, "invalid number literal"); throw comp_error(loc_, "invalid number literal");
if (state_ == state::field || dot || flt) if (dot || flt)
return parser::make_FLOAT(std::string(buffer_.data, buffer_.length), loc_); return parser::make_FLOAT(std::string(buffer_.data, buffer_.length), loc_);
return parser::make_INTEGER(std::string(buffer_.data, buffer_.length), loc_); return parser::make_INTEGER(std::string(buffer_.data, buffer_.length), loc_);
@ -771,18 +751,22 @@ lex_number:
return parser::make_INTEGER(xsk::utils::string::hex_to_dec(buffer_.data), loc_); return parser::make_INTEGER(xsk::utils::string::hex_to_dec(buffer_.data), loc_);
} }
// cant get here!
throw error("UNEXPECTED LEXER INTERNAL ERROR!");
} }
} }
void lexer::advance() void lexer::advance()
{ {
reader_.advance(); reader_.advance();
loc_.end.column++;
// dont wrap comment marks '/\/' '/\*' outside strings if (reader_.current_byte == '\\') [[unlikely]]
if (state_ == state::start && reader_.last_byte == '/') preprocessor_wrap();
return; }
void lexer::preprocessor_wrap()
{
while (reader_.current_byte == '\\') while (reader_.current_byte == '\\')
{ {
if (reader_.bytes_remaining == 1) if (reader_.bytes_remaining == 1)
@ -824,7 +808,7 @@ void lexer::advance()
} }
} }
void lexer::preprocessor(parser::token::token_kind_type token) void lexer::preprocessor_run(parser::token::token_kind_type token)
{ {
if (!clean_) if (!clean_)
throw comp_error(loc_, "invalid token ('#')"); throw comp_error(loc_, "invalid token ('#')");

5
src/h1/xsk/lexer.hpp
View File

@ -44,7 +44,7 @@ struct reader
class lexer class lexer
{ {
enum class state : std::uint8_t { start, string, localize, field, preprocessor }; enum class state : std::uint8_t { start, string, localize, preprocessor };
reader reader_; reader reader_;
buffer buffer_; buffer buffer_;
@ -66,7 +66,8 @@ public:
private: private:
void advance(); void advance();
void preprocessor(parser::token::token_kind_type token); void preprocessor_wrap();
void preprocessor_run(parser::token::token_kind_type token);
}; };
} // namespace xsk::gsc::h1 } // namespace xsk::gsc::h1

1740
src/h1/xsk/parser.cpp
View File

File diff suppressed because it is too large Load Diff

103
src/h1/xsk/parser.hpp
View File

@ -476,6 +476,7 @@ namespace xsk { namespace gsc { namespace h1 {
// expr_game // expr_game
char dummy19[sizeof (ast::expr_game::ptr)]; char dummy19[sizeof (ast::expr_game::ptr)];
// expr_identifier_nosize
// expr_identifier // expr_identifier
char dummy20[sizeof (ast::expr_identifier::ptr)]; char dummy20[sizeof (ast::expr_identifier::ptr)];
@ -620,7 +621,6 @@ namespace xsk { namespace gsc { namespace h1 {
// stmt_while // stmt_while
char dummy66[sizeof (ast::stmt_while::ptr)]; char dummy66[sizeof (ast::stmt_while::ptr)];
// "field"
// "path" // "path"
// "identifier" // "identifier"
// "string literal" // "string literal"
@ -723,7 +723,7 @@ namespace xsk { namespace gsc { namespace h1 {
TRUE = 44, // "true" TRUE = 44, // "true"
FALSE = 45, // "false" FALSE = 45, // "false"
UNDEFINED = 46, // "undefined" UNDEFINED = 46, // "undefined"
SIZE = 47, // ".size" SIZE = 47, // "size"
GAME = 48, // "game" GAME = 48, // "game"
SELF = 49, // "self" SELF = 49, // "self"
ANIM = 50, // "anim" ANIM = 50, // "anim"
@ -773,13 +773,13 @@ namespace xsk { namespace gsc { namespace h1 {
MUL = 94, // "*" MUL = 94, // "*"
DIV = 95, // "/" DIV = 95, // "/"
MOD = 96, // "%" MOD = 96, // "%"
FIELD = 97, // "field" PATH = 97, // "path"
PATH = 98, // "path" IDENTIFIER = 98, // "identifier"
IDENTIFIER = 99, // "identifier" STRING = 99, // "string literal"
STRING = 100, // "string literal" ISTRING = 100, // "localized string"
ISTRING = 101, // "localized string" FLOAT = 101, // "float"
FLOAT = 102, // "float" INTEGER = 102, // "integer"
INTEGER = 103, // "integer" SIZEOF = 103, // SIZEOF
ADD_ARRAY = 104, // ADD_ARRAY ADD_ARRAY = 104, // ADD_ARRAY
THEN = 105, // THEN THEN = 105, // THEN
TERN = 106, // TERN TERN = 106, // TERN
@ -854,7 +854,7 @@ namespace xsk { namespace gsc { namespace h1 {
S_TRUE = 44, // "true" S_TRUE = 44, // "true"
S_FALSE = 45, // "false" S_FALSE = 45, // "false"
S_UNDEFINED = 46, // "undefined" S_UNDEFINED = 46, // "undefined"
S_SIZE = 47, // ".size" S_SIZE = 47, // "size"
S_GAME = 48, // "game" S_GAME = 48, // "game"
S_SELF = 49, // "self" S_SELF = 49, // "self"
S_ANIM = 50, // "anim" S_ANIM = 50, // "anim"
@ -904,13 +904,13 @@ namespace xsk { namespace gsc { namespace h1 {
S_MUL = 94, // "*" S_MUL = 94, // "*"
S_DIV = 95, // "/" S_DIV = 95, // "/"
S_MOD = 96, // "%" S_MOD = 96, // "%"
S_FIELD = 97, // "field" S_PATH = 97, // "path"
S_PATH = 98, // "path" S_IDENTIFIER = 98, // "identifier"
S_IDENTIFIER = 99, // "identifier" S_STRING = 99, // "string literal"
S_STRING = 100, // "string literal" S_ISTRING = 100, // "localized string"
S_ISTRING = 101, // "localized string" S_FLOAT = 101, // "float"
S_FLOAT = 102, // "float" S_INTEGER = 102, // "integer"
S_INTEGER = 103, // "integer" S_SIZEOF = 103, // SIZEOF
S_ADD_ARRAY = 104, // ADD_ARRAY S_ADD_ARRAY = 104, // ADD_ARRAY
S_THEN = 105, // THEN S_THEN = 105, // THEN
S_TERN = 106, // TERN S_TERN = 106, // TERN
@ -993,15 +993,16 @@ namespace xsk { namespace gsc { namespace h1 {
S_expr_level = 183, // expr_level S_expr_level = 183, // expr_level
S_expr_animation = 184, // expr_animation S_expr_animation = 184, // expr_animation
S_expr_animtree = 185, // expr_animtree S_expr_animtree = 185, // expr_animtree
S_expr_identifier = 186, // expr_identifier S_expr_identifier_nosize = 186, // expr_identifier_nosize
S_expr_path = 187, // expr_path S_expr_identifier = 187, // expr_identifier
S_expr_istring = 188, // expr_istring S_expr_path = 188, // expr_path
S_expr_string = 189, // expr_string S_expr_istring = 189, // expr_istring
S_expr_vector = 190, // expr_vector S_expr_string = 190, // expr_string
S_expr_float = 191, // expr_float S_expr_vector = 191, // expr_vector
S_expr_integer = 192, // expr_integer S_expr_float = 192, // expr_float
S_expr_false = 193, // expr_false S_expr_integer = 193, // expr_integer
S_expr_true = 194 // expr_true S_expr_false = 194, // expr_false
S_expr_true = 195 // expr_true
}; };
}; };
@ -1124,6 +1125,7 @@ namespace xsk { namespace gsc { namespace h1 {
value.move< ast::expr_game::ptr > (std::move (that.value)); value.move< ast::expr_game::ptr > (std::move (that.value));
break; break;
case symbol_kind::S_expr_identifier_nosize: // expr_identifier_nosize
case symbol_kind::S_expr_identifier: // expr_identifier case symbol_kind::S_expr_identifier: // expr_identifier
value.move< ast::expr_identifier::ptr > (std::move (that.value)); value.move< ast::expr_identifier::ptr > (std::move (that.value));
break; break;
@ -1315,7 +1317,6 @@ namespace xsk { namespace gsc { namespace h1 {
value.move< ast::stmt_while::ptr > (std::move (that.value)); value.move< ast::stmt_while::ptr > (std::move (that.value));
break; break;
case symbol_kind::S_FIELD: // "field"
case symbol_kind::S_PATH: // "path" case symbol_kind::S_PATH: // "path"
case symbol_kind::S_IDENTIFIER: // "identifier" case symbol_kind::S_IDENTIFIER: // "identifier"
case symbol_kind::S_STRING: // "string literal" case symbol_kind::S_STRING: // "string literal"
@ -2394,6 +2395,7 @@ switch (yykind)
value.template destroy< ast::expr_game::ptr > (); value.template destroy< ast::expr_game::ptr > ();
break; break;
case symbol_kind::S_expr_identifier_nosize: // expr_identifier_nosize
case symbol_kind::S_expr_identifier: // expr_identifier case symbol_kind::S_expr_identifier: // expr_identifier
value.template destroy< ast::expr_identifier::ptr > (); value.template destroy< ast::expr_identifier::ptr > ();
break; break;
@ -2585,7 +2587,6 @@ switch (yykind)
value.template destroy< ast::stmt_while::ptr > (); value.template destroy< ast::stmt_while::ptr > ();
break; break;
case symbol_kind::S_FIELD: // "field"
case symbol_kind::S_PATH: // "path" case symbol_kind::S_PATH: // "path"
case symbol_kind::S_IDENTIFIER: // "identifier" case symbol_kind::S_IDENTIFIER: // "identifier"
case symbol_kind::S_STRING: // "string literal" case symbol_kind::S_STRING: // "string literal"
@ -2691,7 +2692,7 @@ switch (yykind)
{ {
H1_ASSERT (tok == token::H1EOF H1_ASSERT (tok == token::H1EOF
|| (token::H1error <= tok && tok <= token::MOD) || (token::H1error <= tok && tok <= token::MOD)
|| (token::ADD_ARRAY <= tok && tok <= token::POSTDEC)); || (token::SIZEOF <= tok && tok <= token::POSTDEC));
} }
#if 201103L <= YY_CPLUSPLUS #if 201103L <= YY_CPLUSPLUS
symbol_type (int tok, std::string v, location_type l) symbol_type (int tok, std::string v, location_type l)
@ -2701,7 +2702,7 @@ switch (yykind)
: super_type(token_type (tok), v, l) : super_type(token_type (tok), v, l)
#endif #endif
{ {
H1_ASSERT ((token::FIELD <= tok && tok <= token::INTEGER)); H1_ASSERT ((token::PATH <= tok && tok <= token::INTEGER));
} }
}; };
@ -4206,21 +4207,6 @@ switch (yykind)
return symbol_type (token::MOD, l); return symbol_type (token::MOD, l);
} }
#endif #endif
#if 201103L <= YY_CPLUSPLUS
static
symbol_type
make_FIELD (std::string v, location_type l)
{
return symbol_type (token::FIELD, std::move (v), std::move (l));
}
#else
static
symbol_type
make_FIELD (const std::string& v, const location_type& l)
{
return symbol_type (token::FIELD, v, l);
}
#endif
#if 201103L <= YY_CPLUSPLUS #if 201103L <= YY_CPLUSPLUS
static static
symbol_type symbol_type
@ -4311,6 +4297,21 @@ switch (yykind)
return symbol_type (token::INTEGER, v, l); return symbol_type (token::INTEGER, v, l);
} }
#endif #endif
#if 201103L <= YY_CPLUSPLUS
static
symbol_type
make_SIZEOF (location_type l)
{
return symbol_type (token::SIZEOF, std::move (l));
}
#else
static
symbol_type
make_SIZEOF (const location_type& l)
{
return symbol_type (token::SIZEOF, l);
}
#endif
#if 201103L <= YY_CPLUSPLUS #if 201103L <= YY_CPLUSPLUS
static static
symbol_type symbol_type
@ -4790,9 +4791,9 @@ switch (yykind)
/// Constants. /// Constants.
enum enum
{ {
yylast_ = 2300, ///< Last index in yytable_. yylast_ = 2251, ///< Last index in yytable_.
yynnts_ = 82, ///< Number of nonterminal symbols. yynnts_ = 83, ///< Number of nonterminal symbols.
yyfinal_ = 21 ///< Termination state number. yyfinal_ = 22 ///< Termination state number.
}; };
@ -4904,6 +4905,7 @@ switch (yykind)
value.copy< ast::expr_game::ptr > (YY_MOVE (that.value)); value.copy< ast::expr_game::ptr > (YY_MOVE (that.value));
break; break;
case symbol_kind::S_expr_identifier_nosize: // expr_identifier_nosize
case symbol_kind::S_expr_identifier: // expr_identifier case symbol_kind::S_expr_identifier: // expr_identifier
value.copy< ast::expr_identifier::ptr > (YY_MOVE (that.value)); value.copy< ast::expr_identifier::ptr > (YY_MOVE (that.value));
break; break;
@ -5095,7 +5097,6 @@ switch (yykind)
value.copy< ast::stmt_while::ptr > (YY_MOVE (that.value)); value.copy< ast::stmt_while::ptr > (YY_MOVE (that.value));
break; break;
case symbol_kind::S_FIELD: // "field"
case symbol_kind::S_PATH: // "path" case symbol_kind::S_PATH: // "path"
case symbol_kind::S_IDENTIFIER: // "identifier" case symbol_kind::S_IDENTIFIER: // "identifier"
case symbol_kind::S_STRING: // "string literal" case symbol_kind::S_STRING: // "string literal"
@ -5220,6 +5221,7 @@ switch (yykind)
value.move< ast::expr_game::ptr > (YY_MOVE (s.value)); value.move< ast::expr_game::ptr > (YY_MOVE (s.value));
break; break;
case symbol_kind::S_expr_identifier_nosize: // expr_identifier_nosize
case symbol_kind::S_expr_identifier: // expr_identifier case symbol_kind::S_expr_identifier: // expr_identifier
value.move< ast::expr_identifier::ptr > (YY_MOVE (s.value)); value.move< ast::expr_identifier::ptr > (YY_MOVE (s.value));
break; break;
@ -5411,7 +5413,6 @@ switch (yykind)
value.move< ast::stmt_while::ptr > (YY_MOVE (s.value)); value.move< ast::stmt_while::ptr > (YY_MOVE (s.value));
break; break;
case symbol_kind::S_FIELD: // "field"
case symbol_kind::S_PATH: // "path" case symbol_kind::S_PATH: // "path"
case symbol_kind::S_IDENTIFIER: // "identifier" case symbol_kind::S_IDENTIFIER: // "identifier"
case symbol_kind::S_STRING: // "string literal" case symbol_kind::S_STRING: // "string literal"
@ -5484,7 +5485,7 @@ switch (yykind)
#line 13 "parser.ypp" #line 13 "parser.ypp"
} } } // xsk::gsc::h1 } } } // xsk::gsc::h1
#line 5488 "parser.hpp" #line 5489 "parser.hpp"

120
src/h2/xsk/lexer.cpp
View File

@ -60,6 +60,7 @@ const std::unordered_map<std::string_view, parser::token::token_kind_type> keywo
{ "true", parser::token::TRUE }, { "true", parser::token::TRUE },
{ "false", parser::token::FALSE }, { "false", parser::token::FALSE },
{ "undefined", parser::token::UNDEFINED }, { "undefined", parser::token::UNDEFINED },
{ "size", parser::token::SIZE },
{ "game", parser::token::GAME }, { "game", parser::token::GAME },
{ "self", parser::token::SELF }, { "self", parser::token::SELF },
{ "anim", parser::token::ANIM }, { "anim", parser::token::ANIM },
@ -85,8 +86,8 @@ bool buffer::push(char c)
return true; return true;
} }
reader::reader() : state(reader::end), buffer_pos(0), bytes_remaining(0), reader::reader() : state(reader::end), buffer_pos(0),
last_byte(0), current_byte(0) { } bytes_remaining(0), last_byte(0), current_byte(0) {}
void reader::init(const char* data, size_t size) void reader::init(const char* data, size_t size)
{ {
@ -174,7 +175,6 @@ auto lexer::lex() -> parser::symbol_type
{ {
buffer_.length = 0; buffer_.length = 0;
state_ = state::start; state_ = state::start;
loc_.step();
while (true) while (true)
{ {
@ -182,6 +182,7 @@ auto lexer::lex() -> parser::symbol_type
auto& last = reader_.last_byte; auto& last = reader_.last_byte;
auto& curr = reader_.current_byte; auto& curr = reader_.current_byte;
auto path = false; auto path = false;
loc_.step();
if (state == reader::end) if (state == reader::end)
{ {
@ -214,7 +215,7 @@ auto lexer::lex() -> parser::symbol_type
case '\\': case '\\':
throw comp_error(loc_, "invalid token ('\\')"); throw comp_error(loc_, "invalid token ('\\')");
case '/': case '/':
if (curr != '/' && curr != '*' && curr != '#' && curr != '=') if (curr != '=' && curr != '#' && curr != '@' && curr != '*' && curr != '/')
return parser::make_DIV(loc_); return parser::make_DIV(loc_);
advance(); advance();
@ -246,14 +247,35 @@ auto lexer::lex() -> parser::symbol_type
} }
else if (last == '#' && curr == '/') else if (last == '#' && curr == '/')
{ {
reader_.advance(); advance();
break; break;
} }
reader_.advance(); advance();
} }
} }
} }
else if (last == '@')
{
while (true)
{
if (state == reader::end)
throw comp_error(loc_, "unmatched script doc comment start ('/@')");
if (curr == '\n')
{
loc_.lines();
loc_.step();
}
else if (last == '@' && curr == '/')
{
advance();
break;
}
advance();
}
}
else if (last == '*') else if (last == '*')
{ {
while (true) while (true)
@ -268,11 +290,11 @@ auto lexer::lex() -> parser::symbol_type
} }
else if (last == '*' && curr == '/') else if (last == '*' && curr == '/')
{ {
reader_.advance(); advance();
break; break;
} }
reader_.advance(); advance();
} }
} }
else if (last == '/') else if (last == '/')
@ -282,30 +304,10 @@ auto lexer::lex() -> parser::symbol_type
if (state == reader::end) if (state == reader::end)
break; break;
if (last == '\\' && (curr == '\r' || curr == '\n'))
{
reader_.advance();
if (state == reader::end)
break;
if (last == '\r')
{
if (curr != '\n')
throw comp_error(loc_, "invalid token ('\')");
reader_.advance();
}
loc_.lines();
loc_.step();
continue;
}
if (curr == '\n') if (curr == '\n')
break; break;
reader_.advance(); advance();
} }
} }
continue; continue;
@ -315,8 +317,8 @@ auto lexer::lex() -> parser::symbol_type
if (!indev_) if (!indev_)
throw comp_error(loc_, "unmatched devblock end ('#/')"); throw comp_error(loc_, "unmatched devblock end ('#/')");
indev_ = false;
advance(); advance();
indev_ = false;
return parser::make_DEVEND(loc_); return parser::make_DEVEND(loc_);
} }
@ -337,7 +339,7 @@ auto lexer::lex() -> parser::symbol_type
state_ = state::preprocessor; state_ = state::preprocessor;
goto lex_name; goto lex_name;
case '*': case '*':
if (curr != '/' && curr != '=') if (curr != '=' && curr != '/')
return parser::make_MUL(loc_); return parser::make_MUL(loc_);
advance(); advance();
@ -350,13 +352,9 @@ auto lexer::lex() -> parser::symbol_type
state_ = state::string; state_ = state::string;
goto lex_string; goto lex_string;
case '.': case '.':
advance(); if (curr < '0' || curr > '9')
return parser::make_DOT(loc_);
if (state == reader::end) goto lex_number;
throw comp_error(loc_, "unterminated field ('.')");
state_ = state::field;
goto lex_name_or_number;
case '(': case '(':
return parser::make_LPAREN(loc_); return parser::make_LPAREN(loc_);
case ')': case ')':
@ -480,7 +478,6 @@ auto lexer::lex() -> parser::symbol_type
advance(); advance();
return parser::make_ASSIGN_RSHIFT(loc_); return parser::make_ASSIGN_RSHIFT(loc_);
default: default:
lex_name_or_number:
if (last >= '0' && last <= '9') if (last >= '0' && last <= '9')
goto lex_number; goto lex_number;
else if (last == '_' || last >= 'A' && last <= 'Z' || last >= 'a' && last <= 'z') else if (last == '_' || last >= 'A' && last <= 'Z' || last >= 'a' && last <= 'z')
@ -562,23 +559,8 @@ lex_name:
advance(); advance();
} }
if (state_ == state::field) if (state_ == state::preprocessor)
{ {
if (path)
throw comp_error(loc_, "invalid field token '\\'");
if (std::string_view(buffer_.data, buffer_.length) == "size")
{
return parser::make_SIZE(loc_);
}
return parser::make_FIELD(std::string(buffer_.data, buffer_.length), loc_);
}
else if (state_ == state::preprocessor)
{
if (path)
throw comp_error(loc_, "invalid preprocessor directive");
auto token = parser::token::H2UNDEF; auto token = parser::token::H2UNDEF;
if (buffer_.length < 16) if (buffer_.length < 16)
@ -594,7 +576,8 @@ lex_name:
} }
} }
preprocessor(token); preprocessor_run(token);
state_ = state::start; state_ = state::start;
continue; continue;
} }
@ -620,14 +603,11 @@ lex_name:
} }
lex_number: lex_number:
if (state_ == state::field) if (last == '.' || last != '0' || (last == '0' && (curr != 'o' && curr != 'b' && curr != 'x')))
buffer_.push('.');
if (state_ == state::field || last == '.' || last != '0' || (last == '0' && (curr != 'o' && curr != 'b' && curr != 'x')))
{ {
buffer_.push(last); buffer_.push(last);
auto dot = 0; auto dot = last == '.' ? 1 : 0;
auto flt = 0; auto flt = 0;
while (true) while (true)
@ -663,10 +643,10 @@ lex_number:
if (last == '\'') if (last == '\'')
throw comp_error(loc_, "invalid number literal"); throw comp_error(loc_, "invalid number literal");
if (state_ == state::field && dot || dot > 1 || flt > 1 || flt && buffer_.data[buffer_.length - 1] != 'f') if (dot > 1 || flt > 1 || flt && buffer_.data[buffer_.length - 1] != 'f')
throw comp_error(loc_, "invalid number literal"); throw comp_error(loc_, "invalid number literal");
if (state_ == state::field || dot || flt) if (dot || flt)
return parser::make_FLOAT(std::string(buffer_.data, buffer_.length), loc_); return parser::make_FLOAT(std::string(buffer_.data, buffer_.length), loc_);
return parser::make_INTEGER(std::string(buffer_.data, buffer_.length), loc_); return parser::make_INTEGER(std::string(buffer_.data, buffer_.length), loc_);
@ -771,18 +751,22 @@ lex_number:
return parser::make_INTEGER(xsk::utils::string::hex_to_dec(buffer_.data), loc_); return parser::make_INTEGER(xsk::utils::string::hex_to_dec(buffer_.data), loc_);
} }
// cant get here!
throw error("UNEXPECTED LEXER INTERNAL ERROR!");
} }
} }
void lexer::advance() void lexer::advance()
{ {
reader_.advance(); reader_.advance();
loc_.end.column++;
// dont wrap comment marks '/\/' '/\*' outside strings if (reader_.current_byte == '\\') [[unlikely]]
if (state_ == state::start && reader_.last_byte == '/') preprocessor_wrap();
return; }
void lexer::preprocessor_wrap()
{
while (reader_.current_byte == '\\') while (reader_.current_byte == '\\')
{ {
if (reader_.bytes_remaining == 1) if (reader_.bytes_remaining == 1)
@ -824,7 +808,7 @@ void lexer::advance()
} }
} }
void lexer::preprocessor(parser::token::token_kind_type token) void lexer::preprocessor_run(parser::token::token_kind_type token)
{ {
if (!clean_) if (!clean_)
throw comp_error(loc_, "invalid token ('#')"); throw comp_error(loc_, "invalid token ('#')");

5
src/h2/xsk/lexer.hpp
View File

@ -44,7 +44,7 @@ struct reader
class lexer class lexer
{ {
enum class state : std::uint8_t { start, string, localize, field, preprocessor }; enum class state : std::uint8_t { start, string, localize, preprocessor };
reader reader_; reader reader_;
buffer buffer_; buffer buffer_;
@ -66,7 +66,8 @@ public:
private: private:
void advance(); void advance();
void preprocessor(parser::token::token_kind_type token); void preprocessor_wrap();
void preprocessor_run(parser::token::token_kind_type token);
}; };
} // namespace xsk::gsc::h2 } // namespace xsk::gsc::h2

1740
src/h2/xsk/parser.cpp
View File

File diff suppressed because it is too large Load Diff

103
src/h2/xsk/parser.hpp
View File

@ -474,6 +474,7 @@ namespace xsk { namespace gsc { namespace h2 {
// expr_game // expr_game
char dummy19[sizeof (ast::expr_game::ptr)]; char dummy19[sizeof (ast::expr_game::ptr)];
// expr_identifier_nosize
// expr_identifier // expr_identifier
char dummy20[sizeof (ast::expr_identifier::ptr)]; char dummy20[sizeof (ast::expr_identifier::ptr)];
@ -618,7 +619,6 @@ namespace xsk { namespace gsc { namespace h2 {
// stmt_while // stmt_while
char dummy66[sizeof (ast::stmt_while::ptr)]; char dummy66[sizeof (ast::stmt_while::ptr)];
// "field"
// "path" // "path"
// "identifier" // "identifier"
// "string literal" // "string literal"
@ -721,7 +721,7 @@ namespace xsk { namespace gsc { namespace h2 {
TRUE = 44, // "true" TRUE = 44, // "true"
FALSE = 45, // "false" FALSE = 45, // "false"
UNDEFINED = 46, // "undefined" UNDEFINED = 46, // "undefined"
SIZE = 47, // ".size" SIZE = 47, // "size"
GAME = 48, // "game" GAME = 48, // "game"
SELF = 49, // "self" SELF = 49, // "self"
ANIM = 50, // "anim" ANIM = 50, // "anim"
@ -771,13 +771,13 @@ namespace xsk { namespace gsc { namespace h2 {
MUL = 94, // "*" MUL = 94, // "*"
DIV = 95, // "/" DIV = 95, // "/"
MOD = 96, // "%" MOD = 96, // "%"
FIELD = 97, // "field" PATH = 97, // "path"
PATH = 98, // "path" IDENTIFIER = 98, // "identifier"
IDENTIFIER = 99, // "identifier" STRING = 99, // "string literal"
STRING = 100, // "string literal" ISTRING = 100, // "localized string"
ISTRING = 101, // "localized string" FLOAT = 101, // "float"
FLOAT = 102, // "float" INTEGER = 102, // "integer"
INTEGER = 103, // "integer" SIZEOF = 103, // SIZEOF
ADD_ARRAY = 104, // ADD_ARRAY ADD_ARRAY = 104, // ADD_ARRAY
THEN = 105, // THEN THEN = 105, // THEN
TERN = 106, // TERN TERN = 106, // TERN
@ -852,7 +852,7 @@ namespace xsk { namespace gsc { namespace h2 {
S_TRUE = 44, // "true" S_TRUE = 44, // "true"
S_FALSE = 45, // "false" S_FALSE = 45, // "false"
S_UNDEFINED = 46, // "undefined" S_UNDEFINED = 46, // "undefined"
S_SIZE = 47, // ".size" S_SIZE = 47, // "size"
S_GAME = 48, // "game" S_GAME = 48, // "game"
S_SELF = 49, // "self" S_SELF = 49, // "self"
S_ANIM = 50, // "anim" S_ANIM = 50, // "anim"
@ -902,13 +902,13 @@ namespace xsk { namespace gsc { namespace h2 {
S_MUL = 94, // "*" S_MUL = 94, // "*"
S_DIV = 95, // "/" S_DIV = 95, // "/"
S_MOD = 96, // "%" S_MOD = 96, // "%"
S_FIELD = 97, // "field" S_PATH = 97, // "path"
S_PATH = 98, // "path" S_IDENTIFIER = 98, // "identifier"
S_IDENTIFIER = 99, // "identifier" S_STRING = 99, // "string literal"
S_STRING = 100, // "string literal" S_ISTRING = 100, // "localized string"
S_ISTRING = 101, // "localized string" S_FLOAT = 101, // "float"
S_FLOAT = 102, // "float" S_INTEGER = 102, // "integer"
S_INTEGER = 103, // "integer" S_SIZEOF = 103, // SIZEOF
S_ADD_ARRAY = 104, // ADD_ARRAY S_ADD_ARRAY = 104, // ADD_ARRAY
S_THEN = 105, // THEN S_THEN = 105, // THEN
S_TERN = 106, // TERN S_TERN = 106, // TERN
@ -991,15 +991,16 @@ namespace xsk { namespace gsc { namespace h2 {
S_expr_level = 183, // expr_level S_expr_level = 183, // expr_level
S_expr_animation = 184, // expr_animation S_expr_animation = 184, // expr_animation
S_expr_animtree = 185, // expr_animtree S_expr_animtree = 185, // expr_animtree
S_expr_identifier = 186, // expr_identifier S_expr_identifier_nosize = 186, // expr_identifier_nosize
S_expr_path = 187, // expr_path S_expr_identifier = 187, // expr_identifier
S_expr_istring = 188, // expr_istring S_expr_path = 188, // expr_path
S_expr_string = 189, // expr_string S_expr_istring = 189, // expr_istring
S_expr_vector = 190, // expr_vector S_expr_string = 190, // expr_string
S_expr_float = 191, // expr_float S_expr_vector = 191, // expr_vector
S_expr_integer = 192, // expr_integer S_expr_float = 192, // expr_float
S_expr_false = 193, // expr_false S_expr_integer = 193, // expr_integer
S_expr_true = 194 // expr_true S_expr_false = 194, // expr_false
S_expr_true = 195 // expr_true
}; };
}; };
@ -1122,6 +1123,7 @@ namespace xsk { namespace gsc { namespace h2 {
value.move< ast::expr_game::ptr > (std::move (that.value)); value.move< ast::expr_game::ptr > (std::move (that.value));
break; break;
case symbol_kind::S_expr_identifier_nosize: // expr_identifier_nosize
case symbol_kind::S_expr_identifier: // expr_identifier case symbol_kind::S_expr_identifier: // expr_identifier
value.move< ast::expr_identifier::ptr > (std::move (that.value)); value.move< ast::expr_identifier::ptr > (std::move (that.value));
break; break;
@ -1313,7 +1315,6 @@ namespace xsk { namespace gsc { namespace h2 {
value.move< ast::stmt_while::ptr > (std::move (that.value)); value.move< ast::stmt_while::ptr > (std::move (that.value));
break; break;
case symbol_kind::S_FIELD: // "field"
case symbol_kind::S_PATH: // "path" case symbol_kind::S_PATH: // "path"
case symbol_kind::S_IDENTIFIER: // "identifier" case symbol_kind::S_IDENTIFIER: // "identifier"
case symbol_kind::S_STRING: // "string literal" case symbol_kind::S_STRING: // "string literal"
@ -2392,6 +2393,7 @@ switch (yykind)
value.template destroy< ast::expr_game::ptr > (); value.template destroy< ast::expr_game::ptr > ();
break; break;
case symbol_kind::S_expr_identifier_nosize: // expr_identifier_nosize
case symbol_kind::S_expr_identifier: // expr_identifier case symbol_kind::S_expr_identifier: // expr_identifier
value.template destroy< ast::expr_identifier::ptr > (); value.template destroy< ast::expr_identifier::ptr > ();
break; break;
@ -2583,7 +2585,6 @@ switch (yykind)
value.template destroy< ast::stmt_while::ptr > (); value.template destroy< ast::stmt_while::ptr > ();
break; break;
case symbol_kind::S_FIELD: // "field"
case symbol_kind::S_PATH: // "path" case symbol_kind::S_PATH: // "path"
case symbol_kind::S_IDENTIFIER: // "identifier" case symbol_kind::S_IDENTIFIER: // "identifier"
case symbol_kind::S_STRING: // "string literal" case symbol_kind::S_STRING: // "string literal"
@ -2689,7 +2690,7 @@ switch (yykind)
{ {
H2_ASSERT (tok == token::H2EOF H2_ASSERT (tok == token::H2EOF
|| (token::H2error <= tok && tok <= token::MOD) || (token::H2error <= tok && tok <= token::MOD)
|| (token::ADD_ARRAY <= tok && tok <= token::POSTDEC)); || (token::SIZEOF <= tok && tok <= token::POSTDEC));
} }
#if 201103L <= YY_CPLUSPLUS #if 201103L <= YY_CPLUSPLUS
symbol_type (int tok, std::string v, location_type l) symbol_type (int tok, std::string v, location_type l)
@ -2699,7 +2700,7 @@ switch (yykind)
: super_type(token_type (tok), v, l) : super_type(token_type (tok), v, l)
#endif #endif
{ {
H2_ASSERT ((token::FIELD <= tok && tok <= token::INTEGER)); H2_ASSERT ((token::PATH <= tok && tok <= token::INTEGER));
} }
}; };
@ -4204,21 +4205,6 @@ switch (yykind)
return symbol_type (token::MOD, l); return symbol_type (token::MOD, l);
} }
#endif #endif
#if 201103L <= YY_CPLUSPLUS
static
symbol_type
make_FIELD (std::string v, location_type l)
{
return symbol_type (token::FIELD, std::move (v), std::move (l));
}
#else
static
symbol_type
make_FIELD (const std::string& v, const location_type& l)
{
return symbol_type (token::FIELD, v, l);
}
#endif
#if 201103L <= YY_CPLUSPLUS #if 201103L <= YY_CPLUSPLUS
static static
symbol_type symbol_type
@ -4309,6 +4295,21 @@ switch (yykind)
return symbol_type (token::INTEGER, v, l); return symbol_type (token::INTEGER, v, l);
} }
#endif #endif
#if 201103L <= YY_CPLUSPLUS
static
symbol_type
make_SIZEOF (location_type l)
{
return symbol_type (token::SIZEOF, std::move (l));
}
#else
static
symbol_type
make_SIZEOF (const location_type& l)
{
return symbol_type (token::SIZEOF, l);
}
#endif
#if 201103L <= YY_CPLUSPLUS #if 201103L <= YY_CPLUSPLUS
static static
symbol_type symbol_type
@ -4788,9 +4789,9 @@ switch (yykind)
/// Constants. /// Constants.
enum enum
{ {
yylast_ = 2300, ///< Last index in yytable_. yylast_ = 2251, ///< Last index in yytable_.
yynnts_ = 82, ///< Number of nonterminal symbols. yynnts_ = 83, ///< Number of nonterminal symbols.
yyfinal_ = 21 ///< Termination state number. yyfinal_ = 22 ///< Termination state number.
}; };
@ -4902,6 +4903,7 @@ switch (yykind)
value.copy< ast::expr_game::ptr > (YY_MOVE (that.value)); value.copy< ast::expr_game::ptr > (YY_MOVE (that.value));
break; break;
case symbol_kind::S_expr_identifier_nosize: // expr_identifier_nosize
case symbol_kind::S_expr_identifier: // expr_identifier case symbol_kind::S_expr_identifier: // expr_identifier
value.copy< ast::expr_identifier::ptr > (YY_MOVE (that.value)); value.copy< ast::expr_identifier::ptr > (YY_MOVE (that.value));
break; break;
@ -5093,7 +5095,6 @@ switch (yykind)
value.copy< ast::stmt_while::ptr > (YY_MOVE (that.value)); value.copy< ast::stmt_while::ptr > (YY_MOVE (that.value));
break; break;
case symbol_kind::S_FIELD: // "field"
case symbol_kind::S_PATH: // "path" case symbol_kind::S_PATH: // "path"
case symbol_kind::S_IDENTIFIER: // "identifier" case symbol_kind::S_IDENTIFIER: // "identifier"
case symbol_kind::S_STRING: // "string literal" case symbol_kind::S_STRING: // "string literal"
@ -5218,6 +5219,7 @@ switch (yykind)
value.move< ast::expr_game::ptr > (YY_MOVE (s.value)); value.move< ast::expr_game::ptr > (YY_MOVE (s.value));
break; break;
case symbol_kind::S_expr_identifier_nosize: // expr_identifier_nosize
case symbol_kind::S_expr_identifier: // expr_identifier case symbol_kind::S_expr_identifier: // expr_identifier
value.move< ast::expr_identifier::ptr > (YY_MOVE (s.value)); value.move< ast::expr_identifier::ptr > (YY_MOVE (s.value));
break; break;
@ -5409,7 +5411,6 @@ switch (yykind)
value.move< ast::stmt_while::ptr > (YY_MOVE (s.value)); value.move< ast::stmt_while::ptr > (YY_MOVE (s.value));
break; break;
case symbol_kind::S_FIELD: // "field"
case symbol_kind::S_PATH: // "path" case symbol_kind::S_PATH: // "path"
case symbol_kind::S_IDENTIFIER: // "identifier" case symbol_kind::S_IDENTIFIER: // "identifier"
case symbol_kind::S_STRING: // "string literal" case symbol_kind::S_STRING: // "string literal"
@ -5482,7 +5483,7 @@ switch (yykind)
#line 13 "parser.ypp" #line 13 "parser.ypp"
} } } // xsk::gsc::h2 } } } // xsk::gsc::h2
#line 5486 "parser.hpp" #line 5487 "parser.hpp"

120
src/iw5/xsk/lexer.cpp
View File

@ -59,6 +59,7 @@ const std::unordered_map<std::string_view, parser::token::token_kind_type> keywo
{ "true", parser::token::TRUE }, { "true", parser::token::TRUE },
{ "false", parser::token::FALSE }, { "false", parser::token::FALSE },
{ "undefined", parser::token::UNDEFINED }, { "undefined", parser::token::UNDEFINED },
{ "size", parser::token::SIZE },
{ "game", parser::token::GAME }, { "game", parser::token::GAME },
{ "self", parser::token::SELF }, { "self", parser::token::SELF },
{ "anim", parser::token::ANIM }, { "anim", parser::token::ANIM },
@ -84,8 +85,8 @@ bool buffer::push(char c)
return true; return true;
} }
reader::reader() : state(reader::end), buffer_pos(0), bytes_remaining(0), reader::reader() : state(reader::end), buffer_pos(0),
last_byte(0), current_byte(0) { } bytes_remaining(0), last_byte(0), current_byte(0) {}
void reader::init(const char* data, size_t size) void reader::init(const char* data, size_t size)
{ {
@ -173,7 +174,6 @@ auto lexer::lex() -> parser::symbol_type
{ {
buffer_.length = 0; buffer_.length = 0;
state_ = state::start; state_ = state::start;
loc_.step();
while (true) while (true)
{ {
@ -181,6 +181,7 @@ auto lexer::lex() -> parser::symbol_type
auto& last = reader_.last_byte; auto& last = reader_.last_byte;
auto& curr = reader_.current_byte; auto& curr = reader_.current_byte;
auto path = false; auto path = false;
loc_.step();
if (state == reader::end) if (state == reader::end)
{ {
@ -213,7 +214,7 @@ auto lexer::lex() -> parser::symbol_type
case '\\': case '\\':
throw comp_error(loc_, "invalid token ('\\')"); throw comp_error(loc_, "invalid token ('\\')");
case '/': case '/':
if (curr != '/' && curr != '*' && curr != '#' && curr != '=') if (curr != '=' && curr != '#' && curr != '@' && curr != '*' && curr != '/')
return parser::make_DIV(loc_); return parser::make_DIV(loc_);
advance(); advance();
@ -245,14 +246,35 @@ auto lexer::lex() -> parser::symbol_type
} }
else if (last == '#' && curr == '/') else if (last == '#' && curr == '/')
{ {
reader_.advance(); advance();
break; break;
} }
reader_.advance(); advance();
} }
} }
} }
else if (last == '@')
{
while (true)
{
if (state == reader::end)
throw comp_error(loc_, "unmatched script doc comment start ('/@')");
if (curr == '\n')
{
loc_.lines();
loc_.step();
}
else if (last == '@' && curr == '/')
{
advance();
break;
}
advance();
}
}
else if (last == '*') else if (last == '*')
{ {
while (true) while (true)
@ -267,11 +289,11 @@ auto lexer::lex() -> parser::symbol_type
} }
else if (last == '*' && curr == '/') else if (last == '*' && curr == '/')
{ {
reader_.advance(); advance();
break; break;
} }
reader_.advance(); advance();
} }
} }
else if (last == '/') else if (last == '/')
@ -281,30 +303,10 @@ auto lexer::lex() -> parser::symbol_type
if (state == reader::end) if (state == reader::end)
break; break;
if (last == '\\' && (curr == '\r' || curr == '\n'))
{
reader_.advance();
if (state == reader::end)
break;
if (last == '\r')
{
if (curr != '\n')
throw comp_error(loc_, "invalid token ('\')");
reader_.advance();
}
loc_.lines();
loc_.step();
continue;
}
if (curr == '\n') if (curr == '\n')
break; break;
reader_.advance(); advance();
} }
} }
continue; continue;
@ -314,8 +316,8 @@ auto lexer::lex() -> parser::symbol_type
if (!indev_) if (!indev_)
throw comp_error(loc_, "unmatched devblock end ('#/')"); throw comp_error(loc_, "unmatched devblock end ('#/')");
indev_ = false;
advance(); advance();
indev_ = false;
return parser::make_DEVEND(loc_); return parser::make_DEVEND(loc_);
} }
@ -336,7 +338,7 @@ auto lexer::lex() -> parser::symbol_type
state_ = state::preprocessor; state_ = state::preprocessor;
goto lex_name; goto lex_name;
case '*': case '*':
if (curr != '/' && curr != '=') if (curr != '=' && curr != '/')
return parser::make_MUL(loc_); return parser::make_MUL(loc_);
advance(); advance();
@ -349,13 +351,9 @@ auto lexer::lex() -> parser::symbol_type
state_ = state::string; state_ = state::string;
goto lex_string; goto lex_string;
case '.': case '.':
advance(); if (curr < '0' || curr > '9')
return parser::make_DOT(loc_);
if (state == reader::end) goto lex_number;
throw comp_error(loc_, "unterminated field ('.')");
state_ = state::field;
goto lex_name_or_number;
case '(': case '(':
return parser::make_LPAREN(loc_); return parser::make_LPAREN(loc_);
case ')': case ')':
@ -479,7 +477,6 @@ auto lexer::lex() -> parser::symbol_type
advance(); advance();
return parser::make_ASSIGN_RSHIFT(loc_); return parser::make_ASSIGN_RSHIFT(loc_);
default: default:
lex_name_or_number:
if (last >= '0' && last <= '9') if (last >= '0' && last <= '9')
goto lex_number; goto lex_number;
else if (last == '_' || last >= 'A' && last <= 'Z' || last >= 'a' && last <= 'z') else if (last == '_' || last >= 'A' && last <= 'Z' || last >= 'a' && last <= 'z')
@ -561,23 +558,8 @@ lex_name:
advance(); advance();
} }
if (state_ == state::field) if (state_ == state::preprocessor)
{ {
if (path)
throw comp_error(loc_, "invalid field token '\\'");
if (std::string_view(buffer_.data, buffer_.length) == "size")
{
return parser::make_SIZE(loc_);
}
return parser::make_FIELD(std::string(buffer_.data, buffer_.length), loc_);
}
else if (state_ == state::preprocessor)
{
if (path)
throw comp_error(loc_, "invalid preprocessor directive");
auto token = parser::token::IW5UNDEF; auto token = parser::token::IW5UNDEF;
if (buffer_.length < 16) if (buffer_.length < 16)
@ -593,7 +575,8 @@ lex_name:
} }
} }
preprocessor(token); preprocessor_run(token);
state_ = state::start; state_ = state::start;
continue; continue;
} }
@ -619,14 +602,11 @@ lex_name:
} }
lex_number: lex_number:
if (state_ == state::field) if (last == '.' || last != '0' || (last == '0' && (curr != 'o' && curr != 'b' && curr != 'x')))
buffer_.push('.');
if (state_ == state::field || last == '.' || last != '0' || (last == '0' && (curr != 'o' && curr != 'b' && curr != 'x')))
{ {
buffer_.push(last); buffer_.push(last);
auto dot = 0; auto dot = last == '.' ? 1 : 0;
auto flt = 0; auto flt = 0;
while (true) while (true)
@ -662,10 +642,10 @@ lex_number:
if (last == '\'') if (last == '\'')
throw comp_error(loc_, "invalid number literal"); throw comp_error(loc_, "invalid number literal");
if (state_ == state::field && dot || dot > 1 || flt > 1 || flt && buffer_.data[buffer_.length - 1] != 'f') if (dot > 1 || flt > 1 || flt && buffer_.data[buffer_.length - 1] != 'f')
throw comp_error(loc_, "invalid number literal"); throw comp_error(loc_, "invalid number literal");
if (state_ == state::field || dot || flt) if (dot || flt)
return parser::make_FLOAT(std::string(buffer_.data, buffer_.length), loc_); return parser::make_FLOAT(std::string(buffer_.data, buffer_.length), loc_);
return parser::make_INTEGER(std::string(buffer_.data, buffer_.length), loc_); return parser::make_INTEGER(std::string(buffer_.data, buffer_.length), loc_);
@ -770,18 +750,22 @@ lex_number:
return parser::make_INTEGER(xsk::utils::string::hex_to_dec(buffer_.data), loc_); return parser::make_INTEGER(xsk::utils::string::hex_to_dec(buffer_.data), loc_);
} }
// cant get here!
throw error("UNEXPECTED LEXER INTERNAL ERROR!");
} }
} }
void lexer::advance() void lexer::advance()
{ {
reader_.advance(); reader_.advance();
loc_.end.column++;
// dont wrap comment marks '/\/' '/\*' outside strings if (reader_.current_byte == '\\') [[unlikely]]
if (state_ == state::start && reader_.last_byte == '/') preprocessor_wrap();
return; }
void lexer::preprocessor_wrap()
{
while (reader_.current_byte == '\\') while (reader_.current_byte == '\\')
{ {
if (reader_.bytes_remaining == 1) if (reader_.bytes_remaining == 1)
@ -823,7 +807,7 @@ void lexer::advance()
} }
} }
void lexer::preprocessor(parser::token::token_kind_type token) void lexer::preprocessor_run(parser::token::token_kind_type token)
{ {
if (!clean_) if (!clean_)
throw comp_error(loc_, "invalid token ('#')"); throw comp_error(loc_, "invalid token ('#')");

5
src/iw5/xsk/lexer.hpp
View File

@ -44,7 +44,7 @@ struct reader
class lexer class lexer
{ {
enum class state : std::uint8_t { start, string, localize, field, preprocessor }; enum class state : std::uint8_t { start, string, localize, preprocessor };
reader reader_; reader reader_;
buffer buffer_; buffer buffer_;
@ -66,7 +66,8 @@ public:
private: private:
void advance(); void advance();
void preprocessor(parser::token::token_kind_type token); void preprocessor_wrap();
void preprocessor_run(parser::token::token_kind_type token);
}; };
} // namespace xsk::gsc::iw5 } // namespace xsk::gsc::iw5

1804
src/iw5/xsk/parser.cpp
View File

File diff suppressed because it is too large Load Diff

103
src/iw5/xsk/parser.hpp
View File

@ -474,6 +474,7 @@ namespace xsk { namespace gsc { namespace iw5 {
// expr_game // expr_game
char dummy19[sizeof (ast::expr_game::ptr)]; char dummy19[sizeof (ast::expr_game::ptr)];
// expr_identifier_nosize
// expr_identifier // expr_identifier
char dummy20[sizeof (ast::expr_identifier::ptr)]; char dummy20[sizeof (ast::expr_identifier::ptr)];
@ -615,7 +616,6 @@ namespace xsk { namespace gsc { namespace iw5 {
// stmt_while // stmt_while
char dummy65[sizeof (ast::stmt_while::ptr)]; char dummy65[sizeof (ast::stmt_while::ptr)];
// "field"
// "path" // "path"
// "identifier" // "identifier"
// "string literal" // "string literal"
@ -717,7 +717,7 @@ namespace xsk { namespace gsc { namespace iw5 {
TRUE = 43, // "true" TRUE = 43, // "true"
FALSE = 44, // "false" FALSE = 44, // "false"
UNDEFINED = 45, // "undefined" UNDEFINED = 45, // "undefined"
SIZE = 46, // ".size" SIZE = 46, // "size"
GAME = 47, // "game" GAME = 47, // "game"
SELF = 48, // "self" SELF = 48, // "self"
ANIM = 49, // "anim" ANIM = 49, // "anim"
@ -767,13 +767,13 @@ namespace xsk { namespace gsc { namespace iw5 {
MUL = 93, // "*" MUL = 93, // "*"
DIV = 94, // "/" DIV = 94, // "/"
MOD = 95, // "%" MOD = 95, // "%"
FIELD = 96, // "field" PATH = 96, // "path"
PATH = 97, // "path" IDENTIFIER = 97, // "identifier"
IDENTIFIER = 98, // "identifier" STRING = 98, // "string literal"
STRING = 99, // "string literal" ISTRING = 99, // "localized string"
ISTRING = 100, // "localized string" FLOAT = 100, // "float"
FLOAT = 101, // "float" INTEGER = 101, // "integer"
INTEGER = 102, // "integer" SIZEOF = 102, // SIZEOF
ADD_ARRAY = 103, // ADD_ARRAY ADD_ARRAY = 103, // ADD_ARRAY
THEN = 104, // THEN THEN = 104, // THEN
TERN = 105, // TERN TERN = 105, // TERN
@ -847,7 +847,7 @@ namespace xsk { namespace gsc { namespace iw5 {
S_TRUE = 43, // "true" S_TRUE = 43, // "true"
S_FALSE = 44, // "false" S_FALSE = 44, // "false"
S_UNDEFINED = 45, // "undefined" S_UNDEFINED = 45, // "undefined"
S_SIZE = 46, // ".size" S_SIZE = 46, // "size"
S_GAME = 47, // "game" S_GAME = 47, // "game"
S_SELF = 48, // "self" S_SELF = 48, // "self"
S_ANIM = 49, // "anim" S_ANIM = 49, // "anim"
@ -897,13 +897,13 @@ namespace xsk { namespace gsc { namespace iw5 {
S_MUL = 93, // "*" S_MUL = 93, // "*"
S_DIV = 94, // "/" S_DIV = 94, // "/"
S_MOD = 95, // "%" S_MOD = 95, // "%"
S_FIELD = 96, // "field" S_PATH = 96, // "path"
S_PATH = 97, // "path" S_IDENTIFIER = 97, // "identifier"
S_IDENTIFIER = 98, // "identifier" S_STRING = 98, // "string literal"
S_STRING = 99, // "string literal" S_ISTRING = 99, // "localized string"
S_ISTRING = 100, // "localized string" S_FLOAT = 100, // "float"
S_FLOAT = 101, // "float" S_INTEGER = 101, // "integer"
S_INTEGER = 102, // "integer" S_SIZEOF = 102, // SIZEOF
S_ADD_ARRAY = 103, // ADD_ARRAY S_ADD_ARRAY = 103, // ADD_ARRAY
S_THEN = 104, // THEN S_THEN = 104, // THEN
S_TERN = 105, // TERN S_TERN = 105, // TERN
@ -985,15 +985,16 @@ namespace xsk { namespace gsc { namespace iw5 {
S_expr_level = 181, // expr_level S_expr_level = 181, // expr_level
S_expr_animation = 182, // expr_animation S_expr_animation = 182, // expr_animation
S_expr_animtree = 183, // expr_animtree S_expr_animtree = 183, // expr_animtree
S_expr_identifier = 184, // expr_identifier S_expr_identifier_nosize = 184, // expr_identifier_nosize
S_expr_path = 185, // expr_path S_expr_identifier = 185, // expr_identifier
S_expr_istring = 186, // expr_istring S_expr_path = 186, // expr_path
S_expr_string = 187, // expr_string S_expr_istring = 187, // expr_istring
S_expr_vector = 188, // expr_vector S_expr_string = 188, // expr_string
S_expr_float = 189, // expr_float S_expr_vector = 189, // expr_vector
S_expr_integer = 190, // expr_integer S_expr_float = 190, // expr_float
S_expr_false = 191, // expr_false S_expr_integer = 191, // expr_integer
S_expr_true = 192 // expr_true S_expr_false = 192, // expr_false
S_expr_true = 193 // expr_true
}; };
}; };
@ -1116,6 +1117,7 @@ namespace xsk { namespace gsc { namespace iw5 {
value.move< ast::expr_game::ptr > (std::move (that.value)); value.move< ast::expr_game::ptr > (std::move (that.value));
break; break;
case symbol_kind::S_expr_identifier_nosize: // expr_identifier_nosize
case symbol_kind::S_expr_identifier: // expr_identifier case symbol_kind::S_expr_identifier: // expr_identifier
value.move< ast::expr_identifier::ptr > (std::move (that.value)); value.move< ast::expr_identifier::ptr > (std::move (that.value));
break; break;
@ -1303,7 +1305,6 @@ namespace xsk { namespace gsc { namespace iw5 {
value.move< ast::stmt_while::ptr > (std::move (that.value)); value.move< ast::stmt_while::ptr > (std::move (that.value));
break; break;
case symbol_kind::S_FIELD: // "field"
case symbol_kind::S_PATH: // "path" case symbol_kind::S_PATH: // "path"
case symbol_kind::S_IDENTIFIER: // "identifier" case symbol_kind::S_IDENTIFIER: // "identifier"
case symbol_kind::S_STRING: // "string literal" case symbol_kind::S_STRING: // "string literal"
@ -2368,6 +2369,7 @@ switch (yykind)
value.template destroy< ast::expr_game::ptr > (); value.template destroy< ast::expr_game::ptr > ();
break; break;
case symbol_kind::S_expr_identifier_nosize: // expr_identifier_nosize
case symbol_kind::S_expr_identifier: // expr_identifier case symbol_kind::S_expr_identifier: // expr_identifier
value.template destroy< ast::expr_identifier::ptr > (); value.template destroy< ast::expr_identifier::ptr > ();
break; break;
@ -2555,7 +2557,6 @@ switch (yykind)
value.template destroy< ast::stmt_while::ptr > (); value.template destroy< ast::stmt_while::ptr > ();
break; break;
case symbol_kind::S_FIELD: // "field"
case symbol_kind::S_PATH: // "path" case symbol_kind::S_PATH: // "path"
case symbol_kind::S_IDENTIFIER: // "identifier" case symbol_kind::S_IDENTIFIER: // "identifier"
case symbol_kind::S_STRING: // "string literal" case symbol_kind::S_STRING: // "string literal"
@ -2661,7 +2662,7 @@ switch (yykind)
{ {
IW5_ASSERT (tok == token::IW5EOF IW5_ASSERT (tok == token::IW5EOF
|| (token::IW5error <= tok && tok <= token::MOD) || (token::IW5error <= tok && tok <= token::MOD)
|| (token::ADD_ARRAY <= tok && tok <= token::POSTDEC)); || (token::SIZEOF <= tok && tok <= token::POSTDEC));
} }
#if 201103L <= YY_CPLUSPLUS #if 201103L <= YY_CPLUSPLUS
symbol_type (int tok, std::string v, location_type l) symbol_type (int tok, std::string v, location_type l)
@ -2671,7 +2672,7 @@ switch (yykind)
: super_type(token_type (tok), v, l) : super_type(token_type (tok), v, l)
#endif #endif
{ {
IW5_ASSERT ((token::FIELD <= tok && tok <= token::INTEGER)); IW5_ASSERT ((token::PATH <= tok && tok <= token::INTEGER));
} }
}; };
@ -4161,21 +4162,6 @@ switch (yykind)
return symbol_type (token::MOD, l); return symbol_type (token::MOD, l);
} }
#endif #endif
#if 201103L <= YY_CPLUSPLUS
static
symbol_type
make_FIELD (std::string v, location_type l)
{
return symbol_type (token::FIELD, std::move (v), std::move (l));
}
#else
static
symbol_type
make_FIELD (const std::string& v, const location_type& l)
{
return symbol_type (token::FIELD, v, l);
}
#endif
#if 201103L <= YY_CPLUSPLUS #if 201103L <= YY_CPLUSPLUS
static static
symbol_type symbol_type
@ -4266,6 +4252,21 @@ switch (yykind)
return symbol_type (token::INTEGER, v, l); return symbol_type (token::INTEGER, v, l);
} }
#endif #endif
#if 201103L <= YY_CPLUSPLUS
static
symbol_type
make_SIZEOF (location_type l)
{
return symbol_type (token::SIZEOF, std::move (l));
}
#else
static
symbol_type
make_SIZEOF (const location_type& l)
{
return symbol_type (token::SIZEOF, l);
}
#endif
#if 201103L <= YY_CPLUSPLUS #if 201103L <= YY_CPLUSPLUS
static static
symbol_type symbol_type
@ -4745,9 +4746,9 @@ switch (yykind)
/// Constants. /// Constants.
enum enum
{ {
yylast_ = 2274, ///< Last index in yytable_. yylast_ = 2303, ///< Last index in yytable_.
yynnts_ = 81, ///< Number of nonterminal symbols. yynnts_ = 82, ///< Number of nonterminal symbols.
yyfinal_ = 21 ///< Termination state number. yyfinal_ = 22 ///< Termination state number.
}; };
@ -4859,6 +4860,7 @@ switch (yykind)
value.copy< ast::expr_game::ptr > (YY_MOVE (that.value)); value.copy< ast::expr_game::ptr > (YY_MOVE (that.value));
break; break;
case symbol_kind::S_expr_identifier_nosize: // expr_identifier_nosize
case symbol_kind::S_expr_identifier: // expr_identifier case symbol_kind::S_expr_identifier: // expr_identifier
value.copy< ast::expr_identifier::ptr > (YY_MOVE (that.value)); value.copy< ast::expr_identifier::ptr > (YY_MOVE (that.value));
break; break;
@ -5046,7 +5048,6 @@ switch (yykind)
value.copy< ast::stmt_while::ptr > (YY_MOVE (that.value)); value.copy< ast::stmt_while::ptr > (YY_MOVE (that.value));
break; break;
case symbol_kind::S_FIELD: // "field"
case symbol_kind::S_PATH: // "path" case symbol_kind::S_PATH: // "path"
case symbol_kind::S_IDENTIFIER: // "identifier" case symbol_kind::S_IDENTIFIER: // "identifier"
case symbol_kind::S_STRING: // "string literal" case symbol_kind::S_STRING: // "string literal"
@ -5171,6 +5172,7 @@ switch (yykind)
value.move< ast::expr_game::ptr > (YY_MOVE (s.value)); value.move< ast::expr_game::ptr > (YY_MOVE (s.value));
break; break;
case symbol_kind::S_expr_identifier_nosize: // expr_identifier_nosize
case symbol_kind::S_expr_identifier: // expr_identifier case symbol_kind::S_expr_identifier: // expr_identifier
value.move< ast::expr_identifier::ptr > (YY_MOVE (s.value)); value.move< ast::expr_identifier::ptr > (YY_MOVE (s.value));
break; break;
@ -5358,7 +5360,6 @@ switch (yykind)
value.move< ast::stmt_while::ptr > (YY_MOVE (s.value)); value.move< ast::stmt_while::ptr > (YY_MOVE (s.value));
break; break;
case symbol_kind::S_FIELD: // "field"
case symbol_kind::S_PATH: // "path" case symbol_kind::S_PATH: // "path"
case symbol_kind::S_IDENTIFIER: // "identifier" case symbol_kind::S_IDENTIFIER: // "identifier"
case symbol_kind::S_STRING: // "string literal" case symbol_kind::S_STRING: // "string literal"
@ -5431,7 +5432,7 @@ switch (yykind)
#line 13 "parser.ypp" #line 13 "parser.ypp"
} } } // xsk::gsc::iw5 } } } // xsk::gsc::iw5
#line 5435 "parser.hpp" #line 5436 "parser.hpp"

120
src/iw6/xsk/lexer.cpp
View File

@ -59,6 +59,7 @@ const std::unordered_map<std::string_view, parser::token::token_kind_type> keywo
{ "true", parser::token::TRUE }, { "true", parser::token::TRUE },
{ "false", parser::token::FALSE }, { "false", parser::token::FALSE },
{ "undefined", parser::token::UNDEFINED }, { "undefined", parser::token::UNDEFINED },
{ "size", parser::token::SIZE },
{ "game", parser::token::GAME }, { "game", parser::token::GAME },
{ "self", parser::token::SELF }, { "self", parser::token::SELF },
{ "anim", parser::token::ANIM }, { "anim", parser::token::ANIM },
@ -84,8 +85,8 @@ bool buffer::push(char c)
return true; return true;
} }
reader::reader() : state(reader::end), buffer_pos(0), bytes_remaining(0), reader::reader() : state(reader::end), buffer_pos(0),
last_byte(0), current_byte(0) { } bytes_remaining(0), last_byte(0), current_byte(0) {}
void reader::init(const char* data, size_t size) void reader::init(const char* data, size_t size)
{ {
@ -173,7 +174,6 @@ auto lexer::lex() -> parser::symbol_type
{ {
buffer_.length = 0; buffer_.length = 0;
state_ = state::start; state_ = state::start;
loc_.step();
while (true) while (true)
{ {
@ -181,6 +181,7 @@ auto lexer::lex() -> parser::symbol_type
auto& last = reader_.last_byte; auto& last = reader_.last_byte;
auto& curr = reader_.current_byte; auto& curr = reader_.current_byte;
auto path = false; auto path = false;
loc_.step();
if (state == reader::end) if (state == reader::end)
{ {
@ -213,7 +214,7 @@ auto lexer::lex() -> parser::symbol_type
case '\\': case '\\':
throw comp_error(loc_, "invalid token ('\\')"); throw comp_error(loc_, "invalid token ('\\')");
case '/': case '/':
if (curr != '/' && curr != '*' && curr != '#' && curr != '=') if (curr != '=' && curr != '#' && curr != '@' && curr != '*' && curr != '/')
return parser::make_DIV(loc_); return parser::make_DIV(loc_);
advance(); advance();
@ -245,14 +246,35 @@ auto lexer::lex() -> parser::symbol_type
} }
else if (last == '#' && curr == '/') else if (last == '#' && curr == '/')
{ {
reader_.advance(); advance();
break; break;
} }
reader_.advance(); advance();
} }
} }
} }
else if (last == '@')
{
while (true)
{
if (state == reader::end)
throw comp_error(loc_, "unmatched script doc comment start ('/@')");
if (curr == '\n')
{
loc_.lines();
loc_.step();
}
else if (last == '@' && curr == '/')
{
advance();
break;
}
advance();
}
}
else if (last == '*') else if (last == '*')
{ {
while (true) while (true)
@ -267,11 +289,11 @@ auto lexer::lex() -> parser::symbol_type
} }
else if (last == '*' && curr == '/') else if (last == '*' && curr == '/')
{ {
reader_.advance(); advance();
break; break;
} }
reader_.advance(); advance();
} }
} }
else if (last == '/') else if (last == '/')
@ -281,30 +303,10 @@ auto lexer::lex() -> parser::symbol_type
if (state == reader::end) if (state == reader::end)
break; break;
if (last == '\\' && (curr == '\r' || curr == '\n'))
{
reader_.advance();
if (state == reader::end)
break;
if (last == '\r')
{
if (curr != '\n')
throw comp_error(loc_, "invalid token ('\')");
reader_.advance();
}
loc_.lines();
loc_.step();
continue;
}
if (curr == '\n') if (curr == '\n')
break; break;
reader_.advance(); advance();
} }
} }
continue; continue;
@ -314,8 +316,8 @@ auto lexer::lex() -> parser::symbol_type
if (!indev_) if (!indev_)
throw comp_error(loc_, "unmatched devblock end ('#/')"); throw comp_error(loc_, "unmatched devblock end ('#/')");
indev_ = false;
advance(); advance();
indev_ = false;
return parser::make_DEVEND(loc_); return parser::make_DEVEND(loc_);
} }
@ -336,7 +338,7 @@ auto lexer::lex() -> parser::symbol_type
state_ = state::preprocessor; state_ = state::preprocessor;
goto lex_name; goto lex_name;
case '*': case '*':
if (curr != '/' && curr != '=') if (curr != '=' && curr != '/')
return parser::make_MUL(loc_); return parser::make_MUL(loc_);
advance(); advance();
@ -349,13 +351,9 @@ auto lexer::lex() -> parser::symbol_type
state_ = state::string; state_ = state::string;
goto lex_string; goto lex_string;
case '.': case '.':
advance(); if (curr < '0' || curr > '9')
return parser::make_DOT(loc_);
if (state == reader::end) goto lex_number;
throw comp_error(loc_, "unterminated field ('.')");
state_ = state::field;
goto lex_name_or_number;
case '(': case '(':
return parser::make_LPAREN(loc_); return parser::make_LPAREN(loc_);
case ')': case ')':
@ -479,7 +477,6 @@ auto lexer::lex() -> parser::symbol_type
advance(); advance();
return parser::make_ASSIGN_RSHIFT(loc_); return parser::make_ASSIGN_RSHIFT(loc_);
default: default:
lex_name_or_number:
if (last >= '0' && last <= '9') if (last >= '0' && last <= '9')
goto lex_number; goto lex_number;
else if (last == '_' || last >= 'A' && last <= 'Z' || last >= 'a' && last <= 'z') else if (last == '_' || last >= 'A' && last <= 'Z' || last >= 'a' && last <= 'z')
@ -561,23 +558,8 @@ lex_name:
advance(); advance();
} }
if (state_ == state::field) if (state_ == state::preprocessor)
{ {
if (path)
throw comp_error(loc_, "invalid field token '\\'");
if (std::string_view(buffer_.data, buffer_.length) == "size")
{
return parser::make_SIZE(loc_);
}
return parser::make_FIELD(std::string(buffer_.data, buffer_.length), loc_);
}
else if (state_ == state::preprocessor)
{
if (path)
throw comp_error(loc_, "invalid preprocessor directive");
auto token = parser::token::IW6UNDEF; auto token = parser::token::IW6UNDEF;
if (buffer_.length < 16) if (buffer_.length < 16)
@ -593,7 +575,8 @@ lex_name:
} }
} }
preprocessor(token); preprocessor_run(token);
state_ = state::start; state_ = state::start;
continue; continue;
} }
@ -619,14 +602,11 @@ lex_name:
} }
lex_number: lex_number:
if (state_ == state::field) if (last == '.' || last != '0' || (last == '0' && (curr != 'o' && curr != 'b' && curr != 'x')))
buffer_.push('.');
if (state_ == state::field || last == '.' || last != '0' || (last == '0' && (curr != 'o' && curr != 'b' && curr != 'x')))
{ {
buffer_.push(last); buffer_.push(last);
auto dot = 0; auto dot = last == '.' ? 1 : 0;
auto flt = 0; auto flt = 0;
while (true) while (true)
@ -662,10 +642,10 @@ lex_number:
if (last == '\'') if (last == '\'')
throw comp_error(loc_, "invalid number literal"); throw comp_error(loc_, "invalid number literal");
if (state_ == state::field && dot || dot > 1 || flt > 1 || flt && buffer_.data[buffer_.length - 1] != 'f') if (dot > 1 || flt > 1 || flt && buffer_.data[buffer_.length - 1] != 'f')
throw comp_error(loc_, "invalid number literal"); throw comp_error(loc_, "invalid number literal");
if (state_ == state::field || dot || flt) if (dot || flt)
return parser::make_FLOAT(std::string(buffer_.data, buffer_.length), loc_); return parser::make_FLOAT(std::string(buffer_.data, buffer_.length), loc_);
return parser::make_INTEGER(std::string(buffer_.data, buffer_.length), loc_); return parser::make_INTEGER(std::string(buffer_.data, buffer_.length), loc_);
@ -770,18 +750,22 @@ lex_number:
return parser::make_INTEGER(xsk::utils::string::hex_to_dec(buffer_.data), loc_); return parser::make_INTEGER(xsk::utils::string::hex_to_dec(buffer_.data), loc_);
} }
// cant get here!
throw error("UNEXPECTED LEXER INTERNAL ERROR!");
} }
} }
void lexer::advance() void lexer::advance()
{ {
reader_.advance(); reader_.advance();
loc_.end.column++;
// dont wrap comment marks '/\/' '/\*' outside strings if (reader_.current_byte == '\\') [[unlikely]]
if (state_ == state::start && reader_.last_byte == '/') preprocessor_wrap();
return; }
void lexer::preprocessor_wrap()
{
while (reader_.current_byte == '\\') while (reader_.current_byte == '\\')
{ {
if (reader_.bytes_remaining == 1) if (reader_.bytes_remaining == 1)
@ -823,7 +807,7 @@ void lexer::advance()
} }
} }
void lexer::preprocessor(parser::token::token_kind_type token) void lexer::preprocessor_run(parser::token::token_kind_type token)
{ {
if (!clean_) if (!clean_)
throw comp_error(loc_, "invalid token ('#')"); throw comp_error(loc_, "invalid token ('#')");

5
src/iw6/xsk/lexer.hpp
View File

@ -44,7 +44,7 @@ struct reader
class lexer class lexer
{ {
enum class state : std::uint8_t { start, string, localize, field, preprocessor }; enum class state : std::uint8_t { start, string, localize, preprocessor };
reader reader_; reader reader_;
buffer buffer_; buffer buffer_;
@ -66,7 +66,8 @@ public:
private: private:
void advance(); void advance();
void preprocessor(parser::token::token_kind_type token); void preprocessor_wrap();
void preprocessor_run(parser::token::token_kind_type token);
}; };
} // namespace xsk::gsc::iw6 } // namespace xsk::gsc::iw6

1804
src/iw6/xsk/parser.cpp
View File

File diff suppressed because it is too large Load Diff

103
src/iw6/xsk/parser.hpp
View File

@ -474,6 +474,7 @@ namespace xsk { namespace gsc { namespace iw6 {
// expr_game // expr_game
char dummy19[sizeof (ast::expr_game::ptr)]; char dummy19[sizeof (ast::expr_game::ptr)];
// expr_identifier_nosize
// expr_identifier // expr_identifier
char dummy20[sizeof (ast::expr_identifier::ptr)]; char dummy20[sizeof (ast::expr_identifier::ptr)];
@ -615,7 +616,6 @@ namespace xsk { namespace gsc { namespace iw6 {
// stmt_while // stmt_while
char dummy65[sizeof (ast::stmt_while::ptr)]; char dummy65[sizeof (ast::stmt_while::ptr)];
// "field"
// "path" // "path"
// "identifier" // "identifier"
// "string literal" // "string literal"
@ -717,7 +717,7 @@ namespace xsk { namespace gsc { namespace iw6 {
TRUE = 43, // "true" TRUE = 43, // "true"
FALSE = 44, // "false" FALSE = 44, // "false"
UNDEFINED = 45, // "undefined" UNDEFINED = 45, // "undefined"
SIZE = 46, // ".size" SIZE = 46, // "size"
GAME = 47, // "game" GAME = 47, // "game"
SELF = 48, // "self" SELF = 48, // "self"
ANIM = 49, // "anim" ANIM = 49, // "anim"
@ -767,13 +767,13 @@ namespace xsk { namespace gsc { namespace iw6 {
MUL = 93, // "*" MUL = 93, // "*"
DIV = 94, // "/" DIV = 94, // "/"
MOD = 95, // "%" MOD = 95, // "%"
FIELD = 96, // "field" PATH = 96, // "path"
PATH = 97, // "path" IDENTIFIER = 97, // "identifier"
IDENTIFIER = 98, // "identifier" STRING = 98, // "string literal"
STRING = 99, // "string literal" ISTRING = 99, // "localized string"
ISTRING = 100, // "localized string" FLOAT = 100, // "float"
FLOAT = 101, // "float" INTEGER = 101, // "integer"
INTEGER = 102, // "integer" SIZEOF = 102, // SIZEOF
ADD_ARRAY = 103, // ADD_ARRAY ADD_ARRAY = 103, // ADD_ARRAY
THEN = 104, // THEN THEN = 104, // THEN
TERN = 105, // TERN TERN = 105, // TERN
@ -847,7 +847,7 @@ namespace xsk { namespace gsc { namespace iw6 {
S_TRUE = 43, // "true" S_TRUE = 43, // "true"
S_FALSE = 44, // "false" S_FALSE = 44, // "false"
S_UNDEFINED = 45, // "undefined" S_UNDEFINED = 45, // "undefined"
S_SIZE = 46, // ".size" S_SIZE = 46, // "size"
S_GAME = 47, // "game" S_GAME = 47, // "game"
S_SELF = 48, // "self" S_SELF = 48, // "self"
S_ANIM = 49, // "anim" S_ANIM = 49, // "anim"
@ -897,13 +897,13 @@ namespace xsk { namespace gsc { namespace iw6 {
S_MUL = 93, // "*" S_MUL = 93, // "*"
S_DIV = 94, // "/" S_DIV = 94, // "/"
S_MOD = 95, // "%" S_MOD = 95, // "%"
S_FIELD = 96, // "field" S_PATH = 96, // "path"
S_PATH = 97, // "path" S_IDENTIFIER = 97, // "identifier"
S_IDENTIFIER = 98, // "identifier" S_STRING = 98, // "string literal"
S_STRING = 99, // "string literal" S_ISTRING = 99, // "localized string"
S_ISTRING = 100, // "localized string" S_FLOAT = 100, // "float"
S_FLOAT = 101, // "float" S_INTEGER = 101, // "integer"
S_INTEGER = 102, // "integer" S_SIZEOF = 102, // SIZEOF
S_ADD_ARRAY = 103, // ADD_ARRAY S_ADD_ARRAY = 103, // ADD_ARRAY
S_THEN = 104, // THEN S_THEN = 104, // THEN
S_TERN = 105, // TERN S_TERN = 105, // TERN
@ -985,15 +985,16 @@ namespace xsk { namespace gsc { namespace iw6 {
S_expr_level = 181, // expr_level S_expr_level = 181, // expr_level
S_expr_animation = 182, // expr_animation S_expr_animation = 182, // expr_animation
S_expr_animtree = 183, // expr_animtree S_expr_animtree = 183, // expr_animtree
S_expr_identifier = 184, // expr_identifier S_expr_identifier_nosize = 184, // expr_identifier_nosize
S_expr_path = 185, // expr_path S_expr_identifier = 185, // expr_identifier
S_expr_istring = 186, // expr_istring S_expr_path = 186, // expr_path
S_expr_string = 187, // expr_string S_expr_istring = 187, // expr_istring
S_expr_vector = 188, // expr_vector S_expr_string = 188, // expr_string
S_expr_float = 189, // expr_float S_expr_vector = 189, // expr_vector
S_expr_integer = 190, // expr_integer S_expr_float = 190, // expr_float
S_expr_false = 191, // expr_false S_expr_integer = 191, // expr_integer
S_expr_true = 192 // expr_true S_expr_false = 192, // expr_false
S_expr_true = 193 // expr_true
}; };
}; };
@ -1116,6 +1117,7 @@ namespace xsk { namespace gsc { namespace iw6 {
value.move< ast::expr_game::ptr > (std::move (that.value)); value.move< ast::expr_game::ptr > (std::move (that.value));
break; break;
case symbol_kind::S_expr_identifier_nosize: // expr_identifier_nosize
case symbol_kind::S_expr_identifier: // expr_identifier case symbol_kind::S_expr_identifier: // expr_identifier
value.move< ast::expr_identifier::ptr > (std::move (that.value)); value.move< ast::expr_identifier::ptr > (std::move (that.value));
break; break;
@ -1303,7 +1305,6 @@ namespace xsk { namespace gsc { namespace iw6 {
value.move< ast::stmt_while::ptr > (std::move (that.value)); value.move< ast::stmt_while::ptr > (std::move (that.value));
break; break;
case symbol_kind::S_FIELD: // "field"
case symbol_kind::S_PATH: // "path" case symbol_kind::S_PATH: // "path"
case symbol_kind::S_IDENTIFIER: // "identifier" case symbol_kind::S_IDENTIFIER: // "identifier"
case symbol_kind::S_STRING: // "string literal" case symbol_kind::S_STRING: // "string literal"
@ -2368,6 +2369,7 @@ switch (yykind)
value.template destroy< ast::expr_game::ptr > (); value.template destroy< ast::expr_game::ptr > ();
break; break;
case symbol_kind::S_expr_identifier_nosize: // expr_identifier_nosize
case symbol_kind::S_expr_identifier: // expr_identifier case symbol_kind::S_expr_identifier: // expr_identifier
value.template destroy< ast::expr_identifier::ptr > (); value.template destroy< ast::expr_identifier::ptr > ();
break; break;
@ -2555,7 +2557,6 @@ switch (yykind)
value.template destroy< ast::stmt_while::ptr > (); value.template destroy< ast::stmt_while::ptr > ();
break; break;
case symbol_kind::S_FIELD: // "field"
case symbol_kind::S_PATH: // "path" case symbol_kind::S_PATH: // "path"
case symbol_kind::S_IDENTIFIER: // "identifier" case symbol_kind::S_IDENTIFIER: // "identifier"
case symbol_kind::S_STRING: // "string literal" case symbol_kind::S_STRING: // "string literal"
@ -2661,7 +2662,7 @@ switch (yykind)
{ {
IW6_ASSERT (tok == token::IW6EOF IW6_ASSERT (tok == token::IW6EOF
|| (token::IW6error <= tok && tok <= token::MOD) || (token::IW6error <= tok && tok <= token::MOD)
|| (token::ADD_ARRAY <= tok && tok <= token::POSTDEC)); || (token::SIZEOF <= tok && tok <= token::POSTDEC));
} }
#if 201103L <= YY_CPLUSPLUS #if 201103L <= YY_CPLUSPLUS
symbol_type (int tok, std::string v, location_type l) symbol_type (int tok, std::string v, location_type l)
@ -2671,7 +2672,7 @@ switch (yykind)
: super_type(token_type (tok), v, l) : super_type(token_type (tok), v, l)
#endif #endif
{ {
IW6_ASSERT ((token::FIELD <= tok && tok <= token::INTEGER)); IW6_ASSERT ((token::PATH <= tok && tok <= token::INTEGER));
} }
}; };
@ -4161,21 +4162,6 @@ switch (yykind)
return symbol_type (token::MOD, l); return symbol_type (token::MOD, l);
} }
#endif #endif
#if 201103L <= YY_CPLUSPLUS
static
symbol_type
make_FIELD (std::string v, location_type l)
{
return symbol_type (token::FIELD, std::move (v), std::move (l));
}
#else
static
symbol_type
make_FIELD (const std::string& v, const location_type& l)
{
return symbol_type (token::FIELD, v, l);
}
#endif
#if 201103L <= YY_CPLUSPLUS #if 201103L <= YY_CPLUSPLUS
static static
symbol_type symbol_type
@ -4266,6 +4252,21 @@ switch (yykind)
return symbol_type (token::INTEGER, v, l); return symbol_type (token::INTEGER, v, l);
} }
#endif #endif
#if 201103L <= YY_CPLUSPLUS
static
symbol_type
make_SIZEOF (location_type l)
{
return symbol_type (token::SIZEOF, std::move (l));
}
#else
static
symbol_type
make_SIZEOF (const location_type& l)
{
return symbol_type (token::SIZEOF, l);
}
#endif
#if 201103L <= YY_CPLUSPLUS #if 201103L <= YY_CPLUSPLUS
static static
symbol_type symbol_type
@ -4745,9 +4746,9 @@ switch (yykind)
/// Constants. /// Constants.
enum enum
{ {
yylast_ = 2274, ///< Last index in yytable_. yylast_ = 2303, ///< Last index in yytable_.
yynnts_ = 81, ///< Number of nonterminal symbols. yynnts_ = 82, ///< Number of nonterminal symbols.
yyfinal_ = 21 ///< Termination state number. yyfinal_ = 22 ///< Termination state number.
}; };
@ -4859,6 +4860,7 @@ switch (yykind)
value.copy< ast::expr_game::ptr > (YY_MOVE (that.value)); value.copy< ast::expr_game::ptr > (YY_MOVE (that.value));
break; break;
case symbol_kind::S_expr_identifier_nosize: // expr_identifier_nosize
case symbol_kind::S_expr_identifier: // expr_identifier case symbol_kind::S_expr_identifier: // expr_identifier
value.copy< ast::expr_identifier::ptr > (YY_MOVE (that.value)); value.copy< ast::expr_identifier::ptr > (YY_MOVE (that.value));
break; break;
@ -5046,7 +5048,6 @@ switch (yykind)
value.copy< ast::stmt_while::ptr > (YY_MOVE (that.value)); value.copy< ast::stmt_while::ptr > (YY_MOVE (that.value));
break; break;
case symbol_kind::S_FIELD: // "field"
case symbol_kind::S_PATH: // "path" case symbol_kind::S_PATH: // "path"
case symbol_kind::S_IDENTIFIER: // "identifier" case symbol_kind::S_IDENTIFIER: // "identifier"
case symbol_kind::S_STRING: // "string literal" case symbol_kind::S_STRING: // "string literal"
@ -5171,6 +5172,7 @@ switch (yykind)
value.move< ast::expr_game::ptr > (YY_MOVE (s.value)); value.move< ast::expr_game::ptr > (YY_MOVE (s.value));
break; break;
case symbol_kind::S_expr_identifier_nosize: // expr_identifier_nosize
case symbol_kind::S_expr_identifier: // expr_identifier case symbol_kind::S_expr_identifier: // expr_identifier
value.move< ast::expr_identifier::ptr > (YY_MOVE (s.value)); value.move< ast::expr_identifier::ptr > (YY_MOVE (s.value));
break; break;
@ -5358,7 +5360,6 @@ switch (yykind)
value.move< ast::stmt_while::ptr > (YY_MOVE (s.value)); value.move< ast::stmt_while::ptr > (YY_MOVE (s.value));
break; break;
case symbol_kind::S_FIELD: // "field"
case symbol_kind::S_PATH: // "path" case symbol_kind::S_PATH: // "path"
case symbol_kind::S_IDENTIFIER: // "identifier" case symbol_kind::S_IDENTIFIER: // "identifier"
case symbol_kind::S_STRING: // "string literal" case symbol_kind::S_STRING: // "string literal"
@ -5431,7 +5432,7 @@ switch (yykind)
#line 13 "parser.ypp" #line 13 "parser.ypp"
} } } // xsk::gsc::iw6 } } } // xsk::gsc::iw6
#line 5435 "parser.hpp" #line 5436 "parser.hpp"

120
src/iw7/xsk/lexer.cpp
View File

@ -59,6 +59,7 @@ const std::unordered_map<std::string_view, parser::token::token_kind_type> keywo
{ "true", parser::token::TRUE }, { "true", parser::token::TRUE },
{ "false", parser::token::FALSE }, { "false", parser::token::FALSE },
{ "undefined", parser::token::UNDEFINED }, { "undefined", parser::token::UNDEFINED },
{ "size", parser::token::SIZE },
{ "game", parser::token::GAME }, { "game", parser::token::GAME },
{ "self", parser::token::SELF }, { "self", parser::token::SELF },
{ "anim", parser::token::ANIM }, { "anim", parser::token::ANIM },
@ -84,8 +85,8 @@ bool buffer::push(char c)
return true; return true;
} }
reader::reader() : state(reader::end), buffer_pos(0), bytes_remaining(0), reader::reader() : state(reader::end), buffer_pos(0),
last_byte(0), current_byte(0) { } bytes_remaining(0), last_byte(0), current_byte(0) {}
void reader::init(const char* data, size_t size) void reader::init(const char* data, size_t size)
{ {
@ -173,7 +174,6 @@ auto lexer::lex() -> parser::symbol_type
{ {
buffer_.length = 0; buffer_.length = 0;
state_ = state::start; state_ = state::start;
loc_.step();
while (true) while (true)
{ {
@ -181,6 +181,7 @@ auto lexer::lex() -> parser::symbol_type
auto& last = reader_.last_byte; auto& last = reader_.last_byte;
auto& curr = reader_.current_byte; auto& curr = reader_.current_byte;
auto path = false; auto path = false;
loc_.step();
if (state == reader::end) if (state == reader::end)
{ {
@ -213,7 +214,7 @@ auto lexer::lex() -> parser::symbol_type
case '\\': case '\\':
throw comp_error(loc_, "invalid token ('\\')"); throw comp_error(loc_, "invalid token ('\\')");
case '/': case '/':
if (curr != '/' && curr != '*' && curr != '#' && curr != '=') if (curr != '=' && curr != '#' && curr != '@' && curr != '*' && curr != '/')
return parser::make_DIV(loc_); return parser::make_DIV(loc_);
advance(); advance();
@ -245,14 +246,35 @@ auto lexer::lex() -> parser::symbol_type
} }
else if (last == '#' && curr == '/') else if (last == '#' && curr == '/')
{ {
reader_.advance(); advance();
break; break;
} }
reader_.advance(); advance();
} }
} }
} }
else if (last == '@')
{
while (true)
{
if (state == reader::end)
throw comp_error(loc_, "unmatched script doc comment start ('/@')");
if (curr == '\n')
{
loc_.lines();
loc_.step();
}
else if (last == '@' && curr == '/')
{
advance();
break;
}
advance();
}
}
else if (last == '*') else if (last == '*')
{ {
while (true) while (true)
@ -267,11 +289,11 @@ auto lexer::lex() -> parser::symbol_type
} }
else if (last == '*' && curr == '/') else if (last == '*' && curr == '/')
{ {
reader_.advance(); advance();
break; break;
} }
reader_.advance(); advance();
} }
} }
else if (last == '/') else if (last == '/')
@ -281,30 +303,10 @@ auto lexer::lex() -> parser::symbol_type
if (state == reader::end) if (state == reader::end)
break; break;
if (last == '\\' && (curr == '\r' || curr == '\n'))
{
reader_.advance();
if (state == reader::end)
break;
if (last == '\r')
{
if (curr != '\n')
throw comp_error(loc_, "invalid token ('\')");
reader_.advance();
}
loc_.lines();
loc_.step();
continue;
}
if (curr == '\n') if (curr == '\n')
break; break;
reader_.advance(); advance();
} }
} }
continue; continue;
@ -314,8 +316,8 @@ auto lexer::lex() -> parser::symbol_type
if (!indev_) if (!indev_)
throw comp_error(loc_, "unmatched devblock end ('#/')"); throw comp_error(loc_, "unmatched devblock end ('#/')");
indev_ = false;
advance(); advance();
indev_ = false;
return parser::make_DEVEND(loc_); return parser::make_DEVEND(loc_);
} }
@ -336,7 +338,7 @@ auto lexer::lex() -> parser::symbol_type
state_ = state::preprocessor; state_ = state::preprocessor;
goto lex_name; goto lex_name;
case '*': case '*':
if (curr != '/' && curr != '=') if (curr != '=' && curr != '/')
return parser::make_MUL(loc_); return parser::make_MUL(loc_);
advance(); advance();
@ -349,13 +351,9 @@ auto lexer::lex() -> parser::symbol_type
state_ = state::string; state_ = state::string;
goto lex_string; goto lex_string;
case '.': case '.':
advance(); if (curr < '0' || curr > '9')
return parser::make_DOT(loc_);
if (state == reader::end) goto lex_number;
throw comp_error(loc_, "unterminated field ('.')");
state_ = state::field;
goto lex_name_or_number;
case '(': case '(':
return parser::make_LPAREN(loc_); return parser::make_LPAREN(loc_);
case ')': case ')':
@ -479,7 +477,6 @@ auto lexer::lex() -> parser::symbol_type
advance(); advance();
return parser::make_ASSIGN_RSHIFT(loc_); return parser::make_ASSIGN_RSHIFT(loc_);
default: default:
lex_name_or_number:
if (last >= '0' && last <= '9') if (last >= '0' && last <= '9')
goto lex_number; goto lex_number;
else if (last == '_' || last >= 'A' && last <= 'Z' || last >= 'a' && last <= 'z') else if (last == '_' || last >= 'A' && last <= 'Z' || last >= 'a' && last <= 'z')
@ -561,23 +558,8 @@ lex_name:
advance(); advance();
} }
if (state_ == state::field) if (state_ == state::preprocessor)
{ {
if (path)
throw comp_error(loc_, "invalid field token '\\'");
if (std::string_view(buffer_.data, buffer_.length) == "size")
{
return parser::make_SIZE(loc_);
}
return parser::make_FIELD(std::string(buffer_.data, buffer_.length), loc_);
}
else if (state_ == state::preprocessor)
{
if (path)
throw comp_error(loc_, "invalid preprocessor directive");
auto token = parser::token::IW7UNDEF; auto token = parser::token::IW7UNDEF;
if (buffer_.length < 16) if (buffer_.length < 16)
@ -593,7 +575,8 @@ lex_name:
} }
} }
preprocessor(token); preprocessor_run(token);
state_ = state::start; state_ = state::start;
continue; continue;
} }
@ -619,14 +602,11 @@ lex_name:
} }
lex_number: lex_number:
if (state_ == state::field) if (last == '.' || last != '0' || (last == '0' && (curr != 'o' && curr != 'b' && curr != 'x')))
buffer_.push('.');
if (state_ == state::field || last == '.' || last != '0' || (last == '0' && (curr != 'o' && curr != 'b' && curr != 'x')))
{ {
buffer_.push(last); buffer_.push(last);
auto dot = 0; auto dot = last == '.' ? 1 : 0;
auto flt = 0; auto flt = 0;
while (true) while (true)
@ -662,10 +642,10 @@ lex_number:
if (last == '\'') if (last == '\'')
throw comp_error(loc_, "invalid number literal"); throw comp_error(loc_, "invalid number literal");
if (state_ == state::field && dot || dot > 1 || flt > 1 || flt && buffer_.data[buffer_.length - 1] != 'f') if (dot > 1 || flt > 1 || flt && buffer_.data[buffer_.length - 1] != 'f')
throw comp_error(loc_, "invalid number literal"); throw comp_error(loc_, "invalid number literal");
if (state_ == state::field || dot || flt) if (dot || flt)
return parser::make_FLOAT(std::string(buffer_.data, buffer_.length), loc_); return parser::make_FLOAT(std::string(buffer_.data, buffer_.length), loc_);
return parser::make_INTEGER(std::string(buffer_.data, buffer_.length), loc_); return parser::make_INTEGER(std::string(buffer_.data, buffer_.length), loc_);
@ -770,18 +750,22 @@ lex_number:
return parser::make_INTEGER(xsk::utils::string::hex_to_dec(buffer_.data), loc_); return parser::make_INTEGER(xsk::utils::string::hex_to_dec(buffer_.data), loc_);
} }
// cant get here!
throw error("UNEXPECTED LEXER INTERNAL ERROR!");
} }
} }
void lexer::advance() void lexer::advance()
{ {
reader_.advance(); reader_.advance();
loc_.end.column++;
// dont wrap comment marks '/\/' '/\*' outside strings if (reader_.current_byte == '\\') [[unlikely]]
if (state_ == state::start && reader_.last_byte == '/') preprocessor_wrap();
return; }
void lexer::preprocessor_wrap()
{
while (reader_.current_byte == '\\') while (reader_.current_byte == '\\')
{ {
if (reader_.bytes_remaining == 1) if (reader_.bytes_remaining == 1)
@ -823,7 +807,7 @@ void lexer::advance()
} }
} }
void lexer::preprocessor(parser::token::token_kind_type token) void lexer::preprocessor_run(parser::token::token_kind_type token)
{ {
if (!clean_) if (!clean_)
throw comp_error(loc_, "invalid token ('#')"); throw comp_error(loc_, "invalid token ('#')");

5
src/iw7/xsk/lexer.hpp
View File

@ -44,7 +44,7 @@ struct reader
class lexer class lexer
{ {
enum class state : std::uint8_t { start, string, localize, field, preprocessor }; enum class state : std::uint8_t { start, string, localize, preprocessor };
reader reader_; reader reader_;
buffer buffer_; buffer buffer_;
@ -66,7 +66,8 @@ public:
private: private:
void advance(); void advance();
void preprocessor(parser::token::token_kind_type token); void preprocessor_wrap();
void preprocessor_run(parser::token::token_kind_type token);
}; };
} // namespace xsk::gsc::iw7 } // namespace xsk::gsc::iw7

1804
src/iw7/xsk/parser.cpp
View File

File diff suppressed because it is too large Load Diff

103
src/iw7/xsk/parser.hpp
View File

@ -474,6 +474,7 @@ namespace xsk { namespace gsc { namespace iw7 {
// expr_game // expr_game
char dummy19[sizeof (ast::expr_game::ptr)]; char dummy19[sizeof (ast::expr_game::ptr)];
// expr_identifier_nosize
// expr_identifier // expr_identifier
char dummy20[sizeof (ast::expr_identifier::ptr)]; char dummy20[sizeof (ast::expr_identifier::ptr)];
@ -615,7 +616,6 @@ namespace xsk { namespace gsc { namespace iw7 {
// stmt_while // stmt_while
char dummy65[sizeof (ast::stmt_while::ptr)]; char dummy65[sizeof (ast::stmt_while::ptr)];
// "field"
// "path" // "path"
// "identifier" // "identifier"
// "string literal" // "string literal"
@ -717,7 +717,7 @@ namespace xsk { namespace gsc { namespace iw7 {
TRUE = 43, // "true" TRUE = 43, // "true"
FALSE = 44, // "false" FALSE = 44, // "false"
UNDEFINED = 45, // "undefined" UNDEFINED = 45, // "undefined"
SIZE = 46, // ".size" SIZE = 46, // "size"
GAME = 47, // "game" GAME = 47, // "game"
SELF = 48, // "self" SELF = 48, // "self"
ANIM = 49, // "anim" ANIM = 49, // "anim"
@ -767,13 +767,13 @@ namespace xsk { namespace gsc { namespace iw7 {
MUL = 93, // "*" MUL = 93, // "*"
DIV = 94, // "/" DIV = 94, // "/"
MOD = 95, // "%" MOD = 95, // "%"
FIELD = 96, // "field" PATH = 96, // "path"
PATH = 97, // "path" IDENTIFIER = 97, // "identifier"
IDENTIFIER = 98, // "identifier" STRING = 98, // "string literal"
STRING = 99, // "string literal" ISTRING = 99, // "localized string"
ISTRING = 100, // "localized string" FLOAT = 100, // "float"
FLOAT = 101, // "float" INTEGER = 101, // "integer"
INTEGER = 102, // "integer" SIZEOF = 102, // SIZEOF
ADD_ARRAY = 103, // ADD_ARRAY ADD_ARRAY = 103, // ADD_ARRAY
THEN = 104, // THEN THEN = 104, // THEN
TERN = 105, // TERN TERN = 105, // TERN
@ -847,7 +847,7 @@ namespace xsk { namespace gsc { namespace iw7 {
S_TRUE = 43, // "true" S_TRUE = 43, // "true"
S_FALSE = 44, // "false" S_FALSE = 44, // "false"
S_UNDEFINED = 45, // "undefined" S_UNDEFINED = 45, // "undefined"
S_SIZE = 46, // ".size" S_SIZE = 46, // "size"
S_GAME = 47, // "game" S_GAME = 47, // "game"
S_SELF = 48, // "self" S_SELF = 48, // "self"
S_ANIM = 49, // "anim" S_ANIM = 49, // "anim"
@ -897,13 +897,13 @@ namespace xsk { namespace gsc { namespace iw7 {
S_MUL = 93, // "*" S_MUL = 93, // "*"
S_DIV = 94, // "/" S_DIV = 94, // "/"
S_MOD = 95, // "%" S_MOD = 95, // "%"
S_FIELD = 96, // "field" S_PATH = 96, // "path"
S_PATH = 97, // "path" S_IDENTIFIER = 97, // "identifier"
S_IDENTIFIER = 98, // "identifier" S_STRING = 98, // "string literal"
S_STRING = 99, // "string literal" S_ISTRING = 99, // "localized string"
S_ISTRING = 100, // "localized string" S_FLOAT = 100, // "float"
S_FLOAT = 101, // "float" S_INTEGER = 101, // "integer"
S_INTEGER = 102, // "integer" S_SIZEOF = 102, // SIZEOF
S_ADD_ARRAY = 103, // ADD_ARRAY S_ADD_ARRAY = 103, // ADD_ARRAY
S_THEN = 104, // THEN S_THEN = 104, // THEN
S_TERN = 105, // TERN S_TERN = 105, // TERN
@ -985,15 +985,16 @@ namespace xsk { namespace gsc { namespace iw7 {
S_expr_level = 181, // expr_level S_expr_level = 181, // expr_level
S_expr_animation = 182, // expr_animation S_expr_animation = 182, // expr_animation
S_expr_animtree = 183, // expr_animtree S_expr_animtree = 183, // expr_animtree
S_expr_identifier = 184, // expr_identifier S_expr_identifier_nosize = 184, // expr_identifier_nosize
S_expr_path = 185, // expr_path S_expr_identifier = 185, // expr_identifier
S_expr_istring = 186, // expr_istring S_expr_path = 186, // expr_path
S_expr_string = 187, // expr_string S_expr_istring = 187, // expr_istring
S_expr_vector = 188, // expr_vector S_expr_string = 188, // expr_string
S_expr_float = 189, // expr_float S_expr_vector = 189, // expr_vector
S_expr_integer = 190, // expr_integer S_expr_float = 190, // expr_float
S_expr_false = 191, // expr_false S_expr_integer = 191, // expr_integer
S_expr_true = 192 // expr_true S_expr_false = 192, // expr_false
S_expr_true = 193 // expr_true
}; };
}; };
@ -1116,6 +1117,7 @@ namespace xsk { namespace gsc { namespace iw7 {
value.move< ast::expr_game::ptr > (std::move (that.value)); value.move< ast::expr_game::ptr > (std::move (that.value));
break; break;
case symbol_kind::S_expr_identifier_nosize: // expr_identifier_nosize
case symbol_kind::S_expr_identifier: // expr_identifier case symbol_kind::S_expr_identifier: // expr_identifier
value.move< ast::expr_identifier::ptr > (std::move (that.value)); value.move< ast::expr_identifier::ptr > (std::move (that.value));
break; break;
@ -1303,7 +1305,6 @@ namespace xsk { namespace gsc { namespace iw7 {
value.move< ast::stmt_while::ptr > (std::move (that.value)); value.move< ast::stmt_while::ptr > (std::move (that.value));
break; break;
case symbol_kind::S_FIELD: // "field"
case symbol_kind::S_PATH: // "path" case symbol_kind::S_PATH: // "path"
case symbol_kind::S_IDENTIFIER: // "identifier" case symbol_kind::S_IDENTIFIER: // "identifier"
case symbol_kind::S_STRING: // "string literal" case symbol_kind::S_STRING: // "string literal"
@ -2368,6 +2369,7 @@ switch (yykind)
value.template destroy< ast::expr_game::ptr > (); value.template destroy< ast::expr_game::ptr > ();
break; break;
case symbol_kind::S_expr_identifier_nosize: // expr_identifier_nosize
case symbol_kind::S_expr_identifier: // expr_identifier case symbol_kind::S_expr_identifier: // expr_identifier
value.template destroy< ast::expr_identifier::ptr > (); value.template destroy< ast::expr_identifier::ptr > ();
break; break;
@ -2555,7 +2557,6 @@ switch (yykind)
value.template destroy< ast::stmt_while::ptr > (); value.template destroy< ast::stmt_while::ptr > ();
break; break;
case symbol_kind::S_FIELD: // "field"
case symbol_kind::S_PATH: // "path" case symbol_kind::S_PATH: // "path"
case symbol_kind::S_IDENTIFIER: // "identifier" case symbol_kind::S_IDENTIFIER: // "identifier"
case symbol_kind::S_STRING: // "string literal" case symbol_kind::S_STRING: // "string literal"
@ -2661,7 +2662,7 @@ switch (yykind)
{ {
IW7_ASSERT (tok == token::IW7EOF IW7_ASSERT (tok == token::IW7EOF
|| (token::IW7error <= tok && tok <= token::MOD) || (token::IW7error <= tok && tok <= token::MOD)
|| (token::ADD_ARRAY <= tok && tok <= token::POSTDEC)); || (token::SIZEOF <= tok && tok <= token::POSTDEC));
} }
#if 201103L <= YY_CPLUSPLUS #if 201103L <= YY_CPLUSPLUS
symbol_type (int tok, std::string v, location_type l) symbol_type (int tok, std::string v, location_type l)
@ -2671,7 +2672,7 @@ switch (yykind)
: super_type(token_type (tok), v, l) : super_type(token_type (tok), v, l)
#endif #endif
{ {
IW7_ASSERT ((token::FIELD <= tok && tok <= token::INTEGER)); IW7_ASSERT ((token::PATH <= tok && tok <= token::INTEGER));
} }
}; };
@ -4161,21 +4162,6 @@ switch (yykind)
return symbol_type (token::MOD, l); return symbol_type (token::MOD, l);
} }
#endif #endif
#if 201103L <= YY_CPLUSPLUS
static
symbol_type
make_FIELD (std::string v, location_type l)
{
return symbol_type (token::FIELD, std::move (v), std::move (l));
}
#else
static
symbol_type
make_FIELD (const std::string& v, const location_type& l)
{
return symbol_type (token::FIELD, v, l);
}
#endif
#if 201103L <= YY_CPLUSPLUS #if 201103L <= YY_CPLUSPLUS
static static
symbol_type symbol_type
@ -4266,6 +4252,21 @@ switch (yykind)
return symbol_type (token::INTEGER, v, l); return symbol_type (token::INTEGER, v, l);
} }
#endif #endif
#if 201103L <= YY_CPLUSPLUS
static
symbol_type
make_SIZEOF (location_type l)
{
return symbol_type (token::SIZEOF, std::move (l));
}
#else
static
symbol_type
make_SIZEOF (const location_type& l)
{
return symbol_type (token::SIZEOF, l);
}
#endif
#if 201103L <= YY_CPLUSPLUS #if 201103L <= YY_CPLUSPLUS
static static
symbol_type symbol_type
@ -4745,9 +4746,9 @@ switch (yykind)
/// Constants. /// Constants.
enum enum
{ {
yylast_ = 2274, ///< Last index in yytable_. yylast_ = 2303, ///< Last index in yytable_.
yynnts_ = 81, ///< Number of nonterminal symbols. yynnts_ = 82, ///< Number of nonterminal symbols.
yyfinal_ = 21 ///< Termination state number. yyfinal_ = 22 ///< Termination state number.
}; };
@ -4859,6 +4860,7 @@ switch (yykind)
value.copy< ast::expr_game::ptr > (YY_MOVE (that.value)); value.copy< ast::expr_game::ptr > (YY_MOVE (that.value));
break; break;
case symbol_kind::S_expr_identifier_nosize: // expr_identifier_nosize
case symbol_kind::S_expr_identifier: // expr_identifier case symbol_kind::S_expr_identifier: // expr_identifier
value.copy< ast::expr_identifier::ptr > (YY_MOVE (that.value)); value.copy< ast::expr_identifier::ptr > (YY_MOVE (that.value));
break; break;
@ -5046,7 +5048,6 @@ switch (yykind)
value.copy< ast::stmt_while::ptr > (YY_MOVE (that.value)); value.copy< ast::stmt_while::ptr > (YY_MOVE (that.value));
break; break;
case symbol_kind::S_FIELD: // "field"
case symbol_kind::S_PATH: // "path" case symbol_kind::S_PATH: // "path"
case symbol_kind::S_IDENTIFIER: // "identifier" case symbol_kind::S_IDENTIFIER: // "identifier"
case symbol_kind::S_STRING: // "string literal" case symbol_kind::S_STRING: // "string literal"
@ -5171,6 +5172,7 @@ switch (yykind)
value.move< ast::expr_game::ptr > (YY_MOVE (s.value)); value.move< ast::expr_game::ptr > (YY_MOVE (s.value));
break; break;
case symbol_kind::S_expr_identifier_nosize: // expr_identifier_nosize
case symbol_kind::S_expr_identifier: // expr_identifier case symbol_kind::S_expr_identifier: // expr_identifier
value.move< ast::expr_identifier::ptr > (YY_MOVE (s.value)); value.move< ast::expr_identifier::ptr > (YY_MOVE (s.value));
break; break;
@ -5358,7 +5360,6 @@ switch (yykind)
value.move< ast::stmt_while::ptr > (YY_MOVE (s.value)); value.move< ast::stmt_while::ptr > (YY_MOVE (s.value));
break; break;
case symbol_kind::S_FIELD: // "field"
case symbol_kind::S_PATH: // "path" case symbol_kind::S_PATH: // "path"
case symbol_kind::S_IDENTIFIER: // "identifier" case symbol_kind::S_IDENTIFIER: // "identifier"
case symbol_kind::S_STRING: // "string literal" case symbol_kind::S_STRING: // "string literal"
@ -5431,7 +5432,7 @@ switch (yykind)
#line 13 "parser.ypp" #line 13 "parser.ypp"
} } } // xsk::gsc::iw7 } } } // xsk::gsc::iw7
#line 5435 "parser.hpp" #line 5436 "parser.hpp"

120
src/iw8/xsk/lexer.cpp
View File

@ -60,6 +60,7 @@ const std::unordered_map<std::string_view, parser::token::token_kind_type> keywo
{ "true", parser::token::TRUE }, { "true", parser::token::TRUE },
{ "false", parser::token::FALSE }, { "false", parser::token::FALSE },
{ "undefined", parser::token::UNDEFINED }, { "undefined", parser::token::UNDEFINED },
{ "size", parser::token::SIZE },
{ "game", parser::token::GAME }, { "game", parser::token::GAME },
{ "self", parser::token::SELF }, { "self", parser::token::SELF },
{ "anim", parser::token::ANIM }, { "anim", parser::token::ANIM },
@ -87,8 +88,8 @@ bool buffer::push(char c)
return true; return true;
} }
reader::reader() : state(reader::end), buffer_pos(0), bytes_remaining(0), reader::reader() : state(reader::end), buffer_pos(0),
last_byte(0), current_byte(0) { } bytes_remaining(0), last_byte(0), current_byte(0) {}
void reader::init(const char* data, size_t size) void reader::init(const char* data, size_t size)
{ {
@ -176,7 +177,6 @@ auto lexer::lex() -> parser::symbol_type
{ {
buffer_.length = 0; buffer_.length = 0;
state_ = state::start; state_ = state::start;
loc_.step();
while (true) while (true)
{ {
@ -184,6 +184,7 @@ auto lexer::lex() -> parser::symbol_type
auto& last = reader_.last_byte; auto& last = reader_.last_byte;
auto& curr = reader_.current_byte; auto& curr = reader_.current_byte;
auto path = false; auto path = false;
loc_.step();
if (state == reader::end) if (state == reader::end)
{ {
@ -216,7 +217,7 @@ auto lexer::lex() -> parser::symbol_type
case '\\': case '\\':
throw comp_error(loc_, "invalid token ('\\')"); throw comp_error(loc_, "invalid token ('\\')");
case '/': case '/':
if (curr != '/' && curr != '*' && curr != '#' && curr != '=') if (curr != '=' && curr != '#' && curr != '@' && curr != '*' && curr != '/')
return parser::make_DIV(loc_); return parser::make_DIV(loc_);
advance(); advance();
@ -248,14 +249,35 @@ auto lexer::lex() -> parser::symbol_type
} }
else if (last == '#' && curr == '/') else if (last == '#' && curr == '/')
{ {
reader_.advance(); advance();
break; break;
} }
reader_.advance(); advance();
} }
} }
} }
else if (last == '@')
{
while (true)
{
if (state == reader::end)
throw comp_error(loc_, "unmatched script doc comment start ('/@')");
if (curr == '\n')
{
loc_.lines();
loc_.step();
}
else if (last == '@' && curr == '/')
{
advance();
break;
}
advance();
}
}
else if (last == '*') else if (last == '*')
{ {
while (true) while (true)
@ -270,11 +292,11 @@ auto lexer::lex() -> parser::symbol_type
} }
else if (last == '*' && curr == '/') else if (last == '*' && curr == '/')
{ {
reader_.advance(); advance();
break; break;
} }
reader_.advance(); advance();
} }
} }
else if (last == '/') else if (last == '/')
@ -284,30 +306,10 @@ auto lexer::lex() -> parser::symbol_type
if (state == reader::end) if (state == reader::end)
break; break;
if (last == '\\' && (curr == '\r' || curr == '\n'))
{
reader_.advance();
if (state == reader::end)
break;
if (last == '\r')
{
if (curr != '\n')
throw comp_error(loc_, "invalid token ('\')");
reader_.advance();
}
loc_.lines();
loc_.step();
continue;
}
if (curr == '\n') if (curr == '\n')
break; break;
reader_.advance(); advance();
} }
} }
continue; continue;
@ -317,8 +319,8 @@ auto lexer::lex() -> parser::symbol_type
if (!indev_) if (!indev_)
throw comp_error(loc_, "unmatched devblock end ('#/')"); throw comp_error(loc_, "unmatched devblock end ('#/')");
indev_ = false;
advance(); advance();
indev_ = false;
return parser::make_DEVEND(loc_); return parser::make_DEVEND(loc_);
} }
@ -339,7 +341,7 @@ auto lexer::lex() -> parser::symbol_type
state_ = state::preprocessor; state_ = state::preprocessor;
goto lex_name; goto lex_name;
case '*': case '*':
if (curr != '/' && curr != '=') if (curr != '=' && curr != '/')
return parser::make_MUL(loc_); return parser::make_MUL(loc_);
advance(); advance();
@ -352,13 +354,9 @@ auto lexer::lex() -> parser::symbol_type
state_ = state::string; state_ = state::string;
goto lex_string; goto lex_string;
case '.': case '.':
advance(); if (curr < '0' || curr > '9')
return parser::make_DOT(loc_);
if (state == reader::end) goto lex_number;
throw comp_error(loc_, "unterminated field ('.')");
state_ = state::field;
goto lex_name_or_number;
case '(': case '(':
return parser::make_LPAREN(loc_); return parser::make_LPAREN(loc_);
case ')': case ')':
@ -482,7 +480,6 @@ auto lexer::lex() -> parser::symbol_type
advance(); advance();
return parser::make_ASSIGN_RSHIFT(loc_); return parser::make_ASSIGN_RSHIFT(loc_);
default: default:
lex_name_or_number:
if (last >= '0' && last <= '9') if (last >= '0' && last <= '9')
goto lex_number; goto lex_number;
else if (last == '_' || last >= 'A' && last <= 'Z' || last >= 'a' && last <= 'z') else if (last == '_' || last >= 'A' && last <= 'Z' || last >= 'a' && last <= 'z')
@ -564,23 +561,8 @@ lex_name:
advance(); advance();
} }
if (state_ == state::field) if (state_ == state::preprocessor)
{ {
if (path)
throw comp_error(loc_, "invalid field token '\\'");
if (std::string_view(buffer_.data, buffer_.length) == "size")
{
return parser::make_SIZE(loc_);
}
return parser::make_FIELD(std::string(buffer_.data, buffer_.length), loc_);
}
else if (state_ == state::preprocessor)
{
if (path)
throw comp_error(loc_, "invalid preprocessor directive");
auto token = parser::token::IW8UNDEF; auto token = parser::token::IW8UNDEF;
if (buffer_.length < 16) if (buffer_.length < 16)
@ -596,7 +578,8 @@ lex_name:
} }
} }
preprocessor(token); preprocessor_run(token);
state_ = state::start; state_ = state::start;
continue; continue;
} }
@ -633,14 +616,11 @@ lex_name:
} }
lex_number: lex_number:
if (state_ == state::field) if (last == '.' || last != '0' || (last == '0' && (curr != 'o' && curr != 'b' && curr != 'x')))
buffer_.push('.');
if (state_ == state::field || last == '.' || last != '0' || (last == '0' && (curr != 'o' && curr != 'b' && curr != 'x')))
{ {
buffer_.push(last); buffer_.push(last);
auto dot = 0; auto dot = last == '.' ? 1 : 0;
auto flt = 0; auto flt = 0;
while (true) while (true)
@ -676,10 +656,10 @@ lex_number:
if (last == '\'') if (last == '\'')
throw comp_error(loc_, "invalid number literal"); throw comp_error(loc_, "invalid number literal");
if (state_ == state::field && dot || dot > 1 || flt > 1 || flt && buffer_.data[buffer_.length - 1] != 'f') if (dot > 1 || flt > 1 || flt && buffer_.data[buffer_.length - 1] != 'f')
throw comp_error(loc_, "invalid number literal"); throw comp_error(loc_, "invalid number literal");
if (state_ == state::field || dot || flt) if (dot || flt)
return parser::make_FLOAT(std::string(buffer_.data, buffer_.length), loc_); return parser::make_FLOAT(std::string(buffer_.data, buffer_.length), loc_);
return parser::make_INTEGER(std::string(buffer_.data, buffer_.length), loc_); return parser::make_INTEGER(std::string(buffer_.data, buffer_.length), loc_);
@ -784,18 +764,22 @@ lex_number:
return parser::make_INTEGER(xsk::utils::string::hex_to_dec(buffer_.data), loc_); return parser::make_INTEGER(xsk::utils::string::hex_to_dec(buffer_.data), loc_);
} }
// cant get here!
throw error("UNEXPECTED LEXER INTERNAL ERROR!");
} }
} }
void lexer::advance() void lexer::advance()
{ {
reader_.advance(); reader_.advance();
loc_.end.column++;
// dont wrap comment marks '/\/' '/\*' outside strings if (reader_.current_byte == '\\') [[unlikely]]
if (state_ == state::start && reader_.last_byte == '/') preprocessor_wrap();
return; }
void lexer::preprocessor_wrap()
{
while (reader_.current_byte == '\\') while (reader_.current_byte == '\\')
{ {
if (reader_.bytes_remaining == 1) if (reader_.bytes_remaining == 1)
@ -837,7 +821,7 @@ void lexer::advance()
} }
} }
void lexer::preprocessor(parser::token::token_kind_type token) void lexer::preprocessor_run(parser::token::token_kind_type token)
{ {
if (!clean_) if (!clean_)
throw comp_error(loc_, "invalid token ('#')"); throw comp_error(loc_, "invalid token ('#')");

5
src/iw8/xsk/lexer.hpp
View File

@ -44,7 +44,7 @@ struct reader
class lexer class lexer
{ {
enum class state : std::uint8_t { start, string, localize, field, preprocessor }; enum class state : std::uint8_t { start, string, localize, preprocessor };
reader reader_; reader reader_;
buffer buffer_; buffer buffer_;
@ -66,7 +66,8 @@ public:
private: private:
void advance(); void advance();
void preprocessor(parser::token::token_kind_type token); void preprocessor_wrap();
void preprocessor_run(parser::token::token_kind_type token);
}; };
} // namespace xsk::gsc::iw8 } // namespace xsk::gsc::iw8

1815
src/iw8/xsk/parser.cpp
View File

File diff suppressed because it is too large Load Diff

103
src/iw8/xsk/parser.hpp
View File

@ -474,6 +474,7 @@ namespace xsk { namespace gsc { namespace iw8 {
// expr_game // expr_game
char dummy19[sizeof (ast::expr_game::ptr)]; char dummy19[sizeof (ast::expr_game::ptr)];
// expr_identifier_nosize
// expr_identifier // expr_identifier
char dummy20[sizeof (ast::expr_identifier::ptr)]; char dummy20[sizeof (ast::expr_identifier::ptr)];
@ -624,7 +625,6 @@ namespace xsk { namespace gsc { namespace iw8 {
// stmt_while // stmt_while
char dummy68[sizeof (ast::stmt_while::ptr)]; char dummy68[sizeof (ast::stmt_while::ptr)];
// "field"
// "path" // "path"
// "identifier" // "identifier"
// "string literal" // "string literal"
@ -727,7 +727,7 @@ namespace xsk { namespace gsc { namespace iw8 {
TRUE = 44, // "true" TRUE = 44, // "true"
FALSE = 45, // "false" FALSE = 45, // "false"
UNDEFINED = 46, // "undefined" UNDEFINED = 46, // "undefined"
SIZE = 47, // ".size" SIZE = 47, // "size"
GAME = 48, // "game" GAME = 48, // "game"
SELF = 49, // "self" SELF = 49, // "self"
ANIM = 50, // "anim" ANIM = 50, // "anim"
@ -779,13 +779,13 @@ namespace xsk { namespace gsc { namespace iw8 {
MUL = 96, // "*" MUL = 96, // "*"
DIV = 97, // "/" DIV = 97, // "/"
MOD = 98, // "%" MOD = 98, // "%"
FIELD = 99, // "field" PATH = 99, // "path"
PATH = 100, // "path" IDENTIFIER = 100, // "identifier"
IDENTIFIER = 101, // "identifier" STRING = 101, // "string literal"
STRING = 102, // "string literal" ISTRING = 102, // "localized string"
ISTRING = 103, // "localized string" FLOAT = 103, // "float"
FLOAT = 104, // "float" INTEGER = 104, // "integer"
INTEGER = 105, // "integer" SIZEOF = 105, // SIZEOF
ADD_ARRAY = 106, // ADD_ARRAY ADD_ARRAY = 106, // ADD_ARRAY
THEN = 107, // THEN THEN = 107, // THEN
TERN = 108, // TERN TERN = 108, // TERN
@ -860,7 +860,7 @@ namespace xsk { namespace gsc { namespace iw8 {
S_TRUE = 44, // "true" S_TRUE = 44, // "true"
S_FALSE = 45, // "false" S_FALSE = 45, // "false"
S_UNDEFINED = 46, // "undefined" S_UNDEFINED = 46, // "undefined"
S_SIZE = 47, // ".size" S_SIZE = 47, // "size"
S_GAME = 48, // "game" S_GAME = 48, // "game"
S_SELF = 49, // "self" S_SELF = 49, // "self"
S_ANIM = 50, // "anim" S_ANIM = 50, // "anim"
@ -912,13 +912,13 @@ namespace xsk { namespace gsc { namespace iw8 {
S_MUL = 96, // "*" S_MUL = 96, // "*"
S_DIV = 97, // "/" S_DIV = 97, // "/"
S_MOD = 98, // "%" S_MOD = 98, // "%"
S_FIELD = 99, // "field" S_PATH = 99, // "path"
S_PATH = 100, // "path" S_IDENTIFIER = 100, // "identifier"
S_IDENTIFIER = 101, // "identifier" S_STRING = 101, // "string literal"
S_STRING = 102, // "string literal" S_ISTRING = 102, // "localized string"
S_ISTRING = 103, // "localized string" S_FLOAT = 103, // "float"
S_FLOAT = 104, // "float" S_INTEGER = 104, // "integer"
S_INTEGER = 105, // "integer" S_SIZEOF = 105, // SIZEOF
S_ADD_ARRAY = 106, // ADD_ARRAY S_ADD_ARRAY = 106, // ADD_ARRAY
S_THEN = 107, // THEN S_THEN = 107, // THEN
S_TERN = 108, // TERN S_TERN = 108, // TERN
@ -1003,15 +1003,16 @@ namespace xsk { namespace gsc { namespace iw8 {
S_expr_level = 187, // expr_level S_expr_level = 187, // expr_level
S_expr_animation = 188, // expr_animation S_expr_animation = 188, // expr_animation
S_expr_animtree = 189, // expr_animtree S_expr_animtree = 189, // expr_animtree
S_expr_identifier = 190, // expr_identifier S_expr_identifier_nosize = 190, // expr_identifier_nosize
S_expr_path = 191, // expr_path S_expr_identifier = 191, // expr_identifier
S_expr_istring = 192, // expr_istring S_expr_path = 192, // expr_path
S_expr_string = 193, // expr_string S_expr_istring = 193, // expr_istring
S_expr_vector = 194, // expr_vector S_expr_string = 194, // expr_string
S_expr_float = 195, // expr_float S_expr_vector = 195, // expr_vector
S_expr_integer = 196, // expr_integer S_expr_float = 196, // expr_float
S_expr_false = 197, // expr_false S_expr_integer = 197, // expr_integer
S_expr_true = 198 // expr_true S_expr_false = 198, // expr_false
S_expr_true = 199 // expr_true
}; };
}; };
@ -1134,6 +1135,7 @@ namespace xsk { namespace gsc { namespace iw8 {
value.move< ast::expr_game::ptr > (std::move (that.value)); value.move< ast::expr_game::ptr > (std::move (that.value));
break; break;
case symbol_kind::S_expr_identifier_nosize: // expr_identifier_nosize
case symbol_kind::S_expr_identifier: // expr_identifier case symbol_kind::S_expr_identifier: // expr_identifier
value.move< ast::expr_identifier::ptr > (std::move (that.value)); value.move< ast::expr_identifier::ptr > (std::move (that.value));
break; break;
@ -1333,7 +1335,6 @@ namespace xsk { namespace gsc { namespace iw8 {
value.move< ast::stmt_while::ptr > (std::move (that.value)); value.move< ast::stmt_while::ptr > (std::move (that.value));
break; break;
case symbol_kind::S_FIELD: // "field"
case symbol_kind::S_PATH: // "path" case symbol_kind::S_PATH: // "path"
case symbol_kind::S_IDENTIFIER: // "identifier" case symbol_kind::S_IDENTIFIER: // "identifier"
case symbol_kind::S_STRING: // "string literal" case symbol_kind::S_STRING: // "string literal"
@ -2440,6 +2441,7 @@ switch (yykind)
value.template destroy< ast::expr_game::ptr > (); value.template destroy< ast::expr_game::ptr > ();
break; break;
case symbol_kind::S_expr_identifier_nosize: // expr_identifier_nosize
case symbol_kind::S_expr_identifier: // expr_identifier case symbol_kind::S_expr_identifier: // expr_identifier
value.template destroy< ast::expr_identifier::ptr > (); value.template destroy< ast::expr_identifier::ptr > ();
break; break;
@ -2639,7 +2641,6 @@ switch (yykind)
value.template destroy< ast::stmt_while::ptr > (); value.template destroy< ast::stmt_while::ptr > ();
break; break;
case symbol_kind::S_FIELD: // "field"
case symbol_kind::S_PATH: // "path" case symbol_kind::S_PATH: // "path"
case symbol_kind::S_IDENTIFIER: // "identifier" case symbol_kind::S_IDENTIFIER: // "identifier"
case symbol_kind::S_STRING: // "string literal" case symbol_kind::S_STRING: // "string literal"
@ -2745,7 +2746,7 @@ switch (yykind)
{ {
IW8_ASSERT (tok == token::IW8EOF IW8_ASSERT (tok == token::IW8EOF
|| (token::IW8error <= tok && tok <= token::MOD) || (token::IW8error <= tok && tok <= token::MOD)
|| (token::ADD_ARRAY <= tok && tok <= token::POSTDEC)); || (token::SIZEOF <= tok && tok <= token::POSTDEC));
} }
#if 201103L <= YY_CPLUSPLUS #if 201103L <= YY_CPLUSPLUS
symbol_type (int tok, std::string v, location_type l) symbol_type (int tok, std::string v, location_type l)
@ -2755,7 +2756,7 @@ switch (yykind)
: super_type(token_type (tok), v, l) : super_type(token_type (tok), v, l)
#endif #endif
{ {
IW8_ASSERT ((token::FIELD <= tok && tok <= token::INTEGER)); IW8_ASSERT ((token::PATH <= tok && tok <= token::INTEGER));
} }
}; };
@ -4290,21 +4291,6 @@ switch (yykind)
return symbol_type (token::MOD, l); return symbol_type (token::MOD, l);
} }
#endif #endif
#if 201103L <= YY_CPLUSPLUS
static
symbol_type
make_FIELD (std::string v, location_type l)
{
return symbol_type (token::FIELD, std::move (v), std::move (l));
}
#else
static
symbol_type
make_FIELD (const std::string& v, const location_type& l)
{
return symbol_type (token::FIELD, v, l);
}
#endif
#if 201103L <= YY_CPLUSPLUS #if 201103L <= YY_CPLUSPLUS
static static
symbol_type symbol_type
@ -4395,6 +4381,21 @@ switch (yykind)
return symbol_type (token::INTEGER, v, l); return symbol_type (token::INTEGER, v, l);
} }
#endif #endif
#if 201103L <= YY_CPLUSPLUS
static
symbol_type
make_SIZEOF (location_type l)
{
return symbol_type (token::SIZEOF, std::move (l));
}
#else
static
symbol_type
make_SIZEOF (const location_type& l)
{
return symbol_type (token::SIZEOF, l);
}
#endif
#if 201103L <= YY_CPLUSPLUS #if 201103L <= YY_CPLUSPLUS
static static
symbol_type symbol_type
@ -4874,9 +4875,9 @@ switch (yykind)
/// Constants. /// Constants.
enum enum
{ {
yylast_ = 2404, ///< Last index in yytable_. yylast_ = 2278, ///< Last index in yytable_.
yynnts_ = 84, ///< Number of nonterminal symbols. yynnts_ = 85, ///< Number of nonterminal symbols.
yyfinal_ = 21 ///< Termination state number. yyfinal_ = 22 ///< Termination state number.
}; };
@ -4988,6 +4989,7 @@ switch (yykind)
value.copy< ast::expr_game::ptr > (YY_MOVE (that.value)); value.copy< ast::expr_game::ptr > (YY_MOVE (that.value));
break; break;
case symbol_kind::S_expr_identifier_nosize: // expr_identifier_nosize
case symbol_kind::S_expr_identifier: // expr_identifier case symbol_kind::S_expr_identifier: // expr_identifier
value.copy< ast::expr_identifier::ptr > (YY_MOVE (that.value)); value.copy< ast::expr_identifier::ptr > (YY_MOVE (that.value));
break; break;
@ -5187,7 +5189,6 @@ switch (yykind)
value.copy< ast::stmt_while::ptr > (YY_MOVE (that.value)); value.copy< ast::stmt_while::ptr > (YY_MOVE (that.value));
break; break;
case symbol_kind::S_FIELD: // "field"
case symbol_kind::S_PATH: // "path" case symbol_kind::S_PATH: // "path"
case symbol_kind::S_IDENTIFIER: // "identifier" case symbol_kind::S_IDENTIFIER: // "identifier"
case symbol_kind::S_STRING: // "string literal" case symbol_kind::S_STRING: // "string literal"
@ -5312,6 +5313,7 @@ switch (yykind)
value.move< ast::expr_game::ptr > (YY_MOVE (s.value)); value.move< ast::expr_game::ptr > (YY_MOVE (s.value));
break; break;
case symbol_kind::S_expr_identifier_nosize: // expr_identifier_nosize
case symbol_kind::S_expr_identifier: // expr_identifier case symbol_kind::S_expr_identifier: // expr_identifier
value.move< ast::expr_identifier::ptr > (YY_MOVE (s.value)); value.move< ast::expr_identifier::ptr > (YY_MOVE (s.value));
break; break;
@ -5511,7 +5513,6 @@ switch (yykind)
value.move< ast::stmt_while::ptr > (YY_MOVE (s.value)); value.move< ast::stmt_while::ptr > (YY_MOVE (s.value));
break; break;
case symbol_kind::S_FIELD: // "field"
case symbol_kind::S_PATH: // "path" case symbol_kind::S_PATH: // "path"
case symbol_kind::S_IDENTIFIER: // "identifier" case symbol_kind::S_IDENTIFIER: // "identifier"
case symbol_kind::S_STRING: // "string literal" case symbol_kind::S_STRING: // "string literal"
@ -5584,7 +5585,7 @@ switch (yykind)
#line 13 "parser.ypp" #line 13 "parser.ypp"
} } } // xsk::gsc::iw8 } } } // xsk::gsc::iw8
#line 5588 "parser.hpp" #line 5589 "parser.hpp"

120
src/s1/xsk/lexer.cpp
View File

@ -60,6 +60,7 @@ const std::unordered_map<std::string_view, parser::token::token_kind_type> keywo
{ "true", parser::token::TRUE }, { "true", parser::token::TRUE },
{ "false", parser::token::FALSE }, { "false", parser::token::FALSE },
{ "undefined", parser::token::UNDEFINED }, { "undefined", parser::token::UNDEFINED },
{ "size", parser::token::SIZE },
{ "game", parser::token::GAME }, { "game", parser::token::GAME },
{ "self", parser::token::SELF }, { "self", parser::token::SELF },
{ "anim", parser::token::ANIM }, { "anim", parser::token::ANIM },
@ -85,8 +86,8 @@ bool buffer::push(char c)
return true; return true;
} }
reader::reader() : state(reader::end), buffer_pos(0), bytes_remaining(0), reader::reader() : state(reader::end), buffer_pos(0),
last_byte(0), current_byte(0) { } bytes_remaining(0), last_byte(0), current_byte(0) {}
void reader::init(const char* data, size_t size) void reader::init(const char* data, size_t size)
{ {
@ -174,7 +175,6 @@ auto lexer::lex() -> parser::symbol_type
{ {
buffer_.length = 0; buffer_.length = 0;
state_ = state::start; state_ = state::start;
loc_.step();
while (true) while (true)
{ {
@ -182,6 +182,7 @@ auto lexer::lex() -> parser::symbol_type
auto& last = reader_.last_byte; auto& last = reader_.last_byte;
auto& curr = reader_.current_byte; auto& curr = reader_.current_byte;
auto path = false; auto path = false;
loc_.step();
if (state == reader::end) if (state == reader::end)
{ {
@ -214,7 +215,7 @@ auto lexer::lex() -> parser::symbol_type
case '\\': case '\\':
throw comp_error(loc_, "invalid token ('\\')"); throw comp_error(loc_, "invalid token ('\\')");
case '/': case '/':
if (curr != '/' && curr != '*' && curr != '#' && curr != '=') if (curr != '=' && curr != '#' && curr != '@' && curr != '*' && curr != '/')
return parser::make_DIV(loc_); return parser::make_DIV(loc_);
advance(); advance();
@ -246,14 +247,35 @@ auto lexer::lex() -> parser::symbol_type
} }
else if (last == '#' && curr == '/') else if (last == '#' && curr == '/')
{ {
reader_.advance(); advance();
break; break;
} }
reader_.advance(); advance();
} }
} }
} }
else if (last == '@')
{
while (true)
{
if (state == reader::end)
throw comp_error(loc_, "unmatched script doc comment start ('/@')");
if (curr == '\n')
{
loc_.lines();
loc_.step();
}
else if (last == '@' && curr == '/')
{
advance();
break;
}
advance();
}
}
else if (last == '*') else if (last == '*')
{ {
while (true) while (true)
@ -268,11 +290,11 @@ auto lexer::lex() -> parser::symbol_type
} }
else if (last == '*' && curr == '/') else if (last == '*' && curr == '/')
{ {
reader_.advance(); advance();
break; break;
} }
reader_.advance(); advance();
} }
} }
else if (last == '/') else if (last == '/')
@ -282,30 +304,10 @@ auto lexer::lex() -> parser::symbol_type
if (state == reader::end) if (state == reader::end)
break; break;
if (last == '\\' && (curr == '\r' || curr == '\n'))
{
reader_.advance();
if (state == reader::end)
break;
if (last == '\r')
{
if (curr != '\n')
throw comp_error(loc_, "invalid token ('\')");
reader_.advance();
}
loc_.lines();
loc_.step();
continue;
}
if (curr == '\n') if (curr == '\n')
break; break;
reader_.advance(); advance();
} }
} }
continue; continue;
@ -315,8 +317,8 @@ auto lexer::lex() -> parser::symbol_type
if (!indev_) if (!indev_)
throw comp_error(loc_, "unmatched devblock end ('#/')"); throw comp_error(loc_, "unmatched devblock end ('#/')");
indev_ = false;
advance(); advance();
indev_ = false;
return parser::make_DEVEND(loc_); return parser::make_DEVEND(loc_);
} }
@ -337,7 +339,7 @@ auto lexer::lex() -> parser::symbol_type
state_ = state::preprocessor; state_ = state::preprocessor;
goto lex_name; goto lex_name;
case '*': case '*':
if (curr != '/' && curr != '=') if (curr != '=' && curr != '/')
return parser::make_MUL(loc_); return parser::make_MUL(loc_);
advance(); advance();
@ -350,13 +352,9 @@ auto lexer::lex() -> parser::symbol_type
state_ = state::string; state_ = state::string;
goto lex_string; goto lex_string;
case '.': case '.':
advance(); if (curr < '0' || curr > '9')
return parser::make_DOT(loc_);
if (state == reader::end) goto lex_number;
throw comp_error(loc_, "unterminated field ('.')");
state_ = state::field;
goto lex_name_or_number;
case '(': case '(':
return parser::make_LPAREN(loc_); return parser::make_LPAREN(loc_);
case ')': case ')':
@ -480,7 +478,6 @@ auto lexer::lex() -> parser::symbol_type
advance(); advance();
return parser::make_ASSIGN_RSHIFT(loc_); return parser::make_ASSIGN_RSHIFT(loc_);
default: default:
lex_name_or_number:
if (last >= '0' && last <= '9') if (last >= '0' && last <= '9')
goto lex_number; goto lex_number;
else if (last == '_' || last >= 'A' && last <= 'Z' || last >= 'a' && last <= 'z') else if (last == '_' || last >= 'A' && last <= 'Z' || last >= 'a' && last <= 'z')
@ -562,23 +559,8 @@ lex_name:
advance(); advance();
} }
if (state_ == state::field) if (state_ == state::preprocessor)
{ {
if (path)
throw comp_error(loc_, "invalid field token '\\'");
if (std::string_view(buffer_.data, buffer_.length) == "size")
{
return parser::make_SIZE(loc_);
}
return parser::make_FIELD(std::string(buffer_.data, buffer_.length), loc_);
}
else if (state_ == state::preprocessor)
{
if (path)
throw comp_error(loc_, "invalid preprocessor directive");
auto token = parser::token::S1UNDEF; auto token = parser::token::S1UNDEF;
if (buffer_.length < 16) if (buffer_.length < 16)
@ -594,7 +576,8 @@ lex_name:
} }
} }
preprocessor(token); preprocessor_run(token);
state_ = state::start; state_ = state::start;
continue; continue;
} }
@ -620,14 +603,11 @@ lex_name:
} }
lex_number: lex_number:
if (state_ == state::field) if (last == '.' || last != '0' || (last == '0' && (curr != 'o' && curr != 'b' && curr != 'x')))
buffer_.push('.');
if (state_ == state::field || last == '.' || last != '0' || (last == '0' && (curr != 'o' && curr != 'b' && curr != 'x')))
{ {
buffer_.push(last); buffer_.push(last);
auto dot = 0; auto dot = last == '.' ? 1 : 0;
auto flt = 0; auto flt = 0;
while (true) while (true)
@ -663,10 +643,10 @@ lex_number:
if (last == '\'') if (last == '\'')
throw comp_error(loc_, "invalid number literal"); throw comp_error(loc_, "invalid number literal");
if (state_ == state::field && dot || dot > 1 || flt > 1 || flt && buffer_.data[buffer_.length - 1] != 'f') if (dot > 1 || flt > 1 || flt && buffer_.data[buffer_.length - 1] != 'f')
throw comp_error(loc_, "invalid number literal"); throw comp_error(loc_, "invalid number literal");
if (state_ == state::field || dot || flt) if (dot || flt)
return parser::make_FLOAT(std::string(buffer_.data, buffer_.length), loc_); return parser::make_FLOAT(std::string(buffer_.data, buffer_.length), loc_);
return parser::make_INTEGER(std::string(buffer_.data, buffer_.length), loc_); return parser::make_INTEGER(std::string(buffer_.data, buffer_.length), loc_);
@ -771,18 +751,22 @@ lex_number:
return parser::make_INTEGER(xsk::utils::string::hex_to_dec(buffer_.data), loc_); return parser::make_INTEGER(xsk::utils::string::hex_to_dec(buffer_.data), loc_);
} }
// cant get here!
throw error("UNEXPECTED LEXER INTERNAL ERROR!");
} }
} }
void lexer::advance() void lexer::advance()
{ {
reader_.advance(); reader_.advance();
loc_.end.column++;
// dont wrap comment marks '/\/' '/\*' outside strings if (reader_.current_byte == '\\') [[unlikely]]
if (state_ == state::start && reader_.last_byte == '/') preprocessor_wrap();
return; }
void lexer::preprocessor_wrap()
{
while (reader_.current_byte == '\\') while (reader_.current_byte == '\\')
{ {
if (reader_.bytes_remaining == 1) if (reader_.bytes_remaining == 1)
@ -824,7 +808,7 @@ void lexer::advance()
} }
} }
void lexer::preprocessor(parser::token::token_kind_type token) void lexer::preprocessor_run(parser::token::token_kind_type token)
{ {
if (!clean_) if (!clean_)
throw comp_error(loc_, "invalid token ('#')"); throw comp_error(loc_, "invalid token ('#')");

5
src/s1/xsk/lexer.hpp
View File

@ -44,7 +44,7 @@ struct reader
class lexer class lexer
{ {
enum class state : std::uint8_t { start, string, localize, field, preprocessor }; enum class state : std::uint8_t { start, string, localize, preprocessor };
reader reader_; reader reader_;
buffer buffer_; buffer buffer_;
@ -66,7 +66,8 @@ public:
private: private:
void advance(); void advance();
void preprocessor(parser::token::token_kind_type token); void preprocessor_wrap();
void preprocessor_run(parser::token::token_kind_type token);
}; };
} // namespace xsk::gsc::s1 } // namespace xsk::gsc::s1

1740
src/s1/xsk/parser.cpp
View File

File diff suppressed because it is too large Load Diff

103
src/s1/xsk/parser.hpp
View File

@ -474,6 +474,7 @@ namespace xsk { namespace gsc { namespace s1 {
// expr_game // expr_game
char dummy19[sizeof (ast::expr_game::ptr)]; char dummy19[sizeof (ast::expr_game::ptr)];
// expr_identifier_nosize
// expr_identifier // expr_identifier
char dummy20[sizeof (ast::expr_identifier::ptr)]; char dummy20[sizeof (ast::expr_identifier::ptr)];
@ -618,7 +619,6 @@ namespace xsk { namespace gsc { namespace s1 {
// stmt_while // stmt_while
char dummy66[sizeof (ast::stmt_while::ptr)]; char dummy66[sizeof (ast::stmt_while::ptr)];
// "field"
// "path" // "path"
// "identifier" // "identifier"
// "string literal" // "string literal"
@ -721,7 +721,7 @@ namespace xsk { namespace gsc { namespace s1 {
TRUE = 44, // "true" TRUE = 44, // "true"
FALSE = 45, // "false" FALSE = 45, // "false"
UNDEFINED = 46, // "undefined" UNDEFINED = 46, // "undefined"
SIZE = 47, // ".size" SIZE = 47, // "size"
GAME = 48, // "game" GAME = 48, // "game"
SELF = 49, // "self" SELF = 49, // "self"
ANIM = 50, // "anim" ANIM = 50, // "anim"
@ -771,13 +771,13 @@ namespace xsk { namespace gsc { namespace s1 {
MUL = 94, // "*" MUL = 94, // "*"
DIV = 95, // "/" DIV = 95, // "/"
MOD = 96, // "%" MOD = 96, // "%"
FIELD = 97, // "field" PATH = 97, // "path"
PATH = 98, // "path" IDENTIFIER = 98, // "identifier"
IDENTIFIER = 99, // "identifier" STRING = 99, // "string literal"
STRING = 100, // "string literal" ISTRING = 100, // "localized string"
ISTRING = 101, // "localized string" FLOAT = 101, // "float"
FLOAT = 102, // "float" INTEGER = 102, // "integer"
INTEGER = 103, // "integer" SIZEOF = 103, // SIZEOF
ADD_ARRAY = 104, // ADD_ARRAY ADD_ARRAY = 104, // ADD_ARRAY
THEN = 105, // THEN THEN = 105, // THEN
TERN = 106, // TERN TERN = 106, // TERN
@ -852,7 +852,7 @@ namespace xsk { namespace gsc { namespace s1 {
S_TRUE = 44, // "true" S_TRUE = 44, // "true"
S_FALSE = 45, // "false" S_FALSE = 45, // "false"
S_UNDEFINED = 46, // "undefined" S_UNDEFINED = 46, // "undefined"
S_SIZE = 47, // ".size" S_SIZE = 47, // "size"
S_GAME = 48, // "game" S_GAME = 48, // "game"
S_SELF = 49, // "self" S_SELF = 49, // "self"
S_ANIM = 50, // "anim" S_ANIM = 50, // "anim"
@ -902,13 +902,13 @@ namespace xsk { namespace gsc { namespace s1 {
S_MUL = 94, // "*" S_MUL = 94, // "*"
S_DIV = 95, // "/" S_DIV = 95, // "/"
S_MOD = 96, // "%" S_MOD = 96, // "%"
S_FIELD = 97, // "field" S_PATH = 97, // "path"
S_PATH = 98, // "path" S_IDENTIFIER = 98, // "identifier"
S_IDENTIFIER = 99, // "identifier" S_STRING = 99, // "string literal"
S_STRING = 100, // "string literal" S_ISTRING = 100, // "localized string"
S_ISTRING = 101, // "localized string" S_FLOAT = 101, // "float"
S_FLOAT = 102, // "float" S_INTEGER = 102, // "integer"
S_INTEGER = 103, // "integer" S_SIZEOF = 103, // SIZEOF
S_ADD_ARRAY = 104, // ADD_ARRAY S_ADD_ARRAY = 104, // ADD_ARRAY
S_THEN = 105, // THEN S_THEN = 105, // THEN
S_TERN = 106, // TERN S_TERN = 106, // TERN
@ -991,15 +991,16 @@ namespace xsk { namespace gsc { namespace s1 {
S_expr_level = 183, // expr_level S_expr_level = 183, // expr_level
S_expr_animation = 184, // expr_animation S_expr_animation = 184, // expr_animation
S_expr_animtree = 185, // expr_animtree S_expr_animtree = 185, // expr_animtree
S_expr_identifier = 186, // expr_identifier S_expr_identifier_nosize = 186, // expr_identifier_nosize
S_expr_path = 187, // expr_path S_expr_identifier = 187, // expr_identifier
S_expr_istring = 188, // expr_istring S_expr_path = 188, // expr_path
S_expr_string = 189, // expr_string S_expr_istring = 189, // expr_istring
S_expr_vector = 190, // expr_vector S_expr_string = 190, // expr_string
S_expr_float = 191, // expr_float S_expr_vector = 191, // expr_vector
S_expr_integer = 192, // expr_integer S_expr_float = 192, // expr_float
S_expr_false = 193, // expr_false S_expr_integer = 193, // expr_integer
S_expr_true = 194 // expr_true S_expr_false = 194, // expr_false
S_expr_true = 195 // expr_true
}; };
}; };
@ -1122,6 +1123,7 @@ namespace xsk { namespace gsc { namespace s1 {
value.move< ast::expr_game::ptr > (std::move (that.value)); value.move< ast::expr_game::ptr > (std::move (that.value));
break; break;
case symbol_kind::S_expr_identifier_nosize: // expr_identifier_nosize
case symbol_kind::S_expr_identifier: // expr_identifier case symbol_kind::S_expr_identifier: // expr_identifier
value.move< ast::expr_identifier::ptr > (std::move (that.value)); value.move< ast::expr_identifier::ptr > (std::move (that.value));
break; break;
@ -1313,7 +1315,6 @@ namespace xsk { namespace gsc { namespace s1 {
value.move< ast::stmt_while::ptr > (std::move (that.value)); value.move< ast::stmt_while::ptr > (std::move (that.value));
break; break;
case symbol_kind::S_FIELD: // "field"
case symbol_kind::S_PATH: // "path" case symbol_kind::S_PATH: // "path"
case symbol_kind::S_IDENTIFIER: // "identifier" case symbol_kind::S_IDENTIFIER: // "identifier"
case symbol_kind::S_STRING: // "string literal" case symbol_kind::S_STRING: // "string literal"
@ -2392,6 +2393,7 @@ switch (yykind)
value.template destroy< ast::expr_game::ptr > (); value.template destroy< ast::expr_game::ptr > ();
break; break;
case symbol_kind::S_expr_identifier_nosize: // expr_identifier_nosize
case symbol_kind::S_expr_identifier: // expr_identifier case symbol_kind::S_expr_identifier: // expr_identifier
value.template destroy< ast::expr_identifier::ptr > (); value.template destroy< ast::expr_identifier::ptr > ();
break; break;
@ -2583,7 +2585,6 @@ switch (yykind)
value.template destroy< ast::stmt_while::ptr > (); value.template destroy< ast::stmt_while::ptr > ();
break; break;
case symbol_kind::S_FIELD: // "field"
case symbol_kind::S_PATH: // "path" case symbol_kind::S_PATH: // "path"
case symbol_kind::S_IDENTIFIER: // "identifier" case symbol_kind::S_IDENTIFIER: // "identifier"
case symbol_kind::S_STRING: // "string literal" case symbol_kind::S_STRING: // "string literal"
@ -2689,7 +2690,7 @@ switch (yykind)
{ {
S1_ASSERT (tok == token::S1EOF S1_ASSERT (tok == token::S1EOF
|| (token::S1error <= tok && tok <= token::MOD) || (token::S1error <= tok && tok <= token::MOD)
|| (token::ADD_ARRAY <= tok && tok <= token::POSTDEC)); || (token::SIZEOF <= tok && tok <= token::POSTDEC));
} }
#if 201103L <= YY_CPLUSPLUS #if 201103L <= YY_CPLUSPLUS
symbol_type (int tok, std::string v, location_type l) symbol_type (int tok, std::string v, location_type l)
@ -2699,7 +2700,7 @@ switch (yykind)
: super_type(token_type (tok), v, l) : super_type(token_type (tok), v, l)
#endif #endif
{ {
S1_ASSERT ((token::FIELD <= tok && tok <= token::INTEGER)); S1_ASSERT ((token::PATH <= tok && tok <= token::INTEGER));
} }
}; };
@ -4204,21 +4205,6 @@ switch (yykind)
return symbol_type (token::MOD, l); return symbol_type (token::MOD, l);
} }
#endif #endif
#if 201103L <= YY_CPLUSPLUS
static
symbol_type
make_FIELD (std::string v, location_type l)
{
return symbol_type (token::FIELD, std::move (v), std::move (l));
}
#else
static
symbol_type
make_FIELD (const std::string& v, const location_type& l)
{
return symbol_type (token::FIELD, v, l);
}
#endif
#if 201103L <= YY_CPLUSPLUS #if 201103L <= YY_CPLUSPLUS
static static
symbol_type symbol_type
@ -4309,6 +4295,21 @@ switch (yykind)
return symbol_type (token::INTEGER, v, l); return symbol_type (token::INTEGER, v, l);
} }
#endif #endif
#if 201103L <= YY_CPLUSPLUS
static
symbol_type
make_SIZEOF (location_type l)
{
return symbol_type (token::SIZEOF, std::move (l));
}
#else
static
symbol_type
make_SIZEOF (const location_type& l)
{
return symbol_type (token::SIZEOF, l);
}
#endif
#if 201103L <= YY_CPLUSPLUS #if 201103L <= YY_CPLUSPLUS
static static
symbol_type symbol_type
@ -4788,9 +4789,9 @@ switch (yykind)
/// Constants. /// Constants.
enum enum
{ {
yylast_ = 2300, ///< Last index in yytable_. yylast_ = 2251, ///< Last index in yytable_.
yynnts_ = 82, ///< Number of nonterminal symbols. yynnts_ = 83, ///< Number of nonterminal symbols.
yyfinal_ = 21 ///< Termination state number. yyfinal_ = 22 ///< Termination state number.
}; };
@ -4902,6 +4903,7 @@ switch (yykind)
value.copy< ast::expr_game::ptr > (YY_MOVE (that.value)); value.copy< ast::expr_game::ptr > (YY_MOVE (that.value));
break; break;
case symbol_kind::S_expr_identifier_nosize: // expr_identifier_nosize
case symbol_kind::S_expr_identifier: // expr_identifier case symbol_kind::S_expr_identifier: // expr_identifier
value.copy< ast::expr_identifier::ptr > (YY_MOVE (that.value)); value.copy< ast::expr_identifier::ptr > (YY_MOVE (that.value));
break; break;
@ -5093,7 +5095,6 @@ switch (yykind)
value.copy< ast::stmt_while::ptr > (YY_MOVE (that.value)); value.copy< ast::stmt_while::ptr > (YY_MOVE (that.value));
break; break;
case symbol_kind::S_FIELD: // "field"
case symbol_kind::S_PATH: // "path" case symbol_kind::S_PATH: // "path"
case symbol_kind::S_IDENTIFIER: // "identifier" case symbol_kind::S_IDENTIFIER: // "identifier"
case symbol_kind::S_STRING: // "string literal" case symbol_kind::S_STRING: // "string literal"
@ -5218,6 +5219,7 @@ switch (yykind)
value.move< ast::expr_game::ptr > (YY_MOVE (s.value)); value.move< ast::expr_game::ptr > (YY_MOVE (s.value));
break; break;
case symbol_kind::S_expr_identifier_nosize: // expr_identifier_nosize
case symbol_kind::S_expr_identifier: // expr_identifier case symbol_kind::S_expr_identifier: // expr_identifier
value.move< ast::expr_identifier::ptr > (YY_MOVE (s.value)); value.move< ast::expr_identifier::ptr > (YY_MOVE (s.value));
break; break;
@ -5409,7 +5411,6 @@ switch (yykind)
value.move< ast::stmt_while::ptr > (YY_MOVE (s.value)); value.move< ast::stmt_while::ptr > (YY_MOVE (s.value));
break; break;
case symbol_kind::S_FIELD: // "field"
case symbol_kind::S_PATH: // "path" case symbol_kind::S_PATH: // "path"
case symbol_kind::S_IDENTIFIER: // "identifier" case symbol_kind::S_IDENTIFIER: // "identifier"
case symbol_kind::S_STRING: // "string literal" case symbol_kind::S_STRING: // "string literal"
@ -5482,7 +5483,7 @@ switch (yykind)
#line 13 "parser.ypp" #line 13 "parser.ypp"
} } } // xsk::gsc::s1 } } } // xsk::gsc::s1
#line 5486 "parser.hpp" #line 5487 "parser.hpp"

120
src/s2/xsk/lexer.cpp
View File

@ -60,6 +60,7 @@ const std::unordered_map<std::string_view, parser::token::token_kind_type> keywo
{ "true", parser::token::TRUE }, { "true", parser::token::TRUE },
{ "false", parser::token::FALSE }, { "false", parser::token::FALSE },
{ "undefined", parser::token::UNDEFINED }, { "undefined", parser::token::UNDEFINED },
{ "size", parser::token::SIZE },
{ "game", parser::token::GAME }, { "game", parser::token::GAME },
{ "self", parser::token::SELF }, { "self", parser::token::SELF },
{ "anim", parser::token::ANIM }, { "anim", parser::token::ANIM },
@ -85,8 +86,8 @@ bool buffer::push(char c)
return true; return true;
} }
reader::reader() : state(reader::end), buffer_pos(0), bytes_remaining(0), reader::reader() : state(reader::end), buffer_pos(0),
last_byte(0), current_byte(0) { } bytes_remaining(0), last_byte(0), current_byte(0) {}
void reader::init(const char* data, size_t size) void reader::init(const char* data, size_t size)
{ {
@ -174,7 +175,6 @@ auto lexer::lex() -> parser::symbol_type
{ {
buffer_.length = 0; buffer_.length = 0;
state_ = state::start; state_ = state::start;
loc_.step();
while (true) while (true)
{ {
@ -182,6 +182,7 @@ auto lexer::lex() -> parser::symbol_type
auto& last = reader_.last_byte; auto& last = reader_.last_byte;
auto& curr = reader_.current_byte; auto& curr = reader_.current_byte;
auto path = false; auto path = false;
loc_.step();
if (state == reader::end) if (state == reader::end)
{ {
@ -214,7 +215,7 @@ auto lexer::lex() -> parser::symbol_type
case '\\': case '\\':
throw comp_error(loc_, "invalid token ('\\')"); throw comp_error(loc_, "invalid token ('\\')");
case '/': case '/':
if (curr != '/' && curr != '*' && curr != '#' && curr != '=') if (curr != '=' && curr != '#' && curr != '@' && curr != '*' && curr != '/')
return parser::make_DIV(loc_); return parser::make_DIV(loc_);
advance(); advance();
@ -246,14 +247,35 @@ auto lexer::lex() -> parser::symbol_type
} }
else if (last == '#' && curr == '/') else if (last == '#' && curr == '/')
{ {
reader_.advance(); advance();
break; break;
} }
reader_.advance(); advance();
} }
} }
} }
else if (last == '@')
{
while (true)
{
if (state == reader::end)
throw comp_error(loc_, "unmatched script doc comment start ('/@')");
if (curr == '\n')
{
loc_.lines();
loc_.step();
}
else if (last == '@' && curr == '/')
{
advance();
break;
}
advance();
}
}
else if (last == '*') else if (last == '*')
{ {
while (true) while (true)
@ -268,11 +290,11 @@ auto lexer::lex() -> parser::symbol_type
} }
else if (last == '*' && curr == '/') else if (last == '*' && curr == '/')
{ {
reader_.advance(); advance();
break; break;
} }
reader_.advance(); advance();
} }
} }
else if (last == '/') else if (last == '/')
@ -282,30 +304,10 @@ auto lexer::lex() -> parser::symbol_type
if (state == reader::end) if (state == reader::end)
break; break;
if (last == '\\' && (curr == '\r' || curr == '\n'))
{
reader_.advance();
if (state == reader::end)
break;
if (last == '\r')
{
if (curr != '\n')
throw comp_error(loc_, "invalid token ('\')");
reader_.advance();
}
loc_.lines();
loc_.step();
continue;
}
if (curr == '\n') if (curr == '\n')
break; break;
reader_.advance(); advance();
} }
} }
continue; continue;
@ -315,8 +317,8 @@ auto lexer::lex() -> parser::symbol_type
if (!indev_) if (!indev_)
throw comp_error(loc_, "unmatched devblock end ('#/')"); throw comp_error(loc_, "unmatched devblock end ('#/')");
indev_ = false;
advance(); advance();
indev_ = false;
return parser::make_DEVEND(loc_); return parser::make_DEVEND(loc_);
} }
@ -337,7 +339,7 @@ auto lexer::lex() -> parser::symbol_type
state_ = state::preprocessor; state_ = state::preprocessor;
goto lex_name; goto lex_name;
case '*': case '*':
if (curr != '/' && curr != '=') if (curr != '=' && curr != '/')
return parser::make_MUL(loc_); return parser::make_MUL(loc_);
advance(); advance();
@ -350,13 +352,9 @@ auto lexer::lex() -> parser::symbol_type
state_ = state::string; state_ = state::string;
goto lex_string; goto lex_string;
case '.': case '.':
advance(); if (curr < '0' || curr > '9')
return parser::make_DOT(loc_);
if (state == reader::end) goto lex_number;
throw comp_error(loc_, "unterminated field ('.')");
state_ = state::field;
goto lex_name_or_number;
case '(': case '(':
return parser::make_LPAREN(loc_); return parser::make_LPAREN(loc_);
case ')': case ')':
@ -480,7 +478,6 @@ auto lexer::lex() -> parser::symbol_type
advance(); advance();
return parser::make_ASSIGN_RSHIFT(loc_); return parser::make_ASSIGN_RSHIFT(loc_);
default: default:
lex_name_or_number:
if (last >= '0' && last <= '9') if (last >= '0' && last <= '9')
goto lex_number; goto lex_number;
else if (last == '_' || last >= 'A' && last <= 'Z' || last >= 'a' && last <= 'z') else if (last == '_' || last >= 'A' && last <= 'Z' || last >= 'a' && last <= 'z')
@ -562,23 +559,8 @@ lex_name:
advance(); advance();
} }
if (state_ == state::field) if (state_ == state::preprocessor)
{ {
if (path)
throw comp_error(loc_, "invalid field token '\\'");
if (std::string_view(buffer_.data, buffer_.length) == "size")
{
return parser::make_SIZE(loc_);
}
return parser::make_FIELD(std::string(buffer_.data, buffer_.length), loc_);
}
else if (state_ == state::preprocessor)
{
if (path)
throw comp_error(loc_, "invalid preprocessor directive");
auto token = parser::token::S2UNDEF; auto token = parser::token::S2UNDEF;
if (buffer_.length < 16) if (buffer_.length < 16)
@ -594,7 +576,8 @@ lex_name:
} }
} }
preprocessor(token); preprocessor_run(token);
state_ = state::start; state_ = state::start;
continue; continue;
} }
@ -620,14 +603,11 @@ lex_name:
} }
lex_number: lex_number:
if (state_ == state::field) if (last == '.' || last != '0' || (last == '0' && (curr != 'o' && curr != 'b' && curr != 'x')))
buffer_.push('.');
if (state_ == state::field || last == '.' || last != '0' || (last == '0' && (curr != 'o' && curr != 'b' && curr != 'x')))
{ {
buffer_.push(last); buffer_.push(last);
auto dot = 0; auto dot = last == '.' ? 1 : 0;
auto flt = 0; auto flt = 0;
while (true) while (true)
@ -663,10 +643,10 @@ lex_number:
if (last == '\'') if (last == '\'')
throw comp_error(loc_, "invalid number literal"); throw comp_error(loc_, "invalid number literal");
if (state_ == state::field && dot || dot > 1 || flt > 1 || flt && buffer_.data[buffer_.length - 1] != 'f') if (dot > 1 || flt > 1 || flt && buffer_.data[buffer_.length - 1] != 'f')
throw comp_error(loc_, "invalid number literal"); throw comp_error(loc_, "invalid number literal");
if (state_ == state::field || dot || flt) if (dot || flt)
return parser::make_FLOAT(std::string(buffer_.data, buffer_.length), loc_); return parser::make_FLOAT(std::string(buffer_.data, buffer_.length), loc_);
return parser::make_INTEGER(std::string(buffer_.data, buffer_.length), loc_); return parser::make_INTEGER(std::string(buffer_.data, buffer_.length), loc_);
@ -771,18 +751,22 @@ lex_number:
return parser::make_INTEGER(xsk::utils::string::hex_to_dec(buffer_.data), loc_); return parser::make_INTEGER(xsk::utils::string::hex_to_dec(buffer_.data), loc_);
} }
// cant get here!
throw error("UNEXPECTED LEXER INTERNAL ERROR!");
} }
} }
void lexer::advance() void lexer::advance()
{ {
reader_.advance(); reader_.advance();
loc_.end.column++;
// dont wrap comment marks '/\/' '/\*' outside strings if (reader_.current_byte == '\\') [[unlikely]]
if (state_ == state::start && reader_.last_byte == '/') preprocessor_wrap();
return; }
void lexer::preprocessor_wrap()
{
while (reader_.current_byte == '\\') while (reader_.current_byte == '\\')
{ {
if (reader_.bytes_remaining == 1) if (reader_.bytes_remaining == 1)
@ -824,7 +808,7 @@ void lexer::advance()
} }
} }
void lexer::preprocessor(parser::token::token_kind_type token) void lexer::preprocessor_run(parser::token::token_kind_type token)
{ {
if (!clean_) if (!clean_)
throw comp_error(loc_, "invalid token ('#')"); throw comp_error(loc_, "invalid token ('#')");

5
src/s2/xsk/lexer.hpp
View File

@ -44,7 +44,7 @@ struct reader
class lexer class lexer
{ {
enum class state : std::uint8_t { start, string, localize, field, preprocessor }; enum class state : std::uint8_t { start, string, localize, preprocessor };
reader reader_; reader reader_;
buffer buffer_; buffer buffer_;
@ -66,7 +66,8 @@ public:
private: private:
void advance(); void advance();
void preprocessor(parser::token::token_kind_type token); void preprocessor_wrap();
void preprocessor_run(parser::token::token_kind_type token);
}; };
} // namespace xsk::gsc::s2 } // namespace xsk::gsc::s2

1740
src/s2/xsk/parser.cpp
View File

File diff suppressed because it is too large Load Diff

103
src/s2/xsk/parser.hpp
View File

@ -474,6 +474,7 @@ namespace xsk { namespace gsc { namespace s2 {
// expr_game // expr_game
char dummy19[sizeof (ast::expr_game::ptr)]; char dummy19[sizeof (ast::expr_game::ptr)];
// expr_identifier_nosize
// expr_identifier // expr_identifier
char dummy20[sizeof (ast::expr_identifier::ptr)]; char dummy20[sizeof (ast::expr_identifier::ptr)];
@ -618,7 +619,6 @@ namespace xsk { namespace gsc { namespace s2 {
// stmt_while // stmt_while
char dummy66[sizeof (ast::stmt_while::ptr)]; char dummy66[sizeof (ast::stmt_while::ptr)];
// "field"
// "path" // "path"
// "identifier" // "identifier"
// "string literal" // "string literal"
@ -721,7 +721,7 @@ namespace xsk { namespace gsc { namespace s2 {
TRUE = 44, // "true" TRUE = 44, // "true"
FALSE = 45, // "false" FALSE = 45, // "false"
UNDEFINED = 46, // "undefined" UNDEFINED = 46, // "undefined"
SIZE = 47, // ".size" SIZE = 47, // "size"
GAME = 48, // "game" GAME = 48, // "game"
SELF = 49, // "self" SELF = 49, // "self"
ANIM = 50, // "anim" ANIM = 50, // "anim"
@ -771,13 +771,13 @@ namespace xsk { namespace gsc { namespace s2 {
MUL = 94, // "*" MUL = 94, // "*"
DIV = 95, // "/" DIV = 95, // "/"
MOD = 96, // "%" MOD = 96, // "%"
FIELD = 97, // "field" PATH = 97, // "path"
PATH = 98, // "path" IDENTIFIER = 98, // "identifier"
IDENTIFIER = 99, // "identifier" STRING = 99, // "string literal"
STRING = 100, // "string literal" ISTRING = 100, // "localized string"
ISTRING = 101, // "localized string" FLOAT = 101, // "float"
FLOAT = 102, // "float" INTEGER = 102, // "integer"
INTEGER = 103, // "integer" SIZEOF = 103, // SIZEOF
ADD_ARRAY = 104, // ADD_ARRAY ADD_ARRAY = 104, // ADD_ARRAY
THEN = 105, // THEN THEN = 105, // THEN
TERN = 106, // TERN TERN = 106, // TERN
@ -852,7 +852,7 @@ namespace xsk { namespace gsc { namespace s2 {
S_TRUE = 44, // "true" S_TRUE = 44, // "true"
S_FALSE = 45, // "false" S_FALSE = 45, // "false"
S_UNDEFINED = 46, // "undefined" S_UNDEFINED = 46, // "undefined"
S_SIZE = 47, // ".size" S_SIZE = 47, // "size"
S_GAME = 48, // "game" S_GAME = 48, // "game"
S_SELF = 49, // "self" S_SELF = 49, // "self"
S_ANIM = 50, // "anim" S_ANIM = 50, // "anim"
@ -902,13 +902,13 @@ namespace xsk { namespace gsc { namespace s2 {
S_MUL = 94, // "*" S_MUL = 94, // "*"
S_DIV = 95, // "/" S_DIV = 95, // "/"
S_MOD = 96, // "%" S_MOD = 96, // "%"
S_FIELD = 97, // "field" S_PATH = 97, // "path"
S_PATH = 98, // "path" S_IDENTIFIER = 98, // "identifier"
S_IDENTIFIER = 99, // "identifier" S_STRING = 99, // "string literal"
S_STRING = 100, // "string literal" S_ISTRING = 100, // "localized string"
S_ISTRING = 101, // "localized string" S_FLOAT = 101, // "float"
S_FLOAT = 102, // "float" S_INTEGER = 102, // "integer"
S_INTEGER = 103, // "integer" S_SIZEOF = 103, // SIZEOF
S_ADD_ARRAY = 104, // ADD_ARRAY S_ADD_ARRAY = 104, // ADD_ARRAY
S_THEN = 105, // THEN S_THEN = 105, // THEN
S_TERN = 106, // TERN S_TERN = 106, // TERN
@ -991,15 +991,16 @@ namespace xsk { namespace gsc { namespace s2 {
S_expr_level = 183, // expr_level S_expr_level = 183, // expr_level
S_expr_animation = 184, // expr_animation S_expr_animation = 184, // expr_animation
S_expr_animtree = 185, // expr_animtree S_expr_animtree = 185, // expr_animtree
S_expr_identifier = 186, // expr_identifier S_expr_identifier_nosize = 186, // expr_identifier_nosize
S_expr_path = 187, // expr_path S_expr_identifier = 187, // expr_identifier
S_expr_istring = 188, // expr_istring S_expr_path = 188, // expr_path
S_expr_string = 189, // expr_string S_expr_istring = 189, // expr_istring
S_expr_vector = 190, // expr_vector S_expr_string = 190, // expr_string
S_expr_float = 191, // expr_float S_expr_vector = 191, // expr_vector
S_expr_integer = 192, // expr_integer S_expr_float = 192, // expr_float
S_expr_false = 193, // expr_false S_expr_integer = 193, // expr_integer
S_expr_true = 194 // expr_true S_expr_false = 194, // expr_false
S_expr_true = 195 // expr_true
}; };
}; };
@ -1122,6 +1123,7 @@ namespace xsk { namespace gsc { namespace s2 {
value.move< ast::expr_game::ptr > (std::move (that.value)); value.move< ast::expr_game::ptr > (std::move (that.value));
break; break;
case symbol_kind::S_expr_identifier_nosize: // expr_identifier_nosize
case symbol_kind::S_expr_identifier: // expr_identifier case symbol_kind::S_expr_identifier: // expr_identifier
value.move< ast::expr_identifier::ptr > (std::move (that.value)); value.move< ast::expr_identifier::ptr > (std::move (that.value));
break; break;
@ -1313,7 +1315,6 @@ namespace xsk { namespace gsc { namespace s2 {
value.move< ast::stmt_while::ptr > (std::move (that.value)); value.move< ast::stmt_while::ptr > (std::move (that.value));
break; break;
case symbol_kind::S_FIELD: // "field"
case symbol_kind::S_PATH: // "path" case symbol_kind::S_PATH: // "path"
case symbol_kind::S_IDENTIFIER: // "identifier" case symbol_kind::S_IDENTIFIER: // "identifier"
case symbol_kind::S_STRING: // "string literal" case symbol_kind::S_STRING: // "string literal"
@ -2392,6 +2393,7 @@ switch (yykind)
value.template destroy< ast::expr_game::ptr > (); value.template destroy< ast::expr_game::ptr > ();
break; break;
case symbol_kind::S_expr_identifier_nosize: // expr_identifier_nosize
case symbol_kind::S_expr_identifier: // expr_identifier case symbol_kind::S_expr_identifier: // expr_identifier
value.template destroy< ast::expr_identifier::ptr > (); value.template destroy< ast::expr_identifier::ptr > ();
break; break;
@ -2583,7 +2585,6 @@ switch (yykind)
value.template destroy< ast::stmt_while::ptr > (); value.template destroy< ast::stmt_while::ptr > ();
break; break;
case symbol_kind::S_FIELD: // "field"
case symbol_kind::S_PATH: // "path" case symbol_kind::S_PATH: // "path"
case symbol_kind::S_IDENTIFIER: // "identifier" case symbol_kind::S_IDENTIFIER: // "identifier"
case symbol_kind::S_STRING: // "string literal" case symbol_kind::S_STRING: // "string literal"
@ -2689,7 +2690,7 @@ switch (yykind)
{ {
S2_ASSERT (tok == token::S2EOF S2_ASSERT (tok == token::S2EOF
|| (token::S2error <= tok && tok <= token::MOD) || (token::S2error <= tok && tok <= token::MOD)
|| (token::ADD_ARRAY <= tok && tok <= token::POSTDEC)); || (token::SIZEOF <= tok && tok <= token::POSTDEC));
} }
#if 201103L <= YY_CPLUSPLUS #if 201103L <= YY_CPLUSPLUS
symbol_type (int tok, std::string v, location_type l) symbol_type (int tok, std::string v, location_type l)
@ -2699,7 +2700,7 @@ switch (yykind)
: super_type(token_type (tok), v, l) : super_type(token_type (tok), v, l)
#endif #endif
{ {
S2_ASSERT ((token::FIELD <= tok && tok <= token::INTEGER)); S2_ASSERT ((token::PATH <= tok && tok <= token::INTEGER));
} }
}; };
@ -4204,21 +4205,6 @@ switch (yykind)
return symbol_type (token::MOD, l); return symbol_type (token::MOD, l);
} }
#endif #endif
#if 201103L <= YY_CPLUSPLUS
static
symbol_type
make_FIELD (std::string v, location_type l)
{
return symbol_type (token::FIELD, std::move (v), std::move (l));
}
#else
static
symbol_type
make_FIELD (const std::string& v, const location_type& l)
{
return symbol_type (token::FIELD, v, l);
}
#endif
#if 201103L <= YY_CPLUSPLUS #if 201103L <= YY_CPLUSPLUS
static static
symbol_type symbol_type
@ -4309,6 +4295,21 @@ switch (yykind)
return symbol_type (token::INTEGER, v, l); return symbol_type (token::INTEGER, v, l);
} }
#endif #endif
#if 201103L <= YY_CPLUSPLUS
static
symbol_type
make_SIZEOF (location_type l)
{
return symbol_type (token::SIZEOF, std::move (l));
}
#else
static
symbol_type
make_SIZEOF (const location_type& l)
{
return symbol_type (token::SIZEOF, l);
}
#endif
#if 201103L <= YY_CPLUSPLUS #if 201103L <= YY_CPLUSPLUS
static static
symbol_type symbol_type
@ -4788,9 +4789,9 @@ switch (yykind)
/// Constants. /// Constants.
enum enum
{ {
yylast_ = 2300, ///< Last index in yytable_. yylast_ = 2251, ///< Last index in yytable_.
yynnts_ = 82, ///< Number of nonterminal symbols. yynnts_ = 83, ///< Number of nonterminal symbols.
yyfinal_ = 21 ///< Termination state number. yyfinal_ = 22 ///< Termination state number.
}; };
@ -4902,6 +4903,7 @@ switch (yykind)
value.copy< ast::expr_game::ptr > (YY_MOVE (that.value)); value.copy< ast::expr_game::ptr > (YY_MOVE (that.value));
break; break;
case symbol_kind::S_expr_identifier_nosize: // expr_identifier_nosize
case symbol_kind::S_expr_identifier: // expr_identifier case symbol_kind::S_expr_identifier: // expr_identifier
value.copy< ast::expr_identifier::ptr > (YY_MOVE (that.value)); value.copy< ast::expr_identifier::ptr > (YY_MOVE (that.value));
break; break;
@ -5093,7 +5095,6 @@ switch (yykind)
value.copy< ast::stmt_while::ptr > (YY_MOVE (that.value)); value.copy< ast::stmt_while::ptr > (YY_MOVE (that.value));
break; break;
case symbol_kind::S_FIELD: // "field"
case symbol_kind::S_PATH: // "path" case symbol_kind::S_PATH: // "path"
case symbol_kind::S_IDENTIFIER: // "identifier" case symbol_kind::S_IDENTIFIER: // "identifier"
case symbol_kind::S_STRING: // "string literal" case symbol_kind::S_STRING: // "string literal"
@ -5218,6 +5219,7 @@ switch (yykind)
value.move< ast::expr_game::ptr > (YY_MOVE (s.value)); value.move< ast::expr_game::ptr > (YY_MOVE (s.value));
break; break;
case symbol_kind::S_expr_identifier_nosize: // expr_identifier_nosize
case symbol_kind::S_expr_identifier: // expr_identifier case symbol_kind::S_expr_identifier: // expr_identifier
value.move< ast::expr_identifier::ptr > (YY_MOVE (s.value)); value.move< ast::expr_identifier::ptr > (YY_MOVE (s.value));
break; break;
@ -5409,7 +5411,6 @@ switch (yykind)
value.move< ast::stmt_while::ptr > (YY_MOVE (s.value)); value.move< ast::stmt_while::ptr > (YY_MOVE (s.value));
break; break;
case symbol_kind::S_FIELD: // "field"
case symbol_kind::S_PATH: // "path" case symbol_kind::S_PATH: // "path"
case symbol_kind::S_IDENTIFIER: // "identifier" case symbol_kind::S_IDENTIFIER: // "identifier"
case symbol_kind::S_STRING: // "string literal" case symbol_kind::S_STRING: // "string literal"
@ -5482,7 +5483,7 @@ switch (yykind)
#line 13 "parser.ypp" #line 13 "parser.ypp"
} } } // xsk::gsc::s2 } } } // xsk::gsc::s2
#line 5486 "parser.hpp" #line 5487 "parser.hpp"

2
src/s4/xsk/compiler.cpp
View File

@ -1624,7 +1624,7 @@ void compiler::emit_expr_reference(const ast::expr_reference::ptr& expr, const b
void compiler::emit_expr_size(const ast::expr_size::ptr& expr, const block::ptr& blk) void compiler::emit_expr_size(const ast::expr_size::ptr& expr, const block::ptr& blk)
{ {
emit_expr_variable(expr->obj, blk); emit_expr(expr->obj, blk);
emit_opcode(opcode::OP_size); emit_opcode(opcode::OP_size);
} }

120
src/s4/xsk/lexer.cpp
View File

@ -60,6 +60,7 @@ const std::unordered_map<std::string_view, parser::token::token_kind_type> keywo
{ "true", parser::token::TRUE }, { "true", parser::token::TRUE },
{ "false", parser::token::FALSE }, { "false", parser::token::FALSE },
{ "undefined", parser::token::UNDEFINED }, { "undefined", parser::token::UNDEFINED },
{ "size", parser::token::SIZE },
{ "game", parser::token::GAME }, { "game", parser::token::GAME },
{ "self", parser::token::SELF }, { "self", parser::token::SELF },
{ "anim", parser::token::ANIM }, { "anim", parser::token::ANIM },
@ -87,8 +88,8 @@ bool buffer::push(char c)
return true; return true;
} }
reader::reader() : state(reader::end), buffer_pos(0), bytes_remaining(0), reader::reader() : state(reader::end), buffer_pos(0),
last_byte(0), current_byte(0) { } bytes_remaining(0), last_byte(0), current_byte(0) {}
void reader::init(const char* data, size_t size) void reader::init(const char* data, size_t size)
{ {
@ -176,7 +177,6 @@ auto lexer::lex() -> parser::symbol_type
{ {
buffer_.length = 0; buffer_.length = 0;
state_ = state::start; state_ = state::start;
loc_.step();
while (true) while (true)
{ {
@ -184,6 +184,7 @@ auto lexer::lex() -> parser::symbol_type
auto& last = reader_.last_byte; auto& last = reader_.last_byte;
auto& curr = reader_.current_byte; auto& curr = reader_.current_byte;
auto path = false; auto path = false;
loc_.step();
if (state == reader::end) if (state == reader::end)
{ {
@ -216,7 +217,7 @@ auto lexer::lex() -> parser::symbol_type
case '\\': case '\\':
throw comp_error(loc_, "invalid token ('\\')"); throw comp_error(loc_, "invalid token ('\\')");
case '/': case '/':
if (curr != '=' && curr != '#' && curr != '*' && curr != '/') if (curr != '=' && curr != '#' && curr != '@' && curr != '*' && curr != '/')
return parser::make_DIV(loc_); return parser::make_DIV(loc_);
advance(); advance();
@ -248,14 +249,35 @@ auto lexer::lex() -> parser::symbol_type
} }
else if (last == '#' && curr == '/') else if (last == '#' && curr == '/')
{ {
reader_.advance(); advance();
break; break;
} }
reader_.advance(); advance();
} }
} }
} }
else if (last == '@')
{
while (true)
{
if (state == reader::end)
throw comp_error(loc_, "unmatched script doc comment start ('/@')");
if (curr == '\n')
{
loc_.lines();
loc_.step();
}
else if (last == '@' && curr == '/')
{
advance();
break;
}
advance();
}
}
else if (last == '*') else if (last == '*')
{ {
while (true) while (true)
@ -270,11 +292,11 @@ auto lexer::lex() -> parser::symbol_type
} }
else if (last == '*' && curr == '/') else if (last == '*' && curr == '/')
{ {
reader_.advance(); advance();
break; break;
} }
reader_.advance(); advance();
} }
} }
else if (last == '/') else if (last == '/')
@ -284,30 +306,10 @@ auto lexer::lex() -> parser::symbol_type
if (state == reader::end) if (state == reader::end)
break; break;
if (last == '\\' && (curr == '\r' || curr == '\n'))
{
reader_.advance();
if (state == reader::end)
break;
if (last == '\r')
{
if (curr != '\n')
throw comp_error(loc_, "invalid token ('\')");
reader_.advance();
}
loc_.lines();
loc_.step();
continue;
}
if (curr == '\n') if (curr == '\n')
break; break;
reader_.advance(); advance();
} }
} }
continue; continue;
@ -317,8 +319,8 @@ auto lexer::lex() -> parser::symbol_type
if (!indev_) if (!indev_)
throw comp_error(loc_, "unmatched devblock end ('#/')"); throw comp_error(loc_, "unmatched devblock end ('#/')");
indev_ = false;
advance(); advance();
indev_ = false;
return parser::make_DEVEND(loc_); return parser::make_DEVEND(loc_);
} }
@ -339,7 +341,7 @@ auto lexer::lex() -> parser::symbol_type
state_ = state::preprocessor; state_ = state::preprocessor;
goto lex_name; goto lex_name;
case '*': case '*':
if (curr != '/' && curr != '=') if (curr != '=' && curr != '/')
return parser::make_MUL(loc_); return parser::make_MUL(loc_);
advance(); advance();
@ -352,13 +354,9 @@ auto lexer::lex() -> parser::symbol_type
state_ = state::string; state_ = state::string;
goto lex_string; goto lex_string;
case '.': case '.':
advance(); if (curr < '0' || curr > '9')
return parser::make_DOT(loc_);
if (state == reader::end) goto lex_number;
throw comp_error(loc_, "unterminated field ('.')");
state_ = state::field;
goto lex_name_or_number;
case '(': case '(':
return parser::make_LPAREN(loc_); return parser::make_LPAREN(loc_);
case ')': case ')':
@ -482,7 +480,6 @@ auto lexer::lex() -> parser::symbol_type
advance(); advance();
return parser::make_ASSIGN_RSHIFT(loc_); return parser::make_ASSIGN_RSHIFT(loc_);
default: default:
lex_name_or_number:
if (last >= '0' && last <= '9') if (last >= '0' && last <= '9')
goto lex_number; goto lex_number;
else if (last == '_' || last >= 'A' && last <= 'Z' || last >= 'a' && last <= 'z') else if (last == '_' || last >= 'A' && last <= 'Z' || last >= 'a' && last <= 'z')
@ -564,23 +561,8 @@ lex_name:
advance(); advance();
} }
if (state_ == state::field) if (state_ == state::preprocessor)
{ {
if (path)
throw comp_error(loc_, "invalid field token '\\'");
if (std::string_view(buffer_.data, buffer_.length) == "size")
{
return parser::make_SIZE(loc_);
}
return parser::make_FIELD(std::string(buffer_.data, buffer_.length), loc_);
}
else if (state_ == state::preprocessor)
{
if (path)
throw comp_error(loc_, "invalid preprocessor directive");
auto token = parser::token::S4UNDEF; auto token = parser::token::S4UNDEF;
if (buffer_.length < 16) if (buffer_.length < 16)
@ -596,7 +578,8 @@ lex_name:
} }
} }
preprocessor(token); preprocessor_run(token);
state_ = state::start; state_ = state::start;
continue; continue;
} }
@ -633,14 +616,11 @@ lex_name:
} }
lex_number: lex_number:
if (state_ == state::field) if (last == '.' || last != '0' || (last == '0' && (curr != 'o' && curr != 'b' && curr != 'x')))
buffer_.push('.');
if (state_ == state::field || last == '.' || last != '0' || (last == '0' && (curr != 'o' && curr != 'b' && curr != 'x')))
{ {
buffer_.push(last); buffer_.push(last);
auto dot = 0; auto dot = last == '.' ? 1 : 0;
auto flt = 0; auto flt = 0;
while (true) while (true)
@ -676,10 +656,10 @@ lex_number:
if (last == '\'') if (last == '\'')
throw comp_error(loc_, "invalid number literal"); throw comp_error(loc_, "invalid number literal");
if (state_ == state::field && dot || dot > 1 || flt > 1 || flt && buffer_.data[buffer_.length - 1] != 'f') if (dot > 1 || flt > 1 || flt && buffer_.data[buffer_.length - 1] != 'f')
throw comp_error(loc_, "invalid number literal"); throw comp_error(loc_, "invalid number literal");
if (state_ == state::field || dot || flt) if (dot || flt)
return parser::make_FLOAT(std::string(buffer_.data, buffer_.length), loc_); return parser::make_FLOAT(std::string(buffer_.data, buffer_.length), loc_);
return parser::make_INTEGER(std::string(buffer_.data, buffer_.length), loc_); return parser::make_INTEGER(std::string(buffer_.data, buffer_.length), loc_);
@ -784,18 +764,22 @@ lex_number:
return parser::make_INTEGER(xsk::utils::string::hex_to_dec(buffer_.data), loc_); return parser::make_INTEGER(xsk::utils::string::hex_to_dec(buffer_.data), loc_);
} }
// cant get here!
throw error("UNEXPECTED LEXER INTERNAL ERROR!");
} }
} }
void lexer::advance() void lexer::advance()
{ {
reader_.advance(); reader_.advance();
loc_.end.column++;
// dont wrap comment marks '/\/' '/\*' outside strings if (reader_.current_byte == '\\') [[unlikely]]
if (state_ == state::start && reader_.last_byte == '/') preprocessor_wrap();
return; }
void lexer::preprocessor_wrap()
{
while (reader_.current_byte == '\\') while (reader_.current_byte == '\\')
{ {
if (reader_.bytes_remaining == 1) if (reader_.bytes_remaining == 1)
@ -837,7 +821,7 @@ void lexer::advance()
} }
} }
void lexer::preprocessor(parser::token::token_kind_type token) void lexer::preprocessor_run(parser::token::token_kind_type token)
{ {
if (!clean_) if (!clean_)
throw comp_error(loc_, "invalid token ('#')"); throw comp_error(loc_, "invalid token ('#')");

5
src/s4/xsk/lexer.hpp
View File

@ -44,7 +44,7 @@ struct reader
class lexer class lexer
{ {
enum class state : std::uint8_t { start, string, localize, field, preprocessor }; enum class state : std::uint8_t { start, string, localize, preprocessor };
reader reader_; reader reader_;
buffer buffer_; buffer buffer_;
@ -66,7 +66,8 @@ public:
private: private:
void advance(); void advance();
void preprocessor(parser::token::token_kind_type token); void preprocessor_wrap();
void preprocessor_run(parser::token::token_kind_type token);
}; };
} // namespace xsk::gsc::s4 } // namespace xsk::gsc::s4

1815
src/s4/xsk/parser.cpp
View File

File diff suppressed because it is too large Load Diff

103
src/s4/xsk/parser.hpp
View File

@ -474,6 +474,7 @@ namespace xsk { namespace gsc { namespace s4 {
// expr_game // expr_game
char dummy19[sizeof (ast::expr_game::ptr)]; char dummy19[sizeof (ast::expr_game::ptr)];
// expr_identifier_nosize
// expr_identifier // expr_identifier
char dummy20[sizeof (ast::expr_identifier::ptr)]; char dummy20[sizeof (ast::expr_identifier::ptr)];
@ -624,7 +625,6 @@ namespace xsk { namespace gsc { namespace s4 {
// stmt_while // stmt_while
char dummy68[sizeof (ast::stmt_while::ptr)]; char dummy68[sizeof (ast::stmt_while::ptr)];
// "field"
// "path" // "path"
// "identifier" // "identifier"
// "string literal" // "string literal"
@ -727,7 +727,7 @@ namespace xsk { namespace gsc { namespace s4 {
TRUE = 44, // "true" TRUE = 44, // "true"
FALSE = 45, // "false" FALSE = 45, // "false"
UNDEFINED = 46, // "undefined" UNDEFINED = 46, // "undefined"
SIZE = 47, // ".size" SIZE = 47, // "size"
GAME = 48, // "game" GAME = 48, // "game"
SELF = 49, // "self" SELF = 49, // "self"
ANIM = 50, // "anim" ANIM = 50, // "anim"
@ -779,13 +779,13 @@ namespace xsk { namespace gsc { namespace s4 {
MUL = 96, // "*" MUL = 96, // "*"
DIV = 97, // "/" DIV = 97, // "/"
MOD = 98, // "%" MOD = 98, // "%"
FIELD = 99, // "field" PATH = 99, // "path"
PATH = 100, // "path" IDENTIFIER = 100, // "identifier"
IDENTIFIER = 101, // "identifier" STRING = 101, // "string literal"
STRING = 102, // "string literal" ISTRING = 102, // "localized string"
ISTRING = 103, // "localized string" FLOAT = 103, // "float"
FLOAT = 104, // "float" INTEGER = 104, // "integer"
INTEGER = 105, // "integer" SIZEOF = 105, // SIZEOF
ADD_ARRAY = 106, // ADD_ARRAY ADD_ARRAY = 106, // ADD_ARRAY
THEN = 107, // THEN THEN = 107, // THEN
TERN = 108, // TERN TERN = 108, // TERN
@ -860,7 +860,7 @@ namespace xsk { namespace gsc { namespace s4 {
S_TRUE = 44, // "true" S_TRUE = 44, // "true"
S_FALSE = 45, // "false" S_FALSE = 45, // "false"
S_UNDEFINED = 46, // "undefined" S_UNDEFINED = 46, // "undefined"
S_SIZE = 47, // ".size" S_SIZE = 47, // "size"
S_GAME = 48, // "game" S_GAME = 48, // "game"
S_SELF = 49, // "self" S_SELF = 49, // "self"
S_ANIM = 50, // "anim" S_ANIM = 50, // "anim"
@ -912,13 +912,13 @@ namespace xsk { namespace gsc { namespace s4 {
S_MUL = 96, // "*" S_MUL = 96, // "*"
S_DIV = 97, // "/" S_DIV = 97, // "/"
S_MOD = 98, // "%" S_MOD = 98, // "%"
S_FIELD = 99, // "field" S_PATH = 99, // "path"
S_PATH = 100, // "path" S_IDENTIFIER = 100, // "identifier"
S_IDENTIFIER = 101, // "identifier" S_STRING = 101, // "string literal"
S_STRING = 102, // "string literal" S_ISTRING = 102, // "localized string"
S_ISTRING = 103, // "localized string" S_FLOAT = 103, // "float"
S_FLOAT = 104, // "float" S_INTEGER = 104, // "integer"
S_INTEGER = 105, // "integer" S_SIZEOF = 105, // SIZEOF
S_ADD_ARRAY = 106, // ADD_ARRAY S_ADD_ARRAY = 106, // ADD_ARRAY
S_THEN = 107, // THEN S_THEN = 107, // THEN
S_TERN = 108, // TERN S_TERN = 108, // TERN
@ -1003,15 +1003,16 @@ namespace xsk { namespace gsc { namespace s4 {
S_expr_level = 187, // expr_level S_expr_level = 187, // expr_level
S_expr_animation = 188, // expr_animation S_expr_animation = 188, // expr_animation
S_expr_animtree = 189, // expr_animtree S_expr_animtree = 189, // expr_animtree
S_expr_identifier = 190, // expr_identifier S_expr_identifier_nosize = 190, // expr_identifier_nosize
S_expr_path = 191, // expr_path S_expr_identifier = 191, // expr_identifier
S_expr_istring = 192, // expr_istring S_expr_path = 192, // expr_path
S_expr_string = 193, // expr_string S_expr_istring = 193, // expr_istring
S_expr_vector = 194, // expr_vector S_expr_string = 194, // expr_string
S_expr_float = 195, // expr_float S_expr_vector = 195, // expr_vector
S_expr_integer = 196, // expr_integer S_expr_float = 196, // expr_float
S_expr_false = 197, // expr_false S_expr_integer = 197, // expr_integer
S_expr_true = 198 // expr_true S_expr_false = 198, // expr_false
S_expr_true = 199 // expr_true
}; };
}; };
@ -1134,6 +1135,7 @@ namespace xsk { namespace gsc { namespace s4 {
value.move< ast::expr_game::ptr > (std::move (that.value)); value.move< ast::expr_game::ptr > (std::move (that.value));
break; break;
case symbol_kind::S_expr_identifier_nosize: // expr_identifier_nosize
case symbol_kind::S_expr_identifier: // expr_identifier case symbol_kind::S_expr_identifier: // expr_identifier
value.move< ast::expr_identifier::ptr > (std::move (that.value)); value.move< ast::expr_identifier::ptr > (std::move (that.value));
break; break;
@ -1333,7 +1335,6 @@ namespace xsk { namespace gsc { namespace s4 {
value.move< ast::stmt_while::ptr > (std::move (that.value)); value.move< ast::stmt_while::ptr > (std::move (that.value));
break; break;
case symbol_kind::S_FIELD: // "field"
case symbol_kind::S_PATH: // "path" case symbol_kind::S_PATH: // "path"
case symbol_kind::S_IDENTIFIER: // "identifier" case symbol_kind::S_IDENTIFIER: // "identifier"
case symbol_kind::S_STRING: // "string literal" case symbol_kind::S_STRING: // "string literal"
@ -2440,6 +2441,7 @@ switch (yykind)
value.template destroy< ast::expr_game::ptr > (); value.template destroy< ast::expr_game::ptr > ();
break; break;
case symbol_kind::S_expr_identifier_nosize: // expr_identifier_nosize
case symbol_kind::S_expr_identifier: // expr_identifier case symbol_kind::S_expr_identifier: // expr_identifier
value.template destroy< ast::expr_identifier::ptr > (); value.template destroy< ast::expr_identifier::ptr > ();
break; break;
@ -2639,7 +2641,6 @@ switch (yykind)
value.template destroy< ast::stmt_while::ptr > (); value.template destroy< ast::stmt_while::ptr > ();
break; break;
case symbol_kind::S_FIELD: // "field"
case symbol_kind::S_PATH: // "path" case symbol_kind::S_PATH: // "path"
case symbol_kind::S_IDENTIFIER: // "identifier" case symbol_kind::S_IDENTIFIER: // "identifier"
case symbol_kind::S_STRING: // "string literal" case symbol_kind::S_STRING: // "string literal"
@ -2745,7 +2746,7 @@ switch (yykind)
{ {
S4_ASSERT (tok == token::S4EOF S4_ASSERT (tok == token::S4EOF
|| (token::S4error <= tok && tok <= token::MOD) || (token::S4error <= tok && tok <= token::MOD)
|| (token::ADD_ARRAY <= tok && tok <= token::POSTDEC)); || (token::SIZEOF <= tok && tok <= token::POSTDEC));
} }
#if 201103L <= YY_CPLUSPLUS #if 201103L <= YY_CPLUSPLUS
symbol_type (int tok, std::string v, location_type l) symbol_type (int tok, std::string v, location_type l)
@ -2755,7 +2756,7 @@ switch (yykind)
: super_type(token_type (tok), v, l) : super_type(token_type (tok), v, l)
#endif #endif
{ {
S4_ASSERT ((token::FIELD <= tok && tok <= token::INTEGER)); S4_ASSERT ((token::PATH <= tok && tok <= token::INTEGER));
} }
}; };
@ -4290,21 +4291,6 @@ switch (yykind)
return symbol_type (token::MOD, l); return symbol_type (token::MOD, l);
} }
#endif #endif
#if 201103L <= YY_CPLUSPLUS
static
symbol_type
make_FIELD (std::string v, location_type l)
{
return symbol_type (token::FIELD, std::move (v), std::move (l));
}
#else
static
symbol_type
make_FIELD (const std::string& v, const location_type& l)
{
return symbol_type (token::FIELD, v, l);
}
#endif
#if 201103L <= YY_CPLUSPLUS #if 201103L <= YY_CPLUSPLUS
static static
symbol_type symbol_type
@ -4395,6 +4381,21 @@ switch (yykind)
return symbol_type (token::INTEGER, v, l); return symbol_type (token::INTEGER, v, l);
} }
#endif #endif
#if 201103L <= YY_CPLUSPLUS
static
symbol_type
make_SIZEOF (location_type l)
{
return symbol_type (token::SIZEOF, std::move (l));
}
#else
static
symbol_type
make_SIZEOF (const location_type& l)
{
return symbol_type (token::SIZEOF, l);
}
#endif
#if 201103L <= YY_CPLUSPLUS #if 201103L <= YY_CPLUSPLUS
static static
symbol_type symbol_type
@ -4874,9 +4875,9 @@ switch (yykind)
/// Constants. /// Constants.
enum enum
{ {
yylast_ = 2404, ///< Last index in yytable_. yylast_ = 2278, ///< Last index in yytable_.
yynnts_ = 84, ///< Number of nonterminal symbols. yynnts_ = 85, ///< Number of nonterminal symbols.
yyfinal_ = 21 ///< Termination state number. yyfinal_ = 22 ///< Termination state number.
}; };
@ -4988,6 +4989,7 @@ switch (yykind)
value.copy< ast::expr_game::ptr > (YY_MOVE (that.value)); value.copy< ast::expr_game::ptr > (YY_MOVE (that.value));
break; break;
case symbol_kind::S_expr_identifier_nosize: // expr_identifier_nosize
case symbol_kind::S_expr_identifier: // expr_identifier case symbol_kind::S_expr_identifier: // expr_identifier
value.copy< ast::expr_identifier::ptr > (YY_MOVE (that.value)); value.copy< ast::expr_identifier::ptr > (YY_MOVE (that.value));
break; break;
@ -5187,7 +5189,6 @@ switch (yykind)
value.copy< ast::stmt_while::ptr > (YY_MOVE (that.value)); value.copy< ast::stmt_while::ptr > (YY_MOVE (that.value));
break; break;
case symbol_kind::S_FIELD: // "field"
case symbol_kind::S_PATH: // "path" case symbol_kind::S_PATH: // "path"
case symbol_kind::S_IDENTIFIER: // "identifier" case symbol_kind::S_IDENTIFIER: // "identifier"
case symbol_kind::S_STRING: // "string literal" case symbol_kind::S_STRING: // "string literal"
@ -5312,6 +5313,7 @@ switch (yykind)
value.move< ast::expr_game::ptr > (YY_MOVE (s.value)); value.move< ast::expr_game::ptr > (YY_MOVE (s.value));
break; break;
case symbol_kind::S_expr_identifier_nosize: // expr_identifier_nosize
case symbol_kind::S_expr_identifier: // expr_identifier case symbol_kind::S_expr_identifier: // expr_identifier
value.move< ast::expr_identifier::ptr > (YY_MOVE (s.value)); value.move< ast::expr_identifier::ptr > (YY_MOVE (s.value));
break; break;
@ -5511,7 +5513,6 @@ switch (yykind)
value.move< ast::stmt_while::ptr > (YY_MOVE (s.value)); value.move< ast::stmt_while::ptr > (YY_MOVE (s.value));
break; break;
case symbol_kind::S_FIELD: // "field"
case symbol_kind::S_PATH: // "path" case symbol_kind::S_PATH: // "path"
case symbol_kind::S_IDENTIFIER: // "identifier" case symbol_kind::S_IDENTIFIER: // "identifier"
case symbol_kind::S_STRING: // "string literal" case symbol_kind::S_STRING: // "string literal"
@ -5584,7 +5585,7 @@ switch (yykind)
#line 13 "parser.ypp" #line 13 "parser.ypp"
} } } // xsk::gsc::s4 } } } // xsk::gsc::s4
#line 5588 "parser.hpp" #line 5589 "parser.hpp"

2
src/t6/xsk/compiler.cpp
View File

@ -1536,7 +1536,7 @@ void compiler::emit_expr_reference(const ast::expr_reference::ptr& expr)
void compiler::emit_expr_size(const ast::expr_size::ptr& expr) void compiler::emit_expr_size(const ast::expr_size::ptr& expr)
{ {
emit_expr_variable(expr->obj); emit_expr(expr->obj);
emit_opcode(opcode::OP_SizeOf); emit_opcode(opcode::OP_SizeOf);
} }

2
src/t6/xsk/decompiler.cpp
View File

@ -1412,7 +1412,7 @@ void decompiler::decompile_aborts(const ast::stmt_list::ptr& stmt)
void decompiler::decompile_devblocks(const ast::stmt_list::ptr& stmt) void decompiler::decompile_devblocks(const ast::stmt_list::ptr& stmt)
{ {
for (auto i = 0; i < stmt->list.size(); i++) for (auto i = 0u; i < stmt->list.size(); i++)
{ {
if (stmt->list.at(i) == ast::kind::asm_dev) if (stmt->list.at(i) == ast::kind::asm_dev)
{ {

2
src/t6/xsk/disassembler.cpp
View File

@ -542,7 +542,7 @@ void disassembler::disassemble_end_switch(const instruction::ptr& inst)
const auto count = script_->read<std::uint32_t>(); const auto count = script_->read<std::uint32_t>();
inst->data.push_back(utils::string::va("%i", count)); inst->data.push_back(utils::string::va("%i", count));
for (auto i = 0; i < count; i++) for (auto i = 0u; i < count; i++)
{ {
const auto value = script_->read<std::uint32_t>(); const auto value = script_->read<std::uint32_t>();

120
src/t6/xsk/lexer.cpp
View File

@ -55,6 +55,7 @@ const std::unordered_map<std::string_view, parser::token::token_kind_type> keywo
{ "true", parser::token::TRUE }, { "true", parser::token::TRUE },
{ "false", parser::token::FALSE }, { "false", parser::token::FALSE },
{ "undefined", parser::token::UNDEFINED }, { "undefined", parser::token::UNDEFINED },
{ "size", parser::token::SIZE },
{ "game", parser::token::GAME }, { "game", parser::token::GAME },
{ "self", parser::token::SELF }, { "self", parser::token::SELF },
{ "anim", parser::token::ANIM }, { "anim", parser::token::ANIM },
@ -99,8 +100,8 @@ bool buffer::push(char c)
return true; return true;
} }
reader::reader() : state(reader::end), buffer_pos(0), bytes_remaining(0), reader::reader() : state(reader::end), buffer_pos(0),
last_byte(0), current_byte(0) { } bytes_remaining(0), last_byte(0), current_byte(0) {}
void reader::init(const char* data, size_t size) void reader::init(const char* data, size_t size)
{ {
@ -188,7 +189,6 @@ auto lexer::lex() -> parser::symbol_type
{ {
buffer_.length = 0; buffer_.length = 0;
state_ = state::start; state_ = state::start;
loc_.step();
while (true) while (true)
{ {
@ -196,6 +196,7 @@ auto lexer::lex() -> parser::symbol_type
auto& last = reader_.last_byte; auto& last = reader_.last_byte;
auto& curr = reader_.current_byte; auto& curr = reader_.current_byte;
auto path = false; auto path = false;
loc_.step();
if (state == reader::end) if (state == reader::end)
{ {
@ -228,7 +229,7 @@ auto lexer::lex() -> parser::symbol_type
case '\\': case '\\':
throw comp_error(loc_, "invalid token ('\\')"); throw comp_error(loc_, "invalid token ('\\')");
case '/': case '/':
if (curr != '/' && curr != '*' && curr != '#' && curr != '=') if (curr != '=' && curr != '#' && curr != '@' && curr != '*' && curr != '/')
return parser::make_DIV(loc_); return parser::make_DIV(loc_);
advance(); advance();
@ -260,14 +261,35 @@ auto lexer::lex() -> parser::symbol_type
} }
else if (last == '#' && curr == '/') else if (last == '#' && curr == '/')
{ {
reader_.advance(); advance();
break; break;
} }
reader_.advance(); advance();
} }
} }
} }
else if (last == '@')
{
while (true)
{
if (state == reader::end)
throw comp_error(loc_, "unmatched script doc comment start ('/@')");
if (curr == '\n')
{
loc_.lines();
loc_.step();
}
else if (last == '@' && curr == '/')
{
advance();
break;
}
advance();
}
}
else if (last == '*') else if (last == '*')
{ {
while (true) while (true)
@ -282,11 +304,11 @@ auto lexer::lex() -> parser::symbol_type
} }
else if (last == '*' && curr == '/') else if (last == '*' && curr == '/')
{ {
reader_.advance(); advance();
break; break;
} }
reader_.advance(); advance();
} }
} }
else if (last == '/') else if (last == '/')
@ -296,30 +318,10 @@ auto lexer::lex() -> parser::symbol_type
if (state == reader::end) if (state == reader::end)
break; break;
if (last == '\\' && (curr == '\r' || curr == '\n'))
{
reader_.advance();
if (state == reader::end)
break;
if (last == '\r')
{
if (curr != '\n')
throw comp_error(loc_, "invalid token ('\')");
reader_.advance();
}
loc_.lines();
loc_.step();
continue;
}
if (curr == '\n') if (curr == '\n')
break; break;
reader_.advance(); advance();
} }
} }
continue; continue;
@ -329,8 +331,8 @@ auto lexer::lex() -> parser::symbol_type
if (!indev_) if (!indev_)
throw comp_error(loc_, "unmatched devblock end ('#/')"); throw comp_error(loc_, "unmatched devblock end ('#/')");
indev_ = false;
advance(); advance();
indev_ = false;
return parser::make_DEVEND(loc_); return parser::make_DEVEND(loc_);
} }
@ -351,7 +353,7 @@ auto lexer::lex() -> parser::symbol_type
state_ = state::preprocessor; state_ = state::preprocessor;
goto lex_name; goto lex_name;
case '*': case '*':
if (curr != '/' && curr != '=') if (curr != '=' && curr != '/')
return parser::make_MUL(loc_); return parser::make_MUL(loc_);
advance(); advance();
@ -364,13 +366,9 @@ auto lexer::lex() -> parser::symbol_type
state_ = state::string; state_ = state::string;
goto lex_string; goto lex_string;
case '.': case '.':
advance(); if (curr < '0' || curr > '9')
return parser::make_DOT(loc_);
if (state == reader::end) goto lex_number;
throw comp_error(loc_, "unterminated field ('.')");
state_ = state::field;
goto lex_name_or_number;
case '(': case '(':
return parser::make_LPAREN(loc_); return parser::make_LPAREN(loc_);
case ')': case ')':
@ -494,7 +492,6 @@ auto lexer::lex() -> parser::symbol_type
advance(); advance();
return parser::make_ASSIGN_RSHIFT(loc_); return parser::make_ASSIGN_RSHIFT(loc_);
default: default:
lex_name_or_number:
if (last >= '0' && last <= '9') if (last >= '0' && last <= '9')
goto lex_number; goto lex_number;
else if (last == '_' || last >= 'A' && last <= 'Z' || last >= 'a' && last <= 'z') else if (last == '_' || last >= 'A' && last <= 'Z' || last >= 'a' && last <= 'z')
@ -576,23 +573,8 @@ lex_name:
advance(); advance();
} }
if (state_ == state::field) if (state_ == state::preprocessor)
{ {
if (path)
throw comp_error(loc_, "invalid field token '\\'");
if (std::string_view(buffer_.data, buffer_.length) == "size")
{
return parser::make_SIZE(loc_);
}
return parser::make_FIELD(std::string(buffer_.data, buffer_.length), loc_);
}
else if (state_ == state::preprocessor)
{
if (path)
throw comp_error(loc_, "invalid preprocessor directive");
auto token = parser::token::T6UNDEF; auto token = parser::token::T6UNDEF;
if (buffer_.length < 16) if (buffer_.length < 16)
@ -608,7 +590,8 @@ lex_name:
} }
} }
preprocessor(token); preprocessor_run(token);
state_ = state::start; state_ = state::start;
continue; continue;
} }
@ -647,14 +630,11 @@ lex_name:
} }
lex_number: lex_number:
if (state_ == state::field) if (last == '.' || last != '0' || (last == '0' && (curr != 'o' && curr != 'b' && curr != 'x')))
buffer_.push('.');
if (state_ == state::field || last == '.' || last != '0' || (last == '0' && (curr != 'o' && curr != 'b' && curr != 'x')))
{ {
buffer_.push(last); buffer_.push(last);
auto dot = 0; auto dot = last == '.' ? 1 : 0;
auto flt = 0; auto flt = 0;
while (true) while (true)
@ -690,10 +670,10 @@ lex_number:
if (last == '\'') if (last == '\'')
throw comp_error(loc_, "invalid number literal"); throw comp_error(loc_, "invalid number literal");
if (state_ == state::field && dot || dot > 1 || flt > 1 || flt && buffer_.data[buffer_.length - 1] != 'f') if (dot > 1 || flt > 1 || flt && buffer_.data[buffer_.length - 1] != 'f')
throw comp_error(loc_, "invalid number literal"); throw comp_error(loc_, "invalid number literal");
if (state_ == state::field || dot || flt) if (dot || flt)
return parser::make_FLOAT(std::string(buffer_.data, buffer_.length), loc_); return parser::make_FLOAT(std::string(buffer_.data, buffer_.length), loc_);
return parser::make_INTEGER(std::string(buffer_.data, buffer_.length), loc_); return parser::make_INTEGER(std::string(buffer_.data, buffer_.length), loc_);
@ -798,18 +778,22 @@ lex_number:
return parser::make_INTEGER(xsk::utils::string::hex_to_dec(buffer_.data), loc_); return parser::make_INTEGER(xsk::utils::string::hex_to_dec(buffer_.data), loc_);
} }
// cant get here!
throw error("UNEXPECTED LEXER INTERNAL ERROR!");
} }
} }
void lexer::advance() void lexer::advance()
{ {
reader_.advance(); reader_.advance();
loc_.end.column++;
// dont wrap comment marks '/\/' '/\*' outside strings if (reader_.current_byte == '\\') [[unlikely]]
if (state_ == state::start && reader_.last_byte == '/') preprocessor_wrap();
return; }
void lexer::preprocessor_wrap()
{
while (reader_.current_byte == '\\') while (reader_.current_byte == '\\')
{ {
if (reader_.bytes_remaining == 1) if (reader_.bytes_remaining == 1)
@ -851,7 +835,7 @@ void lexer::advance()
} }
} }
void lexer::preprocessor(parser::token::token_kind_type token) void lexer::preprocessor_run(parser::token::token_kind_type token)
{ {
if (!clean_) if (!clean_)
throw comp_error(loc_, "invalid token ('#')"); throw comp_error(loc_, "invalid token ('#')");

5
src/t6/xsk/lexer.hpp
View File

@ -44,7 +44,7 @@ struct reader
class lexer class lexer
{ {
enum class state : std::uint8_t { start, string, localize, field, preprocessor }; enum class state : std::uint8_t { start, string, localize, preprocessor };
reader reader_; reader reader_;
buffer buffer_; buffer buffer_;
@ -66,7 +66,8 @@ public:
private: private:
void advance(); void advance();
void preprocessor(parser::token::token_kind_type token); void preprocessor_wrap();
void preprocessor_run(parser::token::token_kind_type token);
}; };
} // namespace xsk::arc::t6 } // namespace xsk::arc::t6

2099
src/t6/xsk/parser.cpp
View File

File diff suppressed because it is too large Load Diff

109
src/t6/xsk/parser.hpp
View File

@ -519,6 +519,7 @@ namespace xsk { namespace arc { namespace t6 {
// expr_hash // expr_hash
char dummy34[sizeof (ast::expr_hash::ptr)]; char dummy34[sizeof (ast::expr_hash::ptr)];
// expr_identifier_nosize
// expr_identifier // expr_identifier
char dummy35[sizeof (ast::expr_identifier::ptr)]; char dummy35[sizeof (ast::expr_identifier::ptr)];
@ -666,7 +667,6 @@ namespace xsk { namespace arc { namespace t6 {
// stmt_while // stmt_while
char dummy82[sizeof (ast::stmt_while::ptr)]; char dummy82[sizeof (ast::stmt_while::ptr)];
// "field"
// "path" // "path"
// "identifier" // "identifier"
// "string literal" // "string literal"
@ -765,7 +765,7 @@ namespace xsk { namespace arc { namespace t6 {
TRUE = 39, // "true" TRUE = 39, // "true"
FALSE = 40, // "false" FALSE = 40, // "false"
UNDEFINED = 41, // "undefined" UNDEFINED = 41, // "undefined"
SIZE = 42, // ".size" SIZE = 42, // "size"
GAME = 43, // "game" GAME = 43, // "game"
SELF = 44, // "self" SELF = 44, // "self"
ANIM = 45, // "anim" ANIM = 45, // "anim"
@ -834,14 +834,14 @@ namespace xsk { namespace arc { namespace t6 {
MUL = 108, // "*" MUL = 108, // "*"
DIV = 109, // "/" DIV = 109, // "/"
MOD = 110, // "%" MOD = 110, // "%"
FIELD = 111, // "field" PATH = 111, // "path"
PATH = 112, // "path" IDENTIFIER = 112, // "identifier"
IDENTIFIER = 113, // "identifier" STRING = 113, // "string literal"
STRING = 114, // "string literal" ISTRING = 114, // "localized string"
ISTRING = 115, // "localized string" HASH = 115, // "hash"
HASH = 116, // "hash" FLOAT = 116, // "float"
FLOAT = 117, // "float" INTEGER = 117, // "integer"
INTEGER = 118, // "integer" SIZEOF = 118, // SIZEOF
THEN = 119, // THEN THEN = 119, // THEN
TERN = 120, // TERN TERN = 120, // TERN
NEG = 121, // NEG NEG = 121, // NEG
@ -910,7 +910,7 @@ namespace xsk { namespace arc { namespace t6 {
S_TRUE = 39, // "true" S_TRUE = 39, // "true"
S_FALSE = 40, // "false" S_FALSE = 40, // "false"
S_UNDEFINED = 41, // "undefined" S_UNDEFINED = 41, // "undefined"
S_SIZE = 42, // ".size" S_SIZE = 42, // "size"
S_GAME = 43, // "game" S_GAME = 43, // "game"
S_SELF = 44, // "self" S_SELF = 44, // "self"
S_ANIM = 45, // "anim" S_ANIM = 45, // "anim"
@ -979,14 +979,14 @@ namespace xsk { namespace arc { namespace t6 {
S_MUL = 108, // "*" S_MUL = 108, // "*"
S_DIV = 109, // "/" S_DIV = 109, // "/"
S_MOD = 110, // "%" S_MOD = 110, // "%"
S_FIELD = 111, // "field" S_PATH = 111, // "path"
S_PATH = 112, // "path" S_IDENTIFIER = 112, // "identifier"
S_IDENTIFIER = 113, // "identifier" S_STRING = 113, // "string literal"
S_STRING = 114, // "string literal" S_ISTRING = 114, // "localized string"
S_ISTRING = 115, // "localized string" S_HASH = 115, // "hash"
S_HASH = 116, // "hash" S_FLOAT = 116, // "float"
S_FLOAT = 117, // "float" S_INTEGER = 117, // "integer"
S_INTEGER = 118, // "integer" S_SIZEOF = 118, // SIZEOF
S_THEN = 119, // THEN S_THEN = 119, // THEN
S_TERN = 120, // TERN S_TERN = 120, // TERN
S_NEG = 121, // NEG S_NEG = 121, // NEG
@ -1083,16 +1083,17 @@ namespace xsk { namespace arc { namespace t6 {
S_expr_anim = 212, // expr_anim S_expr_anim = 212, // expr_anim
S_expr_level = 213, // expr_level S_expr_level = 213, // expr_level
S_expr_animation = 214, // expr_animation S_expr_animation = 214, // expr_animation
S_expr_identifier = 215, // expr_identifier S_expr_identifier_nosize = 215, // expr_identifier_nosize
S_expr_path = 216, // expr_path S_expr_identifier = 216, // expr_identifier
S_expr_istring = 217, // expr_istring S_expr_path = 217, // expr_path
S_expr_string = 218, // expr_string S_expr_istring = 218, // expr_istring
S_expr_vector = 219, // expr_vector S_expr_string = 219, // expr_string
S_expr_hash = 220, // expr_hash S_expr_vector = 220, // expr_vector
S_expr_float = 221, // expr_float S_expr_hash = 221, // expr_hash
S_expr_integer = 222, // expr_integer S_expr_float = 222, // expr_float
S_expr_false = 223, // expr_false S_expr_integer = 223, // expr_integer
S_expr_true = 224 // expr_true S_expr_false = 224, // expr_false
S_expr_true = 225 // expr_true
}; };
}; };
@ -1275,6 +1276,7 @@ namespace xsk { namespace arc { namespace t6 {
value.move< ast::expr_hash::ptr > (std::move (that.value)); value.move< ast::expr_hash::ptr > (std::move (that.value));
break; break;
case symbol_kind::S_expr_identifier_nosize: // expr_identifier_nosize
case symbol_kind::S_expr_identifier: // expr_identifier case symbol_kind::S_expr_identifier: // expr_identifier
value.move< ast::expr_identifier::ptr > (std::move (that.value)); value.move< ast::expr_identifier::ptr > (std::move (that.value));
break; break;
@ -1470,7 +1472,6 @@ namespace xsk { namespace arc { namespace t6 {
value.move< ast::stmt_while::ptr > (std::move (that.value)); value.move< ast::stmt_while::ptr > (std::move (that.value));
break; break;
case symbol_kind::S_FIELD: // "field"
case symbol_kind::S_PATH: // "path" case symbol_kind::S_PATH: // "path"
case symbol_kind::S_IDENTIFIER: // "identifier" case symbol_kind::S_IDENTIFIER: // "identifier"
case symbol_kind::S_STRING: // "string literal" case symbol_kind::S_STRING: // "string literal"
@ -2834,6 +2835,7 @@ switch (yykind)
value.template destroy< ast::expr_hash::ptr > (); value.template destroy< ast::expr_hash::ptr > ();
break; break;
case symbol_kind::S_expr_identifier_nosize: // expr_identifier_nosize
case symbol_kind::S_expr_identifier: // expr_identifier case symbol_kind::S_expr_identifier: // expr_identifier
value.template destroy< ast::expr_identifier::ptr > (); value.template destroy< ast::expr_identifier::ptr > ();
break; break;
@ -3029,7 +3031,6 @@ switch (yykind)
value.template destroy< ast::stmt_while::ptr > (); value.template destroy< ast::stmt_while::ptr > ();
break; break;
case symbol_kind::S_FIELD: // "field"
case symbol_kind::S_PATH: // "path" case symbol_kind::S_PATH: // "path"
case symbol_kind::S_IDENTIFIER: // "identifier" case symbol_kind::S_IDENTIFIER: // "identifier"
case symbol_kind::S_STRING: // "string literal" case symbol_kind::S_STRING: // "string literal"
@ -3136,7 +3137,7 @@ switch (yykind)
{ {
T6_ASSERT (tok == token::T6EOF T6_ASSERT (tok == token::T6EOF
|| (token::T6error <= tok && tok <= token::MOD) || (token::T6error <= tok && tok <= token::MOD)
|| (token::THEN <= tok && tok <= token::POSTDEC)); || (token::SIZEOF <= tok && tok <= token::POSTDEC));
} }
#if 201103L <= YY_CPLUSPLUS #if 201103L <= YY_CPLUSPLUS
symbol_type (int tok, std::string v, location_type l) symbol_type (int tok, std::string v, location_type l)
@ -3146,7 +3147,7 @@ switch (yykind)
: super_type(token_type (tok), v, l) : super_type(token_type (tok), v, l)
#endif #endif
{ {
T6_ASSERT ((token::FIELD <= tok && tok <= token::INTEGER)); T6_ASSERT ((token::PATH <= tok && tok <= token::INTEGER));
} }
}; };
@ -4861,21 +4862,6 @@ switch (yykind)
return symbol_type (token::MOD, l); return symbol_type (token::MOD, l);
} }
#endif #endif
#if 201103L <= YY_CPLUSPLUS
static
symbol_type
make_FIELD (std::string v, location_type l)
{
return symbol_type (token::FIELD, std::move (v), std::move (l));
}
#else
static
symbol_type
make_FIELD (const std::string& v, const location_type& l)
{
return symbol_type (token::FIELD, v, l);
}
#endif
#if 201103L <= YY_CPLUSPLUS #if 201103L <= YY_CPLUSPLUS
static static
symbol_type symbol_type
@ -4981,6 +4967,21 @@ switch (yykind)
return symbol_type (token::INTEGER, v, l); return symbol_type (token::INTEGER, v, l);
} }
#endif #endif
#if 201103L <= YY_CPLUSPLUS
static
symbol_type
make_SIZEOF (location_type l)
{
return symbol_type (token::SIZEOF, std::move (l));
}
#else
static
symbol_type
make_SIZEOF (const location_type& l)
{
return symbol_type (token::SIZEOF, l);
}
#endif
#if 201103L <= YY_CPLUSPLUS #if 201103L <= YY_CPLUSPLUS
static static
symbol_type symbol_type
@ -5445,9 +5446,9 @@ switch (yykind)
/// Constants. /// Constants.
enum enum
{ {
yylast_ = 2819, ///< Last index in yytable_. yylast_ = 2863, ///< Last index in yytable_.
yynnts_ = 98, ///< Number of nonterminal symbols. yynnts_ = 99, ///< Number of nonterminal symbols.
yyfinal_ = 21 ///< Termination state number. yyfinal_ = 22 ///< Termination state number.
}; };
@ -5619,6 +5620,7 @@ switch (yykind)
value.copy< ast::expr_hash::ptr > (YY_MOVE (that.value)); value.copy< ast::expr_hash::ptr > (YY_MOVE (that.value));
break; break;
case symbol_kind::S_expr_identifier_nosize: // expr_identifier_nosize
case symbol_kind::S_expr_identifier: // expr_identifier case symbol_kind::S_expr_identifier: // expr_identifier
value.copy< ast::expr_identifier::ptr > (YY_MOVE (that.value)); value.copy< ast::expr_identifier::ptr > (YY_MOVE (that.value));
break; break;
@ -5814,7 +5816,6 @@ switch (yykind)
value.copy< ast::stmt_while::ptr > (YY_MOVE (that.value)); value.copy< ast::stmt_while::ptr > (YY_MOVE (that.value));
break; break;
case symbol_kind::S_FIELD: // "field"
case symbol_kind::S_PATH: // "path" case symbol_kind::S_PATH: // "path"
case symbol_kind::S_IDENTIFIER: // "identifier" case symbol_kind::S_IDENTIFIER: // "identifier"
case symbol_kind::S_STRING: // "string literal" case symbol_kind::S_STRING: // "string literal"
@ -6000,6 +6001,7 @@ switch (yykind)
value.move< ast::expr_hash::ptr > (YY_MOVE (s.value)); value.move< ast::expr_hash::ptr > (YY_MOVE (s.value));
break; break;
case symbol_kind::S_expr_identifier_nosize: // expr_identifier_nosize
case symbol_kind::S_expr_identifier: // expr_identifier case symbol_kind::S_expr_identifier: // expr_identifier
value.move< ast::expr_identifier::ptr > (YY_MOVE (s.value)); value.move< ast::expr_identifier::ptr > (YY_MOVE (s.value));
break; break;
@ -6195,7 +6197,6 @@ switch (yykind)
value.move< ast::stmt_while::ptr > (YY_MOVE (s.value)); value.move< ast::stmt_while::ptr > (YY_MOVE (s.value));
break; break;
case symbol_kind::S_FIELD: // "field"
case symbol_kind::S_PATH: // "path" case symbol_kind::S_PATH: // "path"
case symbol_kind::S_IDENTIFIER: // "identifier" case symbol_kind::S_IDENTIFIER: // "identifier"
case symbol_kind::S_STRING: // "string literal" case symbol_kind::S_STRING: // "string literal"
@ -6269,7 +6270,7 @@ switch (yykind)
#line 13 "parser.ypp" #line 13 "parser.ypp"
} } } // xsk::arc::t6 } } } // xsk::arc::t6
#line 6273 "parser.hpp" #line 6274 "parser.hpp"