external/squirrel/squirrel/sqlexer.cpp

   1 /*\r
   2         see copyright notice in squirrel.h\r
   3 */\r
   4 #include "sqpcheader.h"\r
   5 #include <ctype.h>\r
   6 #include <stdlib.h>\r
   7 #include "sqtable.h"\r
   8 #include "sqstring.h"\r
   9 #include "sqcompiler.h"\r
  10 #include "sqlexer.h"\r
  11 \r
  12 #define CUR_CHAR (_currdata)\r
  13 #define RETURN_TOKEN(t) { _prevtoken = _curtoken; _curtoken = t; return t;}\r
  14 #define IS_EOB() (CUR_CHAR <= SQUIRREL_EOB)\r
  15 #define NEXT() {Next();_currentcolumn++;}\r
  16 #define INIT_TEMP_STRING() { _longstr.resize(0);}\r
  17 #define APPEND_CHAR(c) { _longstr.push_back(c);}\r
  18 #define TERMINATE_BUFFER() {_longstr.push_back(_SC('\0'));}\r
  19 #define ADD_KEYWORD(key,id) _keywords->NewSlot( SQString::Create(ss, _SC(#key)) ,SQInteger(id))\r
  20 \r
  21 SQLexer::SQLexer(){}\r
  22 SQLexer::~SQLexer()\r
  23 {\r
  24         _keywords->Release();\r
  25 }\r
  26 \r
  27 void SQLexer::Init(SQSharedState *ss, SQLEXREADFUNC rg, SQUserPointer up,CompilerErrorFunc efunc,void *ed)\r
  28 {\r
  29         _errfunc = efunc;\r
  30         _errtarget = ed;\r
  31         _sharedstate = ss;\r
  32         _keywords = SQTable::Create(ss, 26);\r
  33         ADD_KEYWORD(while, TK_WHILE);\r
  34         ADD_KEYWORD(do, TK_DO);\r
  35         ADD_KEYWORD(if, TK_IF);\r
  36         ADD_KEYWORD(else, TK_ELSE);\r
  37         ADD_KEYWORD(break, TK_BREAK);\r
  38         ADD_KEYWORD(continue, TK_CONTINUE);\r
  39         ADD_KEYWORD(return, TK_RETURN);\r
  40         ADD_KEYWORD(null, TK_NULL);\r
  41         ADD_KEYWORD(function, TK_FUNCTION);\r
  42         ADD_KEYWORD(local, TK_LOCAL);\r
  43         ADD_KEYWORD(for, TK_FOR);\r
  44         ADD_KEYWORD(foreach, TK_FOREACH);\r
  45         ADD_KEYWORD(in, TK_IN);\r
  46         ADD_KEYWORD(typeof, TK_TYPEOF);\r
  47         ADD_KEYWORD(base, TK_BASE);\r
  48         ADD_KEYWORD(delete, TK_DELETE);\r
  49         ADD_KEYWORD(try, TK_TRY);\r
  50         ADD_KEYWORD(catch, TK_CATCH);\r
  51         ADD_KEYWORD(throw, TK_THROW);\r
  52         ADD_KEYWORD(clone, TK_CLONE);\r
  53         ADD_KEYWORD(yield, TK_YIELD);\r
  54         ADD_KEYWORD(resume, TK_RESUME);\r
  55         ADD_KEYWORD(switch, TK_SWITCH);\r
  56         ADD_KEYWORD(case, TK_CASE);\r
  57         ADD_KEYWORD(default, TK_DEFAULT);\r
  58         ADD_KEYWORD(this, TK_THIS);\r
  59         ADD_KEYWORD(class,TK_CLASS);\r
  60         ADD_KEYWORD(extends,TK_EXTENDS);\r
  61         ADD_KEYWORD(constructor,TK_CONSTRUCTOR);\r
  62         ADD_KEYWORD(instanceof,TK_INSTANCEOF);\r
  63         ADD_KEYWORD(true,TK_TRUE);\r
  64         ADD_KEYWORD(false,TK_FALSE);\r
  65         ADD_KEYWORD(static,TK_STATIC);\r
  66         ADD_KEYWORD(enum,TK_ENUM);\r
  67         ADD_KEYWORD(const,TK_CONST);\r
  68 \r
  69         _readf = rg;\r
  70         _up = up;\r
  71         _lasttokenline = _currentline = 1;\r
  72         _currentcolumn = 0;\r
  73         _prevtoken = -1;\r
  74         _reached_eof = SQFalse;\r
  75         Next();\r
  76 }\r
  77 \r
  78 void SQLexer::Error(const SQChar *err)\r
  79 {\r
  80         _errfunc(_errtarget,err);\r
  81 }\r
  82 \r
  83 void SQLexer::Next()\r
  84 {\r
  85         SQInteger t = _readf(_up);\r
  86         if(t > MAX_CHAR) Error(_SC("Invalid character"));\r
  87         if(t != 0) {\r
  88                 _currdata = (LexChar)t;\r
  89                 return;\r
  90         }\r
  91         _currdata = SQUIRREL_EOB;\r
  92         _reached_eof = SQTrue;\r
  93 }\r
  94 \r
  95 const SQChar *SQLexer::Tok2Str(SQInteger tok)\r
  96 {\r
  97         SQObjectPtr itr, key, val;\r
  98         SQInteger nitr;\r
  99         while((nitr = _keywords->Next(false,itr, key, val)) != -1) {\r
 100                 itr = (SQInteger)nitr;\r
 101                 if(((SQInteger)_integer(val)) == tok)\r
 102                         return _stringval(key);\r
 103         }\r
 104         return NULL;\r
 105 }\r
 106 \r
 107 void SQLexer::LexBlockComment()\r
 108 {\r
 109         bool done = false;\r
 110         while(!done) {\r
 111                 switch(CUR_CHAR) {\r
 112                         case _SC('*'): { NEXT(); if(CUR_CHAR == _SC('/')) { done = true; NEXT(); }}; continue;\r
 113                         case _SC('\n'): _currentline++; NEXT(); continue;\r
 114                         case SQUIRREL_EOB: Error(_SC("missing \"*/\" in comment"));\r
 115                         default: NEXT();\r
 116                 }\r
 117         }\r
 118 }\r
 119 void SQLexer::LexLineComment()\r
 120 {\r
 121         do { NEXT(); } while (CUR_CHAR != _SC('\n') && (!IS_EOB()));\r
 122 }\r
 123 \r
 124 SQInteger SQLexer::Lex()\r
 125 {\r
 126         _lasttokenline = _currentline;\r
 127         while(CUR_CHAR != SQUIRREL_EOB) {\r
 128                 switch(CUR_CHAR){\r
 129                 case _SC('\t'): case _SC('\r'): case _SC(' '): NEXT(); continue;\r
 130                 case _SC('\n'):\r
 131                         _currentline++;\r
 132                         _prevtoken=_curtoken;\r
 133                         _curtoken=_SC('\n');\r
 134                         NEXT();\r
 135                         _currentcolumn=1;\r
 136                         continue;\r
 137                 case _SC('#'): LexLineComment(); continue;\r
 138                 case _SC('/'):\r
 139                         NEXT();\r
 140                         switch(CUR_CHAR){\r
 141                         case _SC('*'):\r
 142                                 NEXT();\r
 143                                 LexBlockComment();\r
 144                                 continue;       \r
 145                         case _SC('/'):\r
 146                                 LexLineComment();\r
 147                                 continue;\r
 148                         case _SC('='):\r
 149                                 NEXT();\r
 150                                 RETURN_TOKEN(TK_DIVEQ);\r
 151                                 continue;\r
 152                         case _SC('>'):\r
 153                                 NEXT();\r
 154                                 RETURN_TOKEN(TK_ATTR_CLOSE);\r
 155                                 continue;\r
 156                         default:\r
 157                                 RETURN_TOKEN('/');\r
 158                         }\r
 159                 case _SC('='):\r
 160                         NEXT();\r
 161                         if (CUR_CHAR != _SC('=')){ RETURN_TOKEN('=') }\r
 162                         else { NEXT(); RETURN_TOKEN(TK_EQ); }\r
 163                 case _SC('<'):\r
 164                         NEXT();\r
 165                         switch(CUR_CHAR) {\r
 166                         case _SC('='):\r
 167                                 NEXT(); \r
 168                                 if(CUR_CHAR == _SC('>')) {\r
 169                                         NEXT();\r
 170                                         RETURN_TOKEN(TK_3WAYSCMP); \r
 171                                 }\r
 172                                 RETURN_TOKEN(TK_LE) \r
 173                                 break;\r
 174                         case _SC('-'): NEXT(); RETURN_TOKEN(TK_NEWSLOT); break;\r
 175                         case _SC('<'): NEXT(); RETURN_TOKEN(TK_SHIFTL); break;\r
 176                         case _SC('/'): NEXT(); RETURN_TOKEN(TK_ATTR_OPEN); break;\r
 177                         }\r
 178                         RETURN_TOKEN('<');\r
 179                 case _SC('>'):\r
 180                         NEXT();\r
 181                         if (CUR_CHAR == _SC('=')){ NEXT(); RETURN_TOKEN(TK_GE);}\r
 182                         else if(CUR_CHAR == _SC('>')){ \r
 183                                 NEXT(); \r
 184                                 if(CUR_CHAR == _SC('>')){\r
 185                                         NEXT();\r
 186                                         RETURN_TOKEN(TK_USHIFTR);\r
 187                                 }\r
 188                                 RETURN_TOKEN(TK_SHIFTR);\r
 189                         }\r
 190                         else { RETURN_TOKEN('>') }\r
 191                 case _SC('!'):\r
 192                         NEXT();\r
 193                         if (CUR_CHAR != _SC('=')){ RETURN_TOKEN('!')}\r
 194                         else { NEXT(); RETURN_TOKEN(TK_NE); }\r
 195                 case _SC('@'): {\r
 196                         SQInteger stype;\r
 197                         NEXT();\r
 198                         if(CUR_CHAR != _SC('"')) {\r
 199                                 RETURN_TOKEN('@');\r
 200                         }\r
 201                         if((stype=ReadString('"',true))!=-1) {\r
 202                                 RETURN_TOKEN(stype);\r
 203                         }\r
 204                         Error(_SC("error parsing the string"));\r
 205                                            }\r
 206                 case _SC('"'):\r
 207                 case _SC('\''): {\r
 208                         SQInteger stype;\r
 209                         if((stype=ReadString(CUR_CHAR,false))!=-1){\r
 210                                 RETURN_TOKEN(stype);\r
 211                         }\r
 212                         Error(_SC("error parsing the string"));\r
 213                         }\r
 214                 case _SC('{'): case _SC('}'): case _SC('('): case _SC(')'): case _SC('['): case _SC(']'):\r
 215                 case _SC(';'): case _SC(','): case _SC('?'): case _SC('^'): case _SC('~'):\r
 216                         {SQInteger ret = CUR_CHAR;\r
 217                         NEXT(); RETURN_TOKEN(ret); }\r
 218                 case _SC('.'):\r
 219                         NEXT();\r
 220                         if (CUR_CHAR != _SC('.')){ RETURN_TOKEN('.') }\r
 221                         NEXT();\r
 222                         if (CUR_CHAR != _SC('.')){ Error(_SC("invalid token '..'")); }\r
 223                         NEXT();\r
 224                         RETURN_TOKEN(TK_VARPARAMS);\r
 225                 case _SC('&'):\r
 226                         NEXT();\r
 227                         if (CUR_CHAR != _SC('&')){ RETURN_TOKEN('&') }\r
 228                         else { NEXT(); RETURN_TOKEN(TK_AND); }\r
 229                 case _SC('|'):\r
 230                         NEXT();\r
 231                         if (CUR_CHAR != _SC('|')){ RETURN_TOKEN('|') }\r
 232                         else { NEXT(); RETURN_TOKEN(TK_OR); }\r
 233                 case _SC(':'):\r
 234                         NEXT();\r
 235                         if (CUR_CHAR != _SC(':')){ RETURN_TOKEN(':') }\r
 236                         else { NEXT(); RETURN_TOKEN(TK_DOUBLE_COLON); }\r
 237                 case _SC('*'):\r
 238                         NEXT();\r
 239                         if (CUR_CHAR == _SC('=')){ NEXT(); RETURN_TOKEN(TK_MULEQ);}\r
 240                         else RETURN_TOKEN('*');\r
 241                 case _SC('%'):\r
 242                         NEXT();\r
 243                         if (CUR_CHAR == _SC('=')){ NEXT(); RETURN_TOKEN(TK_MODEQ);}\r
 244                         else RETURN_TOKEN('%');\r
 245                 case _SC('-'):\r
 246                         NEXT();\r
 247                         if (CUR_CHAR == _SC('=')){ NEXT(); RETURN_TOKEN(TK_MINUSEQ);}\r
 248                         else if  (CUR_CHAR == _SC('-')){ NEXT(); RETURN_TOKEN(TK_MINUSMINUS);}\r
 249                         else RETURN_TOKEN('-');\r
 250                 case _SC('+'):\r
 251                         NEXT();\r
 252                         if (CUR_CHAR == _SC('=')){ NEXT(); RETURN_TOKEN(TK_PLUSEQ);}\r
 253                         else if (CUR_CHAR == _SC('+')){ NEXT(); RETURN_TOKEN(TK_PLUSPLUS);}\r
 254                         else RETURN_TOKEN('+');\r
 255                 case SQUIRREL_EOB:\r
 256                         return 0;\r
 257                 default:{\r
 258                                 if (scisdigit(CUR_CHAR)) {\r
 259                                         SQInteger ret = ReadNumber();\r
 260                                         RETURN_TOKEN(ret);\r
 261                                 }\r
 262                                 else if (scisalpha(CUR_CHAR) || CUR_CHAR == _SC('_')) {\r
 263                                         SQInteger t = ReadID();\r
 264                                         RETURN_TOKEN(t);\r
 265                                 }\r
 266                                 else {\r
 267                                         SQInteger c = CUR_CHAR;\r
 268                                         if (sciscntrl((int)c)) Error(_SC("unexpected character(control)"));\r
 269                                         NEXT();\r
 270                                         RETURN_TOKEN(c);  \r
 271                                 }\r
 272                                 RETURN_TOKEN(0);\r
 273                         }\r
 274                 }\r
 275         }\r
 276         return 0;    \r
 277 }\r
 278         \r
 279 SQInteger SQLexer::GetIDType(SQChar *s)\r
 280 {\r
 281         SQObjectPtr t;\r
 282         if(_keywords->Get(SQString::Create(_sharedstate, s), t)) {\r
 283                 return SQInteger(_integer(t));\r
 284         }\r
 285         return TK_IDENTIFIER;\r
 286 }\r
 287 \r
 288 \r
 289 SQInteger SQLexer::ReadString(SQInteger ndelim,bool verbatim)\r
 290 {\r
 291         INIT_TEMP_STRING();\r
 292         NEXT();\r
 293         if(IS_EOB()) return -1;\r
 294         for(;;) {\r
 295                 while(CUR_CHAR != ndelim) {\r
 296                         switch(CUR_CHAR) {\r
 297                         case SQUIRREL_EOB:\r
 298                                 Error(_SC("unfinished string"));\r
 299                                 return -1;\r
 300                         case _SC('\n'): \r
 301                                 if(!verbatim) Error(_SC("newline in a constant")); \r
 302                                 APPEND_CHAR(CUR_CHAR); NEXT(); \r
 303                                 _currentline++;\r
 304                                 break;\r
 305                         case _SC('\\'):\r
 306                                 if(verbatim) {\r
 307                                         APPEND_CHAR('\\'); NEXT(); \r
 308                                 }\r
 309                                 else {\r
 310                                         NEXT();\r
 311                                         switch(CUR_CHAR) {\r
 312                                         case _SC('x'): NEXT(); {\r
 313                                                 if(!isxdigit(CUR_CHAR)) Error(_SC("hexadecimal number expected")); \r
 314                                                 const SQInteger maxdigits = 4;\r
 315                                                 SQChar temp[maxdigits+1];\r
 316                                                 SQInteger n = 0;\r
 317                                                 while(isxdigit(CUR_CHAR) && n < maxdigits) {\r
 318                                                         temp[n] = CUR_CHAR;\r
 319                                                         n++;\r
 320                                                         NEXT();\r
 321                                                 }\r
 322                                                 temp[n] = 0;\r
 323                                                 SQChar *sTemp;\r
 324                                                 APPEND_CHAR((SQChar)scstrtoul(temp,&sTemp,16));\r
 325                                         }\r
 326                                     break;\r
 327                                         case _SC('t'): APPEND_CHAR(_SC('\t')); NEXT(); break;\r
 328                                         case _SC('a'): APPEND_CHAR(_SC('\a')); NEXT(); break;\r
 329                                         case _SC('b'): APPEND_CHAR(_SC('\b')); NEXT(); break;\r
 330                                         case _SC('n'): APPEND_CHAR(_SC('\n')); NEXT(); break;\r
 331                                         case _SC('r'): APPEND_CHAR(_SC('\r')); NEXT(); break;\r
 332                                         case _SC('v'): APPEND_CHAR(_SC('\v')); NEXT(); break;\r
 333                                         case _SC('f'): APPEND_CHAR(_SC('\f')); NEXT(); break;\r
 334                                         case _SC('0'): APPEND_CHAR(_SC('\0')); NEXT(); break;\r
 335                                         case _SC('\\'): APPEND_CHAR(_SC('\\')); NEXT(); break;\r
 336                                         case _SC('"'): APPEND_CHAR(_SC('"')); NEXT(); break;\r
 337                                         case _SC('\''): APPEND_CHAR(_SC('\'')); NEXT(); break;\r
 338                                         default:\r
 339                                                 Error(_SC("unrecognised escaper char"));\r
 340                                         break;\r
 341                                         }\r
 342                                 }\r
 343                                 break;\r
 344                         default:\r
 345                                 APPEND_CHAR(CUR_CHAR);\r
 346                                 NEXT();\r
 347                         }\r
 348                 }\r
 349                 NEXT();\r
 350                 if(verbatim && CUR_CHAR == '"') { //double quotation\r
 351                         APPEND_CHAR(CUR_CHAR);\r
 352                         NEXT();\r
 353                 }\r
 354                 else {\r
 355                         break;\r
 356                 }\r
 357         }\r
 358         TERMINATE_BUFFER();\r
 359         SQInteger len = _longstr.size()-1;\r
 360         if(ndelim == _SC('\'')) {\r
 361                 if(len == 0) Error(_SC("empty constant"));\r
 362                 if(len > 1) Error(_SC("constant too long"));\r
 363                 _nvalue = _longstr[0];\r
 364                 return TK_INTEGER;\r
 365         }\r
 366         _svalue = &_longstr[0];\r
 367         return TK_STRING_LITERAL;\r
 368 }\r
 369 \r
 370 void LexHexadecimal(const SQChar *s,SQUnsignedInteger *res)\r
 371 {\r
 372         *res = 0;\r
 373         while(*s != 0)\r
 374         {\r
 375                 if(scisdigit(*s)) *res = (*res)*16+((*s++)-'0');\r
 376                 else if(scisxdigit(*s)) *res = (*res)*16+(toupper(*s++)-'A'+10);\r
 377                 else { assert(0); }\r
 378         }\r
 379 }\r
 380 \r
 381 void LexInteger(const SQChar *s,SQUnsignedInteger *res)\r
 382 {\r
 383         *res = 0;\r
 384         while(*s != 0)\r
 385         {\r
 386                 *res = (*res)*10+((*s++)-'0');\r
 387         }\r
 388 }\r
 389 \r
 390 SQInteger scisodigit(SQInteger c) { return c >= _SC('0') && c <= _SC('7'); }\r
 391 \r
 392 void LexOctal(const SQChar *s,SQUnsignedInteger *res)\r
 393 {\r
 394         *res = 0;\r
 395         while(*s != 0)\r
 396         {\r
 397                 if(scisodigit(*s)) *res = (*res)*8+((*s++)-'0');\r
 398                 else { assert(0); }\r
 399         }\r
 400 }\r
 401 \r
 402 SQInteger isexponent(SQInteger c) { return c == 'e' || c=='E'; }\r
 403 \r
 404 \r
 405 #define MAX_HEX_DIGITS (sizeof(SQInteger)*2)\r
 406 SQInteger SQLexer::ReadNumber()\r
 407 {\r
 408 #define TINT 1\r
 409 #define TFLOAT 2\r
 410 #define THEX 3\r
 411 #define TSCIENTIFIC 4\r
 412 #define TOCTAL 5\r
 413         SQInteger type = TINT, firstchar = CUR_CHAR;\r
 414         SQChar *sTemp;\r
 415         INIT_TEMP_STRING();\r
 416         NEXT();\r
 417         if(firstchar == _SC('0') && (toupper(CUR_CHAR) == _SC('X') || scisodigit(CUR_CHAR)) ) {\r
 418                 if(scisodigit(CUR_CHAR)) {\r
 419                         type = TOCTAL;\r
 420                         while(scisodigit(CUR_CHAR)) {\r
 421                                 APPEND_CHAR(CUR_CHAR);\r
 422                                 NEXT();\r
 423                         }\r
 424                         if(scisdigit(CUR_CHAR)) Error(_SC("invalid octal number"));\r
 425                 }\r
 426                 else {\r
 427                         NEXT();\r
 428                         type = THEX;\r
 429                         while(isxdigit(CUR_CHAR)) {\r
 430                                 APPEND_CHAR(CUR_CHAR);\r
 431                                 NEXT();\r
 432                         }\r
 433                         if(_longstr.size() > MAX_HEX_DIGITS) Error(_SC("too many digits for an Hex number"));\r
 434                 }\r
 435         }\r
 436         else {\r
 437                 APPEND_CHAR((int)firstchar);\r
 438                 while (CUR_CHAR == _SC('.') || scisdigit(CUR_CHAR) || isexponent(CUR_CHAR)) {\r
 439             if(CUR_CHAR == _SC('.') || isexponent(CUR_CHAR)) type = TFLOAT;\r
 440                         if(isexponent(CUR_CHAR)) {\r
 441                                 if(type != TFLOAT) Error(_SC("invalid numeric format"));\r
 442                                 type = TSCIENTIFIC;\r
 443                                 APPEND_CHAR(CUR_CHAR);\r
 444                                 NEXT();\r
 445                                 if(CUR_CHAR == '+' || CUR_CHAR == '-'){\r
 446                                         APPEND_CHAR(CUR_CHAR);\r
 447                                         NEXT();\r
 448                                 }\r
 449                                 if(!scisdigit(CUR_CHAR)) Error(_SC("exponent expected"));\r
 450                         }\r
 451                         \r
 452                         APPEND_CHAR(CUR_CHAR);\r
 453                         NEXT();\r
 454                 }\r
 455         }\r
 456         TERMINATE_BUFFER();\r
 457         switch(type) {\r
 458         case TSCIENTIFIC:\r
 459         case TFLOAT:\r
 460                 _fvalue = (SQFloat)scstrtod(&_longstr[0],&sTemp);\r
 461                 return TK_FLOAT;\r
 462         case TINT:\r
 463                 LexInteger(&_longstr[0],(SQUnsignedInteger *)&_nvalue);\r
 464                 return TK_INTEGER;\r
 465         case THEX:\r
 466                 LexHexadecimal(&_longstr[0],(SQUnsignedInteger *)&_nvalue);\r
 467                 return TK_INTEGER;\r
 468         case TOCTAL:\r
 469                 LexOctal(&_longstr[0],(SQUnsignedInteger *)&_nvalue);\r
 470                 return TK_INTEGER;\r
 471         }\r
 472         return 0;\r
 473 }\r
 474 \r
 475 SQInteger SQLexer::ReadID()\r
 476 {\r
 477         SQInteger res;\r
 478         INIT_TEMP_STRING();\r
 479         do {\r
 480                 APPEND_CHAR(CUR_CHAR);\r
 481                 NEXT();\r
 482         } while(scisalnum(CUR_CHAR) || CUR_CHAR == _SC('_'));\r
 483         TERMINATE_BUFFER();\r
 484         res = GetIDType(&_longstr[0]);\r
 485         if(res == TK_IDENTIFIER || res == TK_CONSTRUCTOR) {\r
 486                 _svalue = &_longstr[0];\r
 487         }\r
 488         return res;\r
 489 }\r