2 see copyright notice in squirrel.h
\r
4 #include "sqpcheader.h"
\r
8 #include "sqstring.h"
\r
9 #include "sqcompiler.h"
\r
10 #include "sqlexer.h"
\r
12 #define CUR_CHAR (_currdata)
\r
13 #define RETURN_TOKEN(t) { _prevtoken = _curtoken; _curtoken = t; return t;}
\r
14 #define IS_EOB() (CUR_CHAR <= SQUIRREL_EOB)
\r
15 #define NEXT() {Next();_currentcolumn++;}
\r
16 #define INIT_TEMP_STRING() { _longstr.resize(0);}
\r
17 #define APPEND_CHAR(c) { _longstr.push_back(c);}
\r
18 #define TERMINATE_BUFFER() {_longstr.push_back(_SC('\0'));}
\r
19 #define ADD_KEYWORD(key,id) _keywords->NewSlot( SQString::Create(ss, _SC(#key)) ,SQInteger(id))
\r
21 SQLexer::SQLexer(){}
\r
24 _keywords->Release();
\r
27 void SQLexer::Init(SQSharedState *ss, SQLEXREADFUNC rg, SQUserPointer up,CompilerErrorFunc efunc,void *ed)
\r
32 _keywords = SQTable::Create(ss, 26);
\r
33 ADD_KEYWORD(while, TK_WHILE);
\r
34 ADD_KEYWORD(do, TK_DO);
\r
35 ADD_KEYWORD(if, TK_IF);
\r
36 ADD_KEYWORD(else, TK_ELSE);
\r
37 ADD_KEYWORD(break, TK_BREAK);
\r
38 ADD_KEYWORD(continue, TK_CONTINUE);
\r
39 ADD_KEYWORD(return, TK_RETURN);
\r
40 ADD_KEYWORD(null, TK_NULL);
\r
41 ADD_KEYWORD(function, TK_FUNCTION);
\r
42 ADD_KEYWORD(local, TK_LOCAL);
\r
43 ADD_KEYWORD(for, TK_FOR);
\r
44 ADD_KEYWORD(foreach, TK_FOREACH);
\r
45 ADD_KEYWORD(in, TK_IN);
\r
46 ADD_KEYWORD(typeof, TK_TYPEOF);
\r
47 ADD_KEYWORD(base, TK_BASE);
\r
48 ADD_KEYWORD(delete, TK_DELETE);
\r
49 ADD_KEYWORD(try, TK_TRY);
\r
50 ADD_KEYWORD(catch, TK_CATCH);
\r
51 ADD_KEYWORD(throw, TK_THROW);
\r
52 ADD_KEYWORD(clone, TK_CLONE);
\r
53 ADD_KEYWORD(yield, TK_YIELD);
\r
54 ADD_KEYWORD(resume, TK_RESUME);
\r
55 ADD_KEYWORD(switch, TK_SWITCH);
\r
56 ADD_KEYWORD(case, TK_CASE);
\r
57 ADD_KEYWORD(default, TK_DEFAULT);
\r
58 ADD_KEYWORD(this, TK_THIS);
\r
59 ADD_KEYWORD(class,TK_CLASS);
\r
60 ADD_KEYWORD(extends,TK_EXTENDS);
\r
61 ADD_KEYWORD(constructor,TK_CONSTRUCTOR);
\r
62 ADD_KEYWORD(instanceof,TK_INSTANCEOF);
\r
63 ADD_KEYWORD(true,TK_TRUE);
\r
64 ADD_KEYWORD(false,TK_FALSE);
\r
65 ADD_KEYWORD(static,TK_STATIC);
\r
66 ADD_KEYWORD(enum,TK_ENUM);
\r
67 ADD_KEYWORD(const,TK_CONST);
\r
71 _lasttokenline = _currentline = 1;
\r
74 _reached_eof = SQFalse;
\r
78 void SQLexer::Error(const SQChar *err)
\r
80 _errfunc(_errtarget,err);
\r
83 void SQLexer::Next()
\r
85 SQInteger t = _readf(_up);
\r
86 if(t > MAX_CHAR) Error(_SC("Invalid character"));
\r
88 _currdata = (LexChar)t;
\r
91 _currdata = SQUIRREL_EOB;
\r
92 _reached_eof = SQTrue;
\r
95 const SQChar *SQLexer::Tok2Str(SQInteger tok)
\r
97 SQObjectPtr itr, key, val;
\r
99 while((nitr = _keywords->Next(false,itr, key, val)) != -1) {
\r
100 itr = (SQInteger)nitr;
\r
101 if(((SQInteger)_integer(val)) == tok)
\r
102 return _stringval(key);
\r
107 void SQLexer::LexBlockComment()
\r
112 case _SC('*'): { NEXT(); if(CUR_CHAR == _SC('/')) { done = true; NEXT(); }}; continue;
\r
113 case _SC('\n'): _currentline++; NEXT(); continue;
\r
114 case SQUIRREL_EOB: Error(_SC("missing \"*/\" in comment"));
\r
119 void SQLexer::LexLineComment()
\r
121 do { NEXT(); } while (CUR_CHAR != _SC('\n') && (!IS_EOB()));
\r
124 SQInteger SQLexer::Lex()
\r
126 _lasttokenline = _currentline;
\r
127 while(CUR_CHAR != SQUIRREL_EOB) {
\r
129 case _SC('\t'): case _SC('\r'): case _SC(' '): NEXT(); continue;
\r
132 _prevtoken=_curtoken;
\r
133 _curtoken=_SC('\n');
\r
137 case _SC('#'): LexLineComment(); continue;
\r
150 RETURN_TOKEN(TK_DIVEQ);
\r
154 RETURN_TOKEN(TK_ATTR_CLOSE);
\r
161 if (CUR_CHAR != _SC('=')){ RETURN_TOKEN('=') }
\r
162 else { NEXT(); RETURN_TOKEN(TK_EQ); }
\r
168 if(CUR_CHAR == _SC('>')) {
\r
170 RETURN_TOKEN(TK_3WAYSCMP);
\r
172 RETURN_TOKEN(TK_LE)
\r
174 case _SC('-'): NEXT(); RETURN_TOKEN(TK_NEWSLOT); break;
\r
175 case _SC('<'): NEXT(); RETURN_TOKEN(TK_SHIFTL); break;
\r
176 case _SC('/'): NEXT(); RETURN_TOKEN(TK_ATTR_OPEN); break;
\r
181 if (CUR_CHAR == _SC('=')){ NEXT(); RETURN_TOKEN(TK_GE);}
\r
182 else if(CUR_CHAR == _SC('>')){
\r
184 if(CUR_CHAR == _SC('>')){
\r
186 RETURN_TOKEN(TK_USHIFTR);
\r
188 RETURN_TOKEN(TK_SHIFTR);
\r
190 else { RETURN_TOKEN('>') }
\r
193 if (CUR_CHAR != _SC('=')){ RETURN_TOKEN('!')}
\r
194 else { NEXT(); RETURN_TOKEN(TK_NE); }
\r
198 if(CUR_CHAR != _SC('"')) {
\r
201 if((stype=ReadString('"',true))!=-1) {
\r
202 RETURN_TOKEN(stype);
\r
204 Error(_SC("error parsing the string"));
\r
209 if((stype=ReadString(CUR_CHAR,false))!=-1){
\r
210 RETURN_TOKEN(stype);
\r
212 Error(_SC("error parsing the string"));
\r
214 case _SC('{'): case _SC('}'): case _SC('('): case _SC(')'): case _SC('['): case _SC(']'):
\r
215 case _SC(';'): case _SC(','): case _SC('?'): case _SC('^'): case _SC('~'):
\r
216 {SQInteger ret = CUR_CHAR;
\r
217 NEXT(); RETURN_TOKEN(ret); }
\r
220 if (CUR_CHAR != _SC('.')){ RETURN_TOKEN('.') }
\r
222 if (CUR_CHAR != _SC('.')){ Error(_SC("invalid token '..'")); }
\r
224 RETURN_TOKEN(TK_VARPARAMS);
\r
227 if (CUR_CHAR != _SC('&')){ RETURN_TOKEN('&') }
\r
228 else { NEXT(); RETURN_TOKEN(TK_AND); }
\r
231 if (CUR_CHAR != _SC('|')){ RETURN_TOKEN('|') }
\r
232 else { NEXT(); RETURN_TOKEN(TK_OR); }
\r
235 if (CUR_CHAR != _SC(':')){ RETURN_TOKEN(':') }
\r
236 else { NEXT(); RETURN_TOKEN(TK_DOUBLE_COLON); }
\r
239 if (CUR_CHAR == _SC('=')){ NEXT(); RETURN_TOKEN(TK_MULEQ);}
\r
240 else RETURN_TOKEN('*');
\r
243 if (CUR_CHAR == _SC('=')){ NEXT(); RETURN_TOKEN(TK_MODEQ);}
\r
244 else RETURN_TOKEN('%');
\r
247 if (CUR_CHAR == _SC('=')){ NEXT(); RETURN_TOKEN(TK_MINUSEQ);}
\r
248 else if (CUR_CHAR == _SC('-')){ NEXT(); RETURN_TOKEN(TK_MINUSMINUS);}
\r
249 else RETURN_TOKEN('-');
\r
252 if (CUR_CHAR == _SC('=')){ NEXT(); RETURN_TOKEN(TK_PLUSEQ);}
\r
253 else if (CUR_CHAR == _SC('+')){ NEXT(); RETURN_TOKEN(TK_PLUSPLUS);}
\r
254 else RETURN_TOKEN('+');
\r
258 if (scisdigit(CUR_CHAR)) {
\r
259 SQInteger ret = ReadNumber();
\r
262 else if (scisalpha(CUR_CHAR) || CUR_CHAR == _SC('_')) {
\r
263 SQInteger t = ReadID();
\r
267 SQInteger c = CUR_CHAR;
\r
268 if (sciscntrl((int)c)) Error(_SC("unexpected character(control)"));
\r
279 SQInteger SQLexer::GetIDType(SQChar *s)
\r
282 if(_keywords->Get(SQString::Create(_sharedstate, s), t)) {
\r
283 return SQInteger(_integer(t));
\r
285 return TK_IDENTIFIER;
\r
289 SQInteger SQLexer::ReadString(SQInteger ndelim,bool verbatim)
\r
291 INIT_TEMP_STRING();
\r
293 if(IS_EOB()) return -1;
\r
295 while(CUR_CHAR != ndelim) {
\r
298 Error(_SC("unfinished string"));
\r
301 if(!verbatim) Error(_SC("newline in a constant"));
\r
302 APPEND_CHAR(CUR_CHAR); NEXT();
\r
307 APPEND_CHAR('\\'); NEXT();
\r
312 case _SC('x'): NEXT(); {
\r
313 if(!isxdigit(CUR_CHAR)) Error(_SC("hexadecimal number expected"));
\r
314 const SQInteger maxdigits = 4;
\r
315 SQChar temp[maxdigits+1];
\r
317 while(isxdigit(CUR_CHAR) && n < maxdigits) {
\r
318 temp[n] = CUR_CHAR;
\r
324 APPEND_CHAR((SQChar)scstrtoul(temp,&sTemp,16));
\r
327 case _SC('t'): APPEND_CHAR(_SC('\t')); NEXT(); break;
\r
328 case _SC('a'): APPEND_CHAR(_SC('\a')); NEXT(); break;
\r
329 case _SC('b'): APPEND_CHAR(_SC('\b')); NEXT(); break;
\r
330 case _SC('n'): APPEND_CHAR(_SC('\n')); NEXT(); break;
\r
331 case _SC('r'): APPEND_CHAR(_SC('\r')); NEXT(); break;
\r
332 case _SC('v'): APPEND_CHAR(_SC('\v')); NEXT(); break;
\r
333 case _SC('f'): APPEND_CHAR(_SC('\f')); NEXT(); break;
\r
334 case _SC('0'): APPEND_CHAR(_SC('\0')); NEXT(); break;
\r
335 case _SC('\\'): APPEND_CHAR(_SC('\\')); NEXT(); break;
\r
336 case _SC('"'): APPEND_CHAR(_SC('"')); NEXT(); break;
\r
337 case _SC('\''): APPEND_CHAR(_SC('\'')); NEXT(); break;
\r
339 Error(_SC("unrecognised escaper char"));
\r
345 APPEND_CHAR(CUR_CHAR);
\r
350 if(verbatim && CUR_CHAR == '"') { //double quotation
\r
351 APPEND_CHAR(CUR_CHAR);
\r
358 TERMINATE_BUFFER();
\r
359 SQInteger len = _longstr.size()-1;
\r
360 if(ndelim == _SC('\'')) {
\r
361 if(len == 0) Error(_SC("empty constant"));
\r
362 if(len > 1) Error(_SC("constant too long"));
\r
363 _nvalue = _longstr[0];
\r
366 _svalue = &_longstr[0];
\r
367 return TK_STRING_LITERAL;
\r
370 void LexHexadecimal(const SQChar *s,SQUnsignedInteger *res)
\r
375 if(scisdigit(*s)) *res = (*res)*16+((*s++)-'0');
\r
376 else if(scisxdigit(*s)) *res = (*res)*16+(toupper(*s++)-'A'+10);
\r
377 else { assert(0); }
\r
381 void LexInteger(const SQChar *s,SQUnsignedInteger *res)
\r
386 *res = (*res)*10+((*s++)-'0');
\r
390 SQInteger scisodigit(SQInteger c) { return c >= _SC('0') && c <= _SC('7'); }
\r
392 void LexOctal(const SQChar *s,SQUnsignedInteger *res)
\r
397 if(scisodigit(*s)) *res = (*res)*8+((*s++)-'0');
\r
398 else { assert(0); }
\r
402 SQInteger isexponent(SQInteger c) { return c == 'e' || c=='E'; }
\r
405 #define MAX_HEX_DIGITS (sizeof(SQInteger)*2)
\r
406 SQInteger SQLexer::ReadNumber()
\r
411 #define TSCIENTIFIC 4
\r
413 SQInteger type = TINT, firstchar = CUR_CHAR;
\r
415 INIT_TEMP_STRING();
\r
417 if(firstchar == _SC('0') && (toupper(CUR_CHAR) == _SC('X') || scisodigit(CUR_CHAR)) ) {
\r
418 if(scisodigit(CUR_CHAR)) {
\r
420 while(scisodigit(CUR_CHAR)) {
\r
421 APPEND_CHAR(CUR_CHAR);
\r
424 if(scisdigit(CUR_CHAR)) Error(_SC("invalid octal number"));
\r
429 while(isxdigit(CUR_CHAR)) {
\r
430 APPEND_CHAR(CUR_CHAR);
\r
433 if(_longstr.size() > MAX_HEX_DIGITS) Error(_SC("too many digits for an Hex number"));
\r
437 APPEND_CHAR((int)firstchar);
\r
438 while (CUR_CHAR == _SC('.') || scisdigit(CUR_CHAR) || isexponent(CUR_CHAR)) {
\r
439 if(CUR_CHAR == _SC('.') || isexponent(CUR_CHAR)) type = TFLOAT;
\r
440 if(isexponent(CUR_CHAR)) {
\r
441 if(type != TFLOAT) Error(_SC("invalid numeric format"));
\r
442 type = TSCIENTIFIC;
\r
443 APPEND_CHAR(CUR_CHAR);
\r
445 if(CUR_CHAR == '+' || CUR_CHAR == '-'){
\r
446 APPEND_CHAR(CUR_CHAR);
\r
449 if(!scisdigit(CUR_CHAR)) Error(_SC("exponent expected"));
\r
452 APPEND_CHAR(CUR_CHAR);
\r
456 TERMINATE_BUFFER();
\r
460 _fvalue = (SQFloat)scstrtod(&_longstr[0],&sTemp);
\r
463 LexInteger(&_longstr[0],(SQUnsignedInteger *)&_nvalue);
\r
466 LexHexadecimal(&_longstr[0],(SQUnsignedInteger *)&_nvalue);
\r
469 LexOctal(&_longstr[0],(SQUnsignedInteger *)&_nvalue);
\r
475 SQInteger SQLexer::ReadID()
\r
478 INIT_TEMP_STRING();
\r
480 APPEND_CHAR(CUR_CHAR);
\r
482 } while(scisalnum(CUR_CHAR) || CUR_CHAR == _SC('_'));
\r
483 TERMINATE_BUFFER();
\r
484 res = GetIDType(&_longstr[0]);
\r
485 if(res == TK_IDENTIFIER || res == TK_CONSTRUCTOR) {
\r
486 _svalue = &_longstr[0];
\r