Johan Wikman / SQUIRREL3

The Squirrel interpreter. See http://www.squirrel-lang.org/

Dependents:   Squirrel

squirrel/sqlexer.cpp@0:97a4f8cc534c, 2014-12-16 (annotated)

Committer:
jhnwkmn
Date:
Tue Dec 16 10:20:34 2014 +0000
Revision:
0:97a4f8cc534c
Initial import of Squirrel.

Who changed what in which revision?

UserRevisionLine numberNew contents of line
jhnwkmn 0:97a4f8cc534c 1 /*
jhnwkmn 0:97a4f8cc534c 2 see copyright notice in squirrel.h
jhnwkmn 0:97a4f8cc534c 3 */
jhnwkmn 0:97a4f8cc534c 4 #include "sqpcheader.h"
jhnwkmn 0:97a4f8cc534c 5 #include <ctype.h>
jhnwkmn 0:97a4f8cc534c 6 #include <stdlib.h>
jhnwkmn 0:97a4f8cc534c 7 #include "sqtable.h"
jhnwkmn 0:97a4f8cc534c 8 #include "sqstring.h"
jhnwkmn 0:97a4f8cc534c 9 #include "sqcompiler.h"
jhnwkmn 0:97a4f8cc534c 10 #include "sqlexer.h"
jhnwkmn 0:97a4f8cc534c 11
jhnwkmn 0:97a4f8cc534c 12 #define CUR_CHAR (_currdata)
jhnwkmn 0:97a4f8cc534c 13 #define RETURN_TOKEN(t) { _prevtoken = _curtoken; _curtoken = t; return t;}
jhnwkmn 0:97a4f8cc534c 14 #define IS_EOB() (CUR_CHAR <= SQUIRREL_EOB)
jhnwkmn 0:97a4f8cc534c 15 #define NEXT() {Next();_currentcolumn++;}
jhnwkmn 0:97a4f8cc534c 16 #define INIT_TEMP_STRING() { _longstr.resize(0);}
jhnwkmn 0:97a4f8cc534c 17 #define APPEND_CHAR(c) { _longstr.push_back(c);}
jhnwkmn 0:97a4f8cc534c 18 #define TERMINATE_BUFFER() {_longstr.push_back(_SC('\0'));}
jhnwkmn 0:97a4f8cc534c 19 #define ADD_KEYWORD(key,id) _keywords->NewSlot( SQString::Create(ss, _SC(#key)) ,SQInteger(id))
jhnwkmn 0:97a4f8cc534c 20
jhnwkmn 0:97a4f8cc534c 21 SQLexer::SQLexer(){}
jhnwkmn 0:97a4f8cc534c 22 SQLexer::~SQLexer()
jhnwkmn 0:97a4f8cc534c 23 {
jhnwkmn 0:97a4f8cc534c 24 _keywords->Release();
jhnwkmn 0:97a4f8cc534c 25 }
jhnwkmn 0:97a4f8cc534c 26
jhnwkmn 0:97a4f8cc534c 27 void SQLexer::Init(SQSharedState *ss, SQLEXREADFUNC rg, SQUserPointer up,CompilerErrorFunc efunc,void *ed)
jhnwkmn 0:97a4f8cc534c 28 {
jhnwkmn 0:97a4f8cc534c 29 _errfunc = efunc;
jhnwkmn 0:97a4f8cc534c 30 _errtarget = ed;
jhnwkmn 0:97a4f8cc534c 31 _sharedstate = ss;
jhnwkmn 0:97a4f8cc534c 32 _keywords = SQTable::Create(ss, 26);
jhnwkmn 0:97a4f8cc534c 33 ADD_KEYWORD(while, TK_WHILE);
jhnwkmn 0:97a4f8cc534c 34 ADD_KEYWORD(do, TK_DO);
jhnwkmn 0:97a4f8cc534c 35 ADD_KEYWORD(if, TK_IF);
jhnwkmn 0:97a4f8cc534c 36 ADD_KEYWORD(else, TK_ELSE);
jhnwkmn 0:97a4f8cc534c 37 ADD_KEYWORD(break, TK_BREAK);
jhnwkmn 0:97a4f8cc534c 38 ADD_KEYWORD(continue, TK_CONTINUE);
jhnwkmn 0:97a4f8cc534c 39 ADD_KEYWORD(return, TK_RETURN);
jhnwkmn 0:97a4f8cc534c 40 ADD_KEYWORD(null, TK_NULL);
jhnwkmn 0:97a4f8cc534c 41 ADD_KEYWORD(function, TK_FUNCTION);
jhnwkmn 0:97a4f8cc534c 42 ADD_KEYWORD(local, TK_LOCAL);
jhnwkmn 0:97a4f8cc534c 43 ADD_KEYWORD(for, TK_FOR);
jhnwkmn 0:97a4f8cc534c 44 ADD_KEYWORD(foreach, TK_FOREACH);
jhnwkmn 0:97a4f8cc534c 45 ADD_KEYWORD(in, TK_IN);
jhnwkmn 0:97a4f8cc534c 46 ADD_KEYWORD(typeof, TK_TYPEOF);
jhnwkmn 0:97a4f8cc534c 47 ADD_KEYWORD(base, TK_BASE);
jhnwkmn 0:97a4f8cc534c 48 ADD_KEYWORD(delete, TK_DELETE);
jhnwkmn 0:97a4f8cc534c 49 ADD_KEYWORD(try, TK_TRY);
jhnwkmn 0:97a4f8cc534c 50 ADD_KEYWORD(catch, TK_CATCH);
jhnwkmn 0:97a4f8cc534c 51 ADD_KEYWORD(throw, TK_THROW);
jhnwkmn 0:97a4f8cc534c 52 ADD_KEYWORD(clone, TK_CLONE);
jhnwkmn 0:97a4f8cc534c 53 ADD_KEYWORD(yield, TK_YIELD);
jhnwkmn 0:97a4f8cc534c 54 ADD_KEYWORD(resume, TK_RESUME);
jhnwkmn 0:97a4f8cc534c 55 ADD_KEYWORD(switch, TK_SWITCH);
jhnwkmn 0:97a4f8cc534c 56 ADD_KEYWORD(case, TK_CASE);
jhnwkmn 0:97a4f8cc534c 57 ADD_KEYWORD(default, TK_DEFAULT);
jhnwkmn 0:97a4f8cc534c 58 ADD_KEYWORD(this, TK_THIS);
jhnwkmn 0:97a4f8cc534c 59 ADD_KEYWORD(class,TK_CLASS);
jhnwkmn 0:97a4f8cc534c 60 ADD_KEYWORD(extends,TK_EXTENDS);
jhnwkmn 0:97a4f8cc534c 61 ADD_KEYWORD(constructor,TK_CONSTRUCTOR);
jhnwkmn 0:97a4f8cc534c 62 ADD_KEYWORD(instanceof,TK_INSTANCEOF);
jhnwkmn 0:97a4f8cc534c 63 ADD_KEYWORD(true,TK_TRUE);
jhnwkmn 0:97a4f8cc534c 64 ADD_KEYWORD(false,TK_FALSE);
jhnwkmn 0:97a4f8cc534c 65 ADD_KEYWORD(static,TK_STATIC);
jhnwkmn 0:97a4f8cc534c 66 ADD_KEYWORD(enum,TK_ENUM);
jhnwkmn 0:97a4f8cc534c 67 ADD_KEYWORD(const,TK_CONST);
jhnwkmn 0:97a4f8cc534c 68
jhnwkmn 0:97a4f8cc534c 69 _readf = rg;
jhnwkmn 0:97a4f8cc534c 70 _up = up;
jhnwkmn 0:97a4f8cc534c 71 _lasttokenline = _currentline = 1;
jhnwkmn 0:97a4f8cc534c 72 _currentcolumn = 0;
jhnwkmn 0:97a4f8cc534c 73 _prevtoken = -1;
jhnwkmn 0:97a4f8cc534c 74 _reached_eof = SQFalse;
jhnwkmn 0:97a4f8cc534c 75 Next();
jhnwkmn 0:97a4f8cc534c 76 }
jhnwkmn 0:97a4f8cc534c 77
jhnwkmn 0:97a4f8cc534c 78 void SQLexer::Error(const SQChar *err)
jhnwkmn 0:97a4f8cc534c 79 {
jhnwkmn 0:97a4f8cc534c 80 _errfunc(_errtarget,err);
jhnwkmn 0:97a4f8cc534c 81 }
jhnwkmn 0:97a4f8cc534c 82
jhnwkmn 0:97a4f8cc534c 83 void SQLexer::Next()
jhnwkmn 0:97a4f8cc534c 84 {
jhnwkmn 0:97a4f8cc534c 85 SQInteger t = _readf(_up);
jhnwkmn 0:97a4f8cc534c 86 if(t > MAX_CHAR) Error(_SC("Invalid character"));
jhnwkmn 0:97a4f8cc534c 87 if(t != 0) {
jhnwkmn 0:97a4f8cc534c 88 _currdata = (LexChar)t;
jhnwkmn 0:97a4f8cc534c 89 return;
jhnwkmn 0:97a4f8cc534c 90 }
jhnwkmn 0:97a4f8cc534c 91 _currdata = SQUIRREL_EOB;
jhnwkmn 0:97a4f8cc534c 92 _reached_eof = SQTrue;
jhnwkmn 0:97a4f8cc534c 93 }
jhnwkmn 0:97a4f8cc534c 94
jhnwkmn 0:97a4f8cc534c 95 const SQChar *SQLexer::Tok2Str(SQInteger tok)
jhnwkmn 0:97a4f8cc534c 96 {
jhnwkmn 0:97a4f8cc534c 97 SQObjectPtr itr, key, val;
jhnwkmn 0:97a4f8cc534c 98 SQInteger nitr;
jhnwkmn 0:97a4f8cc534c 99 while((nitr = _keywords->Next(false,itr, key, val)) != -1) {
jhnwkmn 0:97a4f8cc534c 100 itr = (SQInteger)nitr;
jhnwkmn 0:97a4f8cc534c 101 if(((SQInteger)_integer(val)) == tok)
jhnwkmn 0:97a4f8cc534c 102 return _stringval(key);
jhnwkmn 0:97a4f8cc534c 103 }
jhnwkmn 0:97a4f8cc534c 104 return NULL;
jhnwkmn 0:97a4f8cc534c 105 }
jhnwkmn 0:97a4f8cc534c 106
jhnwkmn 0:97a4f8cc534c 107 void SQLexer::LexBlockComment()
jhnwkmn 0:97a4f8cc534c 108 {
jhnwkmn 0:97a4f8cc534c 109 bool done = false;
jhnwkmn 0:97a4f8cc534c 110 while(!done) {
jhnwkmn 0:97a4f8cc534c 111 switch(CUR_CHAR) {
jhnwkmn 0:97a4f8cc534c 112 case _SC('*'): { NEXT(); if(CUR_CHAR == _SC('/')) { done = true; NEXT(); }}; continue;
jhnwkmn 0:97a4f8cc534c 113 case _SC('\n'): _currentline++; NEXT(); continue;
jhnwkmn 0:97a4f8cc534c 114 case SQUIRREL_EOB: Error(_SC("missing \"*/\" in comment"));
jhnwkmn 0:97a4f8cc534c 115 default: NEXT();
jhnwkmn 0:97a4f8cc534c 116 }
jhnwkmn 0:97a4f8cc534c 117 }
jhnwkmn 0:97a4f8cc534c 118 }
jhnwkmn 0:97a4f8cc534c 119 void SQLexer::LexLineComment()
jhnwkmn 0:97a4f8cc534c 120 {
jhnwkmn 0:97a4f8cc534c 121 do { NEXT(); } while (CUR_CHAR != _SC('\n') && (!IS_EOB()));
jhnwkmn 0:97a4f8cc534c 122 }
jhnwkmn 0:97a4f8cc534c 123
jhnwkmn 0:97a4f8cc534c 124 SQInteger SQLexer::Lex()
jhnwkmn 0:97a4f8cc534c 125 {
jhnwkmn 0:97a4f8cc534c 126 _lasttokenline = _currentline;
jhnwkmn 0:97a4f8cc534c 127 while(CUR_CHAR != SQUIRREL_EOB) {
jhnwkmn 0:97a4f8cc534c 128 switch(CUR_CHAR){
jhnwkmn 0:97a4f8cc534c 129 case _SC('\t'): case _SC('\r'): case _SC(' '): NEXT(); continue;
jhnwkmn 0:97a4f8cc534c 130 case _SC('\n'):
jhnwkmn 0:97a4f8cc534c 131 _currentline++;
jhnwkmn 0:97a4f8cc534c 132 _prevtoken=_curtoken;
jhnwkmn 0:97a4f8cc534c 133 _curtoken=_SC('\n');
jhnwkmn 0:97a4f8cc534c 134 NEXT();
jhnwkmn 0:97a4f8cc534c 135 _currentcolumn=1;
jhnwkmn 0:97a4f8cc534c 136 continue;
jhnwkmn 0:97a4f8cc534c 137 case _SC('#'): LexLineComment(); continue;
jhnwkmn 0:97a4f8cc534c 138 case _SC('/'):
jhnwkmn 0:97a4f8cc534c 139 NEXT();
jhnwkmn 0:97a4f8cc534c 140 switch(CUR_CHAR){
jhnwkmn 0:97a4f8cc534c 141 case _SC('*'):
jhnwkmn 0:97a4f8cc534c 142 NEXT();
jhnwkmn 0:97a4f8cc534c 143 LexBlockComment();
jhnwkmn 0:97a4f8cc534c 144 continue;
jhnwkmn 0:97a4f8cc534c 145 case _SC('/'):
jhnwkmn 0:97a4f8cc534c 146 LexLineComment();
jhnwkmn 0:97a4f8cc534c 147 continue;
jhnwkmn 0:97a4f8cc534c 148 case _SC('='):
jhnwkmn 0:97a4f8cc534c 149 NEXT();
jhnwkmn 0:97a4f8cc534c 150 RETURN_TOKEN(TK_DIVEQ);
jhnwkmn 0:97a4f8cc534c 151 continue;
jhnwkmn 0:97a4f8cc534c 152 case _SC('>'):
jhnwkmn 0:97a4f8cc534c 153 NEXT();
jhnwkmn 0:97a4f8cc534c 154 RETURN_TOKEN(TK_ATTR_CLOSE);
jhnwkmn 0:97a4f8cc534c 155 continue;
jhnwkmn 0:97a4f8cc534c 156 default:
jhnwkmn 0:97a4f8cc534c 157 RETURN_TOKEN('/');
jhnwkmn 0:97a4f8cc534c 158 }
jhnwkmn 0:97a4f8cc534c 159 case _SC('='):
jhnwkmn 0:97a4f8cc534c 160 NEXT();
jhnwkmn 0:97a4f8cc534c 161 if (CUR_CHAR != _SC('=')){ RETURN_TOKEN('=') }
jhnwkmn 0:97a4f8cc534c 162 else { NEXT(); RETURN_TOKEN(TK_EQ); }
jhnwkmn 0:97a4f8cc534c 163 case _SC('<'):
jhnwkmn 0:97a4f8cc534c 164 NEXT();
jhnwkmn 0:97a4f8cc534c 165 switch(CUR_CHAR) {
jhnwkmn 0:97a4f8cc534c 166 case _SC('='):
jhnwkmn 0:97a4f8cc534c 167 NEXT();
jhnwkmn 0:97a4f8cc534c 168 if(CUR_CHAR == _SC('>')) {
jhnwkmn 0:97a4f8cc534c 169 NEXT();
jhnwkmn 0:97a4f8cc534c 170 RETURN_TOKEN(TK_3WAYSCMP);
jhnwkmn 0:97a4f8cc534c 171 }
jhnwkmn 0:97a4f8cc534c 172 RETURN_TOKEN(TK_LE)
jhnwkmn 0:97a4f8cc534c 173 break;
jhnwkmn 0:97a4f8cc534c 174 case _SC('-'): NEXT(); RETURN_TOKEN(TK_NEWSLOT); break;
jhnwkmn 0:97a4f8cc534c 175 case _SC('<'): NEXT(); RETURN_TOKEN(TK_SHIFTL); break;
jhnwkmn 0:97a4f8cc534c 176 case _SC('/'): NEXT(); RETURN_TOKEN(TK_ATTR_OPEN); break;
jhnwkmn 0:97a4f8cc534c 177 }
jhnwkmn 0:97a4f8cc534c 178 RETURN_TOKEN('<');
jhnwkmn 0:97a4f8cc534c 179 case _SC('>'):
jhnwkmn 0:97a4f8cc534c 180 NEXT();
jhnwkmn 0:97a4f8cc534c 181 if (CUR_CHAR == _SC('=')){ NEXT(); RETURN_TOKEN(TK_GE);}
jhnwkmn 0:97a4f8cc534c 182 else if(CUR_CHAR == _SC('>')){
jhnwkmn 0:97a4f8cc534c 183 NEXT();
jhnwkmn 0:97a4f8cc534c 184 if(CUR_CHAR == _SC('>')){
jhnwkmn 0:97a4f8cc534c 185 NEXT();
jhnwkmn 0:97a4f8cc534c 186 RETURN_TOKEN(TK_USHIFTR);
jhnwkmn 0:97a4f8cc534c 187 }
jhnwkmn 0:97a4f8cc534c 188 RETURN_TOKEN(TK_SHIFTR);
jhnwkmn 0:97a4f8cc534c 189 }
jhnwkmn 0:97a4f8cc534c 190 else { RETURN_TOKEN('>') }
jhnwkmn 0:97a4f8cc534c 191 case _SC('!'):
jhnwkmn 0:97a4f8cc534c 192 NEXT();
jhnwkmn 0:97a4f8cc534c 193 if (CUR_CHAR != _SC('=')){ RETURN_TOKEN('!')}
jhnwkmn 0:97a4f8cc534c 194 else { NEXT(); RETURN_TOKEN(TK_NE); }
jhnwkmn 0:97a4f8cc534c 195 case _SC('@'): {
jhnwkmn 0:97a4f8cc534c 196 SQInteger stype;
jhnwkmn 0:97a4f8cc534c 197 NEXT();
jhnwkmn 0:97a4f8cc534c 198 if(CUR_CHAR != _SC('"')) {
jhnwkmn 0:97a4f8cc534c 199 RETURN_TOKEN('@');
jhnwkmn 0:97a4f8cc534c 200 }
jhnwkmn 0:97a4f8cc534c 201 if((stype=ReadString('"',true))!=-1) {
jhnwkmn 0:97a4f8cc534c 202 RETURN_TOKEN(stype);
jhnwkmn 0:97a4f8cc534c 203 }
jhnwkmn 0:97a4f8cc534c 204 Error(_SC("error parsing the string"));
jhnwkmn 0:97a4f8cc534c 205 }
jhnwkmn 0:97a4f8cc534c 206 case _SC('"'):
jhnwkmn 0:97a4f8cc534c 207 case _SC('\''): {
jhnwkmn 0:97a4f8cc534c 208 SQInteger stype;
jhnwkmn 0:97a4f8cc534c 209 if((stype=ReadString(CUR_CHAR,false))!=-1){
jhnwkmn 0:97a4f8cc534c 210 RETURN_TOKEN(stype);
jhnwkmn 0:97a4f8cc534c 211 }
jhnwkmn 0:97a4f8cc534c 212 Error(_SC("error parsing the string"));
jhnwkmn 0:97a4f8cc534c 213 }
jhnwkmn 0:97a4f8cc534c 214 case _SC('{'): case _SC('}'): case _SC('('): case _SC(')'): case _SC('['): case _SC(']'):
jhnwkmn 0:97a4f8cc534c 215 case _SC(';'): case _SC(','): case _SC('?'): case _SC('^'): case _SC('~'):
jhnwkmn 0:97a4f8cc534c 216 {SQInteger ret = CUR_CHAR;
jhnwkmn 0:97a4f8cc534c 217 NEXT(); RETURN_TOKEN(ret); }
jhnwkmn 0:97a4f8cc534c 218 case _SC('.'):
jhnwkmn 0:97a4f8cc534c 219 NEXT();
jhnwkmn 0:97a4f8cc534c 220 if (CUR_CHAR != _SC('.')){ RETURN_TOKEN('.') }
jhnwkmn 0:97a4f8cc534c 221 NEXT();
jhnwkmn 0:97a4f8cc534c 222 if (CUR_CHAR != _SC('.')){ Error(_SC("invalid token '..'")); }
jhnwkmn 0:97a4f8cc534c 223 NEXT();
jhnwkmn 0:97a4f8cc534c 224 RETURN_TOKEN(TK_VARPARAMS);
jhnwkmn 0:97a4f8cc534c 225 case _SC('&'):
jhnwkmn 0:97a4f8cc534c 226 NEXT();
jhnwkmn 0:97a4f8cc534c 227 if (CUR_CHAR != _SC('&')){ RETURN_TOKEN('&') }
jhnwkmn 0:97a4f8cc534c 228 else { NEXT(); RETURN_TOKEN(TK_AND); }
jhnwkmn 0:97a4f8cc534c 229 case _SC('|'):
jhnwkmn 0:97a4f8cc534c 230 NEXT();
jhnwkmn 0:97a4f8cc534c 231 if (CUR_CHAR != _SC('|')){ RETURN_TOKEN('|') }
jhnwkmn 0:97a4f8cc534c 232 else { NEXT(); RETURN_TOKEN(TK_OR); }
jhnwkmn 0:97a4f8cc534c 233 case _SC(':'):
jhnwkmn 0:97a4f8cc534c 234 NEXT();
jhnwkmn 0:97a4f8cc534c 235 if (CUR_CHAR != _SC(':')){ RETURN_TOKEN(':') }
jhnwkmn 0:97a4f8cc534c 236 else { NEXT(); RETURN_TOKEN(TK_DOUBLE_COLON); }
jhnwkmn 0:97a4f8cc534c 237 case _SC('*'):
jhnwkmn 0:97a4f8cc534c 238 NEXT();
jhnwkmn 0:97a4f8cc534c 239 if (CUR_CHAR == _SC('=')){ NEXT(); RETURN_TOKEN(TK_MULEQ);}
jhnwkmn 0:97a4f8cc534c 240 else RETURN_TOKEN('*');
jhnwkmn 0:97a4f8cc534c 241 case _SC('%'):
jhnwkmn 0:97a4f8cc534c 242 NEXT();
jhnwkmn 0:97a4f8cc534c 243 if (CUR_CHAR == _SC('=')){ NEXT(); RETURN_TOKEN(TK_MODEQ);}
jhnwkmn 0:97a4f8cc534c 244 else RETURN_TOKEN('%');
jhnwkmn 0:97a4f8cc534c 245 case _SC('-'):
jhnwkmn 0:97a4f8cc534c 246 NEXT();
jhnwkmn 0:97a4f8cc534c 247 if (CUR_CHAR == _SC('=')){ NEXT(); RETURN_TOKEN(TK_MINUSEQ);}
jhnwkmn 0:97a4f8cc534c 248 else if (CUR_CHAR == _SC('-')){ NEXT(); RETURN_TOKEN(TK_MINUSMINUS);}
jhnwkmn 0:97a4f8cc534c 249 else RETURN_TOKEN('-');
jhnwkmn 0:97a4f8cc534c 250 case _SC('+'):
jhnwkmn 0:97a4f8cc534c 251 NEXT();
jhnwkmn 0:97a4f8cc534c 252 if (CUR_CHAR == _SC('=')){ NEXT(); RETURN_TOKEN(TK_PLUSEQ);}
jhnwkmn 0:97a4f8cc534c 253 else if (CUR_CHAR == _SC('+')){ NEXT(); RETURN_TOKEN(TK_PLUSPLUS);}
jhnwkmn 0:97a4f8cc534c 254 else RETURN_TOKEN('+');
jhnwkmn 0:97a4f8cc534c 255 case SQUIRREL_EOB:
jhnwkmn 0:97a4f8cc534c 256 return 0;
jhnwkmn 0:97a4f8cc534c 257 default:{
jhnwkmn 0:97a4f8cc534c 258 if (scisdigit(CUR_CHAR)) {
jhnwkmn 0:97a4f8cc534c 259 SQInteger ret = ReadNumber();
jhnwkmn 0:97a4f8cc534c 260 RETURN_TOKEN(ret);
jhnwkmn 0:97a4f8cc534c 261 }
jhnwkmn 0:97a4f8cc534c 262 else if (scisalpha(CUR_CHAR) || CUR_CHAR == _SC('_')) {
jhnwkmn 0:97a4f8cc534c 263 SQInteger t = ReadID();
jhnwkmn 0:97a4f8cc534c 264 RETURN_TOKEN(t);
jhnwkmn 0:97a4f8cc534c 265 }
jhnwkmn 0:97a4f8cc534c 266 else {
jhnwkmn 0:97a4f8cc534c 267 SQInteger c = CUR_CHAR;
jhnwkmn 0:97a4f8cc534c 268 if (sciscntrl((int)c)) Error(_SC("unexpected character(control)"));
jhnwkmn 0:97a4f8cc534c 269 NEXT();
jhnwkmn 0:97a4f8cc534c 270 RETURN_TOKEN(c);
jhnwkmn 0:97a4f8cc534c 271 }
jhnwkmn 0:97a4f8cc534c 272 RETURN_TOKEN(0);
jhnwkmn 0:97a4f8cc534c 273 }
jhnwkmn 0:97a4f8cc534c 274 }
jhnwkmn 0:97a4f8cc534c 275 }
jhnwkmn 0:97a4f8cc534c 276 return 0;
jhnwkmn 0:97a4f8cc534c 277 }
jhnwkmn 0:97a4f8cc534c 278
jhnwkmn 0:97a4f8cc534c 279 SQInteger SQLexer::GetIDType(SQChar *s)
jhnwkmn 0:97a4f8cc534c 280 {
jhnwkmn 0:97a4f8cc534c 281 SQObjectPtr t;
jhnwkmn 0:97a4f8cc534c 282 if(_keywords->Get(SQString::Create(_sharedstate, s), t)) {
jhnwkmn 0:97a4f8cc534c 283 return SQInteger(_integer(t));
jhnwkmn 0:97a4f8cc534c 284 }
jhnwkmn 0:97a4f8cc534c 285 return TK_IDENTIFIER;
jhnwkmn 0:97a4f8cc534c 286 }
jhnwkmn 0:97a4f8cc534c 287
jhnwkmn 0:97a4f8cc534c 288
jhnwkmn 0:97a4f8cc534c 289 SQInteger SQLexer::ReadString(SQInteger ndelim,bool verbatim)
jhnwkmn 0:97a4f8cc534c 290 {
jhnwkmn 0:97a4f8cc534c 291 INIT_TEMP_STRING();
jhnwkmn 0:97a4f8cc534c 292 NEXT();
jhnwkmn 0:97a4f8cc534c 293 if(IS_EOB()) return -1;
jhnwkmn 0:97a4f8cc534c 294 for(;;) {
jhnwkmn 0:97a4f8cc534c 295 while(CUR_CHAR != ndelim) {
jhnwkmn 0:97a4f8cc534c 296 switch(CUR_CHAR) {
jhnwkmn 0:97a4f8cc534c 297 case SQUIRREL_EOB:
jhnwkmn 0:97a4f8cc534c 298 Error(_SC("unfinished string"));
jhnwkmn 0:97a4f8cc534c 299 return -1;
jhnwkmn 0:97a4f8cc534c 300 case _SC('\n'):
jhnwkmn 0:97a4f8cc534c 301 if(!verbatim) Error(_SC("newline in a constant"));
jhnwkmn 0:97a4f8cc534c 302 APPEND_CHAR(CUR_CHAR); NEXT();
jhnwkmn 0:97a4f8cc534c 303 _currentline++;
jhnwkmn 0:97a4f8cc534c 304 break;
jhnwkmn 0:97a4f8cc534c 305 case _SC('\\'):
jhnwkmn 0:97a4f8cc534c 306 if(verbatim) {
jhnwkmn 0:97a4f8cc534c 307 APPEND_CHAR('\\'); NEXT();
jhnwkmn 0:97a4f8cc534c 308 }
jhnwkmn 0:97a4f8cc534c 309 else {
jhnwkmn 0:97a4f8cc534c 310 NEXT();
jhnwkmn 0:97a4f8cc534c 311 switch(CUR_CHAR) {
jhnwkmn 0:97a4f8cc534c 312 case _SC('x'): NEXT(); {
jhnwkmn 0:97a4f8cc534c 313 if(!isxdigit(CUR_CHAR)) Error(_SC("hexadecimal number expected"));
jhnwkmn 0:97a4f8cc534c 314 const SQInteger maxdigits = 4;
jhnwkmn 0:97a4f8cc534c 315 SQChar temp[maxdigits+1];
jhnwkmn 0:97a4f8cc534c 316 SQInteger n = 0;
jhnwkmn 0:97a4f8cc534c 317 while(isxdigit(CUR_CHAR) && n < maxdigits) {
jhnwkmn 0:97a4f8cc534c 318 temp[n] = CUR_CHAR;
jhnwkmn 0:97a4f8cc534c 319 n++;
jhnwkmn 0:97a4f8cc534c 320 NEXT();
jhnwkmn 0:97a4f8cc534c 321 }
jhnwkmn 0:97a4f8cc534c 322 temp[n] = 0;
jhnwkmn 0:97a4f8cc534c 323 SQChar *sTemp;
jhnwkmn 0:97a4f8cc534c 324 APPEND_CHAR((SQChar)scstrtoul(temp,&sTemp,16));
jhnwkmn 0:97a4f8cc534c 325 }
jhnwkmn 0:97a4f8cc534c 326 break;
jhnwkmn 0:97a4f8cc534c 327 case _SC('t'): APPEND_CHAR(_SC('\t')); NEXT(); break;
jhnwkmn 0:97a4f8cc534c 328 case _SC('a'): APPEND_CHAR(_SC('\a')); NEXT(); break;
jhnwkmn 0:97a4f8cc534c 329 case _SC('b'): APPEND_CHAR(_SC('\b')); NEXT(); break;
jhnwkmn 0:97a4f8cc534c 330 case _SC('n'): APPEND_CHAR(_SC('\n')); NEXT(); break;
jhnwkmn 0:97a4f8cc534c 331 case _SC('r'): APPEND_CHAR(_SC('\r')); NEXT(); break;
jhnwkmn 0:97a4f8cc534c 332 case _SC('v'): APPEND_CHAR(_SC('\v')); NEXT(); break;
jhnwkmn 0:97a4f8cc534c 333 case _SC('f'): APPEND_CHAR(_SC('\f')); NEXT(); break;
jhnwkmn 0:97a4f8cc534c 334 case _SC('0'): APPEND_CHAR(_SC('\0')); NEXT(); break;
jhnwkmn 0:97a4f8cc534c 335 case _SC('\\'): APPEND_CHAR(_SC('\\')); NEXT(); break;
jhnwkmn 0:97a4f8cc534c 336 case _SC('"'): APPEND_CHAR(_SC('"')); NEXT(); break;
jhnwkmn 0:97a4f8cc534c 337 case _SC('\''): APPEND_CHAR(_SC('\'')); NEXT(); break;
jhnwkmn 0:97a4f8cc534c 338 default:
jhnwkmn 0:97a4f8cc534c 339 Error(_SC("unrecognised escaper char"));
jhnwkmn 0:97a4f8cc534c 340 break;
jhnwkmn 0:97a4f8cc534c 341 }
jhnwkmn 0:97a4f8cc534c 342 }
jhnwkmn 0:97a4f8cc534c 343 break;
jhnwkmn 0:97a4f8cc534c 344 default:
jhnwkmn 0:97a4f8cc534c 345 APPEND_CHAR(CUR_CHAR);
jhnwkmn 0:97a4f8cc534c 346 NEXT();
jhnwkmn 0:97a4f8cc534c 347 }
jhnwkmn 0:97a4f8cc534c 348 }
jhnwkmn 0:97a4f8cc534c 349 NEXT();
jhnwkmn 0:97a4f8cc534c 350 if(verbatim && CUR_CHAR == '"') { //double quotation
jhnwkmn 0:97a4f8cc534c 351 APPEND_CHAR(CUR_CHAR);
jhnwkmn 0:97a4f8cc534c 352 NEXT();
jhnwkmn 0:97a4f8cc534c 353 }
jhnwkmn 0:97a4f8cc534c 354 else {
jhnwkmn 0:97a4f8cc534c 355 break;
jhnwkmn 0:97a4f8cc534c 356 }
jhnwkmn 0:97a4f8cc534c 357 }
jhnwkmn 0:97a4f8cc534c 358 TERMINATE_BUFFER();
jhnwkmn 0:97a4f8cc534c 359 SQInteger len = _longstr.size()-1;
jhnwkmn 0:97a4f8cc534c 360 if(ndelim == _SC('\'')) {
jhnwkmn 0:97a4f8cc534c 361 if(len == 0) Error(_SC("empty constant"));
jhnwkmn 0:97a4f8cc534c 362 if(len > 1) Error(_SC("constant too long"));
jhnwkmn 0:97a4f8cc534c 363 _nvalue = _longstr[0];
jhnwkmn 0:97a4f8cc534c 364 return TK_INTEGER;
jhnwkmn 0:97a4f8cc534c 365 }
jhnwkmn 0:97a4f8cc534c 366 _svalue = &_longstr[0];
jhnwkmn 0:97a4f8cc534c 367 return TK_STRING_LITERAL;
jhnwkmn 0:97a4f8cc534c 368 }
jhnwkmn 0:97a4f8cc534c 369
jhnwkmn 0:97a4f8cc534c 370 void LexHexadecimal(const SQChar *s,SQUnsignedInteger *res)
jhnwkmn 0:97a4f8cc534c 371 {
jhnwkmn 0:97a4f8cc534c 372 *res = 0;
jhnwkmn 0:97a4f8cc534c 373 while(*s != 0)
jhnwkmn 0:97a4f8cc534c 374 {
jhnwkmn 0:97a4f8cc534c 375 if(scisdigit(*s)) *res = (*res)*16+((*s++)-'0');
jhnwkmn 0:97a4f8cc534c 376 else if(scisxdigit(*s)) *res = (*res)*16+(toupper(*s++)-'A'+10);
jhnwkmn 0:97a4f8cc534c 377 else { assert(0); }
jhnwkmn 0:97a4f8cc534c 378 }
jhnwkmn 0:97a4f8cc534c 379 }
jhnwkmn 0:97a4f8cc534c 380
jhnwkmn 0:97a4f8cc534c 381 void LexInteger(const SQChar *s,SQUnsignedInteger *res)
jhnwkmn 0:97a4f8cc534c 382 {
jhnwkmn 0:97a4f8cc534c 383 *res = 0;
jhnwkmn 0:97a4f8cc534c 384 while(*s != 0)
jhnwkmn 0:97a4f8cc534c 385 {
jhnwkmn 0:97a4f8cc534c 386 *res = (*res)*10+((*s++)-'0');
jhnwkmn 0:97a4f8cc534c 387 }
jhnwkmn 0:97a4f8cc534c 388 }
jhnwkmn 0:97a4f8cc534c 389
jhnwkmn 0:97a4f8cc534c 390 SQInteger scisodigit(SQInteger c) { return c >= _SC('0') && c <= _SC('7'); }
jhnwkmn 0:97a4f8cc534c 391
jhnwkmn 0:97a4f8cc534c 392 void LexOctal(const SQChar *s,SQUnsignedInteger *res)
jhnwkmn 0:97a4f8cc534c 393 {
jhnwkmn 0:97a4f8cc534c 394 *res = 0;
jhnwkmn 0:97a4f8cc534c 395 while(*s != 0)
jhnwkmn 0:97a4f8cc534c 396 {
jhnwkmn 0:97a4f8cc534c 397 if(scisodigit(*s)) *res = (*res)*8+((*s++)-'0');
jhnwkmn 0:97a4f8cc534c 398 else { assert(0); }
jhnwkmn 0:97a4f8cc534c 399 }
jhnwkmn 0:97a4f8cc534c 400 }
jhnwkmn 0:97a4f8cc534c 401
jhnwkmn 0:97a4f8cc534c 402 SQInteger isexponent(SQInteger c) { return c == 'e' || c=='E'; }
jhnwkmn 0:97a4f8cc534c 403
jhnwkmn 0:97a4f8cc534c 404
jhnwkmn 0:97a4f8cc534c 405 #define MAX_HEX_DIGITS (sizeof(SQInteger)*2)
jhnwkmn 0:97a4f8cc534c 406 SQInteger SQLexer::ReadNumber()
jhnwkmn 0:97a4f8cc534c 407 {
jhnwkmn 0:97a4f8cc534c 408 #define TINT 1
jhnwkmn 0:97a4f8cc534c 409 #define TFLOAT 2
jhnwkmn 0:97a4f8cc534c 410 #define THEX 3
jhnwkmn 0:97a4f8cc534c 411 #define TSCIENTIFIC 4
jhnwkmn 0:97a4f8cc534c 412 #define TOCTAL 5
jhnwkmn 0:97a4f8cc534c 413 SQInteger type = TINT, firstchar = CUR_CHAR;
jhnwkmn 0:97a4f8cc534c 414 SQChar *sTemp;
jhnwkmn 0:97a4f8cc534c 415 INIT_TEMP_STRING();
jhnwkmn 0:97a4f8cc534c 416 NEXT();
jhnwkmn 0:97a4f8cc534c 417 if(firstchar == _SC('0') && (toupper(CUR_CHAR) == _SC('X') || scisodigit(CUR_CHAR)) ) {
jhnwkmn 0:97a4f8cc534c 418 if(scisodigit(CUR_CHAR)) {
jhnwkmn 0:97a4f8cc534c 419 type = TOCTAL;
jhnwkmn 0:97a4f8cc534c 420 while(scisodigit(CUR_CHAR)) {
jhnwkmn 0:97a4f8cc534c 421 APPEND_CHAR(CUR_CHAR);
jhnwkmn 0:97a4f8cc534c 422 NEXT();
jhnwkmn 0:97a4f8cc534c 423 }
jhnwkmn 0:97a4f8cc534c 424 if(scisdigit(CUR_CHAR)) Error(_SC("invalid octal number"));
jhnwkmn 0:97a4f8cc534c 425 }
jhnwkmn 0:97a4f8cc534c 426 else {
jhnwkmn 0:97a4f8cc534c 427 NEXT();
jhnwkmn 0:97a4f8cc534c 428 type = THEX;
jhnwkmn 0:97a4f8cc534c 429 while(isxdigit(CUR_CHAR)) {
jhnwkmn 0:97a4f8cc534c 430 APPEND_CHAR(CUR_CHAR);
jhnwkmn 0:97a4f8cc534c 431 NEXT();
jhnwkmn 0:97a4f8cc534c 432 }
jhnwkmn 0:97a4f8cc534c 433 if(_longstr.size() > MAX_HEX_DIGITS) Error(_SC("too many digits for an Hex number"));
jhnwkmn 0:97a4f8cc534c 434 }
jhnwkmn 0:97a4f8cc534c 435 }
jhnwkmn 0:97a4f8cc534c 436 else {
jhnwkmn 0:97a4f8cc534c 437 APPEND_CHAR((int)firstchar);
jhnwkmn 0:97a4f8cc534c 438 while (CUR_CHAR == _SC('.') || scisdigit(CUR_CHAR) || isexponent(CUR_CHAR)) {
jhnwkmn 0:97a4f8cc534c 439 if(CUR_CHAR == _SC('.') || isexponent(CUR_CHAR)) type = TFLOAT;
jhnwkmn 0:97a4f8cc534c 440 if(isexponent(CUR_CHAR)) {
jhnwkmn 0:97a4f8cc534c 441 if(type != TFLOAT) Error(_SC("invalid numeric format"));
jhnwkmn 0:97a4f8cc534c 442 type = TSCIENTIFIC;
jhnwkmn 0:97a4f8cc534c 443 APPEND_CHAR(CUR_CHAR);
jhnwkmn 0:97a4f8cc534c 444 NEXT();
jhnwkmn 0:97a4f8cc534c 445 if(CUR_CHAR == '+' || CUR_CHAR == '-'){
jhnwkmn 0:97a4f8cc534c 446 APPEND_CHAR(CUR_CHAR);
jhnwkmn 0:97a4f8cc534c 447 NEXT();
jhnwkmn 0:97a4f8cc534c 448 }
jhnwkmn 0:97a4f8cc534c 449 if(!scisdigit(CUR_CHAR)) Error(_SC("exponent expected"));
jhnwkmn 0:97a4f8cc534c 450 }
jhnwkmn 0:97a4f8cc534c 451
jhnwkmn 0:97a4f8cc534c 452 APPEND_CHAR(CUR_CHAR);
jhnwkmn 0:97a4f8cc534c 453 NEXT();
jhnwkmn 0:97a4f8cc534c 454 }
jhnwkmn 0:97a4f8cc534c 455 }
jhnwkmn 0:97a4f8cc534c 456 TERMINATE_BUFFER();
jhnwkmn 0:97a4f8cc534c 457 switch(type) {
jhnwkmn 0:97a4f8cc534c 458 case TSCIENTIFIC:
jhnwkmn 0:97a4f8cc534c 459 case TFLOAT:
jhnwkmn 0:97a4f8cc534c 460 _fvalue = (SQFloat)scstrtod(&_longstr[0],&sTemp);
jhnwkmn 0:97a4f8cc534c 461 return TK_FLOAT;
jhnwkmn 0:97a4f8cc534c 462 case TINT:
jhnwkmn 0:97a4f8cc534c 463 LexInteger(&_longstr[0],(SQUnsignedInteger *)&_nvalue);
jhnwkmn 0:97a4f8cc534c 464 return TK_INTEGER;
jhnwkmn 0:97a4f8cc534c 465 case THEX:
jhnwkmn 0:97a4f8cc534c 466 LexHexadecimal(&_longstr[0],(SQUnsignedInteger *)&_nvalue);
jhnwkmn 0:97a4f8cc534c 467 return TK_INTEGER;
jhnwkmn 0:97a4f8cc534c 468 case TOCTAL:
jhnwkmn 0:97a4f8cc534c 469 LexOctal(&_longstr[0],(SQUnsignedInteger *)&_nvalue);
jhnwkmn 0:97a4f8cc534c 470 return TK_INTEGER;
jhnwkmn 0:97a4f8cc534c 471 }
jhnwkmn 0:97a4f8cc534c 472 return 0;
jhnwkmn 0:97a4f8cc534c 473 }
jhnwkmn 0:97a4f8cc534c 474
jhnwkmn 0:97a4f8cc534c 475 SQInteger SQLexer::ReadID()
jhnwkmn 0:97a4f8cc534c 476 {
jhnwkmn 0:97a4f8cc534c 477 SQInteger res;
jhnwkmn 0:97a4f8cc534c 478 INIT_TEMP_STRING();
jhnwkmn 0:97a4f8cc534c 479 do {
jhnwkmn 0:97a4f8cc534c 480 APPEND_CHAR(CUR_CHAR);
jhnwkmn 0:97a4f8cc534c 481 NEXT();
jhnwkmn 0:97a4f8cc534c 482 } while(scisalnum(CUR_CHAR) || CUR_CHAR == _SC('_'));
jhnwkmn 0:97a4f8cc534c 483 TERMINATE_BUFFER();
jhnwkmn 0:97a4f8cc534c 484 res = GetIDType(&_longstr[0]);
jhnwkmn 0:97a4f8cc534c 485 if(res == TK_IDENTIFIER || res == TK_CONSTRUCTOR) {
jhnwkmn 0:97a4f8cc534c 486 _svalue = &_longstr[0];
jhnwkmn 0:97a4f8cc534c 487 }
jhnwkmn 0:97a4f8cc534c 488 return res;
jhnwkmn 0:97a4f8cc534c 489 }