Diff: tokens.cpp

Revision:

0:864f6ee5169b

diff -r 000000000000 -r 864f6ee5169b tokens.cpp
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/tokens.cpp	Sat Apr 09 18:06:27 2011 +0000
@@ -0,0 +1,67 @@
+/*
+ * Source file tokenization
+ */
+ 
+#include <ctype.h>
+#include <stdlib.h>
+
+#include "tokens.h"
+
+Token::Token() : type_(UNKNOWN) {
+}
+
+Token::~Token() {
+    if (isWord()) {
+        free(u.word);
+    }
+}
+
+const char* Token::getWord() const {
+    return isWord() ? u.word : NULL;
+}
+
+int Token::getInteger() const {
+    return isInteger() ? u.value : 0;
+}
+
+void Token::get_token(Token* token, const char* str, int* pos) {
+    // Reset state of the token...
+    if (token->isWord()) {
+        free(token->u.word);
+        token->u.word = NULL;
+    }
+    token->type_ = UNKNOWN;
+
+    // Skip any spaces in the token stream...
+    int i;
+    for (i = *pos; isspace(str[i]); ++i) {
+    }
+    
+    // Now find and return the token...
+    int start = i;
+    if (isalpha(str[i])) {
+        for ( ; isalnum(str[i]); ++i) {
+        }
+        char* word = (char*)malloc(i - start + 1);
+        int k = 0;
+        for (int j = start; j < i; ++j, ++k) {
+            word[k] = toupper(str[j]);
+        }
+        word[k] = 0;
+        
+        token->type_ = WORD;
+        token->u.word = word;
+    } else if (isdigit(str[i])) {
+        int val = str[i] - '0';
+        for (++i; isdigit(str[i]); ++i) {
+            val *= 10;
+            val += str[i] - '0';
+        }
+        
+        token->type_ = INTEGER;
+        token->u.value = val;
+    } else if (str[i] == 0) {
+        token->type_ = EOL;
+    }
+    *pos = i;
+}
\ No newline at end of file