File content as of revision 0:864f6ee5169b:

/*
 * Source file tokenization
 */
 
#include <ctype.h>
#include <stdlib.h>

#include "tokens.h"

Token::Token() : type_(UNKNOWN) {
}

Token::~Token() {
    if (isWord()) {
        free(u.word);
    }
}

const char* Token::getWord() const {
    return isWord() ? u.word : NULL;
}

int Token::getInteger() const {
    return isInteger() ? u.value : 0;
}

void Token::get_token(Token* token, const char* str, int* pos) {
    // Reset state of the token...
    if (token->isWord()) {
        free(token->u.word);
        token->u.word = NULL;
    }
    token->type_ = UNKNOWN;

    // Skip any spaces in the token stream...
    int i;
    for (i = *pos; isspace(str[i]); ++i) {
    }
    
    // Now find and return the token...
    int start = i;
    if (isalpha(str[i])) {
        for ( ; isalnum(str[i]); ++i) {
        }
        char* word = (char*)malloc(i - start + 1);
        int k = 0;
        for (int j = start; j < i; ++j, ++k) {
            word[k] = toupper(str[j]);
        }
        word[k] = 0;
        
        token->type_ = WORD;
        token->u.word = word;
    } else if (isdigit(str[i])) {
        int val = str[i] - '0';
        for (++i; isdigit(str[i]); ++i) {
            val *= 10;
            val += str[i] - '0';
        }
        
        token->type_ = INTEGER;
        token->u.value = val;
    } else if (str[i] == 0) {
        token->type_ = EOL;
    }
    *pos = i;
}