Invercargill/src/lib/Expressions/ExpressionTokenizer.vala

using Invercargill.DataStructures;

namespace Invercargill.Expressions {

    /**
     * Token types for expression parsing.
     */
    public enum TokenType {
        // Literals
        INTEGER,
        LONG_INTEGER,       // integer with L suffix
        UNSIGNED_LONG,      // integer with UL suffix
        FLOAT,
        FLOAT_LITERAL,      // float with f suffix
        STRING,
        CHAR_LITERAL,       // single character in single quotes
        TRUE,
        FALSE,
        NULL_LITERAL,

        // Identifiers and keywords
        IDENTIFIER,

        // Parameter placeholder
        PARAMETER,  // $0, $1, $2, etc.

        // Operators
        PLUS,           // +
        MINUS,          // -
        STAR,           // *
        SLASH,          // /
        PERCENT,        // %
        EQUALS,         // ==
        NOT_EQUALS,     // !=
        LESS_THAN,      // <
        GREATER_THAN,   // >
        LESS_EQUALS,    // <=
        GREATER_EQUALS, // >=
        AND,            // &&
        OR,             // ||
        NOT,            // !
        ASSIGN,         // = (for single equals, used in some contexts)

        // Punctuation
        DOT,            // .
        COMMA,          // ,
        LPAREN,         // (
        RPAREN,         // )
        LBRACKET,       // [
        RBRACKET,       // ]
        QUESTION,       // ?
        COLON,          // :
        ARROW,          // =>

        // Special
        EOF;

        public string to_string() {
            switch (this) {
                case INTEGER: return "INTEGER";
                case LONG_INTEGER: return "LONG_INTEGER";
                case UNSIGNED_LONG: return "UNSIGNED_LONG";
                case FLOAT: return "FLOAT";
                case FLOAT_LITERAL: return "FLOAT_LITERAL";
                case STRING: return "STRING";
                case CHAR_LITERAL: return "CHAR_LITERAL";
                case TRUE: return "TRUE";
                case FALSE: return "FALSE";
                case NULL_LITERAL: return "NULL";
                case IDENTIFIER: return "IDENTIFIER";
                case PARAMETER: return "PARAMETER";
                case PLUS: return "+";
                case MINUS: return "-";
                case STAR: return "*";
                case SLASH: return "/";
                case PERCENT: return "%";
                case EQUALS: return "==";
                case NOT_EQUALS: return "!=";
                case LESS_THAN: return "<";
                case GREATER_THAN: return ">";
                case LESS_EQUALS: return "<=";
                case GREATER_EQUALS: return ">=";
                case AND: return "&&";
                case OR: return "||";
                case NOT: return "!";
                case ASSIGN: return "=";
                case DOT: return ".";
                case COMMA: return ",";
                case LPAREN: return "(";
                case RPAREN: return ")";
                case LBRACKET: return "[";
                case RBRACKET: return "]";
                case QUESTION: return "?";
                case COLON: return ":";
                case ARROW: return "=>";
                case EOF: return "EOF";
                default: return "UNKNOWN";
            }
        }
    }

    /**
     * A token from the expression tokenizer.
     */
    public class Token : Object {
        public TokenType token_type { get; private set; }
        public string value { get; private set; }
        public int position { get; private set; }

        public Token(TokenType token_type, string value, int position) {
            this.token_type = token_type;
            this.value = value;
            this.position = position;
        }

        public string to_string() {
            return @"Token($(token_type.to_string()), \"$value\", pos=$position)";
        }
    }

    /**
     * Tokenizer for expression strings.
     * 
     * Converts an expression string into a stream of tokens for parsing.
     */
    public class ExpressionTokenizer : Object {

        private string _input;
        private int _position;
        private int _length;

        /**
         * Creates a new tokenizer for the given input string.
         * 
         * @param input The expression string to tokenize
         */
        public ExpressionTokenizer(string input) {
            _input = input;
            _position = 0;
            _length = input.length;
        }

        /**
         * Tokenizes the entire input and returns all tokens.
         * 
         * @return A list of all tokens including the EOF token
         * @throws ExpressionError if tokenization fails
         */
        public Series<Token> tokenize_all() throws ExpressionError {
            var tokens = new Series<Token>();
            
            Token token;
            while ((token = next_token()).token_type != TokenType.EOF) {
                tokens.add(token);
            }
            tokens.add(token); // Add EOF token
            
            return tokens;
        }

        /**
         * Gets the next token from the input.
         * 
         * @return The next token, or EOF if at end of input
         * @throws ExpressionError if tokenization fails
         */
        public Token next_token() throws ExpressionError {
            skip_whitespace();

            if (_position >= _length) {
                return new Token(TokenType.EOF, "", _position);
            }

            char c = _input[_position];
            int start_pos = _position;

            // Single character tokens
            switch (c) {
                case '+':
                    _position++;
                    return new Token(TokenType.PLUS, "+", start_pos);
                case '*':
                    _position++;
                    return new Token(TokenType.STAR, "*", start_pos);
                case '/':
                    _position++;
                    return new Token(TokenType.SLASH, "/", start_pos);
                case '%':
                    _position++;
                    return new Token(TokenType.PERCENT, "%", start_pos);
                case '.':
                    _position++;
                    return new Token(TokenType.DOT, ".", start_pos);
                case ',':
                    _position++;
                    return new Token(TokenType.COMMA, ",", start_pos);
                case '(':
                    _position++;
                    return new Token(TokenType.LPAREN, "(", start_pos);
                case ')':
                    _position++;
                    return new Token(TokenType.RPAREN, ")", start_pos);
                case '[':
                    _position++;
                    return new Token(TokenType.LBRACKET, "[", start_pos);
                case ']':
                    _position++;
                    return new Token(TokenType.RBRACKET, "]", start_pos);
                case '?':
                    _position++;
                    return new Token(TokenType.QUESTION, "?", start_pos);
                case ':':
                    _position++;
                    return new Token(TokenType.COLON, ":", start_pos);
            }

            // Two-character operators
            if (_position + 1 < _length) {
                string two_char = _input.substring(_position, 2);
                
                if (two_char == "==" ) {
                    _position += 2;
                    return new Token(TokenType.EQUALS, "==", start_pos);
                }
                if (two_char == "!=") {
                    _position += 2;
                    return new Token(TokenType.NOT_EQUALS, "!=", start_pos);
                }
                if (two_char == "<=") {
                    _position += 2;
                    return new Token(TokenType.LESS_EQUALS, "<=", start_pos);
                }
                if (two_char == ">=") {
                    _position += 2;
                    return new Token(TokenType.GREATER_EQUALS, ">=", start_pos);
                }
                if (two_char == "&&") {
                    _position += 2;
                    return new Token(TokenType.AND, "&&", start_pos);
                }
                if (two_char == "||") {
                    _position += 2;
                    return new Token(TokenType.OR, "||", start_pos);
                }
                if (two_char == "=>") {
                    _position += 2;
                    return new Token(TokenType.ARROW, "=>", start_pos);
                }
            }

            // Single character operators that might be part of two-char
            if (c == '<') {
                _position++;
                return new Token(TokenType.LESS_THAN, "<", start_pos);
            }
            if (c == '>') {
                _position++;
                return new Token(TokenType.GREATER_THAN, ">", start_pos);
            }
            if (c == '!') {
                _position++;
                return new Token(TokenType.NOT, "!", start_pos);
            }
            if (c == '=') {
                _position++;
                return new Token(TokenType.ASSIGN, "=", start_pos);
            }
            if (c == '-') {
                _position++;
                return new Token(TokenType.MINUS, "-", start_pos);
            }

            // Parameter placeholder ($0, $1, etc.)
            if (c == '$') {
                return read_parameter();
            }

            // String literals (double quotes) or char literals (single quotes)
            if (c == '"') {
                return read_string(c);
            }
            if (c == '\'') {
                return read_char_literal();
            }

            // Numbers
            if (c.isdigit()) {
                return read_number();
            }

            // Identifiers and keywords
            if (c.isalpha() || c == '_') {
                return read_identifier();
            }

            throw new ExpressionError.INVALID_SYNTAX(
                @"Unexpected character '$c' at position $(_position)"
            );
        }

        private void skip_whitespace() {
            while (_position < _length && _input[_position].isspace()) {
                _position++;
            }
        }

        private Token read_string(char quote) throws ExpressionError {
            int start_pos = _position;
            _position++; // Skip opening quote

            var sb = new StringBuilder();

            while (_position < _length) {
                char c = _input[_position];

                if (c == quote) {
                    _position++; // Skip closing quote
                    return new Token(TokenType.STRING, sb.str, start_pos);
                }

                if (c == '\\') {
                    _position++;
                    if (_position >= _length) {
                        throw new ExpressionError.INVALID_SYNTAX(
                            @"Unterminated string at position $start_pos"
                        );
                    }
                    char escaped = _input[_position];
                    switch (escaped) {
                        case 'n': sb.append("\n"); break;
                        case 't': sb.append("\t"); break;
                        case 'r': sb.append("\r"); break;
                        case '\\': sb.append("\\"); break;
                        case '"': sb.append("\""); break;
                        case '\'': sb.append("'"); break;
                        case '%': sb.append("%"); break;
                        case 'x':
                            // Hex escape \xNN
                            _position++;
                            if (_position + 1 >= _length) {
                                throw new ExpressionError.INVALID_SYNTAX(
                                    @"Invalid hex escape at position $(_position)"
                                );
                            }
                            string hex = _input.substring(_position, 2);
                            if (!hex[0].isxdigit() || !hex[1].isxdigit()) {
                                throw new ExpressionError.INVALID_SYNTAX(
                                    @"Invalid hex escape '\\x$hex' at position $(_position)"
                                );
                            }
                            int char_val = parse_hex(hex);
                            sb.append_c((char)char_val);
                            _position += 1; // Will be incremented again below
                            break;
                        default:
                            throw new ExpressionError.INVALID_SYNTAX(
                                @"Unknown escape sequence '\\$escaped' at position $(_position)"
                            );
                    }
                    _position++;
                } else {
                    sb.append_c(c);
                    _position++;
                }
            }

            throw new ExpressionError.INVALID_SYNTAX(
                @"Unterminated string starting at position $start_pos"
            );
        }

        private Token read_char_literal() throws ExpressionError {
            int start_pos = _position;
            _position++; // Skip opening quote

            if (_position >= _length) {
                throw new ExpressionError.INVALID_SYNTAX(
                    @"Unterminated character literal at position $start_pos"
                );
            }

            char c = _input[_position];
            char char_value;

            if (c == '\\') {
                // Escape sequence
                _position++;
                if (_position >= _length) {
                    throw new ExpressionError.INVALID_SYNTAX(
                        @"Unterminated character literal at position $start_pos"
                    );
                }
                char escaped = _input[_position];
                switch (escaped) {
                    case 'n': char_value = '\n'; break;
                    case 't': char_value = '\t'; break;
                    case 'r': char_value = '\r'; break;
                    case '\\': char_value = '\\'; break;
                    case '\'': char_value = '\''; break;
                    case 'x':
                        // Hex escape \xNN
                        _position++;
                        if (_position + 1 >= _length) {
                            throw new ExpressionError.INVALID_SYNTAX(
                                @"Invalid hex escape at position $(_position)"
                            );
                        }
                        string hex = _input.substring(_position, 2);
                        if (!hex[0].isxdigit() || !hex[1].isxdigit()) {
                            throw new ExpressionError.INVALID_SYNTAX(
                                @"Invalid hex escape '\\x$hex' at position $(_position)"
                            );
                        }
                        char_value = (char)parse_hex(hex);
                        _position++; // Extra increment for second hex digit
                        break;
                    default:
                        throw new ExpressionError.INVALID_SYNTAX(
                            @"Unknown escape sequence '\\$escaped' in character literal at position $(_position)"
                        );
                }
            } else {
                char_value = c;
            }

            _position++;

            // Expect closing quote
            if (_position >= _length || _input[_position] != '\'') {
                throw new ExpressionError.INVALID_SYNTAX(
                    @"Unterminated character literal starting at position $start_pos"
                );
            }
            _position++; // Skip closing quote

            return new Token(TokenType.CHAR_LITERAL, char_value.to_string(), start_pos);
        }

        private Token read_number() throws ExpressionError {
            int start_pos = _position;
            var sb = new StringBuilder();
            bool has_decimal = false;

            // Read integer part
            while (_position < _length && _input[_position].isdigit()) {
                sb.append_c(_input[_position]);
                _position++;
            }

            // Check for decimal point
            if (_position < _length && _input[_position] == '.') {
                // Look ahead to make sure it's not a property access
                if (_position + 1 < _length && _input[_position + 1].isdigit()) {
                    has_decimal = true;
                    sb.append_c('.');
                    _position++;

                    // Read decimal part
                    while (_position < _length && _input[_position].isdigit()) {
                        sb.append_c(_input[_position]);
                        _position++;
                    }
                }
            }

            string value = sb.str;

            // Check for type suffixes
            if (_position < _length) {
                char suffix = _input[_position].tolower();

                // Check for 'ul' or 'UL' suffix (unsigned long)
                if (suffix == 'u' && _position + 1 < _length && _input[_position + 1].tolower() == 'l') {
                    _position += 2;
                    return new Token(TokenType.UNSIGNED_LONG, value, start_pos);
                }

                // Check for 'l' suffix (long)
                if (suffix == 'l') {
                    _position++;
                    return new Token(TokenType.LONG_INTEGER, value, start_pos);
                }

                // Check for 'f' suffix (float)
                if (suffix == 'f') {
                    _position++;
                    return new Token(TokenType.FLOAT_LITERAL, value, start_pos);
                }
            }

            if (has_decimal) {
                return new Token(TokenType.FLOAT, value, start_pos);
            } else {
                return new Token(TokenType.INTEGER, value, start_pos);
            }
        }

        private Token read_identifier() {
            int start_pos = _position;
            var sb = new StringBuilder();

            while (_position < _length) {
                char c = _input[_position];
                if (c.isalnum() || c == '_') {
                    sb.append_c(c);
                    _position++;
                } else {
                    break;
                }
            }

            string value = sb.str;

            // Check for keywords
            switch (value.down()) {
                case "true":
                    return new Token(TokenType.TRUE, value, start_pos);
                case "false":
                    return new Token(TokenType.FALSE, value, start_pos);
                case "null":
                    return new Token(TokenType.NULL_LITERAL, value, start_pos);
                default:
                    return new Token(TokenType.IDENTIFIER, value, start_pos);
            }
        }

        private Token read_parameter() throws ExpressionError {
            int start_pos = _position;
            _position++; // Skip $

            if (_position >= _length || !_input[_position].isdigit()) {
                throw new ExpressionError.INVALID_SYNTAX(
                    @"Expected digit after '$$' at position $start_pos"
                );
            }

            var sb = new StringBuilder();
            while (_position < _length && _input[_position].isdigit()) {
                sb.append_c(_input[_position]);
                _position++;
            }

            return new Token(TokenType.PARAMETER, sb.str, start_pos);
        }

        private static int parse_hex(string hex) {
            int result = 0;
            for (int i = 0; i < hex.length; i++) {
                char c = hex[i];
                result *= 16;
                if (c >= '0' && c <= '9') {
                    result += c - '0';
                } else if (c >= 'a' && c <= 'f') {
                    result += c - 'a' + 10;
                } else if (c >= 'A' && c <= 'F') {
                    result += c - 'A' + 10;
                }
            }
            return result;
        }
    }

}