// The MIT License (MIT)
//
// Copyright (C) 2026 Scott Duensing
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to
// deal in the Software without restriction, including without limitation the
// rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
// sell copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
// IN THE SOFTWARE.

// lexer.c -- DVX BASIC lexer implementation
//
// Single-pass tokenizer. Keywords are case-insensitive. Identifiers
// preserve their original case for display but comparisons are
// case-insensitive. Line continuations (underscore at end of line)
// are handled transparently.

#include "lexer.h"
#include "opcodes.h"

#include <ctype.h>
#include <errno.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>

// ============================================================
// Keyword table
// ============================================================

typedef struct {
    const char    *text;
    uint8_t        textLen;   // precomputed so lookupKeyword() can length-reject cheaply
    BasTokenTypeE  type;
} KeywordEntryT;

#define KW(s, t) { s, (uint8_t)(sizeof(s) - 1), t }

static const KeywordEntryT sKeywords[] = {
    KW("AND",             TOK_AND),
    KW("APP",             TOK_APP),
    KW("APPEND",          TOK_APPEND),
    KW("AS",              TOK_AS),
    KW("BASE",            TOK_BASE),
    KW("BINARY",          TOK_BINARY),
    KW("BOOLEAN",         TOK_BOOLEAN),
    KW("BYREF",           TOK_BYREF),
    KW("BYVAL",           TOK_BYVAL),
    KW("CALL",            TOK_CALL),
    KW("CASE",            TOK_CASE),
    KW("CHDIR",           TOK_CHDIR),
    KW("CHDRIVE",         TOK_CHDRIVE),
    KW("CLOSE",           TOK_CLOSE),
    KW("CONST",           TOK_CONST),
    KW("CREATECONTROL",   TOK_CREATECONTROL),
    KW("CREATEFORM",      TOK_CREATEFORM),
    KW("CURDIR",          TOK_CURDIR),
    KW("CURDIR$",         TOK_CURDIR),
    KW("DATA",            TOK_DATA),
    KW("DECLARE",         TOK_DECLARE),
    KW("DEF",             TOK_DEF),
    KW("DEFDBL",          TOK_DEFDBL),
    KW("DEFINT",          TOK_DEFINT),
    KW("DEFLNG",          TOK_DEFLNG),
    KW("DEFSNG",          TOK_DEFSNG),
    KW("DEFSTR",          TOK_DEFSTR),
    KW("DIM",             TOK_DIM),
    KW("DIR",             TOK_DIR),
    KW("DIR$",            TOK_DIR),
    KW("DO",              TOK_DO),
    KW("DOEVENTS",        TOK_DOEVENTS),
    KW("DOUBLE",          TOK_DOUBLE),
    KW("ELSE",            TOK_ELSE),
    KW("ELSEIF",          TOK_ELSEIF),
    KW("END",             TOK_END),
    KW("EOF",             TOK_EOF_KW),
    KW("EQV",             TOK_EQV),
    KW("ERASE",           TOK_ERASE),
    KW("ERL",             TOK_ERL),
    KW("ERR",             TOK_ERR),
    KW("ERROR",           TOK_ERROR_KW),
    KW("EXIT",            TOK_EXIT),
    KW("EXPLICIT",        TOK_EXPLICIT),
    KW("FALSE",           TOK_FALSE_KW),
    KW("FILECOPY",        TOK_FILECOPY),
    KW("FILELEN",         TOK_FILELEN),
    KW("FOR",             TOK_FOR),
    KW("FUNCTION",        TOK_FUNCTION),
    KW("GET",             TOK_GET),
    KW("GETATTR",         TOK_GETATTR),
    KW("GOSUB",           TOK_GOSUB),
    KW("GOTO",            TOK_GOTO),
    KW("HIDE",            TOK_HIDE),
    KW("IF",              TOK_IF),
    KW("IMP",             TOK_IMP),
    KW("INIREAD",         TOK_INIREAD),
    KW("INIREAD$",        TOK_INIREAD),
    KW("INIWRITE",        TOK_INIWRITE),
    KW("INPUT",           TOK_INPUT),
    KW("INPUTBOX",        TOK_INPUTBOX),
    KW("INPUTBOX$",       TOK_INPUTBOX),
    KW("INTEGER",         TOK_INTEGER),
    KW("IS",              TOK_IS),
    KW("KILL",            TOK_KILL),
    KW("LBOUND",          TOK_LBOUND),
    KW("LET",             TOK_LET),
    KW("LINE",            TOK_LINE),
    KW("LOAD",            TOK_LOAD),
    KW("LONG",            TOK_LONG),
    KW("LOOP",            TOK_LOOP),
    KW("ME",              TOK_ME),
    KW("MKDIR",           TOK_MKDIR),
    KW("MOD",             TOK_MOD),
    KW("MSGBOX",          TOK_MSGBOX),
    KW("NAME",            TOK_NAME),
    KW("NEXT",            TOK_NEXT),
    KW("NOT",             TOK_NOT),
    KW("NOTHING",         TOK_NOTHING),
    KW("ON",              TOK_ON),
    KW("OPEN",            TOK_OPEN),
    KW("OPTION",          TOK_OPTION),
    KW("OPTIONAL",        TOK_OPTIONAL),
    KW("OR",              TOK_OR),
    KW("OUTPUT",          TOK_OUTPUT),
    KW("PRESERVE",        TOK_PRESERVE),
    KW("PRINT",           TOK_PRINT),
    KW("PUT",             TOK_PUT),
    KW("RANDOM",          TOK_RANDOM),
    KW("RANDOMIZE",       TOK_RANDOMIZE),
    KW("READ",            TOK_READ),
    KW("REDIM",           TOK_REDIM),
    KW("REM",             TOK_REM),
    KW("REMOVECONTROL",   TOK_REMOVECONTROL),
    KW("RESTORE",         TOK_RESTORE),
    KW("RESUME",          TOK_RESUME),
    KW("RETURN",          TOK_RETURN),
    KW("RMDIR",           TOK_RMDIR),
    KW("SEEK",            TOK_SEEK),
    KW("SELECT",          TOK_SELECT),
    KW("SET",             TOK_SET),
    KW("SETATTR",         TOK_SETATTR),
    KW("SETEVENT",        TOK_SETEVENT),
    KW("SHARED",          TOK_SHARED),
    KW("SHELL",           TOK_SHELL),
    KW("SHOW",            TOK_SHOW),
    KW("SINGLE",          TOK_SINGLE),
    KW("SLEEP",           TOK_SLEEP),
    KW("STATIC",          TOK_STATIC),
    KW("STEP",            TOK_STEP),
    KW("STRING",          TOK_STRING_KW),
    KW("SUB",             TOK_SUB),
    KW("SWAP",            TOK_SWAP),
    KW("THEN",            TOK_THEN),
    KW("TIMER",           TOK_TIMER),
    KW("TO",              TOK_TO),
    KW("TRUE",            TOK_TRUE_KW),
    KW("TYPE",            TOK_TYPE),
    KW("UBOUND",          TOK_UBOUND),
    KW("UNLOAD",          TOK_UNLOAD),
    KW("UNTIL",           TOK_UNTIL),
    KW("WEND",            TOK_WEND),
    KW("WHILE",           TOK_WHILE),
    KW("WITH",            TOK_WITH),
    KW("WRITE",           TOK_WRITE),
    KW("XOR",             TOK_XOR),
    { NULL,         0, TOK_ERROR }
};

#undef KW

#define KEYWORD_COUNT (sizeof(sKeywords) / sizeof(sKeywords[0]) - 1)

// ============================================================
// Type-suffix table
// ============================================================
//
// Single source of truth for the BASIC type-suffix characters and the
// type each one implies; basIsTypeSuffixChar and basTypeSuffixType (and
// through them the parser) all consult this table.

typedef struct {
    char    suffix;
    uint8_t dataType;
} TypeSuffixEntryT;

static const TypeSuffixEntryT sTypeSuffixes[] = {
    { '%', BAS_TYPE_INTEGER },
    { '&', BAS_TYPE_LONG    },
    { '!', BAS_TYPE_SINGLE  },
    { '#', BAS_TYPE_DOUBLE  },
    { '$', BAS_TYPE_STRING  },
};

#define TYPE_SUFFIX_COUNT ((int32_t)(sizeof(sTypeSuffixes) / sizeof(sTypeSuffixes[0])))


// Function prototypes (alphabetical)
static char advance(BasLexerT *lex);
static void appendTokenChar(BasLexerT *lex, int32_t *idx, char c);
static bool atEnd(const BasLexerT *lex);
char basAsciiUpper(char c);
bool basIsIdentChar(char c);
bool basIsTypeSuffixChar(char c);
bool basIsValidIdent(const char *name);
void basLexerInit(BasLexerT *lex, const char *source, int32_t sourceLen);
const char *basLexerKeywordAt(int32_t i);
BasKeywordClassE basLexerKeywordClass(int32_t i);
int32_t basLexerKeywordCount(void);
BasTokenTypeE basLexerNext(BasLexerT *lex);
BasTokenTypeE basLexerPeek(const BasLexerT *lex);
const char *basTokenName(BasTokenTypeE type);
int32_t basTypeSuffixType(char c);
static bool lexIntegerLiteral(BasLexerT *lex, int64_t minVal, int64_t maxVal, const char *what, int64_t *outVal);
static BasTokenTypeE lookupKeyword(const char *text, int32_t len);
static BasTokenTypeE makeNewlineToken(BasLexerT *lex);
static char peek(const BasLexerT *lex);
static char peekNext(const BasLexerT *lex);
static void setError(BasLexerT *lex, const char *msg);
static void skipLineComment(BasLexerT *lex);
static void skipWhitespace(BasLexerT *lex);
static BasTokenTypeE tokenizeHexLiteral(BasLexerT *lex);
static BasTokenTypeE tokenizeIdentOrKeyword(BasLexerT *lex);
static BasTokenTypeE tokenizeNumber(BasLexerT *lex);
static BasTokenTypeE tokenizeString(BasLexerT *lex);

static char advance(BasLexerT *lex) {
    if (atEnd(lex)) {
        return '\0';
    }

    char c = lex->source[lex->pos++];

    // LF, or a CR that is not the first half of a CRLF pair, ends a line
    // (the LF of a CRLF pair is counted when it is consumed).
    if (c == '\n' || (c == '\r' && peek(lex) != '\n')) {
        lex->line++;
        lex->col = 1;
    } else {
        lex->col++;
    }

    return c;
}


static void appendTokenChar(BasLexerT *lex, int32_t *idx, char c) {
    if (*idx < BAS_MAX_TOKEN_LEN - 1) {
        lex->token.text[(*idx)++] = c;
    }
}


static bool atEnd(const BasLexerT *lex) {
    return lex->pos >= lex->sourceLen;
}


char basAsciiUpper(char c) {
    if (c >= 'a' && c <= 'z') {
        c -= ('a' - 'A');
    }

    return c;
}


bool basIsIdentChar(char c) {
    return (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || (c >= '0' && c <= '9') || c == '_';
}


bool basIsTypeSuffixChar(char c) {
    return basTypeSuffixType(c) >= 0;
}


bool basIsValidIdent(const char *name) {
    if (!name || !basIsIdentChar(name[0]) || (name[0] >= '0' && name[0] <= '9')) {
        return false;
    }

    for (const char *p = name + 1; *p; p++) {
        if (!basIsIdentChar(*p)) {
            return false;
        }
    }

    return true;
}


void basLexerInit(BasLexerT *lex, const char *source, int32_t sourceLen) {
    memset(lex, 0, sizeof(*lex));
    lex->source    = source;
    lex->sourceLen = (sourceLen < 0) ? (int32_t)strlen(source) : sourceLen;
    lex->pos       = 0;
    lex->line      = 1;
    lex->col       = 1;

    // Prime the first token
    basLexerNext(lex);
}


const char *basLexerKeywordAt(int32_t i) {
    if (i < 0 || i >= (int32_t)KEYWORD_COUNT) {
        return NULL;
    }

    return sKeywords[i].text;
}


BasKeywordClassE basLexerKeywordClass(int32_t i) {
    if (i < 0 || i >= (int32_t)KEYWORD_COUNT) {
        return BAS_KW_CLASS_OTHER;
    }

    switch (sKeywords[i].type) {
        case TOK_BOOLEAN:
        case TOK_DOUBLE:
        case TOK_INTEGER:
        case TOK_LONG:
        case TOK_SINGLE:
        case TOK_STRING_KW:
            return BAS_KW_CLASS_TYPE;

        case TOK_TRUE_KW:
        case TOK_FALSE_KW:
        case TOK_NOTHING:
            return BAS_KW_CLASS_LITERAL;

        default:
            return BAS_KW_CLASS_OTHER;
    }
}


int32_t basLexerKeywordCount(void) {
    return (int32_t)KEYWORD_COUNT;
}


BasTokenTypeE basLexerNext(BasLexerT *lex) {
    skipWhitespace(lex);

    lex->token.line    = lex->line;
    lex->token.col     = lex->col;
    lex->token.textLen = 0;
    lex->token.text[0] = '\0';

    if (atEnd(lex)) {
        lex->token.type = TOK_EOF;
        return TOK_EOF;
    }

    char c = peek(lex);

    // Newline
    if (c == '\n') {
        advance(lex);
        return makeNewlineToken(lex);
    }

    // Carriage return (handle CR, CRLF)
    if (c == '\r') {
        advance(lex);

        if (!atEnd(lex) && peek(lex) == '\n') {
            advance(lex);
        }

        return makeNewlineToken(lex);
    }

    // Comment (apostrophe)
    if (c == '\'') {
        skipLineComment(lex);
        return makeNewlineToken(lex);
    }

    // String literal
    if (c == '"') {
        lex->token.type = tokenizeString(lex);
        return lex->token.type;
    }

    // Number
    if (isdigit((unsigned char)c) || (c == '.' && isdigit((unsigned char)peekNext(lex)))) {
        lex->token.type = tokenizeNumber(lex);
        return lex->token.type;
    }

    // Numeric-base literals: &H hex, &O octal, &B binary.  &B is an
    // extension beyond classic QBASIC; it's convenient for bitmask
    // work in the widget/graphics code.
    if (c == '&') {
        char n = basAsciiUpper(peekNext(lex));

        if (n == 'H' || n == 'O' || n == 'B') {
            lex->token.type = tokenizeHexLiteral(lex);
            return lex->token.type;
        }
    }

    // Identifier or keyword
    if (basIsIdentChar(c) && !isdigit((unsigned char)c)) {
        lex->token.type = tokenizeIdentOrKeyword(lex);
        return lex->token.type;
    }

    // Single and multi-character operators/punctuation
    advance(lex);

    switch (c) {
        case '+':
            lex->token.type = TOK_PLUS;
            break;

        case '-':
            lex->token.type = TOK_MINUS;
            break;

        case '*':
            lex->token.type = TOK_STAR;
            break;

        case '/':
            lex->token.type = TOK_SLASH;
            break;

        case '\\':
            lex->token.type = TOK_BACKSLASH;
            break;

        case '^':
            lex->token.type = TOK_CARET;
            break;

        case '&':
            lex->token.type = TOK_AMPERSAND;
            break;

        case '(':
            lex->token.type = TOK_LPAREN;
            break;

        case ')':
            lex->token.type = TOK_RPAREN;
            break;

        case ',':
            lex->token.type = TOK_COMMA;
            break;

        case ';':
            lex->token.type = TOK_SEMICOLON;
            break;

        case ':':
            lex->token.type = TOK_COLON;
            break;

        case '.':
            lex->token.type = TOK_DOT;
            break;

        case '#':
            lex->token.type = TOK_HASH;
            break;

        case '?':
            lex->token.type = TOK_PRINT;
            break;

        case '=':
            lex->token.type = TOK_EQ;
            break;

        case '<':
            if (!atEnd(lex) && peek(lex) == '>') {
                advance(lex);
                lex->token.type = TOK_NE;
            } else if (!atEnd(lex) && peek(lex) == '=') {
                advance(lex);
                lex->token.type = TOK_LE;
            } else {
                lex->token.type = TOK_LT;
            }
            break;

        case '>':
            if (!atEnd(lex) && peek(lex) == '=') {
                advance(lex);
                lex->token.type = TOK_GE;
            } else {
                lex->token.type = TOK_GT;
            }
            break;

        default:
            setError(lex, "Unexpected character");
            lex->token.type = TOK_ERROR;
            break;
    }

    // Store the operator text
    if (lex->token.type != TOK_ERROR) {
        lex->token.text[0] = c;
        lex->token.textLen = 1;

        if (lex->token.type == TOK_NE || lex->token.type == TOK_LE || lex->token.type == TOK_GE) {
            lex->token.text[1] = lex->source[lex->pos - 1];
            lex->token.textLen = 2;
        }

        lex->token.text[lex->token.textLen] = '\0';
    }

    return lex->token.type;
}


BasTokenTypeE basLexerPeek(const BasLexerT *lex) {
    return lex->token.type;
}


const char *basTokenName(BasTokenTypeE type) {
    switch (type) {
        case TOK_INT_LIT:      return "integer";
        case TOK_LONG_LIT:     return "long";
        case TOK_FLOAT_LIT:    return "float";
        case TOK_STRING_LIT:   return "string";
        case TOK_IDENT:        return "identifier";
        case TOK_DOT:          return "'.'";
        case TOK_COMMA:        return "','";
        case TOK_SEMICOLON:    return "';'";
        case TOK_COLON:        return "':'";
        case TOK_LPAREN:       return "'('";
        case TOK_RPAREN:       return "')'";
        case TOK_HASH:         return "'#'";
        case TOK_PLUS:         return "'+'";
        case TOK_MINUS:        return "'-'";
        case TOK_STAR:         return "'*'";
        case TOK_SLASH:        return "'/'";
        case TOK_BACKSLASH:    return "'\\'";
        case TOK_CARET:        return "'^'";
        case TOK_AMPERSAND:    return "'&'";
        case TOK_EQ:           return "'='";
        case TOK_NE:           return "'<>'";
        case TOK_LT:           return "'<'";
        case TOK_GT:           return "'>'";
        case TOK_LE:           return "'<='";
        case TOK_GE:           return "'>='";
        case TOK_NEWLINE:      return "newline";
        case TOK_EOF:          return "end of file";
        case TOK_ERROR:        return "error";
        default:               break;
    }

    // Keywords
    for (int32_t i = 0; i < (int32_t)KEYWORD_COUNT; i++) {
        if (sKeywords[i].type == type) {
            return sKeywords[i].text;
        }
    }

    return "?";
}


int32_t basTypeSuffixType(char c) {
    for (int32_t i = 0; i < TYPE_SUFFIX_COUNT; i++) {
        if (sTypeSuffixes[i].suffix == c) {
            return sTypeSuffixes[i].dataType;
        }
    }

    return -1;
}


// Convert the decimal digits in token.text to an integer and range-check
// the result against [minVal, maxVal].  On overflow the token becomes
// TOK_ERROR naming `what`, and false is returned.  strtoll is used rather
// than atol so the conversion is identical on the 32-bit DOS build and
// the 64-bit host build of the compiler.
static bool lexIntegerLiteral(BasLexerT *lex, int64_t minVal, int64_t maxVal, const char *what, int64_t *outVal) {
    errno = 0;
    long long val = strtoll(lex->token.text, NULL, 10);

    if (errno == ERANGE || val < minVal || val > maxVal) {
        char buf[BAS_LEX_ERROR_LEN];
        snprintf(buf, sizeof(buf), "%s literal out of range", what);
        setError(lex, buf);
        lex->token.type = TOK_ERROR;
        return false;
    }

    *outVal = (int64_t)val;
    return true;
}


static BasTokenTypeE lookupKeyword(const char *text, int32_t len) {
    // Case-insensitive keyword lookup.  Short-circuits on length mismatch
    // (via cached keyword length) and on the very first character, both of
    // which reject the vast majority of entries before doing a full scan.
    char firstUp = basAsciiUpper(text[0]);

    for (int32_t i = 0; i < (int32_t)KEYWORD_COUNT; i++) {
        const KeywordEntryT *kw = &sKeywords[i];

        if (kw->textLen != len || kw->text[0] != firstUp) {
            continue;
        }

        bool match = true;

        for (int32_t j = 1; j < len; j++) {
            if (basAsciiUpper(text[j]) != kw->text[j]) {
                match = false;
                break;
            }
        }

        if (match) {
            return kw->type;
        }
    }

    return TOK_IDENT;
}


// Fill the current token as a synthesized newline (used for real newlines,
// CR/CRLF, and the end of apostrophe / REM comments).
static BasTokenTypeE makeNewlineToken(BasLexerT *lex) {
    lex->token.type    = TOK_NEWLINE;
    lex->token.text[0] = '\n';
    lex->token.text[1] = '\0';
    lex->token.textLen = 1;
    return TOK_NEWLINE;
}


static char peek(const BasLexerT *lex) {
    if (atEnd(lex)) {
        return '\0';
    }

    return lex->source[lex->pos];
}


static char peekNext(const BasLexerT *lex) {
    if (lex->pos + 1 >= lex->sourceLen) {
        return '\0';
    }

    return lex->source[lex->pos + 1];
}


static void setError(BasLexerT *lex, const char *msg) {
    snprintf(lex->error, sizeof(lex->error), "Line %d, Col %d: %s", (int)lex->line, (int)lex->col, msg);
}


static void skipLineComment(BasLexerT *lex) {
    while (!atEnd(lex) && peek(lex) != '\n' && peek(lex) != '\r') {
        advance(lex);
    }
}


//
// Skips spaces and tabs. Does NOT skip newlines (they are tokens).
// Handles line continuation: underscore followed by newline joins
// the next line to the current logical line.
static void skipWhitespace(BasLexerT *lex) {
    while (!atEnd(lex)) {
        char c = peek(lex);

        if (c == ' ' || c == '\t') {
            advance(lex);
            continue;
        }

        // Line continuation: _ at end of line
        if (c == '_') {
            int32_t savedPos  = lex->pos;
            int32_t savedLine = lex->line;
            int32_t savedCol  = lex->col;
            advance(lex);

            // Skip spaces/tabs after underscore
            while (!atEnd(lex) && (peek(lex) == ' ' || peek(lex) == '\t')) {
                advance(lex);
            }

            // Must be followed by newline
            if (!atEnd(lex) && (peek(lex) == '\n' || peek(lex) == '\r')) {
                advance(lex);

                if (!atEnd(lex) && peek(lex) == '\n' && lex->source[lex->pos - 1] == '\r') {
                    advance(lex);
                }

                continue;  // Continue skipping whitespace on next line
            }

            // Not a continuation -- put back
            lex->pos  = savedPos;
            lex->line = savedLine;
            lex->col  = savedCol;
            break;
        }

        break;
    }
}


static BasTokenTypeE tokenizeHexLiteral(BasLexerT *lex) {
    advance(lex);                       // skip &
    char base = basAsciiUpper(peek(lex));
    advance(lex);                       // skip H/O/B

    int32_t shift;
    int32_t maxDigit;

    if (base == 'O') {
        shift    = 3;
        maxDigit = 7;
    } else if (base == 'B') {
        shift    = 1;
        maxDigit = 1;
    } else {
        shift    = 4;
        maxDigit = 15;
    }

    bool     overflow = false;
    int32_t  idx      = 0;
    uint64_t value    = 0;

    for (;;) {
        if (atEnd(lex)) {
            break;
        }

        char c = peek(lex);
        int32_t digit;

        if (c >= '0' && c <= '9') {
            digit = c - '0';
        } else if (shift == 4 && c >= 'A' && c <= 'F') {
            digit = c - 'A' + 10;
        } else if (shift == 4 && c >= 'a' && c <= 'f') {
            digit = c - 'a' + 10;
        } else {
            break;
        }

        if (digit > maxDigit) {
            break;
        }

        advance(lex);
        appendTokenChar(lex, &idx, c);

        value = (value << shift) | (uint64_t)digit;

        // Latch overflow instead of testing value after the loop: with
        // enough digits the shifts wrap the uint64 back into range, which
        // would hide the overflow.
        if (value > UINT32_MAX) {
            overflow = true;
        }
    }

    // No digits after the &H/&O/&B marker is a typo, not the integer 0.
    if (idx == 0) {
        setError(lex, "Empty hexadecimal/octal/binary literal");
        lex->token.type = TOK_ERROR;
        return TOK_ERROR;
    }

    // The widest runtime integer (LONG) is 32 bits; silently truncating a
    // wider literal would miscompile the constant, so reject it.  Note the
    // literal is NOT typed by width: &HFFFF is 65535 (a LONG), not QBASIC's
    // 16-bit -1 -- the parser picks the push width from the value.
    if (overflow) {
        setError(lex, "Hexadecimal/octal/binary literal exceeds 32 bits");
        lex->token.type = TOK_ERROR;
        return TOK_ERROR;
    }

    lex->token.text[idx] = '\0';
    lex->token.textLen   = idx;

    // Check for trailing & (long suffix)
    if (!atEnd(lex) && peek(lex) == '&') {
        advance(lex);
        lex->token.longVal = (int64_t)value;
        return TOK_LONG_LIT;
    }

    lex->token.intVal = (int32_t)value;
    return TOK_INT_LIT;
}


static BasTokenTypeE tokenizeIdentOrKeyword(BasLexerT *lex) {
    int32_t idx = 0;

    while (!atEnd(lex) && basIsIdentChar(peek(lex))) {
        appendTokenChar(lex, &idx, advance(lex));
    }

    lex->token.text[idx] = '\0';
    lex->token.textLen   = idx;

    // Check for type suffix.  A keyword only ever takes '$' (CURDIR$,
    // DIR$, INPUT$ ...); any other suffix after a keyword belongs to the
    // next token, so PRINT#1 / CLOSE#1 / INPUT#1 lex as keyword + '#'.
    if (!atEnd(lex)) {
        char c = peek(lex);

        if (basIsTypeSuffixChar(c) && (c == '$' || lookupKeyword(lex->token.text, idx) == TOK_IDENT)) {
            advance(lex);
            appendTokenChar(lex, &idx, c);
            lex->token.text[idx] = '\0';
            lex->token.textLen   = idx;
        }
    }

    // Check if this is a keyword
    // For suffix-bearing identifiers, only check the base (without suffix)
    int32_t baseLen = idx;

    if (baseLen > 0) {
        char last = lex->token.text[baseLen - 1];

        if (basIsTypeSuffixChar(last)) {
            baseLen--;
        }
    }

    // Try the full text first (including any type suffix). Suffix-bearing
    // keywords like CURDIR$, DIR$, INIREAD$, INPUTBOX$ are listed in the
    // keyword table with their $ and will match here. If the full text
    // isn't a keyword, fall back to the base name (without suffix).
    BasTokenTypeE kwType = lookupKeyword(lex->token.text, idx);
    bool matchedWithSuffix = (kwType != TOK_IDENT && baseLen != idx);

    if (kwType == TOK_IDENT && baseLen != idx) {
        kwType = lookupKeyword(lex->token.text, baseLen);
    }

    // REM is a comment -- skip to end of line.  Apply the same acceptance
    // gate as below so a suffixed identifier like REM$ stays an identifier
    // instead of swallowing the rest of the line as a comment.
    if (kwType == TOK_REM && (baseLen == idx || matchedWithSuffix)) {
        skipLineComment(lex);
        return makeNewlineToken(lex);
    }

    // Accept the keyword if it's a plain keyword (no suffix on source) or
    // if it explicitly matched a $-suffixed entry in the keyword table.
    if (kwType != TOK_IDENT && (baseLen == idx || matchedWithSuffix)) {
        return kwType;
    }

    return TOK_IDENT;
}


static BasTokenTypeE tokenizeNumber(BasLexerT *lex) {
    int32_t idx        = 0;
    bool    hasDecimal = false;
    bool    hasExp     = false;

    // Integer part
    while (!atEnd(lex) && isdigit((unsigned char)peek(lex))) {
        appendTokenChar(lex, &idx, advance(lex));
    }

    // Decimal part
    if (!atEnd(lex) && peek(lex) == '.' && isdigit((unsigned char)peekNext(lex))) {
        hasDecimal = true;
        appendTokenChar(lex, &idx, advance(lex));  // .

        while (!atEnd(lex) && isdigit((unsigned char)peek(lex))) {
            appendTokenChar(lex, &idx, advance(lex));
        }
    }

    // Exponent.  Accept QBASIC/Fortran 'D' (double) as well as 'E', but
    // always store 'E' in the buffer so atof() can parse it.  Require at
    // least one digit after the marker (and optional sign); otherwise the
    // marker is not part of the number (e.g. "1E" tokenizes as 1 then E).
    char marker = basAsciiUpper(peek(lex));

    if (!atEnd(lex) && (marker == 'E' || marker == 'D')) {
        int32_t savedIdx  = idx;
        int32_t savedPos  = lex->pos;
        int32_t savedLine = lex->line;
        int32_t savedCol  = lex->col;

        advance(lex);                       // consume marker
        appendTokenChar(lex, &idx, 'E');    // always store 'E', never 'D'

        if (!atEnd(lex) && (peek(lex) == '+' || peek(lex) == '-')) {
            appendTokenChar(lex, &idx, advance(lex));
        }

        if (!atEnd(lex) && isdigit((unsigned char)peek(lex))) {
            hasExp = true;

            while (!atEnd(lex) && isdigit((unsigned char)peek(lex))) {
                appendTokenChar(lex, &idx, advance(lex));
            }
        } else {
            // Not a valid exponent -- roll back the consumed marker/sign.
            idx       = savedIdx;
            lex->pos  = savedPos;
            lex->line = savedLine;
            lex->col  = savedCol;
        }
    }

    lex->token.text[idx] = '\0';
    lex->token.textLen   = idx;

    // Check for type suffix
    if (!atEnd(lex)) {
        char    c = peek(lex);
        int64_t val;

        if (c == '%') {
            advance(lex);

            if (!lexIntegerLiteral(lex, INT16_MIN, INT16_MAX, "INTEGER", &val)) {
                return TOK_ERROR;
            }

            lex->token.intVal = (int32_t)val;
            return TOK_INT_LIT;
        }

        if (c == '&') {
            advance(lex);

            if (!lexIntegerLiteral(lex, INT32_MIN, INT32_MAX, "LONG", &val)) {
                return TOK_ERROR;
            }

            lex->token.longVal = val;
            return TOK_LONG_LIT;
        }

        if (c == '!' || c == '#') {
            advance(lex);
            lex->token.dblVal = atof(lex->token.text);
            return TOK_FLOAT_LIT;
        }
    }

    // No suffix: determine type from content
    if (hasDecimal || hasExp) {
        lex->token.dblVal = atof(lex->token.text);
        return TOK_FLOAT_LIT;
    }

    // Unsuffixed integers must fit LONG; anything wider would be silently
    // truncated by the 32-bit runtime.
    int64_t val;
    int64_t maxVal = lex->allowNegMagnitude ? BAS_LONG_NEG_MAGNITUDE : INT32_MAX;

    if (!lexIntegerLiteral(lex, INT32_MIN, maxVal, "Integer", &val)) {
        return TOK_ERROR;
    }

    if (val >= INT16_MIN && val <= INT16_MAX) {
        lex->token.intVal = (int32_t)val;
        return TOK_INT_LIT;
    }

    lex->token.longVal = val;
    return TOK_LONG_LIT;
}


static BasTokenTypeE tokenizeString(BasLexerT *lex) {
    advance(lex);  // skip opening quote

    int32_t idx      = 0;
    bool    overflow = false;

    while (!atEnd(lex) && peek(lex) != '"' && peek(lex) != '\n' && peek(lex) != '\r') {
        // appendTokenChar silently drops chars once the buffer is full;
        // detect that here so over-long literals are diagnosed instead
        // of truncated.  Keep consuming so the lexer skips the whole
        // literal and its closing quote.
        if (idx >= BAS_MAX_STRING_LEN) {
            overflow = true;
        }
        appendTokenChar(lex, &idx, advance(lex));
    }

    if (atEnd(lex) || peek(lex) != '"') {
        setError(lex, "Unterminated string literal");
        lex->token.text[idx] = '\0';
        lex->token.textLen   = idx;
        return TOK_ERROR;
    }

    advance(lex);  // skip closing quote

    lex->token.text[idx] = '\0';
    lex->token.textLen   = idx;

    if (overflow) {
        char buf[BAS_LEX_ERROR_LEN];
        snprintf(buf, sizeof(buf), "String literal too long (maximum is %d characters)", (int)BAS_MAX_STRING_LEN);
        setError(lex, buf);
        return TOK_ERROR;
    }

    return TOK_STRING_LIT;
}