1046 lines
30 KiB
C
1046 lines
30 KiB
C
// The MIT License (MIT)
|
|
//
|
|
// Copyright (C) 2026 Scott Duensing
|
|
//
|
|
// Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
// of this software and associated documentation files (the "Software"), to
|
|
// deal in the Software without restriction, including without limitation the
|
|
// rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
|
|
// sell copies of the Software, and to permit persons to whom the Software is
|
|
// furnished to do so, subject to the following conditions:
|
|
//
|
|
// The above copyright notice and this permission notice shall be included in
|
|
// all copies or substantial portions of the Software.
|
|
//
|
|
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
|
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
|
|
// IN THE SOFTWARE.
|
|
|
|
// lexer.c -- DVX BASIC lexer implementation
|
|
//
|
|
// Single-pass tokenizer. Keywords are case-insensitive. Identifiers
|
|
// preserve their original case for display but comparisons are
|
|
// case-insensitive. Line continuations (underscore at end of line)
|
|
// are handled transparently.
|
|
|
|
#include "lexer.h"
|
|
#include "opcodes.h"
|
|
|
|
#include <ctype.h>
|
|
#include <errno.h>
|
|
#include <stdio.h>
|
|
#include <stdlib.h>
|
|
#include <string.h>
|
|
|
|
// ============================================================
|
|
// Keyword table
|
|
// ============================================================
|
|
|
|
typedef struct {
|
|
const char *text;
|
|
uint8_t textLen; // precomputed so lookupKeyword() can length-reject cheaply
|
|
BasTokenTypeE type;
|
|
} KeywordEntryT;
|
|
|
|
#define KW(s, t) { s, (uint8_t)(sizeof(s) - 1), t }
|
|
|
|
static const KeywordEntryT sKeywords[] = {
|
|
KW("AND", TOK_AND),
|
|
KW("APP", TOK_APP),
|
|
KW("APPEND", TOK_APPEND),
|
|
KW("AS", TOK_AS),
|
|
KW("BASE", TOK_BASE),
|
|
KW("BINARY", TOK_BINARY),
|
|
KW("BOOLEAN", TOK_BOOLEAN),
|
|
KW("BYREF", TOK_BYREF),
|
|
KW("BYVAL", TOK_BYVAL),
|
|
KW("CALL", TOK_CALL),
|
|
KW("CASE", TOK_CASE),
|
|
KW("CHDIR", TOK_CHDIR),
|
|
KW("CHDRIVE", TOK_CHDRIVE),
|
|
KW("CLOSE", TOK_CLOSE),
|
|
KW("CONST", TOK_CONST),
|
|
KW("CREATECONTROL", TOK_CREATECONTROL),
|
|
KW("CREATEFORM", TOK_CREATEFORM),
|
|
KW("CURDIR", TOK_CURDIR),
|
|
KW("CURDIR$", TOK_CURDIR),
|
|
KW("DATA", TOK_DATA),
|
|
KW("DECLARE", TOK_DECLARE),
|
|
KW("DEF", TOK_DEF),
|
|
KW("DEFDBL", TOK_DEFDBL),
|
|
KW("DEFINT", TOK_DEFINT),
|
|
KW("DEFLNG", TOK_DEFLNG),
|
|
KW("DEFSNG", TOK_DEFSNG),
|
|
KW("DEFSTR", TOK_DEFSTR),
|
|
KW("DIM", TOK_DIM),
|
|
KW("DIR", TOK_DIR),
|
|
KW("DIR$", TOK_DIR),
|
|
KW("DO", TOK_DO),
|
|
KW("DOEVENTS", TOK_DOEVENTS),
|
|
KW("DOUBLE", TOK_DOUBLE),
|
|
KW("ELSE", TOK_ELSE),
|
|
KW("ELSEIF", TOK_ELSEIF),
|
|
KW("END", TOK_END),
|
|
KW("EOF", TOK_EOF_KW),
|
|
KW("EQV", TOK_EQV),
|
|
KW("ERASE", TOK_ERASE),
|
|
KW("ERL", TOK_ERL),
|
|
KW("ERR", TOK_ERR),
|
|
KW("ERROR", TOK_ERROR_KW),
|
|
KW("EXIT", TOK_EXIT),
|
|
KW("EXPLICIT", TOK_EXPLICIT),
|
|
KW("FALSE", TOK_FALSE_KW),
|
|
KW("FILECOPY", TOK_FILECOPY),
|
|
KW("FILELEN", TOK_FILELEN),
|
|
KW("FOR", TOK_FOR),
|
|
KW("FUNCTION", TOK_FUNCTION),
|
|
KW("GET", TOK_GET),
|
|
KW("GETATTR", TOK_GETATTR),
|
|
KW("GOSUB", TOK_GOSUB),
|
|
KW("GOTO", TOK_GOTO),
|
|
KW("HIDE", TOK_HIDE),
|
|
KW("IF", TOK_IF),
|
|
KW("IMP", TOK_IMP),
|
|
KW("INIREAD", TOK_INIREAD),
|
|
KW("INIREAD$", TOK_INIREAD),
|
|
KW("INIWRITE", TOK_INIWRITE),
|
|
KW("INPUT", TOK_INPUT),
|
|
KW("INPUTBOX", TOK_INPUTBOX),
|
|
KW("INPUTBOX$", TOK_INPUTBOX),
|
|
KW("INTEGER", TOK_INTEGER),
|
|
KW("IS", TOK_IS),
|
|
KW("KILL", TOK_KILL),
|
|
KW("LBOUND", TOK_LBOUND),
|
|
KW("LET", TOK_LET),
|
|
KW("LINE", TOK_LINE),
|
|
KW("LOAD", TOK_LOAD),
|
|
KW("LONG", TOK_LONG),
|
|
KW("LOOP", TOK_LOOP),
|
|
KW("ME", TOK_ME),
|
|
KW("MKDIR", TOK_MKDIR),
|
|
KW("MOD", TOK_MOD),
|
|
KW("MSGBOX", TOK_MSGBOX),
|
|
KW("NAME", TOK_NAME),
|
|
KW("NEXT", TOK_NEXT),
|
|
KW("NOT", TOK_NOT),
|
|
KW("NOTHING", TOK_NOTHING),
|
|
KW("ON", TOK_ON),
|
|
KW("OPEN", TOK_OPEN),
|
|
KW("OPTION", TOK_OPTION),
|
|
KW("OPTIONAL", TOK_OPTIONAL),
|
|
KW("OR", TOK_OR),
|
|
KW("OUTPUT", TOK_OUTPUT),
|
|
KW("PRESERVE", TOK_PRESERVE),
|
|
KW("PRINT", TOK_PRINT),
|
|
KW("PUT", TOK_PUT),
|
|
KW("RANDOM", TOK_RANDOM),
|
|
KW("RANDOMIZE", TOK_RANDOMIZE),
|
|
KW("READ", TOK_READ),
|
|
KW("REDIM", TOK_REDIM),
|
|
KW("REM", TOK_REM),
|
|
KW("REMOVECONTROL", TOK_REMOVECONTROL),
|
|
KW("RESTORE", TOK_RESTORE),
|
|
KW("RESUME", TOK_RESUME),
|
|
KW("RETURN", TOK_RETURN),
|
|
KW("RMDIR", TOK_RMDIR),
|
|
KW("SEEK", TOK_SEEK),
|
|
KW("SELECT", TOK_SELECT),
|
|
KW("SET", TOK_SET),
|
|
KW("SETATTR", TOK_SETATTR),
|
|
KW("SETEVENT", TOK_SETEVENT),
|
|
KW("SHARED", TOK_SHARED),
|
|
KW("SHELL", TOK_SHELL),
|
|
KW("SHOW", TOK_SHOW),
|
|
KW("SINGLE", TOK_SINGLE),
|
|
KW("SLEEP", TOK_SLEEP),
|
|
KW("STATIC", TOK_STATIC),
|
|
KW("STEP", TOK_STEP),
|
|
KW("STRING", TOK_STRING_KW),
|
|
KW("SUB", TOK_SUB),
|
|
KW("SWAP", TOK_SWAP),
|
|
KW("THEN", TOK_THEN),
|
|
KW("TIMER", TOK_TIMER),
|
|
KW("TO", TOK_TO),
|
|
KW("TRUE", TOK_TRUE_KW),
|
|
KW("TYPE", TOK_TYPE),
|
|
KW("UBOUND", TOK_UBOUND),
|
|
KW("UNLOAD", TOK_UNLOAD),
|
|
KW("UNTIL", TOK_UNTIL),
|
|
KW("WEND", TOK_WEND),
|
|
KW("WHILE", TOK_WHILE),
|
|
KW("WITH", TOK_WITH),
|
|
KW("WRITE", TOK_WRITE),
|
|
KW("XOR", TOK_XOR),
|
|
{ NULL, 0, TOK_ERROR }
|
|
};
|
|
|
|
#undef KW
|
|
|
|
#define KEYWORD_COUNT (sizeof(sKeywords) / sizeof(sKeywords[0]) - 1)
|
|
|
|
// ============================================================
|
|
// Type-suffix table
|
|
// ============================================================
|
|
//
|
|
// Single source of truth for the BASIC type-suffix characters and the
|
|
// type each one implies; basIsTypeSuffixChar and basTypeSuffixType (and
|
|
// through them the parser) all consult this table.
|
|
|
|
typedef struct {
|
|
char suffix;
|
|
uint8_t dataType;
|
|
} TypeSuffixEntryT;
|
|
|
|
static const TypeSuffixEntryT sTypeSuffixes[] = {
|
|
{ '%', BAS_TYPE_INTEGER },
|
|
{ '&', BAS_TYPE_LONG },
|
|
{ '!', BAS_TYPE_SINGLE },
|
|
{ '#', BAS_TYPE_DOUBLE },
|
|
{ '$', BAS_TYPE_STRING },
|
|
};
|
|
|
|
#define TYPE_SUFFIX_COUNT ((int32_t)(sizeof(sTypeSuffixes) / sizeof(sTypeSuffixes[0])))
|
|
|
|
|
|
// Function prototypes (alphabetical)
|
|
static char advance(BasLexerT *lex);
|
|
static void appendTokenChar(BasLexerT *lex, int32_t *idx, char c);
|
|
static bool atEnd(const BasLexerT *lex);
|
|
char basAsciiUpper(char c);
|
|
bool basIsIdentChar(char c);
|
|
bool basIsTypeSuffixChar(char c);
|
|
bool basIsValidIdent(const char *name);
|
|
void basLexerInit(BasLexerT *lex, const char *source, int32_t sourceLen);
|
|
const char *basLexerKeywordAt(int32_t i);
|
|
BasKeywordClassE basLexerKeywordClass(int32_t i);
|
|
int32_t basLexerKeywordCount(void);
|
|
BasTokenTypeE basLexerNext(BasLexerT *lex);
|
|
BasTokenTypeE basLexerPeek(const BasLexerT *lex);
|
|
const char *basTokenName(BasTokenTypeE type);
|
|
int32_t basTypeSuffixType(char c);
|
|
static bool lexIntegerLiteral(BasLexerT *lex, int64_t minVal, int64_t maxVal, const char *what, int64_t *outVal);
|
|
static BasTokenTypeE lookupKeyword(const char *text, int32_t len);
|
|
static BasTokenTypeE makeNewlineToken(BasLexerT *lex);
|
|
static char peek(const BasLexerT *lex);
|
|
static char peekNext(const BasLexerT *lex);
|
|
static void setError(BasLexerT *lex, const char *msg);
|
|
static void skipLineComment(BasLexerT *lex);
|
|
static void skipWhitespace(BasLexerT *lex);
|
|
static BasTokenTypeE tokenizeHexLiteral(BasLexerT *lex);
|
|
static BasTokenTypeE tokenizeIdentOrKeyword(BasLexerT *lex);
|
|
static BasTokenTypeE tokenizeNumber(BasLexerT *lex);
|
|
static BasTokenTypeE tokenizeString(BasLexerT *lex);
|
|
|
|
static char advance(BasLexerT *lex) {
|
|
if (atEnd(lex)) {
|
|
return '\0';
|
|
}
|
|
|
|
char c = lex->source[lex->pos++];
|
|
|
|
// LF, or a CR that is not the first half of a CRLF pair, ends a line
|
|
// (the LF of a CRLF pair is counted when it is consumed).
|
|
if (c == '\n' || (c == '\r' && peek(lex) != '\n')) {
|
|
lex->line++;
|
|
lex->col = 1;
|
|
} else {
|
|
lex->col++;
|
|
}
|
|
|
|
return c;
|
|
}
|
|
|
|
|
|
static void appendTokenChar(BasLexerT *lex, int32_t *idx, char c) {
|
|
if (*idx < BAS_MAX_TOKEN_LEN - 1) {
|
|
lex->token.text[(*idx)++] = c;
|
|
}
|
|
}
|
|
|
|
|
|
static bool atEnd(const BasLexerT *lex) {
|
|
return lex->pos >= lex->sourceLen;
|
|
}
|
|
|
|
|
|
char basAsciiUpper(char c) {
|
|
if (c >= 'a' && c <= 'z') {
|
|
c -= ('a' - 'A');
|
|
}
|
|
|
|
return c;
|
|
}
|
|
|
|
|
|
bool basIsIdentChar(char c) {
|
|
return (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || (c >= '0' && c <= '9') || c == '_';
|
|
}
|
|
|
|
|
|
bool basIsTypeSuffixChar(char c) {
|
|
return basTypeSuffixType(c) >= 0;
|
|
}
|
|
|
|
|
|
bool basIsValidIdent(const char *name) {
|
|
if (!name || !basIsIdentChar(name[0]) || (name[0] >= '0' && name[0] <= '9')) {
|
|
return false;
|
|
}
|
|
|
|
for (const char *p = name + 1; *p; p++) {
|
|
if (!basIsIdentChar(*p)) {
|
|
return false;
|
|
}
|
|
}
|
|
|
|
return true;
|
|
}
|
|
|
|
|
|
void basLexerInit(BasLexerT *lex, const char *source, int32_t sourceLen) {
|
|
memset(lex, 0, sizeof(*lex));
|
|
lex->source = source;
|
|
lex->sourceLen = (sourceLen < 0) ? (int32_t)strlen(source) : sourceLen;
|
|
lex->pos = 0;
|
|
lex->line = 1;
|
|
lex->col = 1;
|
|
|
|
// Prime the first token
|
|
basLexerNext(lex);
|
|
}
|
|
|
|
|
|
const char *basLexerKeywordAt(int32_t i) {
|
|
if (i < 0 || i >= (int32_t)KEYWORD_COUNT) {
|
|
return NULL;
|
|
}
|
|
|
|
return sKeywords[i].text;
|
|
}
|
|
|
|
|
|
BasKeywordClassE basLexerKeywordClass(int32_t i) {
|
|
if (i < 0 || i >= (int32_t)KEYWORD_COUNT) {
|
|
return BAS_KW_CLASS_OTHER;
|
|
}
|
|
|
|
switch (sKeywords[i].type) {
|
|
case TOK_BOOLEAN:
|
|
case TOK_DOUBLE:
|
|
case TOK_INTEGER:
|
|
case TOK_LONG:
|
|
case TOK_SINGLE:
|
|
case TOK_STRING_KW:
|
|
return BAS_KW_CLASS_TYPE;
|
|
|
|
case TOK_TRUE_KW:
|
|
case TOK_FALSE_KW:
|
|
case TOK_NOTHING:
|
|
return BAS_KW_CLASS_LITERAL;
|
|
|
|
default:
|
|
return BAS_KW_CLASS_OTHER;
|
|
}
|
|
}
|
|
|
|
|
|
int32_t basLexerKeywordCount(void) {
|
|
return (int32_t)KEYWORD_COUNT;
|
|
}
|
|
|
|
|
|
BasTokenTypeE basLexerNext(BasLexerT *lex) {
|
|
skipWhitespace(lex);
|
|
|
|
lex->token.line = lex->line;
|
|
lex->token.col = lex->col;
|
|
lex->token.textLen = 0;
|
|
lex->token.text[0] = '\0';
|
|
|
|
if (atEnd(lex)) {
|
|
lex->token.type = TOK_EOF;
|
|
return TOK_EOF;
|
|
}
|
|
|
|
char c = peek(lex);
|
|
|
|
// Newline
|
|
if (c == '\n') {
|
|
advance(lex);
|
|
return makeNewlineToken(lex);
|
|
}
|
|
|
|
// Carriage return (handle CR, CRLF)
|
|
if (c == '\r') {
|
|
advance(lex);
|
|
|
|
if (!atEnd(lex) && peek(lex) == '\n') {
|
|
advance(lex);
|
|
}
|
|
|
|
return makeNewlineToken(lex);
|
|
}
|
|
|
|
// Comment (apostrophe)
|
|
if (c == '\'') {
|
|
skipLineComment(lex);
|
|
return makeNewlineToken(lex);
|
|
}
|
|
|
|
// String literal
|
|
if (c == '"') {
|
|
lex->token.type = tokenizeString(lex);
|
|
return lex->token.type;
|
|
}
|
|
|
|
// Number
|
|
if (isdigit((unsigned char)c) || (c == '.' && isdigit((unsigned char)peekNext(lex)))) {
|
|
lex->token.type = tokenizeNumber(lex);
|
|
return lex->token.type;
|
|
}
|
|
|
|
// Numeric-base literals: &H hex, &O octal, &B binary. &B is an
|
|
// extension beyond classic QBASIC; it's convenient for bitmask
|
|
// work in the widget/graphics code.
|
|
if (c == '&') {
|
|
char n = basAsciiUpper(peekNext(lex));
|
|
|
|
if (n == 'H' || n == 'O' || n == 'B') {
|
|
lex->token.type = tokenizeHexLiteral(lex);
|
|
return lex->token.type;
|
|
}
|
|
}
|
|
|
|
// Identifier or keyword
|
|
if (basIsIdentChar(c) && !isdigit((unsigned char)c)) {
|
|
lex->token.type = tokenizeIdentOrKeyword(lex);
|
|
return lex->token.type;
|
|
}
|
|
|
|
// Single and multi-character operators/punctuation
|
|
advance(lex);
|
|
|
|
switch (c) {
|
|
case '+':
|
|
lex->token.type = TOK_PLUS;
|
|
break;
|
|
|
|
case '-':
|
|
lex->token.type = TOK_MINUS;
|
|
break;
|
|
|
|
case '*':
|
|
lex->token.type = TOK_STAR;
|
|
break;
|
|
|
|
case '/':
|
|
lex->token.type = TOK_SLASH;
|
|
break;
|
|
|
|
case '\\':
|
|
lex->token.type = TOK_BACKSLASH;
|
|
break;
|
|
|
|
case '^':
|
|
lex->token.type = TOK_CARET;
|
|
break;
|
|
|
|
case '&':
|
|
lex->token.type = TOK_AMPERSAND;
|
|
break;
|
|
|
|
case '(':
|
|
lex->token.type = TOK_LPAREN;
|
|
break;
|
|
|
|
case ')':
|
|
lex->token.type = TOK_RPAREN;
|
|
break;
|
|
|
|
case ',':
|
|
lex->token.type = TOK_COMMA;
|
|
break;
|
|
|
|
case ';':
|
|
lex->token.type = TOK_SEMICOLON;
|
|
break;
|
|
|
|
case ':':
|
|
lex->token.type = TOK_COLON;
|
|
break;
|
|
|
|
case '.':
|
|
lex->token.type = TOK_DOT;
|
|
break;
|
|
|
|
case '#':
|
|
lex->token.type = TOK_HASH;
|
|
break;
|
|
|
|
case '?':
|
|
lex->token.type = TOK_PRINT;
|
|
break;
|
|
|
|
case '=':
|
|
lex->token.type = TOK_EQ;
|
|
break;
|
|
|
|
case '<':
|
|
if (!atEnd(lex) && peek(lex) == '>') {
|
|
advance(lex);
|
|
lex->token.type = TOK_NE;
|
|
} else if (!atEnd(lex) && peek(lex) == '=') {
|
|
advance(lex);
|
|
lex->token.type = TOK_LE;
|
|
} else {
|
|
lex->token.type = TOK_LT;
|
|
}
|
|
break;
|
|
|
|
case '>':
|
|
if (!atEnd(lex) && peek(lex) == '=') {
|
|
advance(lex);
|
|
lex->token.type = TOK_GE;
|
|
} else {
|
|
lex->token.type = TOK_GT;
|
|
}
|
|
break;
|
|
|
|
default:
|
|
setError(lex, "Unexpected character");
|
|
lex->token.type = TOK_ERROR;
|
|
break;
|
|
}
|
|
|
|
// Store the operator text
|
|
if (lex->token.type != TOK_ERROR) {
|
|
lex->token.text[0] = c;
|
|
lex->token.textLen = 1;
|
|
|
|
if (lex->token.type == TOK_NE || lex->token.type == TOK_LE || lex->token.type == TOK_GE) {
|
|
lex->token.text[1] = lex->source[lex->pos - 1];
|
|
lex->token.textLen = 2;
|
|
}
|
|
|
|
lex->token.text[lex->token.textLen] = '\0';
|
|
}
|
|
|
|
return lex->token.type;
|
|
}
|
|
|
|
|
|
BasTokenTypeE basLexerPeek(const BasLexerT *lex) {
|
|
return lex->token.type;
|
|
}
|
|
|
|
|
|
const char *basTokenName(BasTokenTypeE type) {
|
|
switch (type) {
|
|
case TOK_INT_LIT: return "integer";
|
|
case TOK_LONG_LIT: return "long";
|
|
case TOK_FLOAT_LIT: return "float";
|
|
case TOK_STRING_LIT: return "string";
|
|
case TOK_IDENT: return "identifier";
|
|
case TOK_DOT: return "'.'";
|
|
case TOK_COMMA: return "','";
|
|
case TOK_SEMICOLON: return "';'";
|
|
case TOK_COLON: return "':'";
|
|
case TOK_LPAREN: return "'('";
|
|
case TOK_RPAREN: return "')'";
|
|
case TOK_HASH: return "'#'";
|
|
case TOK_PLUS: return "'+'";
|
|
case TOK_MINUS: return "'-'";
|
|
case TOK_STAR: return "'*'";
|
|
case TOK_SLASH: return "'/'";
|
|
case TOK_BACKSLASH: return "'\\'";
|
|
case TOK_CARET: return "'^'";
|
|
case TOK_AMPERSAND: return "'&'";
|
|
case TOK_EQ: return "'='";
|
|
case TOK_NE: return "'<>'";
|
|
case TOK_LT: return "'<'";
|
|
case TOK_GT: return "'>'";
|
|
case TOK_LE: return "'<='";
|
|
case TOK_GE: return "'>='";
|
|
case TOK_NEWLINE: return "newline";
|
|
case TOK_EOF: return "end of file";
|
|
case TOK_ERROR: return "error";
|
|
default: break;
|
|
}
|
|
|
|
// Keywords
|
|
for (int32_t i = 0; i < (int32_t)KEYWORD_COUNT; i++) {
|
|
if (sKeywords[i].type == type) {
|
|
return sKeywords[i].text;
|
|
}
|
|
}
|
|
|
|
return "?";
|
|
}
|
|
|
|
|
|
int32_t basTypeSuffixType(char c) {
|
|
for (int32_t i = 0; i < TYPE_SUFFIX_COUNT; i++) {
|
|
if (sTypeSuffixes[i].suffix == c) {
|
|
return sTypeSuffixes[i].dataType;
|
|
}
|
|
}
|
|
|
|
return -1;
|
|
}
|
|
|
|
|
|
// Convert the decimal digits in token.text to an integer and range-check
|
|
// the result against [minVal, maxVal]. On overflow the token becomes
|
|
// TOK_ERROR naming `what`, and false is returned. strtoll is used rather
|
|
// than atol so the conversion is identical on the 32-bit DOS build and
|
|
// the 64-bit host build of the compiler.
|
|
static bool lexIntegerLiteral(BasLexerT *lex, int64_t minVal, int64_t maxVal, const char *what, int64_t *outVal) {
|
|
errno = 0;
|
|
long long val = strtoll(lex->token.text, NULL, 10);
|
|
|
|
if (errno == ERANGE || val < minVal || val > maxVal) {
|
|
char buf[BAS_LEX_ERROR_LEN];
|
|
snprintf(buf, sizeof(buf), "%s literal out of range", what);
|
|
setError(lex, buf);
|
|
lex->token.type = TOK_ERROR;
|
|
return false;
|
|
}
|
|
|
|
*outVal = (int64_t)val;
|
|
return true;
|
|
}
|
|
|
|
|
|
static BasTokenTypeE lookupKeyword(const char *text, int32_t len) {
|
|
// Case-insensitive keyword lookup. Short-circuits on length mismatch
|
|
// (via cached keyword length) and on the very first character, both of
|
|
// which reject the vast majority of entries before doing a full scan.
|
|
char firstUp = basAsciiUpper(text[0]);
|
|
|
|
for (int32_t i = 0; i < (int32_t)KEYWORD_COUNT; i++) {
|
|
const KeywordEntryT *kw = &sKeywords[i];
|
|
|
|
if (kw->textLen != len || kw->text[0] != firstUp) {
|
|
continue;
|
|
}
|
|
|
|
bool match = true;
|
|
|
|
for (int32_t j = 1; j < len; j++) {
|
|
if (basAsciiUpper(text[j]) != kw->text[j]) {
|
|
match = false;
|
|
break;
|
|
}
|
|
}
|
|
|
|
if (match) {
|
|
return kw->type;
|
|
}
|
|
}
|
|
|
|
return TOK_IDENT;
|
|
}
|
|
|
|
|
|
// Fill the current token as a synthesized newline (used for real newlines,
|
|
// CR/CRLF, and the end of apostrophe / REM comments).
|
|
static BasTokenTypeE makeNewlineToken(BasLexerT *lex) {
|
|
lex->token.type = TOK_NEWLINE;
|
|
lex->token.text[0] = '\n';
|
|
lex->token.text[1] = '\0';
|
|
lex->token.textLen = 1;
|
|
return TOK_NEWLINE;
|
|
}
|
|
|
|
|
|
static char peek(const BasLexerT *lex) {
|
|
if (atEnd(lex)) {
|
|
return '\0';
|
|
}
|
|
|
|
return lex->source[lex->pos];
|
|
}
|
|
|
|
|
|
static char peekNext(const BasLexerT *lex) {
|
|
if (lex->pos + 1 >= lex->sourceLen) {
|
|
return '\0';
|
|
}
|
|
|
|
return lex->source[lex->pos + 1];
|
|
}
|
|
|
|
|
|
static void setError(BasLexerT *lex, const char *msg) {
|
|
snprintf(lex->error, sizeof(lex->error), "Line %d, Col %d: %s", (int)lex->line, (int)lex->col, msg);
|
|
}
|
|
|
|
|
|
static void skipLineComment(BasLexerT *lex) {
|
|
while (!atEnd(lex) && peek(lex) != '\n' && peek(lex) != '\r') {
|
|
advance(lex);
|
|
}
|
|
}
|
|
|
|
|
|
//
|
|
// Skips spaces and tabs. Does NOT skip newlines (they are tokens).
|
|
// Handles line continuation: underscore followed by newline joins
|
|
// the next line to the current logical line.
|
|
static void skipWhitespace(BasLexerT *lex) {
|
|
while (!atEnd(lex)) {
|
|
char c = peek(lex);
|
|
|
|
if (c == ' ' || c == '\t') {
|
|
advance(lex);
|
|
continue;
|
|
}
|
|
|
|
// Line continuation: _ at end of line
|
|
if (c == '_') {
|
|
int32_t savedPos = lex->pos;
|
|
int32_t savedLine = lex->line;
|
|
int32_t savedCol = lex->col;
|
|
advance(lex);
|
|
|
|
// Skip spaces/tabs after underscore
|
|
while (!atEnd(lex) && (peek(lex) == ' ' || peek(lex) == '\t')) {
|
|
advance(lex);
|
|
}
|
|
|
|
// Must be followed by newline
|
|
if (!atEnd(lex) && (peek(lex) == '\n' || peek(lex) == '\r')) {
|
|
advance(lex);
|
|
|
|
if (!atEnd(lex) && peek(lex) == '\n' && lex->source[lex->pos - 1] == '\r') {
|
|
advance(lex);
|
|
}
|
|
|
|
continue; // Continue skipping whitespace on next line
|
|
}
|
|
|
|
// Not a continuation -- put back
|
|
lex->pos = savedPos;
|
|
lex->line = savedLine;
|
|
lex->col = savedCol;
|
|
break;
|
|
}
|
|
|
|
break;
|
|
}
|
|
}
|
|
|
|
|
|
static BasTokenTypeE tokenizeHexLiteral(BasLexerT *lex) {
|
|
advance(lex); // skip &
|
|
char base = basAsciiUpper(peek(lex));
|
|
advance(lex); // skip H/O/B
|
|
|
|
int32_t shift;
|
|
int32_t maxDigit;
|
|
|
|
if (base == 'O') {
|
|
shift = 3;
|
|
maxDigit = 7;
|
|
} else if (base == 'B') {
|
|
shift = 1;
|
|
maxDigit = 1;
|
|
} else {
|
|
shift = 4;
|
|
maxDigit = 15;
|
|
}
|
|
|
|
bool overflow = false;
|
|
int32_t idx = 0;
|
|
uint64_t value = 0;
|
|
|
|
for (;;) {
|
|
if (atEnd(lex)) {
|
|
break;
|
|
}
|
|
|
|
char c = peek(lex);
|
|
int32_t digit;
|
|
|
|
if (c >= '0' && c <= '9') {
|
|
digit = c - '0';
|
|
} else if (shift == 4 && c >= 'A' && c <= 'F') {
|
|
digit = c - 'A' + 10;
|
|
} else if (shift == 4 && c >= 'a' && c <= 'f') {
|
|
digit = c - 'a' + 10;
|
|
} else {
|
|
break;
|
|
}
|
|
|
|
if (digit > maxDigit) {
|
|
break;
|
|
}
|
|
|
|
advance(lex);
|
|
appendTokenChar(lex, &idx, c);
|
|
|
|
value = (value << shift) | (uint64_t)digit;
|
|
|
|
// Latch overflow instead of testing value after the loop: with
|
|
// enough digits the shifts wrap the uint64 back into range, which
|
|
// would hide the overflow.
|
|
if (value > UINT32_MAX) {
|
|
overflow = true;
|
|
}
|
|
}
|
|
|
|
// No digits after the &H/&O/&B marker is a typo, not the integer 0.
|
|
if (idx == 0) {
|
|
setError(lex, "Empty hexadecimal/octal/binary literal");
|
|
lex->token.type = TOK_ERROR;
|
|
return TOK_ERROR;
|
|
}
|
|
|
|
// The widest runtime integer (LONG) is 32 bits; silently truncating a
|
|
// wider literal would miscompile the constant, so reject it. Note the
|
|
// literal is NOT typed by width: &HFFFF is 65535 (a LONG), not QBASIC's
|
|
// 16-bit -1 -- the parser picks the push width from the value.
|
|
if (overflow) {
|
|
setError(lex, "Hexadecimal/octal/binary literal exceeds 32 bits");
|
|
lex->token.type = TOK_ERROR;
|
|
return TOK_ERROR;
|
|
}
|
|
|
|
lex->token.text[idx] = '\0';
|
|
lex->token.textLen = idx;
|
|
|
|
// Check for trailing & (long suffix)
|
|
if (!atEnd(lex) && peek(lex) == '&') {
|
|
advance(lex);
|
|
lex->token.longVal = (int64_t)value;
|
|
return TOK_LONG_LIT;
|
|
}
|
|
|
|
lex->token.intVal = (int32_t)value;
|
|
return TOK_INT_LIT;
|
|
}
|
|
|
|
|
|
static BasTokenTypeE tokenizeIdentOrKeyword(BasLexerT *lex) {
|
|
int32_t idx = 0;
|
|
|
|
while (!atEnd(lex) && basIsIdentChar(peek(lex))) {
|
|
appendTokenChar(lex, &idx, advance(lex));
|
|
}
|
|
|
|
lex->token.text[idx] = '\0';
|
|
lex->token.textLen = idx;
|
|
|
|
// Check for type suffix. A keyword only ever takes '$' (CURDIR$,
|
|
// DIR$, INPUT$ ...); any other suffix after a keyword belongs to the
|
|
// next token, so PRINT#1 / CLOSE#1 / INPUT#1 lex as keyword + '#'.
|
|
if (!atEnd(lex)) {
|
|
char c = peek(lex);
|
|
|
|
if (basIsTypeSuffixChar(c) && (c == '$' || lookupKeyword(lex->token.text, idx) == TOK_IDENT)) {
|
|
advance(lex);
|
|
appendTokenChar(lex, &idx, c);
|
|
lex->token.text[idx] = '\0';
|
|
lex->token.textLen = idx;
|
|
}
|
|
}
|
|
|
|
// Check if this is a keyword
|
|
// For suffix-bearing identifiers, only check the base (without suffix)
|
|
int32_t baseLen = idx;
|
|
|
|
if (baseLen > 0) {
|
|
char last = lex->token.text[baseLen - 1];
|
|
|
|
if (basIsTypeSuffixChar(last)) {
|
|
baseLen--;
|
|
}
|
|
}
|
|
|
|
// Try the full text first (including any type suffix). Suffix-bearing
|
|
// keywords like CURDIR$, DIR$, INIREAD$, INPUTBOX$ are listed in the
|
|
// keyword table with their $ and will match here. If the full text
|
|
// isn't a keyword, fall back to the base name (without suffix).
|
|
BasTokenTypeE kwType = lookupKeyword(lex->token.text, idx);
|
|
bool matchedWithSuffix = (kwType != TOK_IDENT && baseLen != idx);
|
|
|
|
if (kwType == TOK_IDENT && baseLen != idx) {
|
|
kwType = lookupKeyword(lex->token.text, baseLen);
|
|
}
|
|
|
|
// REM is a comment -- skip to end of line. Apply the same acceptance
|
|
// gate as below so a suffixed identifier like REM$ stays an identifier
|
|
// instead of swallowing the rest of the line as a comment.
|
|
if (kwType == TOK_REM && (baseLen == idx || matchedWithSuffix)) {
|
|
skipLineComment(lex);
|
|
return makeNewlineToken(lex);
|
|
}
|
|
|
|
// Accept the keyword if it's a plain keyword (no suffix on source) or
|
|
// if it explicitly matched a $-suffixed entry in the keyword table.
|
|
if (kwType != TOK_IDENT && (baseLen == idx || matchedWithSuffix)) {
|
|
return kwType;
|
|
}
|
|
|
|
return TOK_IDENT;
|
|
}
|
|
|
|
|
|
static BasTokenTypeE tokenizeNumber(BasLexerT *lex) {
|
|
int32_t idx = 0;
|
|
bool hasDecimal = false;
|
|
bool hasExp = false;
|
|
|
|
// Integer part
|
|
while (!atEnd(lex) && isdigit((unsigned char)peek(lex))) {
|
|
appendTokenChar(lex, &idx, advance(lex));
|
|
}
|
|
|
|
// Decimal part
|
|
if (!atEnd(lex) && peek(lex) == '.' && isdigit((unsigned char)peekNext(lex))) {
|
|
hasDecimal = true;
|
|
appendTokenChar(lex, &idx, advance(lex)); // .
|
|
|
|
while (!atEnd(lex) && isdigit((unsigned char)peek(lex))) {
|
|
appendTokenChar(lex, &idx, advance(lex));
|
|
}
|
|
}
|
|
|
|
// Exponent. Accept QBASIC/Fortran 'D' (double) as well as 'E', but
|
|
// always store 'E' in the buffer so atof() can parse it. Require at
|
|
// least one digit after the marker (and optional sign); otherwise the
|
|
// marker is not part of the number (e.g. "1E" tokenizes as 1 then E).
|
|
char marker = basAsciiUpper(peek(lex));
|
|
|
|
if (!atEnd(lex) && (marker == 'E' || marker == 'D')) {
|
|
int32_t savedIdx = idx;
|
|
int32_t savedPos = lex->pos;
|
|
int32_t savedLine = lex->line;
|
|
int32_t savedCol = lex->col;
|
|
|
|
advance(lex); // consume marker
|
|
appendTokenChar(lex, &idx, 'E'); // always store 'E', never 'D'
|
|
|
|
if (!atEnd(lex) && (peek(lex) == '+' || peek(lex) == '-')) {
|
|
appendTokenChar(lex, &idx, advance(lex));
|
|
}
|
|
|
|
if (!atEnd(lex) && isdigit((unsigned char)peek(lex))) {
|
|
hasExp = true;
|
|
|
|
while (!atEnd(lex) && isdigit((unsigned char)peek(lex))) {
|
|
appendTokenChar(lex, &idx, advance(lex));
|
|
}
|
|
} else {
|
|
// Not a valid exponent -- roll back the consumed marker/sign.
|
|
idx = savedIdx;
|
|
lex->pos = savedPos;
|
|
lex->line = savedLine;
|
|
lex->col = savedCol;
|
|
}
|
|
}
|
|
|
|
lex->token.text[idx] = '\0';
|
|
lex->token.textLen = idx;
|
|
|
|
// Check for type suffix
|
|
if (!atEnd(lex)) {
|
|
char c = peek(lex);
|
|
int64_t val;
|
|
|
|
if (c == '%') {
|
|
advance(lex);
|
|
|
|
if (!lexIntegerLiteral(lex, INT16_MIN, INT16_MAX, "INTEGER", &val)) {
|
|
return TOK_ERROR;
|
|
}
|
|
|
|
lex->token.intVal = (int32_t)val;
|
|
return TOK_INT_LIT;
|
|
}
|
|
|
|
if (c == '&') {
|
|
advance(lex);
|
|
|
|
if (!lexIntegerLiteral(lex, INT32_MIN, INT32_MAX, "LONG", &val)) {
|
|
return TOK_ERROR;
|
|
}
|
|
|
|
lex->token.longVal = val;
|
|
return TOK_LONG_LIT;
|
|
}
|
|
|
|
if (c == '!' || c == '#') {
|
|
advance(lex);
|
|
lex->token.dblVal = atof(lex->token.text);
|
|
return TOK_FLOAT_LIT;
|
|
}
|
|
}
|
|
|
|
// No suffix: determine type from content
|
|
if (hasDecimal || hasExp) {
|
|
lex->token.dblVal = atof(lex->token.text);
|
|
return TOK_FLOAT_LIT;
|
|
}
|
|
|
|
// Unsuffixed integers must fit LONG; anything wider would be silently
|
|
// truncated by the 32-bit runtime.
|
|
int64_t val;
|
|
int64_t maxVal = lex->allowNegMagnitude ? BAS_LONG_NEG_MAGNITUDE : INT32_MAX;
|
|
|
|
if (!lexIntegerLiteral(lex, INT32_MIN, maxVal, "Integer", &val)) {
|
|
return TOK_ERROR;
|
|
}
|
|
|
|
if (val >= INT16_MIN && val <= INT16_MAX) {
|
|
lex->token.intVal = (int32_t)val;
|
|
return TOK_INT_LIT;
|
|
}
|
|
|
|
lex->token.longVal = val;
|
|
return TOK_LONG_LIT;
|
|
}
|
|
|
|
|
|
static BasTokenTypeE tokenizeString(BasLexerT *lex) {
|
|
advance(lex); // skip opening quote
|
|
|
|
int32_t idx = 0;
|
|
bool overflow = false;
|
|
|
|
while (!atEnd(lex) && peek(lex) != '"' && peek(lex) != '\n' && peek(lex) != '\r') {
|
|
// appendTokenChar silently drops chars once the buffer is full;
|
|
// detect that here so over-long literals are diagnosed instead
|
|
// of truncated. Keep consuming so the lexer skips the whole
|
|
// literal and its closing quote.
|
|
if (idx >= BAS_MAX_STRING_LEN) {
|
|
overflow = true;
|
|
}
|
|
appendTokenChar(lex, &idx, advance(lex));
|
|
}
|
|
|
|
if (atEnd(lex) || peek(lex) != '"') {
|
|
setError(lex, "Unterminated string literal");
|
|
lex->token.text[idx] = '\0';
|
|
lex->token.textLen = idx;
|
|
return TOK_ERROR;
|
|
}
|
|
|
|
advance(lex); // skip closing quote
|
|
|
|
lex->token.text[idx] = '\0';
|
|
lex->token.textLen = idx;
|
|
|
|
if (overflow) {
|
|
char buf[BAS_LEX_ERROR_LEN];
|
|
snprintf(buf, sizeof(buf), "String literal too long (maximum is %d characters)", (int)BAS_MAX_STRING_LEN);
|
|
setError(lex, buf);
|
|
return TOK_ERROR;
|
|
}
|
|
|
|
return TOK_STRING_LIT;
|
|
}
|