292 lines
7.8 KiB
C
292 lines
7.8 KiB
C
// The MIT License (MIT)
|
|
//
|
|
// Copyright (C) 2026 Scott Duensing
|
|
//
|
|
// Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
// of this software and associated documentation files (the "Software"), to
|
|
// deal in the Software without restriction, including without limitation the
|
|
// rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
|
|
// sell copies of the Software, and to permit persons to whom the Software is
|
|
// furnished to do so, subject to the following conditions:
|
|
//
|
|
// The above copyright notice and this permission notice shall be included in
|
|
// all copies or substantial portions of the Software.
|
|
//
|
|
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
|
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
|
|
// IN THE SOFTWARE.
|
|
|
|
// lexer.h -- DVX BASIC lexer (tokenizer)
|
|
//
|
|
// Converts BASIC source text into a stream of tokens. Case-insensitive
|
|
// for keywords. Handles line continuations (_), comments (' and REM),
|
|
// type suffixes (%, &, !, #, $), and string literals.
|
|
//
|
|
// Embeddable: no DVX dependencies, pure C.
|
|
|
|
#ifndef DVXBASIC_LEXER_H
|
|
#define DVXBASIC_LEXER_H
|
|
|
|
#include <stdint.h>
|
|
#include <stdbool.h>
|
|
|
|
// ============================================================
|
|
// Token types
|
|
// ============================================================
|
|
|
|
typedef enum {
|
|
// Literals
|
|
TOK_INT_LIT, // integer literal (123, &HFF)
|
|
TOK_LONG_LIT, // long literal (123&)
|
|
TOK_FLOAT_LIT, // float literal (3.14, 1.5E10)
|
|
TOK_STRING_LIT, // "string literal"
|
|
|
|
// Identifiers and symbols
|
|
TOK_IDENT, // variable/function name
|
|
TOK_DOT, // .
|
|
TOK_COMMA, // ,
|
|
TOK_SEMICOLON, // ;
|
|
TOK_COLON, // :
|
|
TOK_LPAREN, // (
|
|
TOK_RPAREN, // )
|
|
TOK_HASH, // # (file channel)
|
|
|
|
// Operators
|
|
TOK_PLUS, // +
|
|
TOK_MINUS, // -
|
|
TOK_STAR, // *
|
|
TOK_SLASH, // /
|
|
TOK_BACKSLASH, // \ (integer divide)
|
|
TOK_CARET, // ^
|
|
TOK_AMPERSAND, // & (string concat or hex prefix)
|
|
TOK_EQ, // =
|
|
TOK_NE, // <>
|
|
TOK_LT, // <
|
|
TOK_GT, // >
|
|
TOK_LE, // <=
|
|
TOK_GE, // >=
|
|
|
|
// Keywords
|
|
TOK_AND,
|
|
TOK_APP,
|
|
TOK_AS,
|
|
TOK_BASE,
|
|
TOK_BOOLEAN,
|
|
TOK_BYVAL,
|
|
TOK_CALL,
|
|
TOK_CASE,
|
|
TOK_CLOSE,
|
|
TOK_CONST,
|
|
TOK_CREATECONTROL,
|
|
TOK_CREATEFORM,
|
|
TOK_DATA,
|
|
TOK_DECLARE,
|
|
TOK_DEF,
|
|
TOK_DEFDBL,
|
|
TOK_DEFINT,
|
|
TOK_DEFLNG,
|
|
TOK_DEFSNG,
|
|
TOK_DEFSTR,
|
|
TOK_DIM,
|
|
TOK_DO,
|
|
TOK_DOEVENTS,
|
|
TOK_DOUBLE,
|
|
TOK_ELSE,
|
|
TOK_ELSEIF,
|
|
TOK_END,
|
|
TOK_EOF_KW, // EOF (keyword, not end-of-file)
|
|
TOK_EQV,
|
|
TOK_ERASE,
|
|
TOK_ERR,
|
|
TOK_ERROR_KW,
|
|
TOK_EXPLICIT,
|
|
TOK_EXIT,
|
|
TOK_FALSE_KW,
|
|
TOK_FOR,
|
|
TOK_FUNCTION,
|
|
TOK_GET,
|
|
TOK_GOSUB,
|
|
TOK_GOTO,
|
|
TOK_HIDE,
|
|
TOK_IF,
|
|
TOK_IMP,
|
|
TOK_INPUT,
|
|
TOK_INTEGER,
|
|
TOK_IS,
|
|
TOK_LBOUND,
|
|
TOK_LET,
|
|
TOK_LINE,
|
|
TOK_LOAD,
|
|
TOK_LONG,
|
|
TOK_LOOP,
|
|
TOK_ME,
|
|
TOK_MOD,
|
|
TOK_INPUTBOX,
|
|
TOK_MSGBOX,
|
|
TOK_NEXT,
|
|
TOK_NOT,
|
|
TOK_NOTHING,
|
|
TOK_ON,
|
|
TOK_OPEN,
|
|
TOK_OPTIONAL,
|
|
TOK_OPTION,
|
|
TOK_OR,
|
|
TOK_OUTPUT,
|
|
TOK_PRESERVE,
|
|
TOK_PRINT,
|
|
TOK_PUT,
|
|
TOK_RANDOMIZE,
|
|
TOK_READ,
|
|
TOK_REDIM,
|
|
TOK_REM,
|
|
TOK_REMOVECONTROL,
|
|
TOK_RESTORE,
|
|
TOK_RESUME,
|
|
TOK_RETURN,
|
|
TOK_SEEK,
|
|
TOK_SELECT,
|
|
TOK_SET,
|
|
TOK_SETEVENT,
|
|
TOK_SHARED,
|
|
TOK_SHELL,
|
|
TOK_SHOW,
|
|
TOK_SINGLE,
|
|
TOK_SLEEP,
|
|
TOK_INIREAD,
|
|
TOK_INIWRITE,
|
|
TOK_STATIC,
|
|
TOK_STEP,
|
|
TOK_STRING_KW,
|
|
TOK_SUB,
|
|
TOK_SWAP,
|
|
TOK_THEN,
|
|
TOK_TIMER,
|
|
TOK_TO,
|
|
TOK_TRUE_KW,
|
|
TOK_TYPE,
|
|
TOK_UBOUND,
|
|
TOK_UNLOAD,
|
|
TOK_UNTIL,
|
|
TOK_WEND,
|
|
TOK_WHILE,
|
|
TOK_WITH,
|
|
TOK_WRITE,
|
|
TOK_XOR,
|
|
|
|
// Filesystem keywords
|
|
TOK_CHDIR,
|
|
TOK_CHDRIVE,
|
|
TOK_CURDIR,
|
|
TOK_DIR,
|
|
TOK_FILECOPY,
|
|
TOK_FILELEN,
|
|
TOK_GETATTR,
|
|
TOK_KILL,
|
|
TOK_MKDIR,
|
|
TOK_NAME,
|
|
TOK_RMDIR,
|
|
TOK_SETATTR,
|
|
|
|
// File modes
|
|
TOK_APPEND,
|
|
TOK_BINARY,
|
|
TOK_RANDOM,
|
|
|
|
// Special
|
|
TOK_NEWLINE, // end of logical line
|
|
TOK_EOF, // end of source
|
|
TOK_ERROR // lexer error
|
|
} BasTokenTypeE;
|
|
|
|
// ============================================================
|
|
// Token
|
|
// ============================================================
|
|
|
|
#define BAS_MAX_TOKEN_LEN 256
|
|
// Maximum user-visible string-literal length. One byte of the token
|
|
// buffer is reserved for the NUL terminator.
|
|
#define BAS_MAX_STRING_LEN (BAS_MAX_TOKEN_LEN - 1)
|
|
#define BAS_LEX_ERROR_LEN 256
|
|
|
|
typedef struct {
|
|
BasTokenTypeE type;
|
|
int32_t line; // 1-based source line number
|
|
int32_t col; // 1-based column number
|
|
|
|
// Value (depends on type)
|
|
union {
|
|
int32_t intVal;
|
|
int64_t longVal;
|
|
double dblVal;
|
|
};
|
|
|
|
char text[BAS_MAX_TOKEN_LEN]; // raw text of the token
|
|
int32_t textLen;
|
|
} BasTokenT;
|
|
|
|
// ============================================================
|
|
// Lexer state
|
|
// ============================================================
|
|
|
|
typedef struct {
|
|
const char *source; // source text (not owned)
|
|
int32_t sourceLen;
|
|
int32_t pos; // current position in source
|
|
int32_t line; // current line (1-based)
|
|
int32_t col; // current column (1-based)
|
|
BasTokenT token; // current token
|
|
char error[BAS_LEX_ERROR_LEN];
|
|
} BasLexerT;
|
|
|
|
// ============================================================
|
|
// API
|
|
// ============================================================
|
|
|
|
// Initialize lexer with source text. The source must remain valid
|
|
// for the lifetime of the lexer.
|
|
void basLexerInit(BasLexerT *lex, const char *source, int32_t sourceLen);
|
|
|
|
// Advance to the next token. Returns the token type.
|
|
// The token is available in lex->token.
|
|
BasTokenTypeE basLexerNext(BasLexerT *lex);
|
|
|
|
// Peek at the current token type without advancing.
|
|
BasTokenTypeE basLexerPeek(const BasLexerT *lex);
|
|
|
|
// Return human-readable name for a token type.
|
|
const char *basTokenName(BasTokenTypeE type);
|
|
|
|
// True when c is a BASIC type-suffix character (% & ! # $).
|
|
bool basIsTypeSuffixChar(char c);
|
|
|
|
// ============================================================
|
|
// Keyword iteration
|
|
// ============================================================
|
|
//
|
|
// The lexer's internal keyword table is the one source of truth.
|
|
// Other modules (e.g. the IDE syntax highlighter) iterate through
|
|
// it here instead of keeping their own parallel list.
|
|
|
|
typedef enum {
|
|
BAS_KW_CLASS_OTHER = 0, // control-flow / statement / operator keyword
|
|
BAS_KW_CLASS_TYPE = 1, // INTEGER, LONG, SINGLE, DOUBLE, STRING, BOOLEAN
|
|
BAS_KW_CLASS_LITERAL = 2 // TRUE, FALSE, NOTHING
|
|
} BasKeywordClassE;
|
|
|
|
// Number of keywords in the table (excluding only the trailing NULL
|
|
// sentinel). Dollar-suffixed aliases (DIR$, CURDIR$, etc.) are counted
|
|
// as separate entries, consistent with basLexerKeywordAt below.
|
|
int32_t basLexerKeywordCount(void);
|
|
|
|
// Return the text of keyword i (uppercase). i must be in [0, count).
|
|
// Duplicates such as DIR and DIR$ are both returned at their own index.
|
|
const char *basLexerKeywordAt(int32_t i);
|
|
|
|
// Classify the keyword at index i.
|
|
BasKeywordClassE basLexerKeywordClass(int32_t i);
|
|
|
|
#endif // DVXBASIC_LEXER_H
|