// The MIT License (MIT) // // Copyright (C) 2026 Scott Duensing // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to // deal in the Software without restriction, including without limitation the // rights to use, copy, modify, merge, publish, distribute, sublicense, and/or // sell copies of the Software, and to permit persons to whom the Software is // furnished to do so, subject to the following conditions: // // The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. // // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS // IN THE SOFTWARE. // lexer.h -- DVX BASIC lexer (tokenizer) // // Converts BASIC source text into a stream of tokens. Case-insensitive // for keywords. Handles line continuations (_), comments (' and REM), // type suffixes (%, &, !, #, $), and string literals. // // Embeddable: no DVX dependencies, pure C. #ifndef DVXBASIC_LEXER_H #define DVXBASIC_LEXER_H #include #include // ============================================================ // Token types // ============================================================ typedef enum { // Literals TOK_INT_LIT, // integer literal (123, &HFF) TOK_LONG_LIT, // long literal (123&) TOK_FLOAT_LIT, // float literal (3.14, 1.5E10) TOK_STRING_LIT, // "string literal" // Identifiers and symbols TOK_IDENT, // variable/function name TOK_DOT, // . TOK_COMMA, // , TOK_SEMICOLON, // ; TOK_COLON, // : TOK_LPAREN, // ( TOK_RPAREN, // ) TOK_HASH, // # (file channel) // Operators TOK_PLUS, // + TOK_MINUS, // - TOK_STAR, // * TOK_SLASH, // / TOK_BACKSLASH, // \ (integer divide) TOK_CARET, // ^ TOK_AMPERSAND, // & (string concat or hex prefix) TOK_EQ, // = TOK_NE, // <> TOK_LT, // < TOK_GT, // > TOK_LE, // <= TOK_GE, // >= // Keywords TOK_AND, TOK_APP, TOK_AS, TOK_BASE, TOK_BOOLEAN, TOK_BYVAL, TOK_CALL, TOK_CASE, TOK_CLOSE, TOK_CONST, TOK_CREATECONTROL, TOK_CREATEFORM, TOK_DATA, TOK_DECLARE, TOK_DEF, TOK_DEFDBL, TOK_DEFINT, TOK_DEFLNG, TOK_DEFSNG, TOK_DEFSTR, TOK_DIM, TOK_DO, TOK_DOEVENTS, TOK_DOUBLE, TOK_ELSE, TOK_ELSEIF, TOK_END, TOK_EOF_KW, // EOF (keyword, not end-of-file) TOK_EQV, TOK_ERASE, TOK_ERR, TOK_ERROR_KW, TOK_EXPLICIT, TOK_EXIT, TOK_FALSE_KW, TOK_FOR, TOK_FUNCTION, TOK_GET, TOK_GOSUB, TOK_GOTO, TOK_HIDE, TOK_IF, TOK_IMP, TOK_INPUT, TOK_INTEGER, TOK_IS, TOK_LBOUND, TOK_LET, TOK_LINE, TOK_LOAD, TOK_LONG, TOK_LOOP, TOK_ME, TOK_MOD, TOK_INPUTBOX, TOK_MSGBOX, TOK_NEXT, TOK_NOT, TOK_NOTHING, TOK_ON, TOK_OPEN, TOK_OPTIONAL, TOK_OPTION, TOK_OR, TOK_OUTPUT, TOK_PRESERVE, TOK_PRINT, TOK_PUT, TOK_RANDOMIZE, TOK_READ, TOK_REDIM, TOK_REM, TOK_REMOVECONTROL, TOK_RESTORE, TOK_RESUME, TOK_RETURN, TOK_SEEK, TOK_SELECT, TOK_SET, TOK_SETEVENT, TOK_SHARED, TOK_SHELL, TOK_SHOW, TOK_SINGLE, TOK_SLEEP, TOK_INIREAD, TOK_INIWRITE, TOK_STATIC, TOK_STEP, TOK_STRING_KW, TOK_SUB, TOK_SWAP, TOK_THEN, TOK_TIMER, TOK_TO, TOK_TRUE_KW, TOK_TYPE, TOK_UBOUND, TOK_UNLOAD, TOK_UNTIL, TOK_WEND, TOK_WHILE, TOK_WITH, TOK_WRITE, TOK_XOR, // Filesystem keywords TOK_CHDIR, TOK_CHDRIVE, TOK_CURDIR, TOK_DIR, TOK_FILECOPY, TOK_FILELEN, TOK_GETATTR, TOK_KILL, TOK_MKDIR, TOK_NAME, TOK_RMDIR, TOK_SETATTR, // File modes TOK_APPEND, TOK_BINARY, TOK_RANDOM, // Special TOK_NEWLINE, // end of logical line TOK_EOF, // end of source TOK_ERROR // lexer error } BasTokenTypeE; // ============================================================ // Token // ============================================================ #define BAS_MAX_TOKEN_LEN 256 // Maximum user-visible string-literal length. One byte of the token // buffer is reserved for the NUL terminator. #define BAS_MAX_STRING_LEN (BAS_MAX_TOKEN_LEN - 1) #define BAS_LEX_ERROR_LEN 256 typedef struct { BasTokenTypeE type; int32_t line; // 1-based source line number int32_t col; // 1-based column number // Value (depends on type) union { int32_t intVal; int64_t longVal; double dblVal; }; char text[BAS_MAX_TOKEN_LEN]; // raw text of the token int32_t textLen; } BasTokenT; // ============================================================ // Lexer state // ============================================================ typedef struct { const char *source; // source text (not owned) int32_t sourceLen; int32_t pos; // current position in source int32_t line; // current line (1-based) int32_t col; // current column (1-based) BasTokenT token; // current token char error[BAS_LEX_ERROR_LEN]; } BasLexerT; // ============================================================ // API // ============================================================ // Initialize lexer with source text. The source must remain valid // for the lifetime of the lexer. void basLexerInit(BasLexerT *lex, const char *source, int32_t sourceLen); // Advance to the next token. Returns the token type. // The token is available in lex->token. BasTokenTypeE basLexerNext(BasLexerT *lex); // Peek at the current token type without advancing. BasTokenTypeE basLexerPeek(const BasLexerT *lex); // Return human-readable name for a token type. const char *basTokenName(BasTokenTypeE type); // True when c is a BASIC type-suffix character (% & ! # $). bool basIsTypeSuffixChar(char c); // ============================================================ // Keyword iteration // ============================================================ // // The lexer's internal keyword table is the one source of truth. // Other modules (e.g. the IDE syntax highlighter) iterate through // it here instead of keeping their own parallel list. typedef enum { BAS_KW_CLASS_OTHER = 0, // control-flow / statement / operator keyword BAS_KW_CLASS_TYPE = 1, // INTEGER, LONG, SINGLE, DOUBLE, STRING, BOOLEAN BAS_KW_CLASS_LITERAL = 2 // TRUE, FALSE, NOTHING } BasKeywordClassE; // Number of keywords in the table (excluding only the trailing NULL // sentinel). Dollar-suffixed aliases (DIR$, CURDIR$, etc.) are counted // as separate entries, consistent with basLexerKeywordAt below. int32_t basLexerKeywordCount(void); // Return the text of keyword i (uppercase). i must be in [0, count). // Duplicates such as DIR and DIR$ are both returned at their own index. const char *basLexerKeywordAt(int32_t i); // Classify the keyword at index i. BasKeywordClassE basLexerKeywordClass(int32_t i); #endif // DVXBASIC_LEXER_H