366 lines
11 KiB
C
366 lines
11 KiB
C
// The MIT License (MIT)
|
|
//
|
|
// Copyright (C) 2026 Scott Duensing
|
|
//
|
|
// Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
// of this software and associated documentation files (the "Software"), to
|
|
// deal in the Software without restriction, including without limitation the
|
|
// rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
|
|
// sell copies of the Software, and to permit persons to whom the Software is
|
|
// furnished to do so, subject to the following conditions:
|
|
//
|
|
// The above copyright notice and this permission notice shall be included in
|
|
// all copies or substantial portions of the Software.
|
|
//
|
|
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
|
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
|
|
// IN THE SOFTWARE.
|
|
|
|
// lexer.h -- DVX BASIC lexer (tokenizer)
|
|
//
|
|
// Converts BASIC source text into a stream of tokens. Case-insensitive
|
|
// for keywords. Handles line continuations (_), comments (' and REM),
|
|
// type suffixes (%, &, !, #, $), and string literals.
|
|
//
|
|
// Embeddable: no DVX dependencies, pure C.
|
|
|
|
#ifndef DVXBASIC_LEXER_H
|
|
#define DVXBASIC_LEXER_H
|
|
|
|
#include "basKeywords.h"
|
|
|
|
#include <stdint.h>
|
|
#include <stdbool.h>
|
|
|
|
// ============================================================
|
|
// Token types
|
|
// ============================================================
|
|
|
|
typedef enum {
|
|
// Literals
|
|
TOK_INT_LIT, // integer literal (123, &HFF)
|
|
TOK_LONG_LIT, // long literal (123&)
|
|
TOK_FLOAT_LIT, // float literal (3.14, 1.5E10)
|
|
TOK_STRING_LIT, // "string literal"
|
|
TOK_DATE_LIT, // #1/2/2000# date literal (dblVal holds the serial)
|
|
TOK_CUR_LIT, // currency literal (1.5@; longVal holds the amount times BAS_CURRENCY_SCALE)
|
|
|
|
// Identifiers and symbols
|
|
TOK_IDENT, // variable/function name
|
|
TOK_DOT, // .
|
|
TOK_COMMA, // ,
|
|
TOK_SEMICOLON, // ;
|
|
TOK_COLON, // :
|
|
TOK_COLONEQ, // := (a named argument)
|
|
TOK_LPAREN, // (
|
|
TOK_RPAREN, // )
|
|
TOK_HASH, // # (file channel)
|
|
|
|
// Operators
|
|
TOK_PLUS, // +
|
|
TOK_MINUS, // -
|
|
TOK_STAR, // *
|
|
TOK_SLASH, // /
|
|
TOK_BACKSLASH, // \ (integer divide)
|
|
TOK_CARET, // ^
|
|
TOK_AMPERSAND, // & (string concat or hex prefix)
|
|
TOK_EQ, // =
|
|
TOK_NE, // <>
|
|
TOK_LT, // <
|
|
TOK_GT, // >
|
|
TOK_LE, // <=
|
|
TOK_GE, // >=
|
|
|
|
// Keywords
|
|
TOK_AND,
|
|
TOK_APP,
|
|
TOK_AS,
|
|
TOK_BASE,
|
|
TOK_BEEP,
|
|
TOK_BOOLEAN,
|
|
TOK_BYREF,
|
|
TOK_BYVAL,
|
|
TOK_CALL,
|
|
TOK_CASE,
|
|
TOK_CLOSE,
|
|
TOK_CONST,
|
|
TOK_CREATECONTROL,
|
|
TOK_CREATEFORM,
|
|
TOK_DATA,
|
|
TOK_DECLARE,
|
|
TOK_DEF,
|
|
TOK_DEFBYTE,
|
|
TOK_DEFCUR,
|
|
TOK_DEFDBL,
|
|
TOK_DEFINT,
|
|
TOK_DEFLNG,
|
|
TOK_DEFSNG,
|
|
TOK_DEFSTR,
|
|
TOK_DIM,
|
|
TOK_DO,
|
|
TOK_DOEVENTS,
|
|
TOK_DOUBLE,
|
|
TOK_ELSE,
|
|
TOK_ELSEIF,
|
|
TOK_END,
|
|
TOK_ENUM,
|
|
TOK_EOF_KW, // EOF (keyword, not end-of-file)
|
|
TOK_EQV,
|
|
TOK_ERASE,
|
|
TOK_ERL,
|
|
TOK_ERR,
|
|
TOK_ERROR_KW,
|
|
TOK_EXPLICIT,
|
|
TOK_EXIT,
|
|
TOK_FALSE_KW,
|
|
TOK_FOR,
|
|
TOK_FRAMEWAIT,
|
|
TOK_FUNCTION,
|
|
TOK_GET,
|
|
TOK_GOSUB,
|
|
TOK_GOTO,
|
|
TOK_HIDE,
|
|
TOK_IF,
|
|
TOK_IMP,
|
|
TOK_INPUT,
|
|
TOK_INTEGER,
|
|
TOK_IS,
|
|
TOK_LBOUND,
|
|
TOK_LET,
|
|
TOK_LIKE,
|
|
TOK_LINE,
|
|
TOK_LOAD,
|
|
TOK_LONG,
|
|
TOK_LOOP,
|
|
TOK_LPRINT,
|
|
TOK_ME,
|
|
TOK_MOD,
|
|
TOK_INPUTBOX,
|
|
TOK_MSGBOX,
|
|
TOK_NEW,
|
|
TOK_NEXT,
|
|
TOK_NOT,
|
|
TOK_NOTHING,
|
|
TOK_ON,
|
|
TOK_OPEN,
|
|
TOK_OPTIONAL,
|
|
TOK_OPTION,
|
|
TOK_OR,
|
|
TOK_OUTPUT,
|
|
TOK_PARAMARRAY,
|
|
TOK_PLAY,
|
|
TOK_PRESERVE,
|
|
TOK_PRINT,
|
|
TOK_PRIVATE,
|
|
TOK_PROPERTY,
|
|
TOK_PUBLIC,
|
|
TOK_PUT,
|
|
TOK_RANDOMIZE,
|
|
TOK_READ,
|
|
TOK_REDIM,
|
|
TOK_REM,
|
|
TOK_REMOVECONTROL,
|
|
TOK_RESTORE,
|
|
TOK_RESUME,
|
|
TOK_RETURN,
|
|
TOK_SEEK,
|
|
TOK_SELECT,
|
|
TOK_SET,
|
|
TOK_SETEVENT,
|
|
TOK_SHARED,
|
|
TOK_SHELL,
|
|
TOK_SHOW,
|
|
TOK_SINGLE,
|
|
TOK_SLEEP,
|
|
TOK_SOUND,
|
|
TOK_INIREAD,
|
|
TOK_INIWRITE,
|
|
TOK_STATIC,
|
|
TOK_STEP,
|
|
TOK_STRING_KW,
|
|
TOK_SUB,
|
|
TOK_SWAP,
|
|
TOK_THEN,
|
|
TOK_TIMER,
|
|
TOK_TO,
|
|
TOK_TRUE_KW,
|
|
TOK_TYPE,
|
|
TOK_TYPEOF,
|
|
TOK_UBOUND,
|
|
TOK_UNLOAD,
|
|
TOK_UNTIL,
|
|
TOK_WEND,
|
|
TOK_WHILE,
|
|
TOK_WITH,
|
|
TOK_WRITE,
|
|
TOK_XOR,
|
|
|
|
// Filesystem keywords
|
|
TOK_CHDIR,
|
|
TOK_CHDRIVE,
|
|
TOK_CURDIR,
|
|
TOK_DIR,
|
|
TOK_FILECOPY,
|
|
TOK_FILELEN,
|
|
TOK_GETATTR,
|
|
TOK_KILL,
|
|
TOK_MKDIR,
|
|
TOK_NAME,
|
|
TOK_RMDIR,
|
|
TOK_SETATTR,
|
|
|
|
// File modes
|
|
TOK_APPEND,
|
|
TOK_BINARY,
|
|
TOK_RANDOM,
|
|
|
|
// Special
|
|
TOK_NEWLINE, // end of logical line
|
|
TOK_EOF, // end of source
|
|
TOK_ERROR // lexer error
|
|
} BasTokenTypeE;
|
|
|
|
// Keyword tokens occupy one contiguous range of the enum.
|
|
#define TOK_FIRST_KEYWORD TOK_AND
|
|
#define TOK_LAST_KEYWORD TOK_RANDOM
|
|
|
|
// ============================================================
|
|
// Token
|
|
// ============================================================
|
|
|
|
#define BAS_MAX_TOKEN_LEN 256
|
|
// Maximum user-visible string-literal length. One byte of the token
|
|
// buffer is reserved for the NUL terminator.
|
|
#define BAS_MAX_STRING_LEN (BAS_MAX_TOKEN_LEN - 1)
|
|
#define BAS_LEX_ERROR_LEN 256
|
|
|
|
// Magnitude of LONG's most negative value (-2147483648). Only accepted
|
|
// as a literal directly after unary minus, where its negation fits.
|
|
#define BAS_LONG_NEG_MAGNITUDE ((int64_t)INT32_MAX + 1)
|
|
|
|
typedef struct {
|
|
BasTokenTypeE type;
|
|
int32_t line; // 1-based source line number
|
|
int32_t col; // 1-based column number
|
|
int32_t pos; // offset of the token's first character in the source
|
|
|
|
// Value (depends on type)
|
|
union {
|
|
int32_t intVal;
|
|
int64_t longVal;
|
|
double dblVal;
|
|
};
|
|
|
|
char text[BAS_MAX_TOKEN_LEN]; // raw text of the token
|
|
int32_t textLen;
|
|
} BasTokenT;
|
|
|
|
// ============================================================
|
|
// Lexer state
|
|
// ============================================================
|
|
|
|
typedef struct {
|
|
const char *source; // source text (not owned)
|
|
int32_t sourceLen;
|
|
int32_t pos; // current position in source
|
|
int32_t line; // current line (1-based)
|
|
int32_t col; // current column (1-based)
|
|
BasTokenT token; // current token
|
|
char error[BAS_LEX_ERROR_LEN]; // message of the last TOK_ERROR (no position prefix)
|
|
int32_t errLine; // line of the token that raised error
|
|
int32_t errCol; // column of the token that raised error
|
|
bool allowNegMagnitude; // next integer literal may be BAS_LONG_NEG_MAGNITUDE (set by the parser after unary minus)
|
|
} BasLexerT;
|
|
|
|
// ============================================================
|
|
// API
|
|
// ============================================================
|
|
|
|
// The length of the date literal at s (s[0] is '#', len bytes readable),
|
|
// both '#'s counted, or 0 when there is none: text of a date literal's
|
|
// shape (basDateLiteralShapeLen in basDateLit.h: a date or time separator
|
|
// or a word, so file numbers such as CLOSE #1, 2, #3 never read as one)
|
|
// that also reads as a date (see basDateParse). *outSerial gets the
|
|
// date. The lexer's rule, shared with the IDE's syntax colouring; the
|
|
// TextArea widget's colorizer, which cannot link the date parser, uses
|
|
// the shape alone.
|
|
int32_t basLexerDateLiteralLen(const char *s, int32_t len, double *outSerial);
|
|
|
|
// Initialize lexer with source text. The source must remain valid
|
|
// for the lifetime of the lexer.
|
|
void basLexerInit(BasLexerT *lex, const char *source, int32_t sourceLen);
|
|
|
|
// Advance to the next token. Returns the token type.
|
|
// The token is available in lex->token.
|
|
BasTokenTypeE basLexerNext(BasLexerT *lex);
|
|
|
|
// Continue lexing at source offset pos (line / col its position, as an
|
|
// earlier token recorded them) and read the token there.
|
|
BasTokenTypeE basLexerSeek(BasLexerT *lex, int32_t pos, int32_t line, int32_t col);
|
|
|
|
// Skims the rest of the current statement without making tokens: up to
|
|
// the next newline or ':' outside string literals and comments (line
|
|
// continuations followed). The next basLexerNext reads that separator.
|
|
void basLexerSkipStatement(BasLexerT *lex);
|
|
|
|
// Return human-readable name for a token type.
|
|
const char *basTokenName(BasTokenTypeE type);
|
|
|
|
// True when the token is an identifier or keyword spelled as a word (the
|
|
// shape of a property or method name after a dot).
|
|
bool basTokenIsWord(const BasTokenT *tok);
|
|
|
|
// True when c may appear inside an identifier (ASCII letter, digit or
|
|
// underscore; locale independent).
|
|
bool basIsIdentChar(char c);
|
|
|
|
// True when name is a well-formed identifier: starts with an ASCII
|
|
// letter or underscore and contains only identifier characters.
|
|
bool basIsValidIdent(const char *name);
|
|
|
|
// True when c is a BASIC type-suffix character (% & ! # $).
|
|
bool basIsTypeSuffixChar(char c);
|
|
|
|
// Map a type-suffix character to its BAS_TYPE_* code, or -1 when c is
|
|
// not a suffix character. The suffix table in lexer.c is the single
|
|
// source of truth for both this and basIsTypeSuffixChar.
|
|
int32_t basTypeSuffixType(char c);
|
|
|
|
// Round v to the nearest integer, ties to even (the rounding CINT/CLNG
|
|
// use). Returns false when v is not a number or too large to convert.
|
|
bool basRoundHalfEven(double v, int64_t *out);
|
|
|
|
// Fold a single ASCII byte to upper case. Char in, char out (not int
|
|
// via toupper) so bytes >= 0x80 pass through unchanged.
|
|
char basAsciiUpper(char c);
|
|
|
|
// ============================================================
|
|
// Keyword iteration
|
|
// ============================================================
|
|
//
|
|
// The keyword list in basKeywords.h is the one source of truth. Other
|
|
// modules (e.g. the IDE syntax highlighter) iterate through the lexer's
|
|
// table here instead of keeping their own parallel list.
|
|
|
|
// Number of keywords in the table (excluding only the trailing NULL
|
|
// sentinel). Dollar-suffixed aliases (DIR$, CURDIR$, etc.) are counted
|
|
// as separate entries, consistent with basLexerKeywordAt below.
|
|
int32_t basLexerKeywordCount(void);
|
|
|
|
// Return the text of keyword i (uppercase). i must be in [0, count).
|
|
// Duplicates such as DIR and DIR$ are both returned at their own index.
|
|
const char *basLexerKeywordAt(int32_t i);
|
|
|
|
// Classify the keyword at index i.
|
|
BasKeywordClassE basLexerKeywordClass(int32_t i);
|
|
|
|
// True when keyword i is reserved; false for the color-only entries
|
|
// (TOK_IDENT: built-in type names, Empty, the graphics statements ...),
|
|
// which a program may also use as its own names.
|
|
bool basLexerKeywordReserved(int32_t i);
|
|
|
|
#endif // DVXBASIC_LEXER_H
|