joeylib2/tools/scanPtrBug/scanPtrBug.c

400 lines
12 KiB
C

// Part of JoeyLib, Copyright (c) 2026 Scott Duensing, under the MIT License.
// See LICENSE.md for license information.
// SPDX-License-Identifier: MIT
//
// scanPtrBug.c - find, in the w65816 compiler's assembly output, the
// far-pointer bug whose pointer gets its 16-bit offset half from the
// pointer's bank register.
//
// scanPtrBug FILE.s ...
//
// A far pointer arrives as its offset in A and its bank in X, and the
// function's prologue keeps the bank in a direct-page slot with `stx <dp>`
// (the first such store in its first BANK_WINDOW lines). The pointer for a
// [dp],y access is built at $E0, $E4, $E8, or $EC; correct code stores the
// offset there (pla, txa, or lda n,s first), so `lda <bank dp>` feeding
// `sta 0xe0` means the pointer has the bank in its offset half and reads a
// stray address. Only the prologue's bank slot counts: other slots
// legitimately hold offsets. Looking back from each such store, the lines
// the compiler puts between (sty/ldy 0xfa, ldy #, [dp],y accesses, nop) are
// skipped.
//
// A function starts at each label line (an identifier and a colon at the
// start of the line); lines (ending in "\n" or "\r\n") are read with the
// whitespace round them dropped. Prints "FILE: N" and a line for each
// suspect in each file that has any, then the total. Exit 0, or 1 when a
// file cannot be read.
#define _DEFAULT_SOURCE
#include <ctype.h>
#include <errno.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/types.h>
#include "toolFile.h"
#define TOOL_NAME "scanPtrBug"
#define SCAN_EXIT 0
#define READ_EXIT 1
#define BANK_WINDOW 40u
#define NAME_WIDTH 28
#define LINE_WIDTH 6
#define HEX_PREFIX "0x"
#define BANK_LOAD "lda"
#define BANK_SAVE "stx"
#define PTR_STORE "sta"
#define NOP "nop"
// An indirect access's operand: "[0x<hex>], y".
#define INDIRECT_OPEN "[0x"
#define INDIRECT_CLOSE "], y"
// len bytes at text, not NUL-terminated.
typedef struct SpanT {
const char *text;
size_t len;
} SpanT;
// A file's lines (each its own getline buffer), whitespace dropped.
typedef struct ListingT {
char **buffers;
SpanT *lines;
size_t count;
size_t cap;
} ListingT;
// A suspect: the function, the store's line (counted from 1), the bank's
// slot, and the store.
typedef struct HitT {
SpanT function;
size_t line;
SpanT bank;
SpanT store;
} HitT;
// A line the compiler puts between building a pointer's offset and storing
// it: an opcode, whitespace, and its operand's start (NULL: an indirect
// [0x<hex>], y operand).
typedef struct BoilerT {
const char *opcode;
const char *operand;
} BoilerT;
static const BoilerT gBoiler[] = {
{"sty", "0xfa"},
{"ldy", "0xfa"},
{"ldy", "#"},
{"sta", NULL},
{"lda", NULL}
};
// The slots a [dp],y pointer is built in.
static const char *const gPointerSlots[] = {"0xe0", "0xe4", "0xe8", "0xec"};
static size_t hexRun(SpanT line, size_t at);
static bool isBankLoad(SpanT line, SpanT bank);
static bool isBoiler(SpanT line);
static bool isPointerStore(SpanT line);
static size_t labelLen(SpanT line);
static void listingFree(ListingT *listing);
static bool listingRead(const char *path, ListingT *listing);
static size_t nextLabel(const ListingT *text, size_t from);
static size_t operandAt(SpanT line, const char *opcode);
static size_t scan(const ListingT *text, HitT **hits);
static bool startsAt(SpanT line, size_t at, const char *text);
static bool stxBank(SpanT line, SpanT *bank);
// The run of lowercase hex digits ([0-9a-f]) at line[at].
static size_t hexRun(SpanT line, size_t at) {
size_t run;
char c;
run = 0u;
while (at + run < line.len) {
c = line.text[at + run];
if (!isdigit((unsigned char)c) && (c < 'a' || c > 'f')) {
break;
}
run++;
}
return run;
}
// "lda" then a space or a tab, then the bank's slot, and no more.
static bool isBankLoad(SpanT line, SpanT bank) {
size_t opcodeLen;
opcodeLen = strlen(BANK_LOAD);
return line.len == opcodeLen + 1u + bank.len && startsAt(line, 0u, BANK_LOAD) && (line.text[opcodeLen] == ' ' || line.text[opcodeLen] == '\t') && memcmp(line.text + opcodeLen + 1u, bank.text, bank.len) == 0;
}
static bool isBoiler(SpanT line) {
size_t at;
size_t run;
size_t i;
if (startsAt(line, 0u, NOP)) {
return true;
}
for (i = 0; i < sizeof(gBoiler) / sizeof(gBoiler[0]); i++) {
at = operandAt(line, gBoiler[i].opcode);
if (at == 0u) {
continue;
}
if (gBoiler[i].operand != NULL) {
if (startsAt(line, at, gBoiler[i].operand)) {
return true;
}
continue;
}
if (startsAt(line, at, INDIRECT_OPEN)) {
at += strlen(INDIRECT_OPEN);
run = hexRun(line, at);
if (run > 0u && startsAt(line, at + run, INDIRECT_CLOSE)) {
return true;
}
}
}
return false;
}
// "sta", whitespace, and one of the pointer slots, and no more.
static bool isPointerStore(SpanT line) {
size_t at;
size_t i;
at = operandAt(line, PTR_STORE);
if (at == 0u) {
return false;
}
for (i = 0; i < sizeof(gPointerSlots) / sizeof(gPointerSlots[0]); i++) {
if (line.len - at == strlen(gPointerSlots[i]) && startsAt(line, at, gPointerSlots[i])) {
return true;
}
}
return false;
}
// A label line's name: a letter or '_', then letters, digits, and '_',
// before a colon; its length, or 0 when the line is no label.
static size_t labelLen(SpanT line) {
size_t len;
if (line.len == 0u || !(isalpha((unsigned char)line.text[0]) || line.text[0] == '_')) {
return 0u;
}
for (len = 1u; len < line.len && (isalnum((unsigned char)line.text[len]) || line.text[len] == '_'); len++) {
}
return (len < line.len && line.text[len] == ':') ? len : 0u;
}
static void listingFree(ListingT *listing) {
size_t i;
for (i = 0; i < listing->count; i++) {
free(listing->buffers[i]);
}
free(listing->buffers);
free(listing->lines);
memset(listing, 0, sizeof(*listing));
}
// A file's lines, the whitespace round each dropped; false, with errno
// set, when it cannot be read.
static bool listingRead(const char *path, ListingT *listing) {
FILE *f;
SpanT *line;
char *buffer;
size_t bufferCap;
ssize_t got;
bool ok;
int err;
memset(listing, 0, sizeof(*listing));
f = fopen(path, "rb");
if (f == NULL) {
return false;
}
buffer = NULL;
bufferCap = 0u;
while ((got = getline(&buffer, &bufferCap, f)) >= 0) {
if (listing->count == listing->cap) {
listing->cap = listing->cap * 2u + 1u;
listing->buffers = (char **)realloc(listing->buffers, listing->cap * sizeof(char *));
listing->lines = (SpanT *)realloc(listing->lines, listing->cap * sizeof(SpanT));
if (listing->buffers == NULL || listing->lines == NULL) {
toolFail("out of memory");
}
}
line = &listing->lines[listing->count];
line->text = buffer;
line->len = (size_t)got;
while (line->len > 0u && isspace((unsigned char)line->text[0])) {
line->text++;
line->len--;
}
while (line->len > 0u && isspace((unsigned char)line->text[line->len - 1u])) {
line->len--;
}
listing->buffers[listing->count++] = buffer;
buffer = NULL;
bufferCap = 0u;
}
err = errno;
ok = !ferror(f);
free(buffer);
fclose(f);
if (!ok) {
listingFree(listing);
}
errno = err;
return ok;
}
// The first label line from from, or the count when there is none.
static size_t nextLabel(const ListingT *text, size_t from) {
while (from < text->count && labelLen(text->lines[from]) == 0u) {
from++;
}
return from;
}
// Where the operand starts in a line of opcode and whitespace; 0 when the
// line does not start so.
static size_t operandAt(SpanT line, const char *opcode) {
size_t len;
size_t run;
len = strlen(opcode);
if (!startsAt(line, 0u, opcode)) {
return 0u;
}
for (run = 0u; len + run < line.len && isspace((unsigned char)line.text[len + run]); run++) {
}
return run > 0u ? len + run : 0u;
}
// The suspects in a file's lines (stripped), into a malloc'd *hits; their
// count.
static size_t scan(const ListingT *text, HitT **hits) {
SpanT function;
SpanT bank;
size_t count;
size_t cap;
size_t start;
size_t end;
size_t i;
size_t j;
bool banked;
*hits = NULL;
count = 0u;
cap = 0u;
for (start = nextLabel(text, 0u); start < text->count; start = end) {
end = nextLabel(text, start + 1u);
function.text = text->lines[start].text;
function.len = labelLen(text->lines[start]);
banked = false;
for (i = start; i < start + BANK_WINDOW && i < end && !banked; i++) {
banked = stxBank(text->lines[i], &bank);
}
if (!banked) {
continue;
}
// The label line is never a store, so each store has a line before
// it in the function.
for (i = start + 1u; i < end; i++) {
if (!isPointerStore(text->lines[i])) {
continue;
}
for (j = i - 1u; j > start && isBoiler(text->lines[j]); j--) {
}
if (!isBankLoad(text->lines[j], bank)) {
continue;
}
if (count == cap) {
cap = cap * 2u + 1u;
*hits = (HitT *)realloc(*hits, cap * sizeof(HitT));
if (*hits == NULL) {
toolFail("out of memory");
}
}
(*hits)[count].function = function;
(*hits)[count].line = i + 1u;
(*hits)[count].bank = bank;
(*hits)[count].store = text->lines[i];
count++;
}
}
return count;
}
static bool startsAt(SpanT line, size_t at, const char *text) {
size_t len;
len = strlen(text);
return at <= line.len && line.len - at >= len && memcmp(line.text + at, text, len) == 0;
}
// "stx", whitespace, and "0x<hex>", and no more: the bank's slot.
static bool stxBank(SpanT line, SpanT *bank) {
size_t at;
size_t run;
at = operandAt(line, BANK_SAVE);
if (at == 0u || !startsAt(line, at, HEX_PREFIX)) {
return false;
}
run = hexRun(line, at + strlen(HEX_PREFIX));
if (run == 0u || at + strlen(HEX_PREFIX) + run != line.len) {
return false;
}
bank->text = line.text + at;
bank->len = strlen(HEX_PREFIX) + run;
return true;
}
int main(int argc, char **argv) {
ListingT text;
HitT *hits;
size_t count;
size_t total;
size_t i;
int a;
toolFileName(TOOL_NAME);
total = 0u;
for (a = 1; a < argc; a++) {
if (!listingRead(argv[a], &text)) {
fprintf(stderr, "%s: cannot read %s: %s\n", TOOL_NAME, argv[a], strerror(errno));
return READ_EXIT;
}
count = scan(&text, &hits);
if (count > 0u) {
printf("%s: %zu\n", argv[a], count);
}
for (i = 0; i < count; i++) {
printf(" %-*.*s line %-*zu bank=%.*s -> %.*s\n", NAME_WIDTH, (int)hits[i].function.len, hits[i].function.text, LINE_WIDTH, hits[i].line, (int)hits[i].bank.len, hits[i].bank.text, (int)hits[i].store.len, hits[i].store.text);
}
total += count;
free(hits);
listingFree(&text);
}
printf("total suspect pointer builds: %zu\n", total);
return SCAN_EXIT;
}