AI_Slop/65816-llvm-mos
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions
65816-llvm-mos/src/llvm/lib/Target/W65816/W65816StackSlotCleanup.cpp
Scott Duensing 035c393500 Code generation fixes.
2026-08-25 16:48:35 -05:00

2108 lines
91 KiB
C++
Raw Blame History

//===-- W65816StackSlotCleanup.cpp - Remove redundant spill/reload pairs --===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
// Post-RA cleanup that erases redundant STAfi+LDAfi pairs to the same
// stack slot when no instruction in between writes A or that slot.
//
// The greedy register allocator routinely emits this pattern when
// materialising a COPY of $a into a vreg that gets allocated back to
// $a — the spill+reload cycle is a no-op since A already holds the
// stored value. The standard MachineLateInstrsCleanup pass only
// detects identical instructions; it doesn't recognise that
// `LDAfi slot` after `STAfi $a, slot` is a no-op. We do the
// simple per-block scan here.
//
// Conservative: only matches adjacent STAfi+LDAfi pairs (no scan for
// instructions in between). In practice the greedy-allocator-emitted
// pattern is always adjacent or near-adjacent, and the scheduler keeps
// it that way because the LDAfi feeds the next instruction. If
// future codegen breaks this assumption, generalise to a longer scan
// with explicit clobber tracking.
//
//===----------------------------------------------------------------------===//
#include "W65816.h"
#include "W65816InstrInfo.h"
#include "W65816Subtarget.h"
#include "llvm/ADT/BitVector.h"
#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/SmallPtrSet.h"
#include "llvm/CodeGen/MachineFrameInfo.h"
#include "llvm/CodeGen/MachineFunction.h"
#include "llvm/CodeGen/MachineFunctionPass.h"
#include "llvm/CodeGen/MachineInstr.h"
#include "llvm/CodeGen/MachineRegisterInfo.h"
#include "llvm/CodeGen/TargetRegisterInfo.h"
using namespace llvm;
#define DEBUG_TYPE "w65816-stack-slot-cleanup"
namespace {
class W65816StackSlotCleanup : public MachineFunctionPass {
public:
static char ID;
W65816StackSlotCleanup() : MachineFunctionPass(ID) {}
StringRef getPassName() const override {
return "W65816 redundant stack-slot spill/reload elimination";
}
bool runOnMachineFunction(MachineFunction &MF) override;
};
} // namespace
char W65816StackSlotCleanup::ID = 0;
INITIALIZE_PASS(W65816StackSlotCleanup, DEBUG_TYPE,
"W65816 redundant stack-slot spill/reload elimination",
false, false)
FunctionPass *llvm::createW65816StackSlotCleanup() {
return new W65816StackSlotCleanup();
}
// Returns true if MI references frame index FI as one of its operands.
// Used to bail dead-store removal when an intervening instruction
// reads or writes the slot.
static bool referencesFrameIndex(const MachineInstr &MI, int FI) {
for (const MachineOperand &MO : MI.operands())
if (MO.isFI() && MO.getIndex() == FI)
return true;
return false;
}
// Sentinel for "no match" returned by matchAccSlotOp. We can't use
// -1 because FrameIndex numbers for *fixed* (caller-arg) slots are
// negative — fixed-stack.0 is -1, fixed-stack.1 is -2, etc. Earlier
// passes that did `if (slot < 0) continue;` were silently bailing on
// every legitimate fixed-slot LDA/STA, missing many cross-arg-slot
// optimisation opportunities.
static constexpr int NO_SLOT_MATCH = INT_MIN;
// If MI matches `OP $a, FI, 0` where OP == ExpectedOpc, returns the slot
// index (which may be negative for fixed-stack args); else NO_SLOT_MATCH.
// Callers must compare against NO_SLOT_MATCH, NOT against `< 0`.
static int matchAccSlotOp(const MachineInstr &MI, unsigned ExpectedOpc) {
if (MI.getOpcode() != ExpectedOpc ||
MI.getNumOperands() < 3 ||
!MI.getOperand(0).isReg() || MI.getOperand(0).getReg() != W65816::A ||
!MI.getOperand(1).isFI() ||
!MI.getOperand(2).isImm() || MI.getOperand(2).getImm() != 0)
return NO_SLOT_MATCH;
return MI.getOperand(1).getIndex();
}
// Returns true if Opc is a commutative *_fi pseudo (the load-fold form
// where operand 2 is the FI). ADD/AND/OR/EOR / ADCE all qualify; SBC
// and CMP are non-commutative.
static bool isCommutativeFiOp(unsigned Opc) {
return Opc == W65816::ADCfi || Opc == W65816::ADCEfi ||
Opc == W65816::ANDfi || Opc == W65816::ORAfi ||
Opc == W65816::EORfi;
}
// If MI is a commutative *_fi op of the canonical shape `OPfi $a (tied), slot, 0`
// matching slot SlotB, returns true. Used to recognise the OPfi at the
// end of a *_RR inserter expansion.
static bool matchCommutativeFiOpOnSlot(const MachineInstr &MI, int SlotB) {
if (!isCommutativeFiOp(MI.getOpcode()))
return false;
if (MI.getNumOperands() < 4 ||
!MI.getOperand(0).isReg() || MI.getOperand(0).getReg() != W65816::A ||
!MI.getOperand(2).isFI() || MI.getOperand(2).getIndex() != SlotB ||
!MI.getOperand(3).isImm() || MI.getOperand(3).getImm() != 0)
return false;
return true;
}
// Advance It past debug instructions; returns true if landed on a real
// instruction in the block. Templated because callers mix iterator and
// instr_iterator depending on how they got here.
template <class IterT>
static bool advancePastDebug(MachineBasicBlock &MBB, IterT &It) {
while (It != MBB.end() && It->isDebugInstr())
++It;
return It != MBB.end();
}
// Match `STAfi reg1, FI, 0; ... ; STAfi reg2, FI, 0` (kill via overwrite)
// or `STAfi reg, FI, 0; ... ; <return> (no read in between)` (dead store
// at function exit). Both mean the first STAfi is dead. Conservative:
// bails on anything that references the slot, calls, inline asm. The
// slot must be a *local* (non-fixed) FrameIndex — args live across the
// function so we can't kill stores to fixed slots.
static bool tryEliminateDeadStore(MachineBasicBlock &MBB,
MachineInstr &StaMI) {
if (StaMI.getOpcode() != W65816::STAfi)
return false;
if (StaMI.getNumOperands() < 3 ||
!StaMI.getOperand(1).isFI() ||
!StaMI.getOperand(2).isImm() || StaMI.getOperand(2).getImm() != 0)
return false;
// Never eliminate a volatile store — its observability is the
// whole point of marking it volatile. Caught by the SJLJ EH path
// where SjLjEHPrepare emits `store volatile i32 N, fn_ctx.call_site`
// before each invoke; without this check the call_site never gets
// written and the personality routine can't pick the landing pad.
if (StaMI.hasOrderedMemoryRef())
return false;
int StoredFI = StaMI.getOperand(1).getIndex();
// Don't try to kill a store to a fixed (arg) slot — those are
// observable to the caller. Locals/spills are fair game.
const MachineFunction *MF = StaMI.getMF();
if (MF->getFrameInfo().isFixedObjectIndex(StoredFI))
return false;
auto It = std::next(StaMI.getIterator());
while (It != MBB.end()) {
MachineInstr &MI = *It;
if (MI.isDebugInstr()) {
++It;
continue;
}
// A subsequent STAfi to the same slot, offset 0, kills our store.
if (MI.getOpcode() == W65816::STAfi &&
MI.getNumOperands() >= 3 &&
MI.getOperand(1).isFI() &&
MI.getOperand(1).getIndex() == StoredFI &&
MI.getOperand(2).isImm() && MI.getOperand(2).getImm() == 0) {
// Found the killing store. Erase the first.
StaMI.eraseFromParent();
return true;
}
// A return that doesn't read the slot kills the store too — the
// local goes out of scope at function exit.
if (MI.isReturn() && !referencesFrameIndex(MI, StoredFI)) {
StaMI.eraseFromParent();
return true;
}
// Anything else that touches the slot (load, ADC d,S, etc.) means
// the first store IS observed — bail.
if (referencesFrameIndex(MI, StoredFI))
return false;
// Inline asm / branches: too tricky. Calls are OK to walk past —
// local (non-fixed) slots are addressed at offsets the callee
// can't reach (callee's S has been shifted down by JSL's
// 3-byte return frame and any of its own pha/tsc adjustments,
// so its `(4,s)` reads land above our locals). We've already
// bailed on fixed slots above, so reaching here means the slot
// is local and call-safe.
if (MI.isInlineAsm() || MI.isBranch())
return false;
++It;
}
// Walked off the end of the BB without seeing a return/use. Bail
// (could fall through to a successor that reads the slot).
return false;
}
// Returns true if any MachineInstr in MF (other than those in `ignore`)
// references frame index `FI`. Used by Pass -4 / Pass -4c safety
// checks before erasing an init store: a cross-MBB reload would
// otherwise read stale/uninitialized data. See #107.
static bool slotHasOtherRefs(const MachineFunction &MF, int FI,
ArrayRef<const MachineInstr *> ignore) {
for (const MachineBasicBlock &MBB : MF) {
for (const MachineInstr &MI : MBB) {
bool skip = false;
for (const MachineInstr *I : ignore) {
if (&MI == I) { skip = true; break; }
}
if (skip) continue;
for (const MachineOperand &MO : MI.operands()) {
if (MO.isFI() && MO.getIndex() == FI)
return true;
}
}
}
return false;
}
// Match `STAfi reg, FI, 0; ... ; LDAfi destReg, FI, 0` when reg == destReg
// and nothing in between clobbers reg or the slot. Erase the LDAfi.
static bool tryEliminateLoadAfterStore(MachineBasicBlock &MBB,
MachineInstr &StaMI,
const TargetRegisterInfo *TRI) {
if (StaMI.getOpcode() != W65816::STAfi)
return false;
if (StaMI.getNumOperands() < 3 ||
!StaMI.getOperand(0).isReg() ||
!StaMI.getOperand(1).isFI() ||
!StaMI.getOperand(2).isImm() || StaMI.getOperand(2).getImm() != 0)
return false;
Register StoredReg = StaMI.getOperand(0).getReg();
int StoredFI = StaMI.getOperand(1).getIndex();
// Walk forward looking for the matching LDAfi. Bail on any
// instruction that could clobber StoredReg or write the slot.
auto It = std::next(StaMI.getIterator());
while (It != MBB.end()) {
MachineInstr &MI = *It;
if (MI.isDebugInstr()) {
++It;
continue;
}
if (MI.getOpcode() == W65816::LDAfi &&
MI.getNumOperands() >= 3 &&
MI.getOperand(1).isFI() &&
MI.getOperand(1).getIndex() == StoredFI &&
MI.getOperand(2).isImm() && MI.getOperand(2).getImm() == 0 &&
MI.getOperand(0).isReg() &&
MI.getOperand(0).getReg() == StoredReg) {
// A volatile load is observable — never elide, even if the
// value is provably the same as the prior store. But STAfi/
// LDAfi target compiler-managed stack spill slots, which are
// by construction never volatile — `hasOrderedMemoryRef()`
// returns true here only because both lack explicit memops
// (the conservative "no info → treat as ordered" default).
// Check the actual memops if present; absence is fine.
auto isReallyVolatile = [](const MachineInstr &I) {
for (auto *MMO : I.memoperands())
if (MMO->isVolatile() || MMO->isAtomic())
return true;
return false;
};
if (isReallyVolatile(MI) || isReallyVolatile(StaMI))
return false;
// LDA sets N/Z based on the loaded value. Dropping it would
// expose stale N/Z from before the STA→LDA pair to the next
// flag-reading op (e.g. a branch). Only safe to drop if the
// immediately-following op overwrites N/Z.
auto opSetsNZ = [](unsigned Op) {
switch (Op) {
case W65816::LDAfi:
case W65816::LDAi16imm:
case W65816::LDAabs:
case W65816::ANDi16imm: case W65816::ANDabs: case W65816::ANDfi:
case W65816::ORAi16imm: case W65816::ORAabs: case W65816::ORAfi:
case W65816::EORi16imm: case W65816::EORabs: case W65816::EORfi:
case W65816::ADCi16imm: case W65816::ADCabs: case W65816::ADCfi:
case W65816::SBCi16imm: case W65816::SBCabs: case W65816::SBCfi:
case W65816::ADCEi16imm: case W65816::ADCEabs: case W65816::ADCEfi:
case W65816::SBCEi16imm: case W65816::SBCEabs: case W65816::SBCEfi:
case W65816::ASLA16: case W65816::LSRA16:
case W65816::ASLA8: case W65816::LSRA8:
case W65816::INA: case W65816::DEA:
case W65816::INA_PSEUDO: case W65816::DEA_PSEUDO:
case W65816::INA_PSEUDO8: case W65816::DEA_PSEUDO8:
return true;
default:
return false;
}
};
// Walk past further STAfi pseudos (spill stores) — they preserve
// A's flags. Also walk past STA_DP (plain `sta $dp` doesn't touch
// P). Don't walk past STA_DPIndLongY / STA8fi / etc., whose
// inserter expansions introduce SEP/REP and mutate the M flag
// (that's what broke printf historically).
auto NextIt = std::next(MI.getIterator());
while (NextIt != MBB.end()) {
if (NextIt->isDebugInstr()) { ++NextIt; continue; }
if (NextIt->getOpcode() == W65816::STAfi) { ++NextIt; continue; }
if (NextIt->getOpcode() == W65816::STA_DP) { ++NextIt; continue; }
break;
}
if (NextIt == MBB.end() || NextIt->isBranch() || NextIt->isReturn())
return false;
if (!NextIt->definesRegister(W65816::P, TRI) &&
!opSetsNZ(NextIt->getOpcode()))
return false;
// Eliding the LDA leaves $a "dead per verifier" because STAfi's
// tablegen Defs = [A] is a stale over-approximation (the asm
// actually preserves A via PHA/lda/sta/PLA bracket for IMG/X/Y
// sources, and trivially for $a sources — see W65816InstrInfo.td
// and eliminateFrameIndex). Strip the stale implicit-def $a from
// walked-past STAfis (any source) and clear any kill flag on the
// source operand when source is $a, so the verifier sees $a as
// still alive through the chain. We're post-regalloc so this
// doesn't affect regalloc's decisions — only the verifier and any
// downstream passes that read the annotation.
auto stripStaleAdef = [](MachineInstr &MI2) {
if (MI2.getOpcode() != W65816::STAfi) return;
if (MI2.getNumOperands() < 1 || !MI2.getOperand(0).isReg()) return;
if (MI2.getOperand(0).getReg() == W65816::A) {
MI2.getOperand(0).setIsKill(false);
}
for (unsigned i = MI2.getNumOperands(); i-- > 0;) {
const MachineOperand &MO = MI2.getOperand(i);
if (MO.isReg() && MO.isImplicit() && MO.isDef() &&
MO.getReg() == W65816::A) {
MI2.removeOperand(i);
return;
}
}
};
for (auto FixIt = StaMI.getIterator(); FixIt != MI.getIterator(); ++FixIt) {
stripStaleAdef(*FixIt);
}
MI.eraseFromParent();
return true;
}
// Calls clobber A — be safe.
if (MI.isCall())
return false;
// STAfi has `Defs = [A]` in its tablegen def (a stale over-
// approximation from before the eliminateFrameIndex PHA-bracket
// landed for non-A sources). In reality the asm preserves A
// for every source class — A source is trivial, IMG/X/Y sources
// go through PHA/lda/sta/PLA which restores A. So a STAfi to
// a different slot is NOT an A-clobber and shouldn't break the
// load-after-store redundancy. STAfi to the SAME slot DOES
// invalidate (slot value changed), handled below.
bool IsStAFi = (MI.getOpcode() == W65816::STAfi);
if (!IsStAFi && MI.modifiesRegister(StoredReg, TRI))
return false;
if (IsStAFi &&
MI.getNumOperands() >= 2 && MI.getOperand(1).isFI() &&
MI.getOperand(1).getIndex() == StoredFI)
return false;
++It;
}
return false;
}
bool W65816StackSlotCleanup::runOnMachineFunction(MachineFunction &MF) {
const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
bool Changed = false;
// Pass -4: redundant pointer respill. Pattern that the LDAptrOff +
// STAptrOff inserter pair emits when the same pointer is used for
// both a load and a store within a loop body:
//
// LDAfi slot_c ; reload p from its slot (slot_c = p's home)
// STAfi slot_A ; spill p to slot_A (for the indirect Y-load)
// ... LDA (slot_A,Y) ; INC ...
// LDAfi slot_c ; reload p again (same source!)
// STAfi slot_B ; spill p to slot_B (for the indirect Y-store)
// ...; STA (slot_B),Y
//
// M[slot_A] and M[slot_B] both hold p — equal. We can redirect any
// later use of slot_B to slot_A and drop the LDA+STA pair. The
// saving is 2 insns per affected indirect-pair (4 cycles). Only
// safe if slot_A wasn't written in between (it isn't — no STAfi to
// slot_A appears in the loop) and the second LDA reloads from the
// SAME source slot_c.
for (MachineBasicBlock &MBB : MF) {
SmallVector<MachineInstr *, 8> Ldas;
for (MachineInstr &MI : MBB)
if (MI.getOpcode() == W65816::LDAfi)
Ldas.push_back(&MI);
SmallPtrSet<MachineInstr *, 8> Erased;
for (MachineInstr *Lda1 : Ldas) {
if (Erased.count(Lda1)) continue;
int SlotC = matchAccSlotOp(*Lda1, W65816::LDAfi);
if (SlotC == NO_SLOT_MATCH) continue;
auto It = std::next(Lda1->getIterator());
if (!advancePastDebug(MBB, It)) continue;
// Step 2: STAfi slotA.
int SlotA = matchAccSlotOp(*It, W65816::STAfi);
if (SlotA == NO_SLOT_MATCH || SlotA == SlotC) continue;
// Walk forward looking for LDAfi slotC again, with no STAfi
// slotA / slotC in between.
auto Walker = std::next(It);
MachineInstr *Lda2 = nullptr;
while (Walker != MBB.end()) {
MachineInstr &MI = *Walker;
if (MI.isDebugInstr()) { ++Walker; continue; }
if (MI.isCall() || MI.isInlineAsm() || MI.isBranch() ||
MI.isReturn())
break;
// STA to slotA or slotC: M might no longer hold the same value.
if (MI.getOpcode() == W65816::STAfi &&
MI.getNumOperands() >= 2 && MI.getOperand(1).isFI()) {
int Slot = MI.getOperand(1).getIndex();
if (Slot == SlotA || Slot == SlotC) break;
}
// Found another LDA from slotC?
if (matchAccSlotOp(MI, W65816::LDAfi) == SlotC) {
Lda2 = &MI;
break;
}
++Walker;
}
if (!Lda2) continue;
auto It2 = std::next(Lda2->getIterator());
if (!advancePastDebug(MBB, It2)) continue;
// Step 4: STAfi slotB.
int SlotB = matchAccSlotOp(*It2, W65816::STAfi);
if (SlotB == NO_SLOT_MATCH || SlotB == SlotA || SlotB == SlotC) continue;
MachineInstr &Sta2 = *It2;
// Walk further to find the indirect use (LDAfi_indY / STAfi_indY)
// referencing slotB. Bail on STA to slotA before then.
auto It3 = std::next(Sta2.getIterator());
MachineInstr *IndYTarget = nullptr;
while (It3 != MBB.end()) {
MachineInstr &MI = *It3;
if (MI.isDebugInstr()) { ++It3; continue; }
if (MI.isCall() || MI.isBranch() || MI.isReturn() ||
MI.isInlineAsm())
break;
// Slot A or C overwritten — bail.
if (MI.getOpcode() == W65816::STAfi &&
MI.getNumOperands() >= 2 && MI.getOperand(1).isFI()) {
int Slot = MI.getOperand(1).getIndex();
if (Slot == SlotA || Slot == SlotC) break;
}
// Indirect-Y operand: operand 1 (load) or 1 (store) holds
// the FI pointer slot. Match LDAfi_indY/STAfi_indY using
// slotB.
if (MI.getOpcode() == W65816::LDAfi_indY ||
MI.getOpcode() == W65816::STAfi_indY) {
for (unsigned i = 0; i < MI.getNumOperands(); ++i) {
if (MI.getOperand(i).isFI() &&
MI.getOperand(i).getIndex() == SlotB) {
IndYTarget = &MI;
break;
}
}
if (IndYTarget) break;
}
++It3;
}
if (!IndYTarget) continue;
// Function-wide safety check (#107): slotB must have no other
// refs besides Sta2 (which we erase) and IndYTarget (which we
// rewrite). Cross-MBB reads would otherwise see stale data.
const MachineInstr *ignoreA[] = {&Sta2, IndYTarget};
if (slotHasOtherRefs(MF, SlotB, ignoreA))
continue;
// Apply rewrite: IndYTarget's slotB → slotA.
for (unsigned i = 0; i < IndYTarget->getNumOperands(); ++i) {
if (IndYTarget->getOperand(i).isFI() &&
IndYTarget->getOperand(i).getIndex() == SlotB) {
IndYTarget->getOperand(i).setIndex(SlotA);
break;
}
}
// Mark Lda2 as erased so the outer worklist iteration skips
// it (it's an LDAfi and was added to Ldas). Sta2 isn't in
// any worklist so erasing it directly is safe.
Erased.insert(Lda2);
Lda2->eraseFromParent();
Sta2.eraseFromParent();
Changed = true;
}
}
// Pass -4b: redundant pair of consecutive STAfi. Pattern:
//
// STAfi $a, slotA, 0
// STAfi $a, slotB, 0 ; same value, different slot
// ... use slotB as indirect-Y address ...
//
// Both STAs spill $a's current value, so M[slotA] == M[slotB]. We
// can rewrite later indirect-Y uses of slotB to slotA and drop the
// second STA. Pattern shows up when an i32 pointer is loaded via
// two indirect-Y reads (offsets 0 and 2); the inserter spills the
// pointer twice (once per access).
for (MachineBasicBlock &MBB : MF) {
SmallVector<MachineInstr *, 8> Stas;
for (MachineInstr &MI : MBB)
if (MI.getOpcode() == W65816::STAfi)
Stas.push_back(&MI);
SmallPtrSet<MachineInstr *, 8> Erased;
for (MachineInstr *Sta1 : Stas) {
if (Erased.count(Sta1)) continue;
int SlotA = matchAccSlotOp(*Sta1, W65816::STAfi);
if (SlotA == NO_SLOT_MATCH) continue;
auto It = std::next(Sta1->getIterator());
if (!advancePastDebug(MBB, It)) continue;
// Step 2: another STAfi $a, slotB.
int SlotB = matchAccSlotOp(*It, W65816::STAfi);
if (SlotB == NO_SLOT_MATCH || SlotB == SlotA) continue;
MachineInstr &Sta2 = *It;
// Walk forward redirecting EVERY slotB reference to slotA, until
// we hit a write to slotA (kills the equivalence) or a slotB write
// (re-binds slotB to a new value). Bail on calls/branches/asm.
// Track whether we rewrote anything; if so, drop Sta2.
auto It2 = std::next(Sta2.getIterator());
bool Rewrote = false;
while (It2 != MBB.end()) {
MachineInstr &MI = *It2;
if (MI.isDebugInstr()) { ++It2; continue; }
if (MI.isCall() || MI.isBranch() || MI.isReturn() ||
MI.isInlineAsm()) break;
// STA to slotA changes M[slotA]; M[slotA] no longer equals
// M[slotB] — bail (any further slotB ref reads the unchanged
// M[slotB], which is now distinct).
if (MI.getOpcode() == W65816::STAfi &&
MI.getNumOperands() >= 2 && MI.getOperand(1).isFI() &&
MI.getOperand(1).getIndex() == SlotA)
break;
// STA to slotB rebinds slotB; subsequent reads of slotB read
// the new value, not slotA. Stop here — the redirects we've
// done so far are still valid (they read the pre-write value).
bool StaToB = (MI.getOpcode() == W65816::STAfi ||
MI.getOpcode() == W65816::STA8fi) &&
MI.getNumOperands() >= 2 && MI.getOperand(1).isFI() &&
MI.getOperand(1).getIndex() == SlotB;
if (StaToB) break;
// Any *fi op or indirect-Y referencing slotB → redirect.
if (MI.getOpcode() == W65816::LDAfi_indY ||
MI.getOpcode() == W65816::STAfi_indY ||
MI.getOpcode() == W65816::LDAfi ||
MI.getOpcode() == W65816::ADCfi ||
MI.getOpcode() == W65816::ADCEfi ||
MI.getOpcode() == W65816::SBCfi ||
MI.getOpcode() == W65816::SBCEfi ||
MI.getOpcode() == W65816::ANDfi ||
MI.getOpcode() == W65816::ORAfi ||
MI.getOpcode() == W65816::EORfi ||
MI.getOpcode() == W65816::CMPfi) {
for (unsigned i = 0; i < MI.getNumOperands(); ++i) {
if (MI.getOperand(i).isFI() &&
MI.getOperand(i).getIndex() == SlotB) {
MI.getOperand(i).setIndex(SlotA);
Rewrote = true;
break;
}
}
}
++It2;
}
// Drop Sta2 only if slotB has no remaining references anywhere
// in the function — otherwise we'd break a use we couldn't see.
// (Sta1 stays; SlotA still has the value, and Sta1 is its def.)
if (Rewrote) {
bool SlotBStillUsed = false;
for (MachineBasicBlock &MBBO : MF) {
for (MachineInstr &MIO : MBBO) {
if (&MIO == &Sta2) continue;
for (const MachineOperand &MO : MIO.operands()) {
if (MO.isFI() && MO.getIndex() == SlotB) {
SlotBStillUsed = true; break;
}
}
if (SlotBStillUsed) break;
}
if (SlotBStillUsed) break;
}
if (!SlotBStillUsed) {
Erased.insert(&Sta2);
Sta2.eraseFromParent();
}
Changed = true;
}
}
}
// Pass -4c: redundant single pointer respill. Pattern:
//
// LDAfi $a, slotC, 0 ; A = M[slotC] (slotC is "p")
// STAfi $a, slotB, 0 ; slotB = M[slotC] = "p"
// ... non-A-clobbering, no STA to slotC ...
// LDAfi_indY/STAfi_indY ..., slotB, 0
//
// M[slotB] just mirrors M[slotC], so the indirect-Y access can read
// slotC directly. After the rewrite, if slotB has no remaining uses
// in the MBB, the LDA+STA respill is dead and we erase both. This is
// the loop-counter / pointer-iteration shape that Pass -4 (the
// double-respill variant) doesn't catch when only one indirect-Y
// happens before the pointer increment.
for (MachineBasicBlock &MBB : MF) {
SmallVector<MachineInstr *, 8> Ldas;
for (MachineInstr &MI : MBB)
if (MI.getOpcode() == W65816::LDAfi)
Ldas.push_back(&MI);
SmallPtrSet<MachineInstr *, 8> Erased;
for (MachineInstr *Lda : Ldas) {
if (Erased.count(Lda)) continue;
int SlotC = matchAccSlotOp(*Lda, W65816::LDAfi);
if (SlotC == NO_SLOT_MATCH) continue;
auto It = std::next(Lda->getIterator());
if (!advancePastDebug(MBB, It)) continue;
int SlotB = matchAccSlotOp(*It, W65816::STAfi);
if (SlotB == NO_SLOT_MATCH || SlotB == SlotC) continue;
MachineInstr &Sta = *It;
// Walk forward through the MBB collecting all indirect-Y uses of
// slotB (LDAfi_indY / STAfi_indY referencing it as the pointer
// operand). Bail if we see any *other* reference to slotB (a
// direct LDAfi/STAfi/etc.) — that means the slot has uses other
// than as an indirect-Y pointer and we can't safely rewrite all
// of them. Also bail on STA to slotC (kills the equivalence).
SmallVector<MachineInstr *, 4> IndYUses;
bool OtherUse = false;
auto It2 = std::next(Sta.getIterator());
while (It2 != MBB.end()) {
MachineInstr &MI = *It2;
if (MI.isDebugInstr()) { ++It2; continue; }
if (MI.isCall() || MI.isBranch() || MI.isReturn() ||
MI.isInlineAsm()) break;
if (MI.getOpcode() == W65816::STAfi &&
MI.getNumOperands() >= 2 && MI.getOperand(1).isFI() &&
MI.getOperand(1).getIndex() == SlotC)
break;
bool IsIndY = (MI.getOpcode() == W65816::LDAfi_indY ||
MI.getOpcode() == W65816::STAfi_indY);
bool RefsSlotB = false;
for (unsigned i = 0; i < MI.getNumOperands(); ++i) {
if (MI.getOperand(i).isFI() &&
MI.getOperand(i).getIndex() == SlotB) {
RefsSlotB = true;
break;
}
}
if (RefsSlotB) {
if (IsIndY)
IndYUses.push_back(&MI);
else
{ OtherUse = true; break; }
}
++It2;
}
if (OtherUse || IndYUses.empty()) continue;
// After IndYUses, scan rest of MBB for any further reference to
// slotB; if none, all uses of slotB are in our IndYUses list and
// we can safely redirect them all + erase the LDA+STA.
auto LastIt = std::next(IndYUses.back()->getIterator());
bool LaterUse = false;
for (auto It3 = LastIt; It3 != MBB.end(); ++It3) {
for (const MachineOperand &MO : It3->operands()) {
if (MO.isFI() && MO.getIndex() == SlotB) { LaterUse = true; break; }
}
if (LaterUse) break;
}
if (LaterUse) continue;
// Function-wide safety check (#107): slotB must have no other
// refs besides Sta (erased) and the IndY uses (rewritten).
// The IndYUses list already collects all in-MBB IndY refs;
// anything else (including cross-MBB) would expect the init.
SmallVector<const MachineInstr *, 8> ignore;
ignore.push_back(&Sta);
for (MachineInstr *I : IndYUses)
ignore.push_back(I);
if (slotHasOtherRefs(MF, SlotB, ignore))
continue;
// Apply rewrites: redirect every IndY use of slotB → slotC.
for (MachineInstr *IndY : IndYUses) {
for (unsigned i = 0; i < IndY->getNumOperands(); ++i) {
if (IndY->getOperand(i).isFI() &&
IndY->getOperand(i).getIndex() == SlotB) {
IndY->getOperand(i).setIndex(SlotC);
break;
}
}
}
Erased.insert(Lda);
Lda->eraseFromParent();
Sta.eraseFromParent();
Changed = true;
}
}
// Pass -3: hoist `LDX #imm` (constant materialisation into the X
// register) out from between a flag-defining op and the consuming
// Bxx. LDX physically updates N and Z, but our pseudo lacks
// `Defs = [P]` so the scheduler can place it in the test window.
// SAFE because:
// - LDX writes X; CMP/ORA/etc. read A. Hoisting can't change
// what the CMP sees.
// - The LDX's source is an immediate — no operand dependency.
// - Moving LDX before the CMP just means CMP overwrites the
// flags LDX set, which is what we want.
// Only LDX-style — `LDA #imm` is NOT safe because CMP reads A and
// the hoist would change A's value. Tracked in
// memory/project_known_issue_lda_flags.md.
for (MachineBasicBlock &MBB : MF) {
SmallVector<MachineInstr *, 4> Branches;
for (MachineInstr &MI : MBB) {
unsigned Opc = MI.getOpcode();
if (Opc == W65816::BEQ || Opc == W65816::BNE ||
Opc == W65816::BMI || Opc == W65816::BPL)
Branches.push_back(&MI);
}
for (MachineInstr *Br : Branches) {
SmallVector<MachineInstr *, 4> ToHoist;
MachineInstr *Test = nullptr;
for (auto It = std::prev(Br->getIterator()); ; --It) {
MachineInstr &MI = *It;
if (MI.isDebugInstr()) {
if (It == MBB.begin()) break;
continue;
}
// STA preserves flags (the MC variants STA_StackRel /
// STA_StackRelIndY only appear post-PEI and are listed here
// defensively; pre-PEI we see STAfi / STAfi_indY / STA8fi
// pseudos). STA8fi expands to SEP/STA/REP, which preserves
// N/Z (only M is touched).
if (MI.getOpcode() == W65816::STA_StackRel ||
MI.getOpcode() == W65816::STA_StackRelIndY ||
MI.getOpcode() == W65816::STAfi ||
MI.getOpcode() == W65816::STAfi_indY ||
MI.getOpcode() == W65816::STA8fi) {
if (It == MBB.begin()) break;
continue;
}
// LDX #imm: candidate to hoist.
if (MI.getOpcode() == W65816::LDXi16imm &&
MI.getNumOperands() >= 2 && MI.getOperand(1).isImm()) {
ToHoist.push_back(&MI);
if (It == MBB.begin()) break;
continue;
}
// First "real" instruction we hit walking back is the flag-
// defining test (CMP, ORA, etc.) — stop here.
Test = &MI;
break;
}
if (!Test || ToHoist.empty()) continue;
for (auto *MI : ToHoist) {
MI->removeFromParent();
MBB.insert(Test->getIterator(), MI);
Changed = true;
}
}
}
// Pass -2.5: BR_CC flag-corruption mitigation via PHP/PLP. When a
// flag-test (CMP/ORA/etc.) is followed by P-corrupting ops (LDA/LDX
// /AND/etc.) and then a flag-testing branch (Bxx), the branch would
// test the corrupting op's N/Z instead of the test's. This is a
// real correctness bug — `while (n > 0)` always exits on first
// iteration; `eq_test(0)` returns 0; etc. Wrap the corrupting span
// with PHP (push flags) / PLP (pop flags), preserving the test's
// flags across the corruption. Costs 2 bytes / 8 cycles per
// affected pattern, but it's the difference between buggy and
// correct code. The 4-block SELECT_CC inserter handles its case
// structurally; this catches the BR_CC paths the inserter can't
// touch. Only inserts when:
// - The branch tests N or Z (BEQ/BNE/BMI/BPL); BCC/BCS test C
// and LDA doesn't touch C, so they're not affected.
// - There's at least one P-corrupting instruction between the
// flag-defining test and the Bxx.
//
// NOTE: Bxx instructions now declare `Uses = [P]` (W65816InstrInfo.td),
// so the pre-RA scheduler / regalloc / peephole won't legally insert
// a P-corrupting op between a CMP and the consuming Bxx. This pass
// is now mostly defensive — it stays in place to catch any post-RA
// pass that might still violate the dep, and to wrap the rare cases
// where the IR-level test is a load (LDA flag side-effect) rather
// than an explicit CMP.
// In VLA functions, FI store pseudos (STAfi, STA8fi, STAfi_indY)
// expand at PEI to a 4-MC sequence ending in `LDY $F8` (Y-restore),
// which clobbers N/Z. The PHP/PLP wrap pass runs pre-PEI; treating
// those pseudos as flag-preserving leaves the trailing LDY outside
// the wrap, so a downstream BEQ/BNE reads the LDY's flags instead of
// the test's. Treat them as corrupting in VLA functions so the wrap
// covers the whole expansion.
// VLAFunc: narrow predicate used by the flag-preserving / lda-like
// helpers (broadening it to UsesFPRel broke dadd's i64-ABI libcall
// flow — the STAfi pseudos in non-VLA large-frame functions don't
// need to be marked corrupting for the wrap-detection walk).
// UsesFPRel: broader FrameLowering-matching predicate used by the
// pseudo-bump's offset-routing check (FP-rel ops must NOT be bumped,
// SP-rel ops MUST be bumped; we replicate eliminateFrameIndex's
// routing decision below to choose).
bool VLAFunc = MF.getFrameInfo().hasVarSizedObjects();
bool UsesFPRel = MF.getFrameInfo().hasVarSizedObjects() ||
MF.getFrameInfo().estimateStackSize(MF) > 200;
for (MachineBasicBlock &MBB : MF) {
SmallVector<MachineInstr *, 4> Branches;
for (MachineInstr &MI : MBB) {
unsigned Opc = MI.getOpcode();
// BCC/BCS/BVC/BVS are collected too, but ONLY for the carry/overflow
// corrupters (isCarryCorrupting below): an LDA-like op between a CMP
// and a BCC leaves C alone and needs no wrap, which is why they were
// never listed here before. ADDframe is different - see its note.
if (Opc == W65816::BEQ || Opc == W65816::BNE ||
Opc == W65816::BMI || Opc == W65816::BPL ||
Opc == W65816::BCC || Opc == W65816::BCS ||
Opc == W65816::BVC || Opc == W65816::BVS)
Branches.push_back(&MI);
}
auto isFlagPreserving = [VLAFunc](unsigned Opc) {
if (VLAFunc) {
// FI store pseudos are flag-corrupting under VLA expansion.
if (Opc == W65816::STAfi || Opc == W65816::STAfi_indY ||
Opc == W65816::STA8fi)
return false;
}
return Opc == W65816::STA_StackRel ||
Opc == W65816::STA_StackRelIndY ||
Opc == W65816::STAfi ||
Opc == W65816::STAfi_indY ||
Opc == W65816::STA8fi ||
Opc == W65816::STA_DP ||
Opc == W65816::STA_Abs ||
Opc == W65816::STA_Long ||
Opc == W65816::STX_DP ||
Opc == W65816::STX_Abs ||
Opc == W65816::STY_DP ||
Opc == W65816::STY_Abs;
};
auto isFlagDefining = [](const MachineInstr &MI) {
// Anything that physically writes A, X, Y, or P updates N/Z (or
// P-bits for CMP). We treat any non-store, non-stack-mgmt op
// that's not a branch as flag-defining. STA family preserves;
// PHA/PLY don't touch flags either; everything else might.
unsigned Opc = MI.getOpcode();
switch (Opc) {
case W65816::PHA: case W65816::PHX: case W65816::PHY:
case W65816::PHP: case W65816::PHB: case W65816::PHD:
case W65816::PHK:
case W65816::TCS: case W65816::TXS:
case W65816::TCD:
case W65816::JSLpseudo: case W65816::JSLpseudo32:
case W65816::JSL_Long:
case W65816::JSR_Abs:
case W65816::JMP_Abs:
case W65816::BRA:
case W65816::RTL: case W65816::RTS:
case W65816::REP: case W65816::SEP:
case W65816::CLC: case W65816::SEC:
case W65816::CLV: case W65816::CLI: case W65816::SEI:
case W65816::CLD: case W65816::SED:
return false;
default:
return !MI.isBranch() && !MI.isReturn();
}
};
auto isLdaLike = [VLAFunc](unsigned Opc) {
if (VLAFunc) {
// STAfi-family: see isFlagPreserving comment. They expand to a
// sequence whose final LDY $F8 corrupts N/Z; treat as corrupting.
if (Opc == W65816::STAfi || Opc == W65816::STAfi_indY ||
Opc == W65816::STA8fi)
return true;
}
// COPY between physregs: lowers in AsmPrinter to one of TXA/TYA/
// LDA $D? (for IMG↔A bridges) etc. — all of which set N/Z based
// on the loaded value. Treating COPY as flag-defining caused the
// wrap pass to identify a PHI-elim COPY as the "Test" and wrap
// too narrow a range, so the cb-test LDA's flags were trampled
// by intervening A-loads before reaching the BEQ. Including
// COPY in the corrupting set forces the pass to walk past these
// PHI-elim copies to find the real test (a CMP).
if (Opc == TargetOpcode::COPY) return true;
// ADDframe (the LEA of a stack slot) expands at PEI to `TSC; CLC;
// ADC #disp`, which rewrites N/Z (TSC, ADC), C (CLC, ADC) and V
// (ADC). It has no Defs=[P] (deliberately, like the loads above)
// and isReMaterializable, so post-RA passes are free to place it
// between a CMP and its Bxx - which Machine Copy Propagation did
// in RetroNet's rnClient.c: `cmp #3; tsc; clc; adc #0xc1; bne`
// made an `act.kind == RN_WA_SUBMIT` test branch on (SP+0xC1)!=0,
// i.e. never true, so the form was never submitted on the IIgs.
// Before this line the walk below treated ADDframe as the flag-
// DEFINING test (isFlagDefining's default) and declined to wrap.
if (Opc == W65816::ADDframe) return true;
// Pure load / register-transfer instructions: only side effect on
// flags is N/Z from the loaded/transferred value. Never a "test"
// — they just move data. Treated as corruption when between the
// real test and a flag-using branch.
return Opc == W65816::LDAi16imm ||
Opc == W65816::LDAi8imm ||
// LDAi16imm_bank lowers to `lda $BE` (LDA_DP of the program-
// bank byte). Under GNO/ME the program bank is non-zero, so
// this load sets N/Z != the CMP's result. Between a CMP and a
// BEQ/BNE (e.g. the bank half of an i32 `select`/`if(!p)`
// pointer pick), it corrupts the test and the wrong half is
// selected -- the `%s`-of-stack-string garbling bug. Plain
// LDAi16imm was already listed; its _bank sibling must be too.
Opc == W65816::LDAi16imm_bank ||
Opc == W65816::LDXi16imm ||
Opc == W65816::LDA_StackRel ||
Opc == W65816::LDA_StackRelIndY ||
Opc == W65816::LDA_DP ||
Opc == W65816::LDA_Abs ||
Opc == W65816::LDA_Long ||
Opc == W65816::LDA_Imm16 || Opc == W65816::LDA_Imm8 ||
Opc == W65816::LDX_Imm16 || Opc == W65816::LDX_Imm8 ||
Opc == W65816::LDX_DP || Opc == W65816::LDX_Abs ||
Opc == W65816::LDY_Imm16 || Opc == W65816::LDY_Imm8 ||
Opc == W65816::LDY_DP || Opc == W65816::LDY_Abs ||
// Pseudo wrappers that lower to LDA #imm.
Opc == W65816::LDAfi ||
Opc == W65816::LDAfi_indY ||
// Pure-logic NZ-setters: AND / ORA / EOR against an immediate
// or absolute operand. These physically set N/Z but
// deliberately do NOT model Defs=[P] (see W65816InstrInfo.td:
// the bitwise ops "don't read/write the carry flag"), so a
// conditional branch can NEVER take its modeled $p from one of
// them — at this pass's point every branch's flags come from a
// real CMP (Defs=[P]); the CMP-vs-AND/ORA fold that drops the
// `cmp #0` runs later. So when one of these is scheduled
// BETWEEN a loop-guard CMP and its branch (e.g. a loop-
// invariant signed-compare bias `contactY ^ 0x8000` hoisted
// into a zero-trip loop's preheader), it silently tramples the
// CMP's Z: the guard then branches on the wrong value, enters a
// should-be-skipped loop, and spins ~65536 iterations reading
// out-of-bounds data (the reported course_support_piece hang).
// Treating them as corruption walks the wrap past them to the
// real CMP. NOTE: the *fi variants (ANDfi/ORAfi/EORfi) and
// ADC/SBC DO model Defs=[P] and can legitimately be the branch's
// flag source, so they are handled by $p liveness and are NOT
// listed here; same for INA/DEA and the shifts, which the DEC/
// INC and shift-carry idioms may read directly.
Opc == W65816::EORi16imm || Opc == W65816::EORabs ||
Opc == W65816::ANDi16imm || Opc == W65816::ANDabs ||
Opc == W65816::ORAi16imm || Opc == W65816::ORAabs ||
// Register transfers — TAX/TXA/TAY/TYA/TXY/TYX update N/Z
// based on the transferred value. They're "data movement"
// not "comparison"; treat as corruption so the wrap pass
// walks past them to the real test. Without this, a loop
// like `for (i...) { ...; t = X; ... }` ends up testing
// (t != 0) instead of (i != 0) and runs forever.
Opc == W65816::TAX || Opc == W65816::TXA ||
Opc == W65816::TAY || Opc == W65816::TYA ||
Opc == W65816::TXY || Opc == W65816::TYX;
};
auto isStackRel = [](unsigned Opc) {
// Stack-relative ops read/write at S+disp. PHP decrements S by 1,
// so any STA/LDA d,S between PHP and PLP would land at the wrong
// address (off by 1). We must keep these OUTSIDE the wrap.
// Includes both the post-lowered MC opcodes (LDA_StackRel etc.)
// AND the pseudo *fi opcodes (LDAfi etc.) — eliminateFrameIndex
// hasn't run yet when the wrap pass executes, so it's the pseudos
// that are actually in the IR.
return Opc == W65816::STA_StackRel ||
Opc == W65816::STA_StackRelIndY ||
Opc == W65816::LDA_StackRel ||
Opc == W65816::LDA_StackRelIndY ||
Opc == W65816::ADC_StackRel ||
Opc == W65816::SBC_StackRel ||
Opc == W65816::AND_StackRel ||
Opc == W65816::ORA_StackRel ||
Opc == W65816::EOR_StackRel ||
Opc == W65816::CMP_StackRel ||
Opc == W65816::LDAfi ||
Opc == W65816::LDAfi_indY ||
Opc == W65816::STAfi ||
Opc == W65816::STAfi_indY ||
Opc == W65816::STA8fi ||
Opc == W65816::ADCfi ||
Opc == W65816::ADCEfi ||
Opc == W65816::SBCfi ||
Opc == W65816::SBCEfi ||
Opc == W65816::ANDfi ||
Opc == W65816::ORAfi ||
Opc == W65816::EORfi ||
Opc == W65816::CMPfi ||
Opc == W65816::ADDframe;
};
// The subset of corrupters that also rewrite C/V, and so must be
// wrapped even for a BCC/BCS/BVC/BVS. Only ADDframe today: every
// other entry in isLdaLike touches N/Z alone.
auto isCarryCorrupting = [](unsigned Opc) {
return Opc == W65816::ADDframe;
};
for (MachineInstr *Br : Branches) {
unsigned BrOpc = Br->getOpcode();
bool TestsNZ = BrOpc == W65816::BEQ || BrOpc == W65816::BNE ||
BrOpc == W65816::BMI || BrOpc == W65816::BPL;
// Walk back from Br looking for the pattern:
// <test>; (mix of preserving + corrupting ops); Br
// where <test> is a flag-defining op (CMP/ORA/AND/ADC/...) and
// there's at least one corrupting (LDA-like / TXA-like) op between
// <test> and Br. Wrap the corrupting region with PHP/PLP so Br
// sees <test>'s flags.
//
// Wrap boundaries:
// PHP goes just before the FIRST corrupting op (not just after
// Test) so any preserving stack-rel STAs before the first
// corruption stay outside the wrap and use the un-decremented S.
// PLP goes just after the LAST corrupting op for the same
// reason — preserving stack-rel STAs that follow stay outside.
// This is critical: PHP changes S by 1, so a `sta 1,s` inside
// the wrap writes at the same address PHP just saved P to,
// corrupting the saved flags. Caught by an iterative fib loop
// that ran forever because PLP loaded a corrupt P value.
MachineInstr *Test = nullptr;
MachineInstr *FirstCorrupt = nullptr;
MachineInstr *LastCorrupt = nullptr;
for (auto It = std::prev(Br->getIterator()); ; --It) {
MachineInstr &MI = *It;
if (!MI.isDebugInstr()) {
if (isFlagPreserving(MI.getOpcode())) {
// skip
} else if (isCarryCorrupting(MI.getOpcode()) ||
(TestsNZ && isLdaLike(MI.getOpcode()))) {
if (!LastCorrupt) LastCorrupt = &MI;
FirstCorrupt = &MI;
} else if (isLdaLike(MI.getOpcode())) {
// An N/Z-only corrupter under a C/V branch: harmless, walk on.
} else if (isFlagDefining(MI)) {
Test = &MI;
break;
} else {
// Opaque (call, unrelated terminator) — stop.
break;
}
}
if (It == MBB.begin()) break;
}
if (!Test || !FirstCorrupt) continue;
// Stack-relative ops inside the wrap need their displacements
// bumped by +1 to compensate for PHP's S decrement. Without
// this, `lda 5,s` between PHP and PLP reads at (orig_S-1)+5
// = orig_S+4, one byte too low. The pseudo *fi ops carry an
// ImmOffset operand that gets folded into the final disp by
// eliminateFrameIndex; bumping ImmOffset by 1 produces the
// right post-lowered disp. For already-lowered MC ops
// (LDA_StackRel etc), bump the disp operand directly.
//
// CAVEAT for FP-relative functions (see UsesFPRel declaration above):
// FI accesses go through FP-relative addressing (eliminateFrameIndex
// routes through expandFarFI when FrameLowering captured FP). FP
// was captured BEFORE PHP, so (FP),Y reads aren't affected by PHP's
// S decrement. Don't bump pseudo *fi ImmOffsets in that case
// (already-lowered MC StackRel ops still need the bump — those are
// SP-rel).
const TargetInstrInfo *TII = MF.getSubtarget().getInstrInfo();
DebugLoc DL = Test->getDebugLoc();
BuildMI(MBB, FirstCorrupt->getIterator(), DL, TII->get(W65816::PHP));
for (auto It = FirstCorrupt->getIterator();
It != std::next(LastCorrupt->getIterator()); ++It) {
if (It->isDebugInstr() || !isStackRel(It->getOpcode())) continue;
// Pseudo *fi ops: operand layout is (def, FI, ImmOffset, ...).
// Bump the Imm at index 2. MC StackRel ops: operand 0 is the
// disp Imm (set by eliminateFrameIndex); bump that.
unsigned Opc = It->getOpcode();
bool IsPseudo = Opc == W65816::LDAfi || Opc == W65816::LDAfi_indY ||
Opc == W65816::STAfi || Opc == W65816::STAfi_indY ||
Opc == W65816::STA8fi ||
Opc == W65816::ADCfi || Opc == W65816::ADCEfi ||
Opc == W65816::SBCfi || Opc == W65816::SBCEfi ||
Opc == W65816::ANDfi || Opc == W65816::ORAfi ||
Opc == W65816::EORfi || Opc == W65816::CMPfi ||
Opc == W65816::ADDframe;
// For pseudo *fi ops in FP-rel functions: only SOME will end up
// SP-rel after PEI (offsets in [0,255]); the rest go through
// expandFarFI → `[$F6],Y`. FP-rel access is unaffected by PHP's
// S decrement and must NOT be bumped; SP-rel access IS affected
// and MUST be bumped. Replicate eliminateFrameIndex's offset
// calculation here to decide. Without this, large-frame
// functions that mix both addressing modes (e.g. sha256-style
// i32-libcall loops) get their FP-rel pseudos bumped, which
// shifts reads/writes by one byte and corrupts state at
// iteration N proportional to the i32-libcall count.
// ADDframe has no FP-relative expansion (eliminateFrameIndex always
// emits TSC + ADC #disp), so it is SP-relative in EVERY function and
// always needs the bump - exempting it here would leave its address
// one byte low inside the wrap in a large-frame function.
if (IsPseudo && UsesFPRel && Opc != W65816::ADDframe) {
const MachineFrameInfo &MFI = MF.getFrameInfo();
if (It->getOperand(1).isFI()) {
int FI = It->getOperand(1).getIndex();
int FrameOffset = MFI.getObjectOffset(FI);
int ImmOffset = It->getOperand(2).isImm()
? (int)It->getOperand(2).getImm() : 0;
int LoweredOff = FrameOffset + ImmOffset +
(int)MFI.getStackSize();
if (FrameOffset < 0) LoweredOff += 1;
// Out-of-range or VLA → FP-rel → no bump.
if (LoweredOff < 0 || LoweredOff > 0xFF ||
MFI.hasVarSizedObjects())
continue;
// Else SP-rel: fall through and bump ImmOffset.
}
}
unsigned ImmIdx = IsPseudo ? 2 : 0;
if (ImmIdx < It->getNumOperands() && It->getOperand(ImmIdx).isImm()) {
int64_t v = It->getOperand(ImmIdx).getImm();
It->getOperand(ImmIdx).setImm(v + 1);
}
}
BuildMI(MBB, std::next(LastCorrupt->getIterator()), DL,
TII->get(W65816::PLP));
Changed = true;
}
}
// Pass -2c: relaxed mem-to-mem copy elimination across arbitrary
// instructions. Pattern:
//
// LDAfi $a, slotA, 0 ; A = M[slotA]
// STAfi $a, slotB, 0 ; M[slotB] = M[slotA]
// ... arbitrary instructions, possibly including JSL, ALU, etc.,
// as long as nothing writes slotA or slotB ...
// OPfi $a, slotB, 0 ; reads M[slotB]
//
// Rewrite OPfi to read slotA and drop the LDA-STA pair if slotB has
// no other uses anywhere in the function. Catches the "loop-carry"
// shape in `for (i = 0; i < n; i++) sum += ...` where each iteration
// re-spills sum to a separate adc-input slot. Pass -2 (the strict
// adjacent variant) doesn't catch this because of the JSL / ALU
// ops between Sta and the OPfi.
for (MachineBasicBlock &MBB : MF) {
SmallVector<MachineInstr *, 8> Ldas;
for (MachineInstr &MI : MBB)
if (MI.getOpcode() == W65816::LDAfi)
Ldas.push_back(&MI);
SmallPtrSet<MachineInstr *, 8> Erased;
for (MachineInstr *Lda : Ldas) {
if (Erased.count(Lda)) continue;
int SlotA = matchAccSlotOp(*Lda, W65816::LDAfi);
if (SlotA == NO_SLOT_MATCH) continue;
auto It = std::next(Lda->getIterator());
if (!advancePastDebug(MBB, It)) continue;
int SlotB = matchAccSlotOp(*It, W65816::STAfi);
if (SlotB == NO_SLOT_MATCH || SlotB == SlotA) continue;
MachineInstr &Sta = *It;
// Walk forward. Find the FIRST *fi op whose pointer-FI operand
// is slotB and rewrite it. We allow calls in between — local
// (non-fixed) slots are below-S and not reachable by the callee
// (the callee's stack-rel offsets are above its own SP). Fixed
// slots are also unreachable for the same reason. Bail on
// branches, asm, returns; on STAs writing slotA or slotB.
auto It2 = std::next(Sta.getIterator());
MachineInstr *OpfiTarget = nullptr;
unsigned RewriteIdx = 0;
while (It2 != MBB.end()) {
MachineInstr &MI = *It2;
if (MI.isDebugInstr()) { ++It2; continue; }
if (MI.isInlineAsm() || MI.isBranch() || MI.isReturn()) break;
bool StaToA = (MI.getOpcode() == W65816::STAfi ||
MI.getOpcode() == W65816::STA8fi) &&
MI.getNumOperands() >= 2 && MI.getOperand(1).isFI() &&
MI.getOperand(1).getIndex() == SlotA;
if (StaToA) break;
bool StaToB = (MI.getOpcode() == W65816::STAfi ||
MI.getOpcode() == W65816::STA8fi) &&
MI.getNumOperands() >= 2 && MI.getOperand(1).isFI() &&
MI.getOperand(1).getIndex() == SlotB;
if (StaToB) break;
unsigned Opc = MI.getOpcode();
bool IsOpFi = (Opc == W65816::ADCfi || Opc == W65816::ADCEfi ||
Opc == W65816::SBCfi || Opc == W65816::SBCEfi ||
Opc == W65816::ANDfi || Opc == W65816::ORAfi ||
Opc == W65816::EORfi || Opc == W65816::CMPfi);
if (IsOpFi) {
unsigned FiIdx = (Opc == W65816::CMPfi) ? 1 : 2;
if (MI.getNumOperands() >= FiIdx + 2 &&
MI.getOperand(FiIdx).isFI() &&
MI.getOperand(FiIdx).getIndex() == SlotB &&
MI.getOperand(FiIdx + 1).isImm() &&
MI.getOperand(FiIdx + 1).getImm() == 0) {
OpfiTarget = &MI;
RewriteIdx = FiIdx;
break;
}
}
// LDAfi $a, slotB, 0 — final reload before a DPF0 stage. The
// LDAptr32 inserter emits this shape per pointer half:
// LDAfi $a, srcSlot ; STAfi $a, freshSlot ; [other STAfi]
// LDAfi $a, freshSlot ; STA_DP $E0
// Pass -2c can elide the round-trip by retargeting the final LDAfi
// to read srcSlot (=SlotA) directly, dropping the LDA+STA pair.
// The final LDAfi is a pure A-write so dropping the earlier Lda1
// can't break any intervening A-reader.
if (Opc == W65816::LDAfi) {
if (MI.getNumOperands() >= 3 &&
MI.getOperand(0).isReg() &&
MI.getOperand(0).getReg() == W65816::A &&
MI.getOperand(1).isFI() &&
MI.getOperand(1).getIndex() == SlotB &&
MI.getOperand(2).isImm() &&
MI.getOperand(2).getImm() == 0) {
OpfiTarget = &MI;
RewriteIdx = 1;
break;
}
}
++It2;
}
if (!OpfiTarget) continue;
// Verify slotB has no OTHER references in this function (besides
// Sta and OpfiTarget). If it does, we can't safely drop Sta.
const MachineInstr *ignoreS2c[] = {&Sta, OpfiTarget};
if (slotHasOtherRefs(MF, SlotB, ignoreS2c))
continue;
// Apply rewrite.
OpfiTarget->getOperand(RewriteIdx).setIndex(SlotA);
Erased.insert(Lda);
Lda->eraseFromParent();
Sta.eraseFromParent();
Changed = true;
}
}
// Pass -2: collapse `LDAfi slotA; STAfi slotB; LDAfi slotC; OPfi slotB`
// to `LDAfi slotC; OPfi slotA`. This is the "memory-to-memory copy
// through A" pattern the inserter + regalloc emit when both operands
// of OR_RR/AND_RR/EOR_RR/CMP_RR are already-spilled vregs. We're not
// using OP commutativity here — after `STAfi $a, slotB` we have
// M[slotB] == M[slotA], so reading slotA in place of slotB is a
// value-identity rewrite that's safe even for non-commutative OPs
// (CMP, SBC). slotA must not be written between the STAfi we erase
// and the OPfi we rewrite — the only intervening instruction is the
// single LDAfi in step 3, which doesn't write any slot.
for (MachineBasicBlock &MBB : MF) {
SmallVector<MachineInstr *, 4> Worklist;
for (MachineInstr &MI : MBB)
if (MI.getOpcode() == W65816::LDAfi)
Worklist.push_back(&MI);
for (MachineInstr *Lda1 : Worklist) {
// Step 1: LDAfi $a, slotA.
int SlotA = matchAccSlotOp(*Lda1, W65816::LDAfi);
if (SlotA == NO_SLOT_MATCH) continue;
auto It = std::next(Lda1->getIterator());
if (!advancePastDebug(MBB, It)) continue;
// Step 2: STAfi $a, slotB.
int SlotB = matchAccSlotOp(*It, W65816::STAfi);
if (SlotB == NO_SLOT_MATCH || SlotA == SlotB) continue;
MachineInstr &Sta = *It;
++It;
if (!advancePastDebug(MBB, It)) continue;
// Step 3: LDAfi $a, slotC (loading the OPfi's tied input).
int SlotC = matchAccSlotOp(*It, W65816::LDAfi);
if (SlotC == NO_SLOT_MATCH || SlotC == SlotB) continue;
++It;
if (!advancePastDebug(MBB, It)) continue;
// Step 4: OPfi $a (tied), slotB — accept any *_fi op whose 2nd
// operand is the FI we want to redirect. Commutative ops always
// match. CMPfi / SBCfi are non-commutative but the rewrite still
// preserves the comparison since M[slotA] == M[slotB] here.
MachineInstr &Op = *It;
unsigned Opc = Op.getOpcode();
bool IsFiOp = isCommutativeFiOp(Opc) ||
Opc == W65816::CMPfi ||
Opc == W65816::SBCfi ||
Opc == W65816::SBCEfi;
if (!IsFiOp) continue;
// Operand layout: CMPfi has (outs), (ins Acc16:$lhs, memfi:$addr)
// → operand 0 = $lhs, operand 1+2 = memfi. All other *fi ops
// are (outs Acc16:$dst), (ins Acc16:$src, memfi) → operand 0 =
// $dst, 1 = $src, 2+3 = memfi. Pick the right FI operand index.
unsigned FiIdx = (Opc == W65816::CMPfi) ? 1 : 2;
if (Op.getNumOperands() < FiIdx + 2 ||
!Op.getOperand(0).isReg() || Op.getOperand(0).getReg() != W65816::A ||
!Op.getOperand(FiIdx).isFI() ||
Op.getOperand(FiIdx).getIndex() != SlotB ||
!Op.getOperand(FiIdx + 1).isImm() ||
Op.getOperand(FiIdx + 1).getImm() != 0)
continue;
// Function-wide safety check: slotB must have no other refs
// besides Sta (which we erase) and Op (which we rewrite to use
// slotA instead). Without this, deleting Sta orphans any other
// reader of slotB (e.g. a loop-header LDAfi reading the same
// copy that this entry-side STA initialises) — surfaces as the
// qsort #70 miscompile where greedy parks a value into a slot
// both for the immediate CMPfi and for a downstream loop-body
// reload, and Pass -2 silently kills the only initialiser.
const MachineInstr *ignoreS2[] = {&Sta, &Op};
if (slotHasOtherRefs(MF, SlotB, ignoreS2))
continue;
// Rewrite OP to use slotA, drop Lda1+Sta.
Op.getOperand(FiIdx).setIndex(SlotA);
Lda1->eraseFromParent();
Sta.eraseFromParent();
Changed = true;
}
}
// Pass -1: redundant double-spill in *_RR custom-inserter expansions.
// The OR_RR / AND_RR / EOR_RR / ADC[E]fi / SBC[E]fi inserter spills
// its Src2 to a fresh slot so the OPfi can load-fold from there.
// When Src2 came from $x (an i32-first-arg-in-A:X hi half) and Src1
// came from $a, the regalloc winds up emitting:
//
// STAfi $a, slot_a ; regalloc-allocated spill of $a (Src1)
// COPY $a = $x ; TXA — reuse $a for Src2
// STAfi $a, slot_b ; inserter-allocated spill of Src2 (now in $a)
// LDAfi $a, slot_a ; reload Src1 (the tied input of OPfi)
// OPfi $a (tied), slot_b
//
// Slot_a holds the original Src1 value; slot_b holds Src2's value.
// OPfi reads slot_b but Src1 is already in $a — so semantically
// we could use slot_a (which already holds Src1's spilled value)
// by swapping which operand the OPfi load-folds:
//
// STAfi $a, slot_a
// COPY $a = $x
// OPfi $a (tied), slot_a ; uses slot_a; OP is commutative
//
// Saves: the STAfi to slot_b and the LDAfi from slot_a. Only
// valid for *commutative* ops (ADD/AND/OR/EOR — and ADCE/ADCfi
// since carry semantics are the same regardless of operand order).
// SBC/CMP/SUB are non-commutative; skip them.
for (MachineBasicBlock &MBB : MF) {
SmallVector<MachineInstr *, 4> Worklist;
for (MachineInstr &MI : MBB)
if (MI.getOpcode() == W65816::STAfi)
Worklist.push_back(&MI);
for (MachineInstr *Sta1 : Worklist) {
// Step 1: STAfi $a, slot_a (the regalloc-allocated spill of Src1).
int SlotA = matchAccSlotOp(*Sta1, W65816::STAfi);
if (SlotA == NO_SLOT_MATCH) continue;
auto It = std::next(Sta1->getIterator());
if (!advancePastDebug(MBB, It)) continue;
// Step 2: COPY $a = <something> (TXA, etc.).
MachineInstr &Copy = *It;
if (!Copy.isCopy() || Copy.getOperand(0).getReg() != W65816::A)
continue;
++It;
if (!advancePastDebug(MBB, It)) continue;
// Step 3: STAfi $a, slot_b (inserter-allocated spill of Src2).
int SlotB = matchAccSlotOp(*It, W65816::STAfi);
if (SlotB == NO_SLOT_MATCH || SlotA == SlotB) continue;
MachineInstr &Sta2 = *It;
++It;
if (!advancePastDebug(MBB, It)) continue;
// Step 4: LDAfi $a, slot_a (reload Src1).
int SlotL = matchAccSlotOp(*It, W65816::LDAfi);
if (SlotL != SlotA) continue;
MachineInstr &Lda = *It;
++It;
if (!advancePastDebug(MBB, It)) continue;
// Step 5: OPfi $a tied, slot_b — must be commutative.
MachineInstr &Op = *It;
if (!matchCommutativeFiOpOnSlot(Op, SlotB)) continue;
// Function-wide safety check: slot_b must have no other refs
// besides Sta2 (which we erase) and Op (which we rewrite).
// Same class of bug as #70's Pass -2: an inserter-spawned
// spill slot can get reused as a regalloc spill home, and
// erasing the only initialiser would orphan downstream reads.
const MachineInstr *ignoreS1[] = {&Sta2, &Op};
if (slotHasOtherRefs(MF, SlotB, ignoreS1))
continue;
// Rewrite Op to use slot_a instead of slot_b, erase Sta2 + Lda.
Op.getOperand(2).setIndex(SlotA);
Sta2.eraseFromParent();
Lda.eraseFromParent();
Changed = true;
}
}
// Pass 0: rewrite `LDAi16imm $a, imm` immediately followed by
// `COPY $x = $a` (with no intervening A clobber) into
// `LDXi16imm $x, imm`. Run BEFORE the spill/reload cleanups so
// the disappearing A clobber unblocks subsequent STAfi+LDAfi
// pair removal.
for (MachineBasicBlock &MBB : MF) {
SmallVector<MachineInstr *, 4> Worklist;
for (MachineInstr &MI : MBB)
if (MI.getOpcode() == W65816::LDAi16imm)
Worklist.push_back(&MI);
for (MachineInstr *Lda : Worklist) {
if (Lda->getNumOperands() < 2 || !Lda->getOperand(0).isReg() ||
Lda->getOperand(0).getReg() != W65816::A)
continue;
auto It = std::next(Lda->getIterator());
while (It != MBB.end() && It->isDebugInstr())
++It;
if (It == MBB.end())
continue;
MachineInstr &Next = *It;
if (!Next.isCopy())
continue;
Register DstReg = Next.getOperand(0).getReg();
Register SrcReg = Next.getOperand(1).getReg();
if (DstReg != W65816::X || SrcReg != W65816::A)
continue;
const MachineOperand &ImmMO = Lda->getOperand(1);
const TargetInstrInfo *TII = MF.getSubtarget().getInstrInfo();
MachineInstrBuilder Mib =
BuildMI(MBB, Lda->getIterator(), Lda->getDebugLoc(),
TII->get(W65816::LDXi16imm), W65816::X);
if (ImmMO.isImm())
Mib.addImm(ImmMO.getImm());
else
Mib.add(ImmMO);
Lda->eraseFromParent();
Next.eraseFromParent();
Changed = true;
}
}
// Pass 1: redundant LDAfi after STAfi (load-after-same-store with
// matching register). Two-pass over Stores worklist to avoid
// iterator invalidation when we erase the LDAfi mid-walk.
for (MachineBasicBlock &MBB : MF) {
SmallVector<MachineInstr *, 8> Stores;
for (MachineInstr &MI : MBB)
if (MI.getOpcode() == W65816::STAfi)
Stores.push_back(&MI);
for (MachineInstr *StaMI : Stores)
if (tryEliminateLoadAfterStore(MBB, *StaMI, TRI))
Changed = true;
}
// Pass 1b: redundant reload of the same slot. Pattern:
// LDAfi $a, slotX, 0
// STAfi $a, slotY, 0 ; STA preserves A and doesn't touch slotX
// ... (any non-A-defining, non-slotX-storing instructions)
// LDAfi $a, slotX, 0 ; <-- redundant: A still holds slotX's value
// Walk forward from each LDAfi looking for a matching second LDAfi
// with no intervening A-def or slotX-store. Drops the second LDAfi.
// This catches the fib-loop pattern where the regalloc emits
// LDA X; STA Y; LDA X; ADC Z (the second LDA is dead).
for (MachineBasicBlock &MBB : MF) {
SmallVector<MachineInstr *, 8> Loads;
for (MachineInstr &MI : MBB)
if (MI.getOpcode() == W65816::LDAfi)
Loads.push_back(&MI);
for (MachineInstr *LdaMI : Loads) {
int SlotX = matchAccSlotOp(*LdaMI, W65816::LDAfi);
if (SlotX == NO_SLOT_MATCH) continue;
auto It = std::next(LdaMI->getIterator());
while (It != MBB.end()) {
MachineInstr &MI = *It;
if (MI.isDebugInstr()) { ++It; continue; }
// Found another LDAfi $a from the same slot. LDA sets N/Z;
// dropping it could leave a stale N/Z visible to a following
// branch. Only drop if the immediately-following instruction
// overwrites N/Z (CMP, ADC, AND, ORA, EOR, BIT, etc. — anything
// that defines P). In practice the second LDA is followed by
// a CLC+ADC or similar arithmetic, so this almost always fires.
if (matchAccSlotOp(MI, W65816::LDAfi) == SlotX) {
auto NextIt = std::next(It);
while (NextIt != MBB.end() && NextIt->isDebugInstr()) ++NextIt;
// If we can't see a follower or the follower is a flag-using
// branch, leave the LDA alone.
if (NextIt == MBB.end() || NextIt->isBranch())
break;
MI.eraseFromParent();
Changed = true;
break;
}
// Calls clobber A.
if (MI.isCall()) break;
// STAfi PRESERVES A in the asm (A source: store-only; non-A
// source: PHA bracket round-trip). The pseudo declares
// Defs = [A] as a stale over-approximation, so we explicitly
// skip STAfi when checking for A-clobber. STAfi to slotX
// (same slot) DOES change M[slotX] — bail in that case below.
if (MI.getOpcode() != W65816::STAfi &&
MI.modifiesRegister(W65816::A, TRI)) break;
// STAfi to slotX would change M[slotX] — bail.
if (MI.getOpcode() == W65816::STAfi &&
MI.getNumOperands() >= 2 && MI.getOperand(1).isFI() &&
MI.getOperand(1).getIndex() == SlotX)
break;
// Inline asm / branch boundaries.
if (MI.isInlineAsm() || MI.isBranch() || MI.isReturn())
break;
++It;
}
}
}
// Pass 1d: redundant `LDY_Imm16 #N` (Y already holds N). The
// LDAptrOff/STAptrOff inserters each emit an `LDY #0` (or `LDY #off`)
// before their indirect access; back-to-back load-then-store of the
// same pointer ends up with two `LDY #0` in a row. Drop the second
// when nothing in between writes Y. Like Pass 1b, bail if the
// following instruction is a branch (Y's flag side-effects matter
// for branches that test N/Z).
for (MachineBasicBlock &MBB : MF) {
SmallVector<MachineInstr *, 8> Ldys;
for (MachineInstr &MI : MBB)
if (MI.getOpcode() == W65816::LDY_Imm16)
Ldys.push_back(&MI);
SmallPtrSet<MachineInstr *, 8> ErasedY;
for (MachineInstr *Ldy : Ldys) {
if (ErasedY.count(Ldy)) continue;
if (Ldy->getNumOperands() < 1 || !Ldy->getOperand(0).isImm())
continue;
int64_t Imm = Ldy->getOperand(0).getImm();
// Walk forward erasing every subsequent matching LDY_Imm16 #Imm
// until something invalidates the held Y value (call, Y-def, asm,
// branch). Multiple LDYs in a row collapse on the first source.
auto It = std::next(Ldy->getIterator());
while (It != MBB.end()) {
MachineInstr &MI = *It;
if (MI.isDebugInstr()) { ++It; continue; }
if (MI.getOpcode() == W65816::LDY_Imm16 &&
MI.getNumOperands() >= 1 && MI.getOperand(0).isImm() &&
MI.getOperand(0).getImm() == Imm) {
// Bail on branch follower (flag-sensitive — LDY sets N/Z).
auto NextIt = std::next(It);
while (NextIt != MBB.end() && NextIt->isDebugInstr()) ++NextIt;
if (NextIt == MBB.end() || NextIt->isBranch()) break;
// Erase and continue walking — there may be more dups.
auto Erased_It = It;
++It;
ErasedY.insert(&*Erased_It);
Erased_It->eraseFromParent();
Changed = true;
continue;
}
if (MI.isCall()) break;
if (MI.modifiesRegister(W65816::Y, TRI)) break;
// killsRegister: an instruction with `implicit killed $y` USES Y
// and that's the LAST use — Y is dead after. We must NOT treat
// a subsequent LDY_Imm16 #N as redundant after a kill, because
// the held value is conceptually gone. Caught by `addOff(p,i)
// { p[i-1] += p[i]; }` where LDY -2 ; LDA_indY (kills Y) ; ... ;
// LDY -2 ; STA_indY needs the second LDY to reinitialize Y.
if (MI.killsRegister(W65816::Y, TRI)) break;
if (MI.isInlineAsm() || MI.isBranch() || MI.isReturn()) break;
++It;
}
}
}
// Pass 1c: drop redundant `CMPi16imm $a, 0` that follows an op which
// already set N/Z based on $a's new value (ORA/AND/EOR/ADC/SBC/LDA/...
// anything that defines $a). Pattern is emitted by the i32-equals-0
// path (i32 (lo|hi) == 0): the OR sets Z, then the SETCC compares
// against 0. The second compare is provably redundant because $a
// hasn't changed since the previous flag-defining op.
// Intra-MBB only — cross-MBB recursion into predecessors was tried
// (catches SETCC merge blocks where each pred ends with `lda #c`)
// but proved too brittle: predecessors ending with JSLpseudo declare
// implicit-def $a but the return-value flags aren't reliably set,
// and other corner cases break smoke.
auto isATransparent = [](const MachineInstr &MI) {
// Stores that don't touch A or P-bits-other-than-via-A. (Byte
// stores that internally SEP/REP wrap toggle the M flag, but that
// doesn't affect N/Z based on A's current value.) Also call-stack
// pseudos (ADJCALLSTACKDOWN / UP) which are zero-effect at this
// point in the pipeline (PEI eliminates UP; DOWN is always nil).
switch (MI.getOpcode()) {
case W65816::STAfi:
case W65816::STAfi_indY:
case W65816::STA8fi:
case W65816::STAabs:
case W65816::STA8abs:
case W65816::STAptr:
case W65816::STBptr:
case W65816::STAptrOff:
case W65816::STBptrOff:
case W65816::ADJCALLSTACKDOWN:
// DOWN expands to nothing (PUSH16 chain already shifted SP).
// UP is NOT transparent: when PEI doesn't process it, AsmPrinter
// emits a TSC/CLC/ADC/TCS sequence that clobbers A and flags.
return true;
default:
return false;
}
};
// Returns true iff walking back from `Start` (exclusive) finds an
// A-modifier as the first non-skip op. Skips debug ops and
// A-transparent stores; stops at the first real op. Templated to
// accept either iterator or const_iterator (Cmps came from a non-
// const iteration; predecessors are walked via const_iterator).
auto walkbackBefore = [&](auto Start, auto Begin) -> bool {
auto It = Start;
while (It != Begin) {
--It;
if (It->isDebugInstr()) continue;
if (isATransparent(*It)) continue;
return It->modifiesRegister(W65816::A, TRI);
}
return false;
};
// Pass 1c can only eliminate CMPi16imm $a, 0 if the preceding
// A-modifier reliably sets N/Z to reflect A's final value. LDAfi
// under FP-rel expansion (`sty $fa ; ldy #imm ; lda [$f6],y ; ldy $fa`)
// ends with `ldy` that clobbers N/Z based on OLD Y, not loaded A — so
// in FP-rel functions (VLA / huge frame), the CMP is load-bearing.
// Skip the whole pass for such functions (saves us from the sum_n
// VLA regression that the PHP-wrap-aware variant tripped).
bool ssCleanupSPRelOnly = !UsesFPRel;
for (MachineBasicBlock &MBB : MF) {
SmallVector<MachineInstr *, 8> Cmps;
for (MachineInstr &MI : MBB)
if (MI.getOpcode() == W65816::CMPi16imm)
Cmps.push_back(&MI);
for (MachineInstr *Cmp : Cmps) {
// Shape: CMPi16imm $a, 0.
if (Cmp->getNumOperands() < 2 ||
!Cmp->getOperand(0).isReg() ||
Cmp->getOperand(0).getReg() != W65816::A ||
!Cmp->getOperand(1).isImm() ||
Cmp->getOperand(1).getImm() != 0)
continue;
bool Found = walkbackBefore(Cmp->getIterator(), MBB.begin());
if (!Found) continue;
// Only eliminate if there are NO LdaLike instructions between
// this CMP and the next Bxx (or end of MBB). Otherwise the
// CMP is the only flag-setting marker between the test value
// and the consuming branch — without it, the Bxx ends up
// testing the latest LdaLike's N/Z (typically a PHI-elim COPY
// or stack reload that has nothing to do with the original
// condition). Caused __adddf3's renormalize while-loop to
// skip its body even though `mr & ~mask` was non-zero.
bool SafeToErase = true;
bool insidePHPWrap = false;
for (auto It = std::next(Cmp->getIterator());
It != Cmp->getParent()->end(); ++It) {
if (It->isDebugInstr()) continue;
if (It->isBranch() || It->isReturn()) break;
// PHP/PLP-wrap-aware: only safe when LDAfi-expansion sets N/Z
// reliably (SP-rel functions, not FP-rel).
if (ssCleanupSPRelOnly && It->getOpcode() == W65816::PHP) {
// PHP must be IMMEDIATELY after CMP to capture CMP's flags.
if (&*It != &*std::next(Cmp->getIterator())) {
SafeToErase = false;
break;
}
insidePHPWrap = true;
continue;
}
if (It->getOpcode() == W65816::PLP) {
insidePHPWrap = false;
continue;
}
if (insidePHPWrap) continue;
if (It->getOpcode() == TargetOpcode::COPY) {
SafeToErase = false;
break;
}
unsigned Opc = It->getOpcode();
// Conservative: any LDA/LDX/LDY/transfer disqualifies erasure.
// Stores and stack-mgmt are flag-preserving and OK.
switch (Opc) {
case W65816::STAfi: case W65816::STAfi_indY: case W65816::STA8fi:
case W65816::STA_StackRel: case W65816::STA_StackRelIndY:
case W65816::STA_DP: case W65816::STA_Abs: case W65816::STA_Long:
case W65816::STX_DP: case W65816::STX_Abs:
case W65816::STY_DP: case W65816::STY_Abs:
case W65816::ADJCALLSTACKDOWN: case W65816::ADJCALLSTACKUP:
case W65816::PHA: case W65816::PHX: case W65816::PHY:
continue;
}
// Anything else (LDA, transfer, ALU op...): bail.
SafeToErase = false;
break;
}
if (SafeToErase) {
Cmp->eraseFromParent();
Changed = true;
}
}
}
// Pass 1e: redundant `ANDi16imm $a, $a, 0xFF`. An i8 value zero-
// extended to i16 has high byte = 0; subsequent AND #$FF is a no-op
// and just adds a 3-byte instruction. This pattern is emitted twice
// by the (zextload-then-spill-twice) shape in *cmp helpers — see
// memcmp_local in the smoke-tests. Drop the second AND when:
// - first AND was `ANDi16imm $a, $a, 0xFF`
// - no A-defining op between them (STAfi, CMP*, etc. are fine)
// - second AND is also `ANDi16imm $a, $a, 0xFF`
// Flag-safe: both ANDs set N=0, Z=(A==0); after the first, the second
// produces identical flags, so dropping it leaves any following Bxx
// with the same N/Z values.
for (MachineBasicBlock &MBB : MF) {
SmallVector<MachineInstr *, 8> Ands;
for (MachineInstr &MI : MBB)
if (MI.getOpcode() == W65816::ANDi16imm &&
MI.getNumOperands() >= 3 && MI.getOperand(2).isImm() &&
MI.getOperand(2).getImm() == 0xFF)
Ands.push_back(&MI);
SmallPtrSet<MachineInstr *, 8> Erased;
for (MachineInstr *And : Ands) {
if (Erased.count(And)) continue;
auto It = std::next(And->getIterator());
while (It != MBB.end()) {
MachineInstr &MI = *It;
if (MI.isDebugInstr()) { ++It; continue; }
// Match: another `AND #$FF` with A unchanged.
if (MI.getOpcode() == W65816::ANDi16imm &&
MI.getNumOperands() >= 3 && MI.getOperand(2).isImm() &&
MI.getOperand(2).getImm() == 0xFF) {
Erased.insert(&MI);
MI.eraseFromParent();
Changed = true;
break;
}
if (MI.isCall() || MI.isInlineAsm() || MI.isBranch() ||
MI.isReturn()) break;
if (MI.modifiesRegister(W65816::A, TRI)) break;
++It;
}
}
}
// Pass 1g: redundant AND #$FF after reload of a masked slot. Pattern:
//
// ANDi16imm $a, $a, 0xFF ; A := A & 0xFF (high byte = 0)
// STAfi $a, slotN, 0 ; M[slotN] = A — slot's high byte is 0
// ... ; no STAfi to slotN, no A defs
// LDAfi $a, slotN, 0 ; A := M[slotN] — high byte still 0
// ANDi16imm $a, $a, 0xFF ; <-- redundant: A's high byte is 0
//
// Drop the second AND. Pass 1e (back-to-back AND #FF) bails on any
// A-defining op in between, so it can't see across the LDA reload.
// This pass is the "through-memory" complement. Found in find_byte
// and other char-iteration loops where the regalloc emits an extra
// mask-then-spill-then-reload-then-mask cycle around the comparison.
for (MachineBasicBlock &MBB : MF) {
SmallVector<MachineInstr *, 8> FirstAnds;
for (MachineInstr &MI : MBB)
if (MI.getOpcode() == W65816::ANDi16imm &&
MI.getNumOperands() >= 3 && MI.getOperand(2).isImm() &&
MI.getOperand(2).getImm() == 0xFF)
FirstAnds.push_back(&MI);
SmallPtrSet<MachineInstr *, 8> Erased;
for (MachineInstr *And1 : FirstAnds) {
if (Erased.count(And1)) continue;
auto It = std::next(And1->getIterator());
if (!advancePastDebug(MBB, It)) continue;
// Step 2: STAfi $a, slotN.
int SlotN = matchAccSlotOp(*It, W65816::STAfi);
if (SlotN == NO_SLOT_MATCH) continue;
// Step 3: walk forward looking for LDAfi from slotN. We allow
// arbitrary A modifications in between because the LDAfi reload
// re-establishes A as the masked value (M[slotN] still has high
// byte = 0 from the And1+Sta we just saw). We ONLY need slotN
// itself to be unchanged. Bail on calls (callee can clobber any
// local slot indirectly), branches/returns/asm.
auto It2 = std::next(It);
MachineInstr *Lda = nullptr;
while (It2 != MBB.end()) {
MachineInstr &MI = *It2;
if (MI.isDebugInstr()) { ++It2; continue; }
if (MI.isCall() || MI.isInlineAsm() || MI.isBranch() ||
MI.isReturn()) break;
if (MI.getOpcode() == W65816::STAfi &&
MI.getNumOperands() >= 2 && MI.getOperand(1).isFI() &&
MI.getOperand(1).getIndex() == SlotN)
break;
if (matchAccSlotOp(MI, W65816::LDAfi) == SlotN) {
Lda = &MI;
break;
}
++It2;
}
if (!Lda) continue;
// Step 4: must be followed by `ANDi16imm $a, $a, 0xFF`.
auto It3 = std::next(Lda->getIterator());
if (!advancePastDebug(MBB, It3)) continue;
if (It3->getOpcode() != W65816::ANDi16imm ||
It3->getNumOperands() < 3 || !It3->getOperand(2).isImm() ||
It3->getOperand(2).getImm() != 0xFF)
continue;
MachineInstr &And2 = *It3;
Erased.insert(&And2);
And2.eraseFromParent();
Changed = true;
}
}
// Pass 2a: function-wide dead-slot stores. If a *local* (non-fixed)
// FrameIndex is never read anywhere in the function (no LDAfi from
// it, no *fi op consuming it, no indirect-Y use of it as a pointer
// slot), then every STAfi/STA8fi that writes to it is dead. This
// catches the cross-MBB pattern Pass 2 misses (Pass 2 walks within a
// single MBB and bails on branches).
//
// Conservative: read = any opcode whose listed write-operands don't
// include this FI. We approximate by treating the operand at the
// STAfi/STA8fi's "addr" position (op 1, the FI; op 2, the imm offset)
// as the *only* write. Every other reference is treated as a read.
{
MachineFrameInfo &MFI = MF.getFrameInfo();
DenseMap<int, unsigned> Reads;
DenseMap<int, unsigned> Writes;
SmallVector<MachineInstr *, 32> Stores;
for (MachineBasicBlock &MBB : MF) {
for (MachineInstr &MI : MBB) {
bool IsStaFi = (MI.getOpcode() == W65816::STAfi ||
MI.getOpcode() == W65816::STA8fi);
for (unsigned i = 0; i < MI.getNumOperands(); ++i) {
const MachineOperand &MO = MI.getOperand(i);
if (!MO.isFI()) continue;
int FI = MO.getIndex();
if (MFI.isFixedObjectIndex(FI)) continue;
// For STAfi/STA8fi, the FI operand at i==1 is the *write*
// target; everything else is a read of this FI.
if (IsStaFi && i == 1)
Writes[FI]++;
else
Reads[FI]++;
}
if (IsStaFi)
Stores.push_back(&MI);
}
}
for (MachineInstr *Sta : Stores) {
if (Sta->getNumOperands() < 2 || !Sta->getOperand(1).isFI()) continue;
// Volatile stores are observable side effects — never elide.
// Caught by SJLJ EH: SjLjEHPrepare emits `store volatile i32 N,
// fn_ctx.call_site` before each invoke; the function context's
// call_site field is "never read" within main but IS read by
// the runtime via gActive — Pass 2a's local liveness can't see
// that, so volatile is the right gate.
if (Sta->hasOrderedMemoryRef()) continue;
int FI = Sta->getOperand(1).getIndex();
if (Reads.count(FI) == 0 && Writes[FI] >= 1) {
Sta->eraseFromParent();
Changed = true;
}
}
}
// Pass 1f: drop adjacent PHP/PLP pairs. Pass -2.5 inserts PHP/PLP
// around LDA-style ops to protect a CMP's flags from being clobbered
// by the LDA before the consuming branch. Pass 1 (load-after-store
// elimination) sometimes deletes the LDA *between* the wrap because
// it's a redundant reload — the spilled value is already in A. After
// that deletion, PHP and PLP are back-to-back with nothing between,
// and the pair is a no-op. Drop both.
for (MachineBasicBlock &MBB : MF) {
SmallVector<MachineInstr *, 8> Phps;
for (MachineInstr &MI : MBB)
if (MI.getOpcode() == W65816::PHP)
Phps.push_back(&MI);
for (MachineInstr *Php : Phps) {
auto It = std::next(Php->getIterator());
while (It != MBB.end() && It->isDebugInstr()) ++It;
if (It == MBB.end() || It->getOpcode() != W65816::PLP) continue;
MachineInstr *Plp = &*It;
Php->eraseFromParent();
Plp->eraseFromParent();
Changed = true;
}
}
// Pass 1h: collapse dead TAX/TXA bridge around DPF0 staging. When
// SpillToX bridges one of the LDAptr32 inserter's STAfi/LDAfi pairs
// (saving 6 cyc), then Pass -2c collapses the OTHER pair entirely,
// the TAX/TXA bridge becomes pure overhead. Pattern:
//
// LDAfi $a, slotA, 0 ; (1) load ptr_hi (originally for X-bridge)
// TAX ; (2) X = ptr_hi
// LDAfi $a, slotB, 0 ; (3) load ptr_lo (the Pass -2c rewrite target)
// STA_DP imm1 ; (4) $E0 = ptr_lo
// TXA ; (5) recover ptr_hi to A
// STA_DP imm2 ; (6) $E2 = ptr_hi
//
// Reorder to drop the bridge:
//
// LDAfi $a, slotB, 0 ; (3) load ptr_lo
// STA_DP imm1 ; (4) $E0 = ptr_lo
// LDAfi $a, slotA, 0 ; (1) load ptr_hi (moved down)
// STA_DP imm2 ; (6) $E2 = ptr_hi
//
// Saves 2 instructions (~6 cyc) per LDAptr32 expansion. Safe iff X
// is dead after TXA (no read before redef). Common in tight loops
// that deref two pointers per iter (memcmp, strcpy, dotProduct).
for (MachineBasicBlock &MBB : MF) {
SmallVector<MachineInstr *, 8> Ldas;
for (MachineInstr &MI : MBB) {
if (MI.getOpcode() == W65816::LDAfi) {
Ldas.push_back(&MI);
}
}
SmallPtrSet<MachineInstr *, 8> ErasedH;
for (MachineInstr *Lda1 : Ldas) {
if (ErasedH.count(Lda1)) {
continue;
}
int SlotA = matchAccSlotOp(*Lda1, W65816::LDAfi);
if (SlotA == NO_SLOT_MATCH) {
continue;
}
auto It = std::next(Lda1->getIterator());
if (!advancePastDebug(MBB, It)) {
continue;
}
// Optional `ADCEfi $a (tied), slot, 0` between Lda1 and TAX.
// strcpy emits `LDAfi hi_part ; ADCEfi $a, $stack.9 ; TAX` to
// compute ptr_hi via carry-propagation from a prior ADCfi (lo).
// The ADCEfi's carry-in comes from an earlier ADCfi; intervening
// LDA/STA preserve C, so reordering this pair past STA_DP is safe.
MachineInstr *MaybeAdcE = nullptr;
if (It->getOpcode() == W65816::ADCEfi &&
It->getNumOperands() >= 4 &&
It->getOperand(0).isReg() &&
It->getOperand(0).getReg() == W65816::A) {
MaybeAdcE = &*It;
++It;
if (!advancePastDebug(MBB, It)) {
continue;
}
}
if (It->getOpcode() != W65816::TAX) {
continue;
}
MachineInstr *Tax = &*It;
++It;
if (!advancePastDebug(MBB, It)) {
continue;
}
int SlotB = matchAccSlotOp(*It, W65816::LDAfi);
if (SlotB == NO_SLOT_MATCH || SlotB == SlotA) {
continue;
}
MachineInstr *Lda2 = &*It;
++It;
if (!advancePastDebug(MBB, It)) {
continue;
}
if (It->getOpcode() != W65816::STA_DP) {
continue;
}
MachineInstr *Sta1 = &*It;
++It;
if (!advancePastDebug(MBB, It)) {
continue;
}
if (It->getOpcode() != W65816::TXA) {
continue;
}
MachineInstr *Txa = &*It;
++It;
if (!advancePastDebug(MBB, It)) {
continue;
}
if (It->getOpcode() != W65816::STA_DP) {
continue;
}
MachineInstr *Sta2 = &*It;
// Verify X is dead after Txa within this MBB. Tighter check
// than just "until branch/return": if we reach end of MBB or a
// branch without seeing an X-read or X-def, look at successor
// MBBs. X is dead iff no successor's livein set includes X.
// Critical for tight loops (strLen, byte-walk loops) where the
// loop terminator is a branch back to the same MBB.
bool XDeadAfter = true;
bool MustCheckSuccessors = false;
auto checkIt = std::next(Sta2->getIterator());
while (checkIt != MBB.end()) {
if (checkIt->isDebugInstr()) {
++checkIt;
continue;
}
if (checkIt->readsRegister(W65816::X, TRI)) {
XDeadAfter = false;
break;
}
if (checkIt->modifiesRegister(W65816::X, TRI)) {
// X redefined before any read — dead range ended.
break;
}
if (checkIt->isCall() || checkIt->isInlineAsm()) {
XDeadAfter = false;
break;
}
if (checkIt->isBranch() || checkIt->isReturn()) {
// Hit the terminator without redef — defer to successor
// liveins to determine if X is live-out.
MustCheckSuccessors = true;
break;
}
++checkIt;
}
if (checkIt == MBB.end()) {
MustCheckSuccessors = true;
}
if (XDeadAfter && MustCheckSuccessors) {
// X is dead iff no successor lists it as a livein.
for (MachineBasicBlock *Succ : MBB.successors()) {
if (Succ->isLiveIn(W65816::X)) {
XDeadAfter = false;
break;
}
}
}
if (!XDeadAfter) {
continue;
}
// Apply: move Lda1 (and optional ADCEfi) to between Sta1 and
// Sta2, then erase Tax/Txa. Splicing both to Sta2's iterator
// preserves their relative order — second splice ends up nearer
// Sta2, so Lda1 first then MaybeAdcE.
MBB.splice(Sta2->getIterator(), &MBB, Lda1->getIterator());
if (MaybeAdcE) {
MBB.splice(Sta2->getIterator(), &MBB, MaybeAdcE->getIterator());
}
Tax->eraseFromParent();
Txa->eraseFromParent();
ErasedH.insert(Lda1);
Changed = true;
}
}
// Pass 1z: re-run Pass 1's load-after-store elimination after Pass 1h.
// Pass 1h's splice may have created newly-adjacent STAfi+LDAfi pairs
// when the moved instructions exposed a producer (e.g., ADCfi) whose
// result was already in A at the LDAfi point. Common in strcpy's
// pointer-arithmetic: `ADCfi $a, lo_off ; STAfi $a, slot ; LDAfi $a,
// slot ; STA_DP` collapses to `ADCfi ; STAfi ; STA_DP` after the LDA
// is recognised as redundant.
for (MachineBasicBlock &MBB : MF) {
SmallVector<MachineInstr *, 8> Stores;
for (MachineInstr &MI : MBB) {
if (MI.getOpcode() == W65816::STAfi) {
Stores.push_back(&MI);
}
}
for (MachineInstr *StaMI : Stores) {
if (tryEliminateLoadAfterStore(MBB, *StaMI, TRI)) {
Changed = true;
}
}
}
// Pass 2: dead stores (STAfi to slot followed by another STAfi to
// the same slot with no intervening read). This catches the
// arg0_lo "preserve" spill that the regalloc emits even though the
// value is consumed by the very next instruction.
for (MachineBasicBlock &MBB : MF) {
SmallVector<MachineInstr *, 8> Stores;
for (MachineInstr &MI : MBB)
if (MI.getOpcode() == W65816::STAfi)
Stores.push_back(&MI);
for (MachineInstr *StaMI : Stores)
if (tryEliminateDeadStore(MBB, *StaMI))
Changed = true;
}
// Pass 3: zero-size unused local frame objects so the
// PrologueEpilogue pass shrinks the prologue PHAs / TSC reservation.
// Walk the MIR collecting which FIs are still referenced; any
// *non-fixed* (local) FI with no remaining reference is dead. We
// can't safely remove it (RemoveStackObject can shift indexes); we
// just zero-size it via setObjectSize, which is enough for the
// frame layout pass to skip it.
MachineFrameInfo &MFI = MF.getFrameInfo();
if (MFI.getNumObjects() > 0) {
BitVector Used(MFI.getObjectIndexEnd() - MFI.getObjectIndexBegin());
auto Mark = [&](int FI) {
int Idx = FI - MFI.getObjectIndexBegin();
if (Idx >= 0 && Idx < (int)Used.size())
Used.set(Idx);
};
for (MachineBasicBlock &MBB : MF)
for (MachineInstr &MI : MBB)
for (MachineOperand &MO : MI.operands())
if (MO.isFI())
Mark(MO.getIndex());
for (int FI = MFI.getObjectIndexBegin();
FI < MFI.getObjectIndexEnd(); ++FI) {
// Skip fixed (arg) slots — those are "owned" by the caller.
if (MFI.isFixedObjectIndex(FI))
continue;
int Idx = FI - MFI.getObjectIndexBegin();
if (Idx < 0 || Idx >= (int)Used.size() || Used.test(Idx))
continue;
// Already zero-sized? Skip.
if (MFI.getObjectSize(FI) == 0)
continue;
// Don't touch dead-stripped objects either.
if (MFI.isDeadObjectIndex(FI))
continue;
MFI.setObjectSize(FI, 0);
Changed = true;
}
}
return Changed;
}
Reference in a new issue View git blame Copy permalink