//===-- W65816StackSlotCleanup.cpp - Remove redundant spill/reload pairs --===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // // Post-RA cleanup that erases redundant STAfi+LDAfi pairs to the same // stack slot when no instruction in between writes A or that slot. // // The greedy register allocator routinely emits this pattern when // materialising a COPY of $a into a vreg that gets allocated back to // $a — the spill+reload cycle is a no-op since A already holds the // stored value. The standard MachineLateInstrsCleanup pass only // detects identical instructions; it doesn't recognise that // `LDAfi slot` after `STAfi $a, slot` is a no-op. We do the // simple per-block scan here. // // Conservative: only matches adjacent STAfi+LDAfi pairs (no scan for // instructions in between). In practice the greedy-allocator-emitted // pattern is always adjacent or near-adjacent, and the scheduler keeps // it that way because the LDAfi feeds the next instruction. If // future codegen breaks this assumption, generalise to a longer scan // with explicit clobber tracking. // //===----------------------------------------------------------------------===// #include "W65816.h" #include "W65816InstrInfo.h" #include "W65816Subtarget.h" #include "llvm/ADT/BitVector.h" #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/SmallPtrSet.h" #include "llvm/CodeGen/MachineFrameInfo.h" #include "llvm/CodeGen/MachineFunction.h" #include "llvm/CodeGen/MachineFunctionPass.h" #include "llvm/CodeGen/MachineInstr.h" #include "llvm/CodeGen/MachineRegisterInfo.h" #include "llvm/CodeGen/TargetRegisterInfo.h" using namespace llvm; #define DEBUG_TYPE "w65816-stack-slot-cleanup" namespace { class W65816StackSlotCleanup : public MachineFunctionPass { public: static char ID; W65816StackSlotCleanup() : MachineFunctionPass(ID) {} StringRef getPassName() const override { return "W65816 redundant stack-slot spill/reload elimination"; } bool runOnMachineFunction(MachineFunction &MF) override; }; } // namespace char W65816StackSlotCleanup::ID = 0; INITIALIZE_PASS(W65816StackSlotCleanup, DEBUG_TYPE, "W65816 redundant stack-slot spill/reload elimination", false, false) FunctionPass *llvm::createW65816StackSlotCleanup() { return new W65816StackSlotCleanup(); } // Returns true if MI references frame index FI as one of its operands. // Used to bail dead-store removal when an intervening instruction // reads or writes the slot. static bool referencesFrameIndex(const MachineInstr &MI, int FI) { for (const MachineOperand &MO : MI.operands()) if (MO.isFI() && MO.getIndex() == FI) return true; return false; } // Sentinel for "no match" returned by matchAccSlotOp. We can't use // -1 because FrameIndex numbers for *fixed* (caller-arg) slots are // negative — fixed-stack.0 is -1, fixed-stack.1 is -2, etc. Earlier // passes that did `if (slot < 0) continue;` were silently bailing on // every legitimate fixed-slot LDA/STA, missing many cross-arg-slot // optimisation opportunities. static constexpr int NO_SLOT_MATCH = INT_MIN; // If MI matches `OP $a, FI, 0` where OP == ExpectedOpc, returns the slot // index (which may be negative for fixed-stack args); else NO_SLOT_MATCH. // Callers must compare against NO_SLOT_MATCH, NOT against `< 0`. static int matchAccSlotOp(const MachineInstr &MI, unsigned ExpectedOpc) { if (MI.getOpcode() != ExpectedOpc || MI.getNumOperands() < 3 || !MI.getOperand(0).isReg() || MI.getOperand(0).getReg() != W65816::A || !MI.getOperand(1).isFI() || !MI.getOperand(2).isImm() || MI.getOperand(2).getImm() != 0) return NO_SLOT_MATCH; return MI.getOperand(1).getIndex(); } // Returns true if Opc is a commutative *_fi pseudo (the load-fold form // where operand 2 is the FI). ADD/AND/OR/EOR / ADCE all qualify; SBC // and CMP are non-commutative. static bool isCommutativeFiOp(unsigned Opc) { return Opc == W65816::ADCfi || Opc == W65816::ADCEfi || Opc == W65816::ANDfi || Opc == W65816::ORAfi || Opc == W65816::EORfi; } // If MI is a commutative *_fi op of the canonical shape `OPfi $a (tied), slot, 0` // matching slot SlotB, returns true. Used to recognise the OPfi at the // end of a *_RR inserter expansion. static bool matchCommutativeFiOpOnSlot(const MachineInstr &MI, int SlotB) { if (!isCommutativeFiOp(MI.getOpcode())) return false; if (MI.getNumOperands() < 4 || !MI.getOperand(0).isReg() || MI.getOperand(0).getReg() != W65816::A || !MI.getOperand(2).isFI() || MI.getOperand(2).getIndex() != SlotB || !MI.getOperand(3).isImm() || MI.getOperand(3).getImm() != 0) return false; return true; } // Advance It past debug instructions; returns true if landed on a real // instruction in the block. Templated because callers mix iterator and // instr_iterator depending on how they got here. template static bool advancePastDebug(MachineBasicBlock &MBB, IterT &It) { while (It != MBB.end() && It->isDebugInstr()) ++It; return It != MBB.end(); } // Match `STAfi reg1, FI, 0; ... ; STAfi reg2, FI, 0` (kill via overwrite) // or `STAfi reg, FI, 0; ... ; (no read in between)` (dead store // at function exit). Both mean the first STAfi is dead. Conservative: // bails on anything that references the slot, calls, inline asm. The // slot must be a *local* (non-fixed) FrameIndex — args live across the // function so we can't kill stores to fixed slots. static bool tryEliminateDeadStore(MachineBasicBlock &MBB, MachineInstr &StaMI) { if (StaMI.getOpcode() != W65816::STAfi) return false; if (StaMI.getNumOperands() < 3 || !StaMI.getOperand(1).isFI() || !StaMI.getOperand(2).isImm() || StaMI.getOperand(2).getImm() != 0) return false; // Never eliminate a volatile store — its observability is the // whole point of marking it volatile. Caught by the SJLJ EH path // where SjLjEHPrepare emits `store volatile i32 N, fn_ctx.call_site` // before each invoke; without this check the call_site never gets // written and the personality routine can't pick the landing pad. if (StaMI.hasOrderedMemoryRef()) return false; int StoredFI = StaMI.getOperand(1).getIndex(); // Don't try to kill a store to a fixed (arg) slot — those are // observable to the caller. Locals/spills are fair game. const MachineFunction *MF = StaMI.getMF(); if (MF->getFrameInfo().isFixedObjectIndex(StoredFI)) return false; auto It = std::next(StaMI.getIterator()); while (It != MBB.end()) { MachineInstr &MI = *It; if (MI.isDebugInstr()) { ++It; continue; } // A subsequent STAfi to the same slot, offset 0, kills our store. if (MI.getOpcode() == W65816::STAfi && MI.getNumOperands() >= 3 && MI.getOperand(1).isFI() && MI.getOperand(1).getIndex() == StoredFI && MI.getOperand(2).isImm() && MI.getOperand(2).getImm() == 0) { // Found the killing store. Erase the first. StaMI.eraseFromParent(); return true; } // A return that doesn't read the slot kills the store too — the // local goes out of scope at function exit. if (MI.isReturn() && !referencesFrameIndex(MI, StoredFI)) { StaMI.eraseFromParent(); return true; } // Anything else that touches the slot (load, ADC d,S, etc.) means // the first store IS observed — bail. if (referencesFrameIndex(MI, StoredFI)) return false; // Inline asm / branches: too tricky. Calls are OK to walk past — // local (non-fixed) slots are addressed at offsets the callee // can't reach (callee's S has been shifted down by JSL's // 3-byte return frame and any of its own pha/tsc adjustments, // so its `(4,s)` reads land above our locals). We've already // bailed on fixed slots above, so reaching here means the slot // is local and call-safe. if (MI.isInlineAsm() || MI.isBranch()) return false; ++It; } // Walked off the end of the BB without seeing a return/use. Bail // (could fall through to a successor that reads the slot). return false; } // Returns true if any MachineInstr in MF (other than those in `ignore`) // references frame index `FI`. Used by Pass -4 / Pass -4c safety // checks before erasing an init store: a cross-MBB reload would // otherwise read stale/uninitialized data. See #107. static bool slotHasOtherRefs(const MachineFunction &MF, int FI, ArrayRef ignore) { for (const MachineBasicBlock &MBB : MF) { for (const MachineInstr &MI : MBB) { bool skip = false; for (const MachineInstr *I : ignore) { if (&MI == I) { skip = true; break; } } if (skip) continue; for (const MachineOperand &MO : MI.operands()) { if (MO.isFI() && MO.getIndex() == FI) return true; } } } return false; } // Match `STAfi reg, FI, 0; ... ; LDAfi destReg, FI, 0` when reg == destReg // and nothing in between clobbers reg or the slot. Erase the LDAfi. static bool tryEliminateLoadAfterStore(MachineBasicBlock &MBB, MachineInstr &StaMI, const TargetRegisterInfo *TRI) { if (StaMI.getOpcode() != W65816::STAfi) return false; if (StaMI.getNumOperands() < 3 || !StaMI.getOperand(0).isReg() || !StaMI.getOperand(1).isFI() || !StaMI.getOperand(2).isImm() || StaMI.getOperand(2).getImm() != 0) return false; Register StoredReg = StaMI.getOperand(0).getReg(); int StoredFI = StaMI.getOperand(1).getIndex(); // Walk forward looking for the matching LDAfi. Bail on any // instruction that could clobber StoredReg or write the slot. auto It = std::next(StaMI.getIterator()); while (It != MBB.end()) { MachineInstr &MI = *It; if (MI.isDebugInstr()) { ++It; continue; } if (MI.getOpcode() == W65816::LDAfi && MI.getNumOperands() >= 3 && MI.getOperand(1).isFI() && MI.getOperand(1).getIndex() == StoredFI && MI.getOperand(2).isImm() && MI.getOperand(2).getImm() == 0 && MI.getOperand(0).isReg() && MI.getOperand(0).getReg() == StoredReg) { // A volatile load is observable — never elide, even if the // value is provably the same as the prior store. But STAfi/ // LDAfi target compiler-managed stack spill slots, which are // by construction never volatile — `hasOrderedMemoryRef()` // returns true here only because both lack explicit memops // (the conservative "no info → treat as ordered" default). // Check the actual memops if present; absence is fine. auto isReallyVolatile = [](const MachineInstr &I) { for (auto *MMO : I.memoperands()) if (MMO->isVolatile() || MMO->isAtomic()) return true; return false; }; if (isReallyVolatile(MI) || isReallyVolatile(StaMI)) return false; // LDA sets N/Z based on the loaded value. Dropping it would // expose stale N/Z from before the STA→LDA pair to the next // flag-reading op (e.g. a branch). Only safe to drop if the // immediately-following op overwrites N/Z. auto opSetsNZ = [](unsigned Op) { switch (Op) { case W65816::LDAfi: case W65816::LDAi16imm: case W65816::LDAabs: case W65816::ANDi16imm: case W65816::ANDabs: case W65816::ANDfi: case W65816::ORAi16imm: case W65816::ORAabs: case W65816::ORAfi: case W65816::EORi16imm: case W65816::EORabs: case W65816::EORfi: case W65816::ADCi16imm: case W65816::ADCabs: case W65816::ADCfi: case W65816::SBCi16imm: case W65816::SBCabs: case W65816::SBCfi: case W65816::ADCEi16imm: case W65816::ADCEabs: case W65816::ADCEfi: case W65816::SBCEi16imm: case W65816::SBCEabs: case W65816::SBCEfi: case W65816::ASLA16: case W65816::LSRA16: case W65816::ASLA8: case W65816::LSRA8: case W65816::INA: case W65816::DEA: case W65816::INA_PSEUDO: case W65816::DEA_PSEUDO: case W65816::INA_PSEUDO8: case W65816::DEA_PSEUDO8: return true; default: return false; } }; // Walk past further STAfi pseudos (spill stores) — they preserve // A's flags. Also walk past STA_DP (plain `sta $dp` doesn't touch // P). Don't walk past STA_DPIndLongY / STA8fi / etc., whose // inserter expansions introduce SEP/REP and mutate the M flag // (that's what broke printf historically). auto NextIt = std::next(MI.getIterator()); while (NextIt != MBB.end()) { if (NextIt->isDebugInstr()) { ++NextIt; continue; } if (NextIt->getOpcode() == W65816::STAfi) { ++NextIt; continue; } if (NextIt->getOpcode() == W65816::STA_DP) { ++NextIt; continue; } break; } if (NextIt == MBB.end() || NextIt->isBranch() || NextIt->isReturn()) return false; if (!NextIt->definesRegister(W65816::P, TRI) && !opSetsNZ(NextIt->getOpcode())) return false; // Eliding the LDA leaves $a "dead per verifier" because STAfi's // tablegen Defs = [A] is a stale over-approximation (the asm // actually preserves A via PHA/lda/sta/PLA bracket for IMG/X/Y // sources, and trivially for $a sources — see W65816InstrInfo.td // and eliminateFrameIndex). Strip the stale implicit-def $a from // walked-past STAfis (any source) and clear any kill flag on the // source operand when source is $a, so the verifier sees $a as // still alive through the chain. We're post-regalloc so this // doesn't affect regalloc's decisions — only the verifier and any // downstream passes that read the annotation. auto stripStaleAdef = [](MachineInstr &MI2) { if (MI2.getOpcode() != W65816::STAfi) return; if (MI2.getNumOperands() < 1 || !MI2.getOperand(0).isReg()) return; if (MI2.getOperand(0).getReg() == W65816::A) { MI2.getOperand(0).setIsKill(false); } for (unsigned i = MI2.getNumOperands(); i-- > 0;) { const MachineOperand &MO = MI2.getOperand(i); if (MO.isReg() && MO.isImplicit() && MO.isDef() && MO.getReg() == W65816::A) { MI2.removeOperand(i); return; } } }; for (auto FixIt = StaMI.getIterator(); FixIt != MI.getIterator(); ++FixIt) { stripStaleAdef(*FixIt); } MI.eraseFromParent(); return true; } // Calls clobber A — be safe. if (MI.isCall()) return false; // STAfi has `Defs = [A]` in its tablegen def (a stale over- // approximation from before the eliminateFrameIndex PHA-bracket // landed for non-A sources). In reality the asm preserves A // for every source class — A source is trivial, IMG/X/Y sources // go through PHA/lda/sta/PLA which restores A. So a STAfi to // a different slot is NOT an A-clobber and shouldn't break the // load-after-store redundancy. STAfi to the SAME slot DOES // invalidate (slot value changed), handled below. bool IsStAFi = (MI.getOpcode() == W65816::STAfi); if (!IsStAFi && MI.modifiesRegister(StoredReg, TRI)) return false; if (IsStAFi && MI.getNumOperands() >= 2 && MI.getOperand(1).isFI() && MI.getOperand(1).getIndex() == StoredFI) return false; ++It; } return false; } bool W65816StackSlotCleanup::runOnMachineFunction(MachineFunction &MF) { const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo(); bool Changed = false; // Pass -4: redundant pointer respill. Pattern that the LDAptrOff + // STAptrOff inserter pair emits when the same pointer is used for // both a load and a store within a loop body: // // LDAfi slot_c ; reload p from its slot (slot_c = p's home) // STAfi slot_A ; spill p to slot_A (for the indirect Y-load) // ... LDA (slot_A,Y) ; INC ... // LDAfi slot_c ; reload p again (same source!) // STAfi slot_B ; spill p to slot_B (for the indirect Y-store) // ...; STA (slot_B),Y // // M[slot_A] and M[slot_B] both hold p — equal. We can redirect any // later use of slot_B to slot_A and drop the LDA+STA pair. The // saving is 2 insns per affected indirect-pair (4 cycles). Only // safe if slot_A wasn't written in between (it isn't — no STAfi to // slot_A appears in the loop) and the second LDA reloads from the // SAME source slot_c. for (MachineBasicBlock &MBB : MF) { SmallVector Ldas; for (MachineInstr &MI : MBB) if (MI.getOpcode() == W65816::LDAfi) Ldas.push_back(&MI); SmallPtrSet Erased; for (MachineInstr *Lda1 : Ldas) { if (Erased.count(Lda1)) continue; int SlotC = matchAccSlotOp(*Lda1, W65816::LDAfi); if (SlotC == NO_SLOT_MATCH) continue; auto It = std::next(Lda1->getIterator()); if (!advancePastDebug(MBB, It)) continue; // Step 2: STAfi slotA. int SlotA = matchAccSlotOp(*It, W65816::STAfi); if (SlotA == NO_SLOT_MATCH || SlotA == SlotC) continue; // Walk forward looking for LDAfi slotC again, with no STAfi // slotA / slotC in between. auto Walker = std::next(It); MachineInstr *Lda2 = nullptr; while (Walker != MBB.end()) { MachineInstr &MI = *Walker; if (MI.isDebugInstr()) { ++Walker; continue; } if (MI.isCall() || MI.isInlineAsm() || MI.isBranch() || MI.isReturn()) break; // STA to slotA or slotC: M might no longer hold the same value. if (MI.getOpcode() == W65816::STAfi && MI.getNumOperands() >= 2 && MI.getOperand(1).isFI()) { int Slot = MI.getOperand(1).getIndex(); if (Slot == SlotA || Slot == SlotC) break; } // Found another LDA from slotC? if (matchAccSlotOp(MI, W65816::LDAfi) == SlotC) { Lda2 = &MI; break; } ++Walker; } if (!Lda2) continue; auto It2 = std::next(Lda2->getIterator()); if (!advancePastDebug(MBB, It2)) continue; // Step 4: STAfi slotB. int SlotB = matchAccSlotOp(*It2, W65816::STAfi); if (SlotB == NO_SLOT_MATCH || SlotB == SlotA || SlotB == SlotC) continue; MachineInstr &Sta2 = *It2; // Walk further to find the indirect use (LDAfi_indY / STAfi_indY) // referencing slotB. Bail on STA to slotA before then. auto It3 = std::next(Sta2.getIterator()); MachineInstr *IndYTarget = nullptr; while (It3 != MBB.end()) { MachineInstr &MI = *It3; if (MI.isDebugInstr()) { ++It3; continue; } if (MI.isCall() || MI.isBranch() || MI.isReturn() || MI.isInlineAsm()) break; // Slot A or C overwritten — bail. if (MI.getOpcode() == W65816::STAfi && MI.getNumOperands() >= 2 && MI.getOperand(1).isFI()) { int Slot = MI.getOperand(1).getIndex(); if (Slot == SlotA || Slot == SlotC) break; } // Indirect-Y operand: operand 1 (load) or 1 (store) holds // the FI pointer slot. Match LDAfi_indY/STAfi_indY using // slotB. if (MI.getOpcode() == W65816::LDAfi_indY || MI.getOpcode() == W65816::STAfi_indY) { for (unsigned i = 0; i < MI.getNumOperands(); ++i) { if (MI.getOperand(i).isFI() && MI.getOperand(i).getIndex() == SlotB) { IndYTarget = &MI; break; } } if (IndYTarget) break; } ++It3; } if (!IndYTarget) continue; // Function-wide safety check (#107): slotB must have no other // refs besides Sta2 (which we erase) and IndYTarget (which we // rewrite). Cross-MBB reads would otherwise see stale data. const MachineInstr *ignoreA[] = {&Sta2, IndYTarget}; if (slotHasOtherRefs(MF, SlotB, ignoreA)) continue; // Apply rewrite: IndYTarget's slotB → slotA. for (unsigned i = 0; i < IndYTarget->getNumOperands(); ++i) { if (IndYTarget->getOperand(i).isFI() && IndYTarget->getOperand(i).getIndex() == SlotB) { IndYTarget->getOperand(i).setIndex(SlotA); break; } } // Mark Lda2 as erased so the outer worklist iteration skips // it (it's an LDAfi and was added to Ldas). Sta2 isn't in // any worklist so erasing it directly is safe. Erased.insert(Lda2); Lda2->eraseFromParent(); Sta2.eraseFromParent(); Changed = true; } } // Pass -4b: redundant pair of consecutive STAfi. Pattern: // // STAfi $a, slotA, 0 // STAfi $a, slotB, 0 ; same value, different slot // ... use slotB as indirect-Y address ... // // Both STAs spill $a's current value, so M[slotA] == M[slotB]. We // can rewrite later indirect-Y uses of slotB to slotA and drop the // second STA. Pattern shows up when an i32 pointer is loaded via // two indirect-Y reads (offsets 0 and 2); the inserter spills the // pointer twice (once per access). for (MachineBasicBlock &MBB : MF) { SmallVector Stas; for (MachineInstr &MI : MBB) if (MI.getOpcode() == W65816::STAfi) Stas.push_back(&MI); SmallPtrSet Erased; for (MachineInstr *Sta1 : Stas) { if (Erased.count(Sta1)) continue; int SlotA = matchAccSlotOp(*Sta1, W65816::STAfi); if (SlotA == NO_SLOT_MATCH) continue; auto It = std::next(Sta1->getIterator()); if (!advancePastDebug(MBB, It)) continue; // Step 2: another STAfi $a, slotB. int SlotB = matchAccSlotOp(*It, W65816::STAfi); if (SlotB == NO_SLOT_MATCH || SlotB == SlotA) continue; MachineInstr &Sta2 = *It; // Walk forward redirecting EVERY slotB reference to slotA, until // we hit a write to slotA (kills the equivalence) or a slotB write // (re-binds slotB to a new value). Bail on calls/branches/asm. // Track whether we rewrote anything; if so, drop Sta2. auto It2 = std::next(Sta2.getIterator()); bool Rewrote = false; while (It2 != MBB.end()) { MachineInstr &MI = *It2; if (MI.isDebugInstr()) { ++It2; continue; } if (MI.isCall() || MI.isBranch() || MI.isReturn() || MI.isInlineAsm()) break; // STA to slotA changes M[slotA]; M[slotA] no longer equals // M[slotB] — bail (any further slotB ref reads the unchanged // M[slotB], which is now distinct). if (MI.getOpcode() == W65816::STAfi && MI.getNumOperands() >= 2 && MI.getOperand(1).isFI() && MI.getOperand(1).getIndex() == SlotA) break; // STA to slotB rebinds slotB; subsequent reads of slotB read // the new value, not slotA. Stop here — the redirects we've // done so far are still valid (they read the pre-write value). bool StaToB = (MI.getOpcode() == W65816::STAfi || MI.getOpcode() == W65816::STA8fi) && MI.getNumOperands() >= 2 && MI.getOperand(1).isFI() && MI.getOperand(1).getIndex() == SlotB; if (StaToB) break; // Any *fi op or indirect-Y referencing slotB → redirect. if (MI.getOpcode() == W65816::LDAfi_indY || MI.getOpcode() == W65816::STAfi_indY || MI.getOpcode() == W65816::LDAfi || MI.getOpcode() == W65816::ADCfi || MI.getOpcode() == W65816::ADCEfi || MI.getOpcode() == W65816::SBCfi || MI.getOpcode() == W65816::SBCEfi || MI.getOpcode() == W65816::ANDfi || MI.getOpcode() == W65816::ORAfi || MI.getOpcode() == W65816::EORfi || MI.getOpcode() == W65816::CMPfi) { for (unsigned i = 0; i < MI.getNumOperands(); ++i) { if (MI.getOperand(i).isFI() && MI.getOperand(i).getIndex() == SlotB) { MI.getOperand(i).setIndex(SlotA); Rewrote = true; break; } } } ++It2; } // Drop Sta2 only if slotB has no remaining references anywhere // in the function — otherwise we'd break a use we couldn't see. // (Sta1 stays; SlotA still has the value, and Sta1 is its def.) if (Rewrote) { bool SlotBStillUsed = false; for (MachineBasicBlock &MBBO : MF) { for (MachineInstr &MIO : MBBO) { if (&MIO == &Sta2) continue; for (const MachineOperand &MO : MIO.operands()) { if (MO.isFI() && MO.getIndex() == SlotB) { SlotBStillUsed = true; break; } } if (SlotBStillUsed) break; } if (SlotBStillUsed) break; } if (!SlotBStillUsed) { Erased.insert(&Sta2); Sta2.eraseFromParent(); } Changed = true; } } } // Pass -4c: redundant single pointer respill. Pattern: // // LDAfi $a, slotC, 0 ; A = M[slotC] (slotC is "p") // STAfi $a, slotB, 0 ; slotB = M[slotC] = "p" // ... non-A-clobbering, no STA to slotC ... // LDAfi_indY/STAfi_indY ..., slotB, 0 // // M[slotB] just mirrors M[slotC], so the indirect-Y access can read // slotC directly. After the rewrite, if slotB has no remaining uses // in the MBB, the LDA+STA respill is dead and we erase both. This is // the loop-counter / pointer-iteration shape that Pass -4 (the // double-respill variant) doesn't catch when only one indirect-Y // happens before the pointer increment. for (MachineBasicBlock &MBB : MF) { SmallVector Ldas; for (MachineInstr &MI : MBB) if (MI.getOpcode() == W65816::LDAfi) Ldas.push_back(&MI); SmallPtrSet Erased; for (MachineInstr *Lda : Ldas) { if (Erased.count(Lda)) continue; int SlotC = matchAccSlotOp(*Lda, W65816::LDAfi); if (SlotC == NO_SLOT_MATCH) continue; auto It = std::next(Lda->getIterator()); if (!advancePastDebug(MBB, It)) continue; int SlotB = matchAccSlotOp(*It, W65816::STAfi); if (SlotB == NO_SLOT_MATCH || SlotB == SlotC) continue; MachineInstr &Sta = *It; // Walk forward through the MBB collecting all indirect-Y uses of // slotB (LDAfi_indY / STAfi_indY referencing it as the pointer // operand). Bail if we see any *other* reference to slotB (a // direct LDAfi/STAfi/etc.) — that means the slot has uses other // than as an indirect-Y pointer and we can't safely rewrite all // of them. Also bail on STA to slotC (kills the equivalence). SmallVector IndYUses; bool OtherUse = false; auto It2 = std::next(Sta.getIterator()); while (It2 != MBB.end()) { MachineInstr &MI = *It2; if (MI.isDebugInstr()) { ++It2; continue; } if (MI.isCall() || MI.isBranch() || MI.isReturn() || MI.isInlineAsm()) break; if (MI.getOpcode() == W65816::STAfi && MI.getNumOperands() >= 2 && MI.getOperand(1).isFI() && MI.getOperand(1).getIndex() == SlotC) break; bool IsIndY = (MI.getOpcode() == W65816::LDAfi_indY || MI.getOpcode() == W65816::STAfi_indY); bool RefsSlotB = false; for (unsigned i = 0; i < MI.getNumOperands(); ++i) { if (MI.getOperand(i).isFI() && MI.getOperand(i).getIndex() == SlotB) { RefsSlotB = true; break; } } if (RefsSlotB) { if (IsIndY) IndYUses.push_back(&MI); else { OtherUse = true; break; } } ++It2; } if (OtherUse || IndYUses.empty()) continue; // After IndYUses, scan rest of MBB for any further reference to // slotB; if none, all uses of slotB are in our IndYUses list and // we can safely redirect them all + erase the LDA+STA. auto LastIt = std::next(IndYUses.back()->getIterator()); bool LaterUse = false; for (auto It3 = LastIt; It3 != MBB.end(); ++It3) { for (const MachineOperand &MO : It3->operands()) { if (MO.isFI() && MO.getIndex() == SlotB) { LaterUse = true; break; } } if (LaterUse) break; } if (LaterUse) continue; // Function-wide safety check (#107): slotB must have no other // refs besides Sta (erased) and the IndY uses (rewritten). // The IndYUses list already collects all in-MBB IndY refs; // anything else (including cross-MBB) would expect the init. SmallVector ignore; ignore.push_back(&Sta); for (MachineInstr *I : IndYUses) ignore.push_back(I); if (slotHasOtherRefs(MF, SlotB, ignore)) continue; // Apply rewrites: redirect every IndY use of slotB → slotC. for (MachineInstr *IndY : IndYUses) { for (unsigned i = 0; i < IndY->getNumOperands(); ++i) { if (IndY->getOperand(i).isFI() && IndY->getOperand(i).getIndex() == SlotB) { IndY->getOperand(i).setIndex(SlotC); break; } } } Erased.insert(Lda); Lda->eraseFromParent(); Sta.eraseFromParent(); Changed = true; } } // Pass -3: hoist `LDX #imm` (constant materialisation into the X // register) out from between a flag-defining op and the consuming // Bxx. LDX physically updates N and Z, but our pseudo lacks // `Defs = [P]` so the scheduler can place it in the test window. // SAFE because: // - LDX writes X; CMP/ORA/etc. read A. Hoisting can't change // what the CMP sees. // - The LDX's source is an immediate — no operand dependency. // - Moving LDX before the CMP just means CMP overwrites the // flags LDX set, which is what we want. // Only LDX-style — `LDA #imm` is NOT safe because CMP reads A and // the hoist would change A's value. Tracked in // memory/project_known_issue_lda_flags.md. for (MachineBasicBlock &MBB : MF) { SmallVector Branches; for (MachineInstr &MI : MBB) { unsigned Opc = MI.getOpcode(); if (Opc == W65816::BEQ || Opc == W65816::BNE || Opc == W65816::BMI || Opc == W65816::BPL) Branches.push_back(&MI); } for (MachineInstr *Br : Branches) { SmallVector ToHoist; MachineInstr *Test = nullptr; for (auto It = std::prev(Br->getIterator()); ; --It) { MachineInstr &MI = *It; if (MI.isDebugInstr()) { if (It == MBB.begin()) break; continue; } // STA preserves flags (the MC variants STA_StackRel / // STA_StackRelIndY only appear post-PEI and are listed here // defensively; pre-PEI we see STAfi / STAfi_indY / STA8fi // pseudos). STA8fi expands to SEP/STA/REP, which preserves // N/Z (only M is touched). if (MI.getOpcode() == W65816::STA_StackRel || MI.getOpcode() == W65816::STA_StackRelIndY || MI.getOpcode() == W65816::STAfi || MI.getOpcode() == W65816::STAfi_indY || MI.getOpcode() == W65816::STA8fi) { if (It == MBB.begin()) break; continue; } // LDX #imm: candidate to hoist. if (MI.getOpcode() == W65816::LDXi16imm && MI.getNumOperands() >= 2 && MI.getOperand(1).isImm()) { ToHoist.push_back(&MI); if (It == MBB.begin()) break; continue; } // First "real" instruction we hit walking back is the flag- // defining test (CMP, ORA, etc.) — stop here. Test = &MI; break; } if (!Test || ToHoist.empty()) continue; for (auto *MI : ToHoist) { MI->removeFromParent(); MBB.insert(Test->getIterator(), MI); Changed = true; } } } // Pass -2.5: BR_CC flag-corruption mitigation via PHP/PLP. When a // flag-test (CMP/ORA/etc.) is followed by P-corrupting ops (LDA/LDX // /AND/etc.) and then a flag-testing branch (Bxx), the branch would // test the corrupting op's N/Z instead of the test's. This is a // real correctness bug — `while (n > 0)` always exits on first // iteration; `eq_test(0)` returns 0; etc. Wrap the corrupting span // with PHP (push flags) / PLP (pop flags), preserving the test's // flags across the corruption. Costs 2 bytes / 8 cycles per // affected pattern, but it's the difference between buggy and // correct code. The 4-block SELECT_CC inserter handles its case // structurally; this catches the BR_CC paths the inserter can't // touch. Only inserts when: // - The branch tests N or Z (BEQ/BNE/BMI/BPL); BCC/BCS test C // and LDA doesn't touch C, so they're not affected. // - There's at least one P-corrupting instruction between the // flag-defining test and the Bxx. // // NOTE: Bxx instructions now declare `Uses = [P]` (W65816InstrInfo.td), // so the pre-RA scheduler / regalloc / peephole won't legally insert // a P-corrupting op between a CMP and the consuming Bxx. This pass // is now mostly defensive — it stays in place to catch any post-RA // pass that might still violate the dep, and to wrap the rare cases // where the IR-level test is a load (LDA flag side-effect) rather // than an explicit CMP. // In VLA functions, FI store pseudos (STAfi, STA8fi, STAfi_indY) // expand at PEI to a 4-MC sequence ending in `LDY $F8` (Y-restore), // which clobbers N/Z. The PHP/PLP wrap pass runs pre-PEI; treating // those pseudos as flag-preserving leaves the trailing LDY outside // the wrap, so a downstream BEQ/BNE reads the LDY's flags instead of // the test's. Treat them as corrupting in VLA functions so the wrap // covers the whole expansion. // VLAFunc: narrow predicate used by the flag-preserving / lda-like // helpers (broadening it to UsesFPRel broke dadd's i64-ABI libcall // flow — the STAfi pseudos in non-VLA large-frame functions don't // need to be marked corrupting for the wrap-detection walk). // UsesFPRel: broader FrameLowering-matching predicate used by the // pseudo-bump's offset-routing check (FP-rel ops must NOT be bumped, // SP-rel ops MUST be bumped; we replicate eliminateFrameIndex's // routing decision below to choose). bool VLAFunc = MF.getFrameInfo().hasVarSizedObjects(); bool UsesFPRel = MF.getFrameInfo().hasVarSizedObjects() || MF.getFrameInfo().estimateStackSize(MF) > 200; for (MachineBasicBlock &MBB : MF) { SmallVector Branches; for (MachineInstr &MI : MBB) { unsigned Opc = MI.getOpcode(); // BCC/BCS/BVC/BVS are collected too, but ONLY for the carry/overflow // corrupters (isCarryCorrupting below): an LDA-like op between a CMP // and a BCC leaves C alone and needs no wrap, which is why they were // never listed here before. ADDframe is different - see its note. if (Opc == W65816::BEQ || Opc == W65816::BNE || Opc == W65816::BMI || Opc == W65816::BPL || Opc == W65816::BCC || Opc == W65816::BCS || Opc == W65816::BVC || Opc == W65816::BVS) Branches.push_back(&MI); } auto isFlagPreserving = [VLAFunc](unsigned Opc) { if (VLAFunc) { // FI store pseudos are flag-corrupting under VLA expansion. if (Opc == W65816::STAfi || Opc == W65816::STAfi_indY || Opc == W65816::STA8fi) return false; } return Opc == W65816::STA_StackRel || Opc == W65816::STA_StackRelIndY || Opc == W65816::STAfi || Opc == W65816::STAfi_indY || Opc == W65816::STA8fi || Opc == W65816::STA_DP || Opc == W65816::STA_Abs || Opc == W65816::STA_Long || Opc == W65816::STX_DP || Opc == W65816::STX_Abs || Opc == W65816::STY_DP || Opc == W65816::STY_Abs; }; auto isFlagDefining = [](const MachineInstr &MI) { // Anything that physically writes A, X, Y, or P updates N/Z (or // P-bits for CMP). We treat any non-store, non-stack-mgmt op // that's not a branch as flag-defining. STA family preserves; // PHA/PLY don't touch flags either; everything else might. unsigned Opc = MI.getOpcode(); switch (Opc) { case W65816::PHA: case W65816::PHX: case W65816::PHY: case W65816::PHP: case W65816::PHB: case W65816::PHD: case W65816::PHK: case W65816::TCS: case W65816::TXS: case W65816::TCD: case W65816::JSLpseudo: case W65816::JSLpseudo32: case W65816::JSL_Long: case W65816::JSR_Abs: case W65816::JMP_Abs: case W65816::BRA: case W65816::RTL: case W65816::RTS: case W65816::REP: case W65816::SEP: case W65816::CLC: case W65816::SEC: case W65816::CLV: case W65816::CLI: case W65816::SEI: case W65816::CLD: case W65816::SED: return false; default: return !MI.isBranch() && !MI.isReturn(); } }; auto isLdaLike = [VLAFunc](unsigned Opc) { if (VLAFunc) { // STAfi-family: see isFlagPreserving comment. They expand to a // sequence whose final LDY $F8 corrupts N/Z; treat as corrupting. if (Opc == W65816::STAfi || Opc == W65816::STAfi_indY || Opc == W65816::STA8fi) return true; } // COPY between physregs: lowers in AsmPrinter to one of TXA/TYA/ // LDA $D? (for IMG↔A bridges) etc. — all of which set N/Z based // on the loaded value. Treating COPY as flag-defining caused the // wrap pass to identify a PHI-elim COPY as the "Test" and wrap // too narrow a range, so the cb-test LDA's flags were trampled // by intervening A-loads before reaching the BEQ. Including // COPY in the corrupting set forces the pass to walk past these // PHI-elim copies to find the real test (a CMP). if (Opc == TargetOpcode::COPY) return true; // ADDframe (the LEA of a stack slot) expands at PEI to `TSC; CLC; // ADC #disp`, which rewrites N/Z (TSC, ADC), C (CLC, ADC) and V // (ADC). It has no Defs=[P] (deliberately, like the loads above) // and isReMaterializable, so post-RA passes are free to place it // between a CMP and its Bxx - which Machine Copy Propagation did // in RetroNet's rnClient.c: `cmp #3; tsc; clc; adc #0xc1; bne` // made an `act.kind == RN_WA_SUBMIT` test branch on (SP+0xC1)!=0, // i.e. never true, so the form was never submitted on the IIgs. // Before this line the walk below treated ADDframe as the flag- // DEFINING test (isFlagDefining's default) and declined to wrap. if (Opc == W65816::ADDframe) return true; // Pure load / register-transfer instructions: only side effect on // flags is N/Z from the loaded/transferred value. Never a "test" // — they just move data. Treated as corruption when between the // real test and a flag-using branch. return Opc == W65816::LDAi16imm || Opc == W65816::LDAi8imm || // LDAi16imm_bank lowers to `lda $BE` (LDA_DP of the program- // bank byte). Under GNO/ME the program bank is non-zero, so // this load sets N/Z != the CMP's result. Between a CMP and a // BEQ/BNE (e.g. the bank half of an i32 `select`/`if(!p)` // pointer pick), it corrupts the test and the wrong half is // selected -- the `%s`-of-stack-string garbling bug. Plain // LDAi16imm was already listed; its _bank sibling must be too. Opc == W65816::LDAi16imm_bank || Opc == W65816::LDXi16imm || Opc == W65816::LDA_StackRel || Opc == W65816::LDA_StackRelIndY || Opc == W65816::LDA_DP || Opc == W65816::LDA_Abs || Opc == W65816::LDA_Long || Opc == W65816::LDA_Imm16 || Opc == W65816::LDA_Imm8 || Opc == W65816::LDX_Imm16 || Opc == W65816::LDX_Imm8 || Opc == W65816::LDX_DP || Opc == W65816::LDX_Abs || Opc == W65816::LDY_Imm16 || Opc == W65816::LDY_Imm8 || Opc == W65816::LDY_DP || Opc == W65816::LDY_Abs || // Pseudo wrappers that lower to LDA #imm. Opc == W65816::LDAfi || Opc == W65816::LDAfi_indY || // Pure-logic NZ-setters: AND / ORA / EOR against an immediate // or absolute operand. These physically set N/Z but // deliberately do NOT model Defs=[P] (see W65816InstrInfo.td: // the bitwise ops "don't read/write the carry flag"), so a // conditional branch can NEVER take its modeled $p from one of // them — at this pass's point every branch's flags come from a // real CMP (Defs=[P]); the CMP-vs-AND/ORA fold that drops the // `cmp #0` runs later. So when one of these is scheduled // BETWEEN a loop-guard CMP and its branch (e.g. a loop- // invariant signed-compare bias `contactY ^ 0x8000` hoisted // into a zero-trip loop's preheader), it silently tramples the // CMP's Z: the guard then branches on the wrong value, enters a // should-be-skipped loop, and spins ~65536 iterations reading // out-of-bounds data (the reported course_support_piece hang). // Treating them as corruption walks the wrap past them to the // real CMP. NOTE: the *fi variants (ANDfi/ORAfi/EORfi) and // ADC/SBC DO model Defs=[P] and can legitimately be the branch's // flag source, so they are handled by $p liveness and are NOT // listed here; same for INA/DEA and the shifts, which the DEC/ // INC and shift-carry idioms may read directly. Opc == W65816::EORi16imm || Opc == W65816::EORabs || Opc == W65816::ANDi16imm || Opc == W65816::ANDabs || Opc == W65816::ORAi16imm || Opc == W65816::ORAabs || // Register transfers — TAX/TXA/TAY/TYA/TXY/TYX update N/Z // based on the transferred value. They're "data movement" // not "comparison"; treat as corruption so the wrap pass // walks past them to the real test. Without this, a loop // like `for (i...) { ...; t = X; ... }` ends up testing // (t != 0) instead of (i != 0) and runs forever. Opc == W65816::TAX || Opc == W65816::TXA || Opc == W65816::TAY || Opc == W65816::TYA || Opc == W65816::TXY || Opc == W65816::TYX; }; auto isStackRel = [](unsigned Opc) { // Stack-relative ops read/write at S+disp. PHP decrements S by 1, // so any STA/LDA d,S between PHP and PLP would land at the wrong // address (off by 1). We must keep these OUTSIDE the wrap. // Includes both the post-lowered MC opcodes (LDA_StackRel etc.) // AND the pseudo *fi opcodes (LDAfi etc.) — eliminateFrameIndex // hasn't run yet when the wrap pass executes, so it's the pseudos // that are actually in the IR. return Opc == W65816::STA_StackRel || Opc == W65816::STA_StackRelIndY || Opc == W65816::LDA_StackRel || Opc == W65816::LDA_StackRelIndY || Opc == W65816::ADC_StackRel || Opc == W65816::SBC_StackRel || Opc == W65816::AND_StackRel || Opc == W65816::ORA_StackRel || Opc == W65816::EOR_StackRel || Opc == W65816::CMP_StackRel || Opc == W65816::LDAfi || Opc == W65816::LDAfi_indY || Opc == W65816::STAfi || Opc == W65816::STAfi_indY || Opc == W65816::STA8fi || Opc == W65816::ADCfi || Opc == W65816::ADCEfi || Opc == W65816::SBCfi || Opc == W65816::SBCEfi || Opc == W65816::ANDfi || Opc == W65816::ORAfi || Opc == W65816::EORfi || Opc == W65816::CMPfi || Opc == W65816::ADDframe; }; // The subset of corrupters that also rewrite C/V, and so must be // wrapped even for a BCC/BCS/BVC/BVS. Only ADDframe today: every // other entry in isLdaLike touches N/Z alone. auto isCarryCorrupting = [](unsigned Opc) { return Opc == W65816::ADDframe; }; for (MachineInstr *Br : Branches) { unsigned BrOpc = Br->getOpcode(); bool TestsNZ = BrOpc == W65816::BEQ || BrOpc == W65816::BNE || BrOpc == W65816::BMI || BrOpc == W65816::BPL; // Walk back from Br looking for the pattern: // ; (mix of preserving + corrupting ops); Br // where is a flag-defining op (CMP/ORA/AND/ADC/...) and // there's at least one corrupting (LDA-like / TXA-like) op between // and Br. Wrap the corrupting region with PHP/PLP so Br // sees 's flags. // // Wrap boundaries: // PHP goes just before the FIRST corrupting op (not just after // Test) so any preserving stack-rel STAs before the first // corruption stay outside the wrap and use the un-decremented S. // PLP goes just after the LAST corrupting op for the same // reason — preserving stack-rel STAs that follow stay outside. // This is critical: PHP changes S by 1, so a `sta 1,s` inside // the wrap writes at the same address PHP just saved P to, // corrupting the saved flags. Caught by an iterative fib loop // that ran forever because PLP loaded a corrupt P value. MachineInstr *Test = nullptr; MachineInstr *FirstCorrupt = nullptr; MachineInstr *LastCorrupt = nullptr; for (auto It = std::prev(Br->getIterator()); ; --It) { MachineInstr &MI = *It; if (!MI.isDebugInstr()) { if (isFlagPreserving(MI.getOpcode())) { // skip } else if (isCarryCorrupting(MI.getOpcode()) || (TestsNZ && isLdaLike(MI.getOpcode()))) { if (!LastCorrupt) LastCorrupt = &MI; FirstCorrupt = &MI; } else if (isLdaLike(MI.getOpcode())) { // An N/Z-only corrupter under a C/V branch: harmless, walk on. } else if (isFlagDefining(MI)) { Test = &MI; break; } else { // Opaque (call, unrelated terminator) — stop. break; } } if (It == MBB.begin()) break; } if (!Test || !FirstCorrupt) continue; // Stack-relative ops inside the wrap need their displacements // bumped by +1 to compensate for PHP's S decrement. Without // this, `lda 5,s` between PHP and PLP reads at (orig_S-1)+5 // = orig_S+4, one byte too low. The pseudo *fi ops carry an // ImmOffset operand that gets folded into the final disp by // eliminateFrameIndex; bumping ImmOffset by 1 produces the // right post-lowered disp. For already-lowered MC ops // (LDA_StackRel etc), bump the disp operand directly. // // CAVEAT for FP-relative functions (see UsesFPRel declaration above): // FI accesses go through FP-relative addressing (eliminateFrameIndex // routes through expandFarFI when FrameLowering captured FP). FP // was captured BEFORE PHP, so (FP),Y reads aren't affected by PHP's // S decrement. Don't bump pseudo *fi ImmOffsets in that case // (already-lowered MC StackRel ops still need the bump — those are // SP-rel). const TargetInstrInfo *TII = MF.getSubtarget().getInstrInfo(); DebugLoc DL = Test->getDebugLoc(); BuildMI(MBB, FirstCorrupt->getIterator(), DL, TII->get(W65816::PHP)); for (auto It = FirstCorrupt->getIterator(); It != std::next(LastCorrupt->getIterator()); ++It) { if (It->isDebugInstr() || !isStackRel(It->getOpcode())) continue; // Pseudo *fi ops: operand layout is (def, FI, ImmOffset, ...). // Bump the Imm at index 2. MC StackRel ops: operand 0 is the // disp Imm (set by eliminateFrameIndex); bump that. unsigned Opc = It->getOpcode(); bool IsPseudo = Opc == W65816::LDAfi || Opc == W65816::LDAfi_indY || Opc == W65816::STAfi || Opc == W65816::STAfi_indY || Opc == W65816::STA8fi || Opc == W65816::ADCfi || Opc == W65816::ADCEfi || Opc == W65816::SBCfi || Opc == W65816::SBCEfi || Opc == W65816::ANDfi || Opc == W65816::ORAfi || Opc == W65816::EORfi || Opc == W65816::CMPfi || Opc == W65816::ADDframe; // For pseudo *fi ops in FP-rel functions: only SOME will end up // SP-rel after PEI (offsets in [0,255]); the rest go through // expandFarFI → `[$F6],Y`. FP-rel access is unaffected by PHP's // S decrement and must NOT be bumped; SP-rel access IS affected // and MUST be bumped. Replicate eliminateFrameIndex's offset // calculation here to decide. Without this, large-frame // functions that mix both addressing modes (e.g. sha256-style // i32-libcall loops) get their FP-rel pseudos bumped, which // shifts reads/writes by one byte and corrupts state at // iteration N proportional to the i32-libcall count. // ADDframe has no FP-relative expansion (eliminateFrameIndex always // emits TSC + ADC #disp), so it is SP-relative in EVERY function and // always needs the bump - exempting it here would leave its address // one byte low inside the wrap in a large-frame function. if (IsPseudo && UsesFPRel && Opc != W65816::ADDframe) { const MachineFrameInfo &MFI = MF.getFrameInfo(); if (It->getOperand(1).isFI()) { int FI = It->getOperand(1).getIndex(); int FrameOffset = MFI.getObjectOffset(FI); int ImmOffset = It->getOperand(2).isImm() ? (int)It->getOperand(2).getImm() : 0; int LoweredOff = FrameOffset + ImmOffset + (int)MFI.getStackSize(); if (FrameOffset < 0) LoweredOff += 1; // Out-of-range or VLA → FP-rel → no bump. if (LoweredOff < 0 || LoweredOff > 0xFF || MFI.hasVarSizedObjects()) continue; // Else SP-rel: fall through and bump ImmOffset. } } unsigned ImmIdx = IsPseudo ? 2 : 0; if (ImmIdx < It->getNumOperands() && It->getOperand(ImmIdx).isImm()) { int64_t v = It->getOperand(ImmIdx).getImm(); It->getOperand(ImmIdx).setImm(v + 1); } } BuildMI(MBB, std::next(LastCorrupt->getIterator()), DL, TII->get(W65816::PLP)); Changed = true; } } // Pass -2c: relaxed mem-to-mem copy elimination across arbitrary // instructions. Pattern: // // LDAfi $a, slotA, 0 ; A = M[slotA] // STAfi $a, slotB, 0 ; M[slotB] = M[slotA] // ... arbitrary instructions, possibly including JSL, ALU, etc., // as long as nothing writes slotA or slotB ... // OPfi $a, slotB, 0 ; reads M[slotB] // // Rewrite OPfi to read slotA and drop the LDA-STA pair if slotB has // no other uses anywhere in the function. Catches the "loop-carry" // shape in `for (i = 0; i < n; i++) sum += ...` where each iteration // re-spills sum to a separate adc-input slot. Pass -2 (the strict // adjacent variant) doesn't catch this because of the JSL / ALU // ops between Sta and the OPfi. for (MachineBasicBlock &MBB : MF) { SmallVector Ldas; for (MachineInstr &MI : MBB) if (MI.getOpcode() == W65816::LDAfi) Ldas.push_back(&MI); SmallPtrSet Erased; for (MachineInstr *Lda : Ldas) { if (Erased.count(Lda)) continue; int SlotA = matchAccSlotOp(*Lda, W65816::LDAfi); if (SlotA == NO_SLOT_MATCH) continue; auto It = std::next(Lda->getIterator()); if (!advancePastDebug(MBB, It)) continue; int SlotB = matchAccSlotOp(*It, W65816::STAfi); if (SlotB == NO_SLOT_MATCH || SlotB == SlotA) continue; MachineInstr &Sta = *It; // Walk forward. Find the FIRST *fi op whose pointer-FI operand // is slotB and rewrite it. We allow calls in between — local // (non-fixed) slots are below-S and not reachable by the callee // (the callee's stack-rel offsets are above its own SP). Fixed // slots are also unreachable for the same reason. Bail on // branches, asm, returns; on STAs writing slotA or slotB. auto It2 = std::next(Sta.getIterator()); MachineInstr *OpfiTarget = nullptr; unsigned RewriteIdx = 0; while (It2 != MBB.end()) { MachineInstr &MI = *It2; if (MI.isDebugInstr()) { ++It2; continue; } if (MI.isInlineAsm() || MI.isBranch() || MI.isReturn()) break; bool StaToA = (MI.getOpcode() == W65816::STAfi || MI.getOpcode() == W65816::STA8fi) && MI.getNumOperands() >= 2 && MI.getOperand(1).isFI() && MI.getOperand(1).getIndex() == SlotA; if (StaToA) break; bool StaToB = (MI.getOpcode() == W65816::STAfi || MI.getOpcode() == W65816::STA8fi) && MI.getNumOperands() >= 2 && MI.getOperand(1).isFI() && MI.getOperand(1).getIndex() == SlotB; if (StaToB) break; unsigned Opc = MI.getOpcode(); bool IsOpFi = (Opc == W65816::ADCfi || Opc == W65816::ADCEfi || Opc == W65816::SBCfi || Opc == W65816::SBCEfi || Opc == W65816::ANDfi || Opc == W65816::ORAfi || Opc == W65816::EORfi || Opc == W65816::CMPfi); if (IsOpFi) { unsigned FiIdx = (Opc == W65816::CMPfi) ? 1 : 2; if (MI.getNumOperands() >= FiIdx + 2 && MI.getOperand(FiIdx).isFI() && MI.getOperand(FiIdx).getIndex() == SlotB && MI.getOperand(FiIdx + 1).isImm() && MI.getOperand(FiIdx + 1).getImm() == 0) { OpfiTarget = &MI; RewriteIdx = FiIdx; break; } } // LDAfi $a, slotB, 0 — final reload before a DPF0 stage. The // LDAptr32 inserter emits this shape per pointer half: // LDAfi $a, srcSlot ; STAfi $a, freshSlot ; [other STAfi] // LDAfi $a, freshSlot ; STA_DP $E0 // Pass -2c can elide the round-trip by retargeting the final LDAfi // to read srcSlot (=SlotA) directly, dropping the LDA+STA pair. // The final LDAfi is a pure A-write so dropping the earlier Lda1 // can't break any intervening A-reader. if (Opc == W65816::LDAfi) { if (MI.getNumOperands() >= 3 && MI.getOperand(0).isReg() && MI.getOperand(0).getReg() == W65816::A && MI.getOperand(1).isFI() && MI.getOperand(1).getIndex() == SlotB && MI.getOperand(2).isImm() && MI.getOperand(2).getImm() == 0) { OpfiTarget = &MI; RewriteIdx = 1; break; } } ++It2; } if (!OpfiTarget) continue; // Verify slotB has no OTHER references in this function (besides // Sta and OpfiTarget). If it does, we can't safely drop Sta. const MachineInstr *ignoreS2c[] = {&Sta, OpfiTarget}; if (slotHasOtherRefs(MF, SlotB, ignoreS2c)) continue; // Apply rewrite. OpfiTarget->getOperand(RewriteIdx).setIndex(SlotA); Erased.insert(Lda); Lda->eraseFromParent(); Sta.eraseFromParent(); Changed = true; } } // Pass -2: collapse `LDAfi slotA; STAfi slotB; LDAfi slotC; OPfi slotB` // to `LDAfi slotC; OPfi slotA`. This is the "memory-to-memory copy // through A" pattern the inserter + regalloc emit when both operands // of OR_RR/AND_RR/EOR_RR/CMP_RR are already-spilled vregs. We're not // using OP commutativity here — after `STAfi $a, slotB` we have // M[slotB] == M[slotA], so reading slotA in place of slotB is a // value-identity rewrite that's safe even for non-commutative OPs // (CMP, SBC). slotA must not be written between the STAfi we erase // and the OPfi we rewrite — the only intervening instruction is the // single LDAfi in step 3, which doesn't write any slot. for (MachineBasicBlock &MBB : MF) { SmallVector Worklist; for (MachineInstr &MI : MBB) if (MI.getOpcode() == W65816::LDAfi) Worklist.push_back(&MI); for (MachineInstr *Lda1 : Worklist) { // Step 1: LDAfi $a, slotA. int SlotA = matchAccSlotOp(*Lda1, W65816::LDAfi); if (SlotA == NO_SLOT_MATCH) continue; auto It = std::next(Lda1->getIterator()); if (!advancePastDebug(MBB, It)) continue; // Step 2: STAfi $a, slotB. int SlotB = matchAccSlotOp(*It, W65816::STAfi); if (SlotB == NO_SLOT_MATCH || SlotA == SlotB) continue; MachineInstr &Sta = *It; ++It; if (!advancePastDebug(MBB, It)) continue; // Step 3: LDAfi $a, slotC (loading the OPfi's tied input). int SlotC = matchAccSlotOp(*It, W65816::LDAfi); if (SlotC == NO_SLOT_MATCH || SlotC == SlotB) continue; ++It; if (!advancePastDebug(MBB, It)) continue; // Step 4: OPfi $a (tied), slotB — accept any *_fi op whose 2nd // operand is the FI we want to redirect. Commutative ops always // match. CMPfi / SBCfi are non-commutative but the rewrite still // preserves the comparison since M[slotA] == M[slotB] here. MachineInstr &Op = *It; unsigned Opc = Op.getOpcode(); bool IsFiOp = isCommutativeFiOp(Opc) || Opc == W65816::CMPfi || Opc == W65816::SBCfi || Opc == W65816::SBCEfi; if (!IsFiOp) continue; // Operand layout: CMPfi has (outs), (ins Acc16:$lhs, memfi:$addr) // → operand 0 = $lhs, operand 1+2 = memfi. All other *fi ops // are (outs Acc16:$dst), (ins Acc16:$src, memfi) → operand 0 = // $dst, 1 = $src, 2+3 = memfi. Pick the right FI operand index. unsigned FiIdx = (Opc == W65816::CMPfi) ? 1 : 2; if (Op.getNumOperands() < FiIdx + 2 || !Op.getOperand(0).isReg() || Op.getOperand(0).getReg() != W65816::A || !Op.getOperand(FiIdx).isFI() || Op.getOperand(FiIdx).getIndex() != SlotB || !Op.getOperand(FiIdx + 1).isImm() || Op.getOperand(FiIdx + 1).getImm() != 0) continue; // Function-wide safety check: slotB must have no other refs // besides Sta (which we erase) and Op (which we rewrite to use // slotA instead). Without this, deleting Sta orphans any other // reader of slotB (e.g. a loop-header LDAfi reading the same // copy that this entry-side STA initialises) — surfaces as the // qsort #70 miscompile where greedy parks a value into a slot // both for the immediate CMPfi and for a downstream loop-body // reload, and Pass -2 silently kills the only initialiser. const MachineInstr *ignoreS2[] = {&Sta, &Op}; if (slotHasOtherRefs(MF, SlotB, ignoreS2)) continue; // Rewrite OP to use slotA, drop Lda1+Sta. Op.getOperand(FiIdx).setIndex(SlotA); Lda1->eraseFromParent(); Sta.eraseFromParent(); Changed = true; } } // Pass -1: redundant double-spill in *_RR custom-inserter expansions. // The OR_RR / AND_RR / EOR_RR / ADC[E]fi / SBC[E]fi inserter spills // its Src2 to a fresh slot so the OPfi can load-fold from there. // When Src2 came from $x (an i32-first-arg-in-A:X hi half) and Src1 // came from $a, the regalloc winds up emitting: // // STAfi $a, slot_a ; regalloc-allocated spill of $a (Src1) // COPY $a = $x ; TXA — reuse $a for Src2 // STAfi $a, slot_b ; inserter-allocated spill of Src2 (now in $a) // LDAfi $a, slot_a ; reload Src1 (the tied input of OPfi) // OPfi $a (tied), slot_b // // Slot_a holds the original Src1 value; slot_b holds Src2's value. // OPfi reads slot_b but Src1 is already in $a — so semantically // we could use slot_a (which already holds Src1's spilled value) // by swapping which operand the OPfi load-folds: // // STAfi $a, slot_a // COPY $a = $x // OPfi $a (tied), slot_a ; uses slot_a; OP is commutative // // Saves: the STAfi to slot_b and the LDAfi from slot_a. Only // valid for *commutative* ops (ADD/AND/OR/EOR — and ADCE/ADCfi // since carry semantics are the same regardless of operand order). // SBC/CMP/SUB are non-commutative; skip them. for (MachineBasicBlock &MBB : MF) { SmallVector Worklist; for (MachineInstr &MI : MBB) if (MI.getOpcode() == W65816::STAfi) Worklist.push_back(&MI); for (MachineInstr *Sta1 : Worklist) { // Step 1: STAfi $a, slot_a (the regalloc-allocated spill of Src1). int SlotA = matchAccSlotOp(*Sta1, W65816::STAfi); if (SlotA == NO_SLOT_MATCH) continue; auto It = std::next(Sta1->getIterator()); if (!advancePastDebug(MBB, It)) continue; // Step 2: COPY $a = (TXA, etc.). MachineInstr &Copy = *It; if (!Copy.isCopy() || Copy.getOperand(0).getReg() != W65816::A) continue; ++It; if (!advancePastDebug(MBB, It)) continue; // Step 3: STAfi $a, slot_b (inserter-allocated spill of Src2). int SlotB = matchAccSlotOp(*It, W65816::STAfi); if (SlotB == NO_SLOT_MATCH || SlotA == SlotB) continue; MachineInstr &Sta2 = *It; ++It; if (!advancePastDebug(MBB, It)) continue; // Step 4: LDAfi $a, slot_a (reload Src1). int SlotL = matchAccSlotOp(*It, W65816::LDAfi); if (SlotL != SlotA) continue; MachineInstr &Lda = *It; ++It; if (!advancePastDebug(MBB, It)) continue; // Step 5: OPfi $a tied, slot_b — must be commutative. MachineInstr &Op = *It; if (!matchCommutativeFiOpOnSlot(Op, SlotB)) continue; // Function-wide safety check: slot_b must have no other refs // besides Sta2 (which we erase) and Op (which we rewrite). // Same class of bug as #70's Pass -2: an inserter-spawned // spill slot can get reused as a regalloc spill home, and // erasing the only initialiser would orphan downstream reads. const MachineInstr *ignoreS1[] = {&Sta2, &Op}; if (slotHasOtherRefs(MF, SlotB, ignoreS1)) continue; // Rewrite Op to use slot_a instead of slot_b, erase Sta2 + Lda. Op.getOperand(2).setIndex(SlotA); Sta2.eraseFromParent(); Lda.eraseFromParent(); Changed = true; } } // Pass 0: rewrite `LDAi16imm $a, imm` immediately followed by // `COPY $x = $a` (with no intervening A clobber) into // `LDXi16imm $x, imm`. Run BEFORE the spill/reload cleanups so // the disappearing A clobber unblocks subsequent STAfi+LDAfi // pair removal. for (MachineBasicBlock &MBB : MF) { SmallVector Worklist; for (MachineInstr &MI : MBB) if (MI.getOpcode() == W65816::LDAi16imm) Worklist.push_back(&MI); for (MachineInstr *Lda : Worklist) { if (Lda->getNumOperands() < 2 || !Lda->getOperand(0).isReg() || Lda->getOperand(0).getReg() != W65816::A) continue; auto It = std::next(Lda->getIterator()); while (It != MBB.end() && It->isDebugInstr()) ++It; if (It == MBB.end()) continue; MachineInstr &Next = *It; if (!Next.isCopy()) continue; Register DstReg = Next.getOperand(0).getReg(); Register SrcReg = Next.getOperand(1).getReg(); if (DstReg != W65816::X || SrcReg != W65816::A) continue; const MachineOperand &ImmMO = Lda->getOperand(1); const TargetInstrInfo *TII = MF.getSubtarget().getInstrInfo(); MachineInstrBuilder Mib = BuildMI(MBB, Lda->getIterator(), Lda->getDebugLoc(), TII->get(W65816::LDXi16imm), W65816::X); if (ImmMO.isImm()) Mib.addImm(ImmMO.getImm()); else Mib.add(ImmMO); Lda->eraseFromParent(); Next.eraseFromParent(); Changed = true; } } // Pass 1: redundant LDAfi after STAfi (load-after-same-store with // matching register). Two-pass over Stores worklist to avoid // iterator invalidation when we erase the LDAfi mid-walk. for (MachineBasicBlock &MBB : MF) { SmallVector Stores; for (MachineInstr &MI : MBB) if (MI.getOpcode() == W65816::STAfi) Stores.push_back(&MI); for (MachineInstr *StaMI : Stores) if (tryEliminateLoadAfterStore(MBB, *StaMI, TRI)) Changed = true; } // Pass 1b: redundant reload of the same slot. Pattern: // LDAfi $a, slotX, 0 // STAfi $a, slotY, 0 ; STA preserves A and doesn't touch slotX // ... (any non-A-defining, non-slotX-storing instructions) // LDAfi $a, slotX, 0 ; <-- redundant: A still holds slotX's value // Walk forward from each LDAfi looking for a matching second LDAfi // with no intervening A-def or slotX-store. Drops the second LDAfi. // This catches the fib-loop pattern where the regalloc emits // LDA X; STA Y; LDA X; ADC Z (the second LDA is dead). for (MachineBasicBlock &MBB : MF) { SmallVector Loads; for (MachineInstr &MI : MBB) if (MI.getOpcode() == W65816::LDAfi) Loads.push_back(&MI); for (MachineInstr *LdaMI : Loads) { int SlotX = matchAccSlotOp(*LdaMI, W65816::LDAfi); if (SlotX == NO_SLOT_MATCH) continue; auto It = std::next(LdaMI->getIterator()); while (It != MBB.end()) { MachineInstr &MI = *It; if (MI.isDebugInstr()) { ++It; continue; } // Found another LDAfi $a from the same slot. LDA sets N/Z; // dropping it could leave a stale N/Z visible to a following // branch. Only drop if the immediately-following instruction // overwrites N/Z (CMP, ADC, AND, ORA, EOR, BIT, etc. — anything // that defines P). In practice the second LDA is followed by // a CLC+ADC or similar arithmetic, so this almost always fires. if (matchAccSlotOp(MI, W65816::LDAfi) == SlotX) { auto NextIt = std::next(It); while (NextIt != MBB.end() && NextIt->isDebugInstr()) ++NextIt; // If we can't see a follower or the follower is a flag-using // branch, leave the LDA alone. if (NextIt == MBB.end() || NextIt->isBranch()) break; MI.eraseFromParent(); Changed = true; break; } // Calls clobber A. if (MI.isCall()) break; // STAfi PRESERVES A in the asm (A source: store-only; non-A // source: PHA bracket round-trip). The pseudo declares // Defs = [A] as a stale over-approximation, so we explicitly // skip STAfi when checking for A-clobber. STAfi to slotX // (same slot) DOES change M[slotX] — bail in that case below. if (MI.getOpcode() != W65816::STAfi && MI.modifiesRegister(W65816::A, TRI)) break; // STAfi to slotX would change M[slotX] — bail. if (MI.getOpcode() == W65816::STAfi && MI.getNumOperands() >= 2 && MI.getOperand(1).isFI() && MI.getOperand(1).getIndex() == SlotX) break; // Inline asm / branch boundaries. if (MI.isInlineAsm() || MI.isBranch() || MI.isReturn()) break; ++It; } } } // Pass 1d: redundant `LDY_Imm16 #N` (Y already holds N). The // LDAptrOff/STAptrOff inserters each emit an `LDY #0` (or `LDY #off`) // before their indirect access; back-to-back load-then-store of the // same pointer ends up with two `LDY #0` in a row. Drop the second // when nothing in between writes Y. Like Pass 1b, bail if the // following instruction is a branch (Y's flag side-effects matter // for branches that test N/Z). for (MachineBasicBlock &MBB : MF) { SmallVector Ldys; for (MachineInstr &MI : MBB) if (MI.getOpcode() == W65816::LDY_Imm16) Ldys.push_back(&MI); SmallPtrSet ErasedY; for (MachineInstr *Ldy : Ldys) { if (ErasedY.count(Ldy)) continue; if (Ldy->getNumOperands() < 1 || !Ldy->getOperand(0).isImm()) continue; int64_t Imm = Ldy->getOperand(0).getImm(); // Walk forward erasing every subsequent matching LDY_Imm16 #Imm // until something invalidates the held Y value (call, Y-def, asm, // branch). Multiple LDYs in a row collapse on the first source. auto It = std::next(Ldy->getIterator()); while (It != MBB.end()) { MachineInstr &MI = *It; if (MI.isDebugInstr()) { ++It; continue; } if (MI.getOpcode() == W65816::LDY_Imm16 && MI.getNumOperands() >= 1 && MI.getOperand(0).isImm() && MI.getOperand(0).getImm() == Imm) { // Bail on branch follower (flag-sensitive — LDY sets N/Z). auto NextIt = std::next(It); while (NextIt != MBB.end() && NextIt->isDebugInstr()) ++NextIt; if (NextIt == MBB.end() || NextIt->isBranch()) break; // Erase and continue walking — there may be more dups. auto Erased_It = It; ++It; ErasedY.insert(&*Erased_It); Erased_It->eraseFromParent(); Changed = true; continue; } if (MI.isCall()) break; if (MI.modifiesRegister(W65816::Y, TRI)) break; // killsRegister: an instruction with `implicit killed $y` USES Y // and that's the LAST use — Y is dead after. We must NOT treat // a subsequent LDY_Imm16 #N as redundant after a kill, because // the held value is conceptually gone. Caught by `addOff(p,i) // { p[i-1] += p[i]; }` where LDY -2 ; LDA_indY (kills Y) ; ... ; // LDY -2 ; STA_indY needs the second LDY to reinitialize Y. if (MI.killsRegister(W65816::Y, TRI)) break; if (MI.isInlineAsm() || MI.isBranch() || MI.isReturn()) break; ++It; } } } // Pass 1c: drop redundant `CMPi16imm $a, 0` that follows an op which // already set N/Z based on $a's new value (ORA/AND/EOR/ADC/SBC/LDA/... // anything that defines $a). Pattern is emitted by the i32-equals-0 // path (i32 (lo|hi) == 0): the OR sets Z, then the SETCC compares // against 0. The second compare is provably redundant because $a // hasn't changed since the previous flag-defining op. // Intra-MBB only — cross-MBB recursion into predecessors was tried // (catches SETCC merge blocks where each pred ends with `lda #c`) // but proved too brittle: predecessors ending with JSLpseudo declare // implicit-def $a but the return-value flags aren't reliably set, // and other corner cases break smoke. auto isATransparent = [](const MachineInstr &MI) { // Stores that don't touch A or P-bits-other-than-via-A. (Byte // stores that internally SEP/REP wrap toggle the M flag, but that // doesn't affect N/Z based on A's current value.) Also call-stack // pseudos (ADJCALLSTACKDOWN / UP) which are zero-effect at this // point in the pipeline (PEI eliminates UP; DOWN is always nil). switch (MI.getOpcode()) { case W65816::STAfi: case W65816::STAfi_indY: case W65816::STA8fi: case W65816::STAabs: case W65816::STA8abs: case W65816::STAptr: case W65816::STBptr: case W65816::STAptrOff: case W65816::STBptrOff: case W65816::ADJCALLSTACKDOWN: // DOWN expands to nothing (PUSH16 chain already shifted SP). // UP is NOT transparent: when PEI doesn't process it, AsmPrinter // emits a TSC/CLC/ADC/TCS sequence that clobbers A and flags. return true; default: return false; } }; // Returns true iff walking back from `Start` (exclusive) finds an // A-modifier as the first non-skip op. Skips debug ops and // A-transparent stores; stops at the first real op. Templated to // accept either iterator or const_iterator (Cmps came from a non- // const iteration; predecessors are walked via const_iterator). auto walkbackBefore = [&](auto Start, auto Begin) -> bool { auto It = Start; while (It != Begin) { --It; if (It->isDebugInstr()) continue; if (isATransparent(*It)) continue; return It->modifiesRegister(W65816::A, TRI); } return false; }; // Pass 1c can only eliminate CMPi16imm $a, 0 if the preceding // A-modifier reliably sets N/Z to reflect A's final value. LDAfi // under FP-rel expansion (`sty $fa ; ldy #imm ; lda [$f6],y ; ldy $fa`) // ends with `ldy` that clobbers N/Z based on OLD Y, not loaded A — so // in FP-rel functions (VLA / huge frame), the CMP is load-bearing. // Skip the whole pass for such functions (saves us from the sum_n // VLA regression that the PHP-wrap-aware variant tripped). bool ssCleanupSPRelOnly = !UsesFPRel; for (MachineBasicBlock &MBB : MF) { SmallVector Cmps; for (MachineInstr &MI : MBB) if (MI.getOpcode() == W65816::CMPi16imm) Cmps.push_back(&MI); for (MachineInstr *Cmp : Cmps) { // Shape: CMPi16imm $a, 0. if (Cmp->getNumOperands() < 2 || !Cmp->getOperand(0).isReg() || Cmp->getOperand(0).getReg() != W65816::A || !Cmp->getOperand(1).isImm() || Cmp->getOperand(1).getImm() != 0) continue; bool Found = walkbackBefore(Cmp->getIterator(), MBB.begin()); if (!Found) continue; // Only eliminate if there are NO LdaLike instructions between // this CMP and the next Bxx (or end of MBB). Otherwise the // CMP is the only flag-setting marker between the test value // and the consuming branch — without it, the Bxx ends up // testing the latest LdaLike's N/Z (typically a PHI-elim COPY // or stack reload that has nothing to do with the original // condition). Caused __adddf3's renormalize while-loop to // skip its body even though `mr & ~mask` was non-zero. bool SafeToErase = true; bool insidePHPWrap = false; for (auto It = std::next(Cmp->getIterator()); It != Cmp->getParent()->end(); ++It) { if (It->isDebugInstr()) continue; if (It->isBranch() || It->isReturn()) break; // PHP/PLP-wrap-aware: only safe when LDAfi-expansion sets N/Z // reliably (SP-rel functions, not FP-rel). if (ssCleanupSPRelOnly && It->getOpcode() == W65816::PHP) { // PHP must be IMMEDIATELY after CMP to capture CMP's flags. if (&*It != &*std::next(Cmp->getIterator())) { SafeToErase = false; break; } insidePHPWrap = true; continue; } if (It->getOpcode() == W65816::PLP) { insidePHPWrap = false; continue; } if (insidePHPWrap) continue; if (It->getOpcode() == TargetOpcode::COPY) { SafeToErase = false; break; } unsigned Opc = It->getOpcode(); // Conservative: any LDA/LDX/LDY/transfer disqualifies erasure. // Stores and stack-mgmt are flag-preserving and OK. switch (Opc) { case W65816::STAfi: case W65816::STAfi_indY: case W65816::STA8fi: case W65816::STA_StackRel: case W65816::STA_StackRelIndY: case W65816::STA_DP: case W65816::STA_Abs: case W65816::STA_Long: case W65816::STX_DP: case W65816::STX_Abs: case W65816::STY_DP: case W65816::STY_Abs: case W65816::ADJCALLSTACKDOWN: case W65816::ADJCALLSTACKUP: case W65816::PHA: case W65816::PHX: case W65816::PHY: continue; } // Anything else (LDA, transfer, ALU op...): bail. SafeToErase = false; break; } if (SafeToErase) { Cmp->eraseFromParent(); Changed = true; } } } // Pass 1e: redundant `ANDi16imm $a, $a, 0xFF`. An i8 value zero- // extended to i16 has high byte = 0; subsequent AND #$FF is a no-op // and just adds a 3-byte instruction. This pattern is emitted twice // by the (zextload-then-spill-twice) shape in *cmp helpers — see // memcmp_local in the smoke-tests. Drop the second AND when: // - first AND was `ANDi16imm $a, $a, 0xFF` // - no A-defining op between them (STAfi, CMP*, etc. are fine) // - second AND is also `ANDi16imm $a, $a, 0xFF` // Flag-safe: both ANDs set N=0, Z=(A==0); after the first, the second // produces identical flags, so dropping it leaves any following Bxx // with the same N/Z values. for (MachineBasicBlock &MBB : MF) { SmallVector Ands; for (MachineInstr &MI : MBB) if (MI.getOpcode() == W65816::ANDi16imm && MI.getNumOperands() >= 3 && MI.getOperand(2).isImm() && MI.getOperand(2).getImm() == 0xFF) Ands.push_back(&MI); SmallPtrSet Erased; for (MachineInstr *And : Ands) { if (Erased.count(And)) continue; auto It = std::next(And->getIterator()); while (It != MBB.end()) { MachineInstr &MI = *It; if (MI.isDebugInstr()) { ++It; continue; } // Match: another `AND #$FF` with A unchanged. if (MI.getOpcode() == W65816::ANDi16imm && MI.getNumOperands() >= 3 && MI.getOperand(2).isImm() && MI.getOperand(2).getImm() == 0xFF) { Erased.insert(&MI); MI.eraseFromParent(); Changed = true; break; } if (MI.isCall() || MI.isInlineAsm() || MI.isBranch() || MI.isReturn()) break; if (MI.modifiesRegister(W65816::A, TRI)) break; ++It; } } } // Pass 1g: redundant AND #$FF after reload of a masked slot. Pattern: // // ANDi16imm $a, $a, 0xFF ; A := A & 0xFF (high byte = 0) // STAfi $a, slotN, 0 ; M[slotN] = A — slot's high byte is 0 // ... ; no STAfi to slotN, no A defs // LDAfi $a, slotN, 0 ; A := M[slotN] — high byte still 0 // ANDi16imm $a, $a, 0xFF ; <-- redundant: A's high byte is 0 // // Drop the second AND. Pass 1e (back-to-back AND #FF) bails on any // A-defining op in between, so it can't see across the LDA reload. // This pass is the "through-memory" complement. Found in find_byte // and other char-iteration loops where the regalloc emits an extra // mask-then-spill-then-reload-then-mask cycle around the comparison. for (MachineBasicBlock &MBB : MF) { SmallVector FirstAnds; for (MachineInstr &MI : MBB) if (MI.getOpcode() == W65816::ANDi16imm && MI.getNumOperands() >= 3 && MI.getOperand(2).isImm() && MI.getOperand(2).getImm() == 0xFF) FirstAnds.push_back(&MI); SmallPtrSet Erased; for (MachineInstr *And1 : FirstAnds) { if (Erased.count(And1)) continue; auto It = std::next(And1->getIterator()); if (!advancePastDebug(MBB, It)) continue; // Step 2: STAfi $a, slotN. int SlotN = matchAccSlotOp(*It, W65816::STAfi); if (SlotN == NO_SLOT_MATCH) continue; // Step 3: walk forward looking for LDAfi from slotN. We allow // arbitrary A modifications in between because the LDAfi reload // re-establishes A as the masked value (M[slotN] still has high // byte = 0 from the And1+Sta we just saw). We ONLY need slotN // itself to be unchanged. Bail on calls (callee can clobber any // local slot indirectly), branches/returns/asm. auto It2 = std::next(It); MachineInstr *Lda = nullptr; while (It2 != MBB.end()) { MachineInstr &MI = *It2; if (MI.isDebugInstr()) { ++It2; continue; } if (MI.isCall() || MI.isInlineAsm() || MI.isBranch() || MI.isReturn()) break; if (MI.getOpcode() == W65816::STAfi && MI.getNumOperands() >= 2 && MI.getOperand(1).isFI() && MI.getOperand(1).getIndex() == SlotN) break; if (matchAccSlotOp(MI, W65816::LDAfi) == SlotN) { Lda = &MI; break; } ++It2; } if (!Lda) continue; // Step 4: must be followed by `ANDi16imm $a, $a, 0xFF`. auto It3 = std::next(Lda->getIterator()); if (!advancePastDebug(MBB, It3)) continue; if (It3->getOpcode() != W65816::ANDi16imm || It3->getNumOperands() < 3 || !It3->getOperand(2).isImm() || It3->getOperand(2).getImm() != 0xFF) continue; MachineInstr &And2 = *It3; Erased.insert(&And2); And2.eraseFromParent(); Changed = true; } } // Pass 2a: function-wide dead-slot stores. If a *local* (non-fixed) // FrameIndex is never read anywhere in the function (no LDAfi from // it, no *fi op consuming it, no indirect-Y use of it as a pointer // slot), then every STAfi/STA8fi that writes to it is dead. This // catches the cross-MBB pattern Pass 2 misses (Pass 2 walks within a // single MBB and bails on branches). // // Conservative: read = any opcode whose listed write-operands don't // include this FI. We approximate by treating the operand at the // STAfi/STA8fi's "addr" position (op 1, the FI; op 2, the imm offset) // as the *only* write. Every other reference is treated as a read. { MachineFrameInfo &MFI = MF.getFrameInfo(); DenseMap Reads; DenseMap Writes; SmallVector Stores; for (MachineBasicBlock &MBB : MF) { for (MachineInstr &MI : MBB) { bool IsStaFi = (MI.getOpcode() == W65816::STAfi || MI.getOpcode() == W65816::STA8fi); for (unsigned i = 0; i < MI.getNumOperands(); ++i) { const MachineOperand &MO = MI.getOperand(i); if (!MO.isFI()) continue; int FI = MO.getIndex(); if (MFI.isFixedObjectIndex(FI)) continue; // For STAfi/STA8fi, the FI operand at i==1 is the *write* // target; everything else is a read of this FI. if (IsStaFi && i == 1) Writes[FI]++; else Reads[FI]++; } if (IsStaFi) Stores.push_back(&MI); } } for (MachineInstr *Sta : Stores) { if (Sta->getNumOperands() < 2 || !Sta->getOperand(1).isFI()) continue; // Volatile stores are observable side effects — never elide. // Caught by SJLJ EH: SjLjEHPrepare emits `store volatile i32 N, // fn_ctx.call_site` before each invoke; the function context's // call_site field is "never read" within main but IS read by // the runtime via gActive — Pass 2a's local liveness can't see // that, so volatile is the right gate. if (Sta->hasOrderedMemoryRef()) continue; int FI = Sta->getOperand(1).getIndex(); if (Reads.count(FI) == 0 && Writes[FI] >= 1) { Sta->eraseFromParent(); Changed = true; } } } // Pass 1f: drop adjacent PHP/PLP pairs. Pass -2.5 inserts PHP/PLP // around LDA-style ops to protect a CMP's flags from being clobbered // by the LDA before the consuming branch. Pass 1 (load-after-store // elimination) sometimes deletes the LDA *between* the wrap because // it's a redundant reload — the spilled value is already in A. After // that deletion, PHP and PLP are back-to-back with nothing between, // and the pair is a no-op. Drop both. for (MachineBasicBlock &MBB : MF) { SmallVector Phps; for (MachineInstr &MI : MBB) if (MI.getOpcode() == W65816::PHP) Phps.push_back(&MI); for (MachineInstr *Php : Phps) { auto It = std::next(Php->getIterator()); while (It != MBB.end() && It->isDebugInstr()) ++It; if (It == MBB.end() || It->getOpcode() != W65816::PLP) continue; MachineInstr *Plp = &*It; Php->eraseFromParent(); Plp->eraseFromParent(); Changed = true; } } // Pass 1h: collapse dead TAX/TXA bridge around DPF0 staging. When // SpillToX bridges one of the LDAptr32 inserter's STAfi/LDAfi pairs // (saving 6 cyc), then Pass -2c collapses the OTHER pair entirely, // the TAX/TXA bridge becomes pure overhead. Pattern: // // LDAfi $a, slotA, 0 ; (1) load ptr_hi (originally for X-bridge) // TAX ; (2) X = ptr_hi // LDAfi $a, slotB, 0 ; (3) load ptr_lo (the Pass -2c rewrite target) // STA_DP imm1 ; (4) $E0 = ptr_lo // TXA ; (5) recover ptr_hi to A // STA_DP imm2 ; (6) $E2 = ptr_hi // // Reorder to drop the bridge: // // LDAfi $a, slotB, 0 ; (3) load ptr_lo // STA_DP imm1 ; (4) $E0 = ptr_lo // LDAfi $a, slotA, 0 ; (1) load ptr_hi (moved down) // STA_DP imm2 ; (6) $E2 = ptr_hi // // Saves 2 instructions (~6 cyc) per LDAptr32 expansion. Safe iff X // is dead after TXA (no read before redef). Common in tight loops // that deref two pointers per iter (memcmp, strcpy, dotProduct). for (MachineBasicBlock &MBB : MF) { SmallVector Ldas; for (MachineInstr &MI : MBB) { if (MI.getOpcode() == W65816::LDAfi) { Ldas.push_back(&MI); } } SmallPtrSet ErasedH; for (MachineInstr *Lda1 : Ldas) { if (ErasedH.count(Lda1)) { continue; } int SlotA = matchAccSlotOp(*Lda1, W65816::LDAfi); if (SlotA == NO_SLOT_MATCH) { continue; } auto It = std::next(Lda1->getIterator()); if (!advancePastDebug(MBB, It)) { continue; } // Optional `ADCEfi $a (tied), slot, 0` between Lda1 and TAX. // strcpy emits `LDAfi hi_part ; ADCEfi $a, $stack.9 ; TAX` to // compute ptr_hi via carry-propagation from a prior ADCfi (lo). // The ADCEfi's carry-in comes from an earlier ADCfi; intervening // LDA/STA preserve C, so reordering this pair past STA_DP is safe. MachineInstr *MaybeAdcE = nullptr; if (It->getOpcode() == W65816::ADCEfi && It->getNumOperands() >= 4 && It->getOperand(0).isReg() && It->getOperand(0).getReg() == W65816::A) { MaybeAdcE = &*It; ++It; if (!advancePastDebug(MBB, It)) { continue; } } if (It->getOpcode() != W65816::TAX) { continue; } MachineInstr *Tax = &*It; ++It; if (!advancePastDebug(MBB, It)) { continue; } int SlotB = matchAccSlotOp(*It, W65816::LDAfi); if (SlotB == NO_SLOT_MATCH || SlotB == SlotA) { continue; } MachineInstr *Lda2 = &*It; ++It; if (!advancePastDebug(MBB, It)) { continue; } if (It->getOpcode() != W65816::STA_DP) { continue; } MachineInstr *Sta1 = &*It; ++It; if (!advancePastDebug(MBB, It)) { continue; } if (It->getOpcode() != W65816::TXA) { continue; } MachineInstr *Txa = &*It; ++It; if (!advancePastDebug(MBB, It)) { continue; } if (It->getOpcode() != W65816::STA_DP) { continue; } MachineInstr *Sta2 = &*It; // Verify X is dead after Txa within this MBB. Tighter check // than just "until branch/return": if we reach end of MBB or a // branch without seeing an X-read or X-def, look at successor // MBBs. X is dead iff no successor's livein set includes X. // Critical for tight loops (strLen, byte-walk loops) where the // loop terminator is a branch back to the same MBB. bool XDeadAfter = true; bool MustCheckSuccessors = false; auto checkIt = std::next(Sta2->getIterator()); while (checkIt != MBB.end()) { if (checkIt->isDebugInstr()) { ++checkIt; continue; } if (checkIt->readsRegister(W65816::X, TRI)) { XDeadAfter = false; break; } if (checkIt->modifiesRegister(W65816::X, TRI)) { // X redefined before any read — dead range ended. break; } if (checkIt->isCall() || checkIt->isInlineAsm()) { XDeadAfter = false; break; } if (checkIt->isBranch() || checkIt->isReturn()) { // Hit the terminator without redef — defer to successor // liveins to determine if X is live-out. MustCheckSuccessors = true; break; } ++checkIt; } if (checkIt == MBB.end()) { MustCheckSuccessors = true; } if (XDeadAfter && MustCheckSuccessors) { // X is dead iff no successor lists it as a livein. for (MachineBasicBlock *Succ : MBB.successors()) { if (Succ->isLiveIn(W65816::X)) { XDeadAfter = false; break; } } } if (!XDeadAfter) { continue; } // Apply: move Lda1 (and optional ADCEfi) to between Sta1 and // Sta2, then erase Tax/Txa. Splicing both to Sta2's iterator // preserves their relative order — second splice ends up nearer // Sta2, so Lda1 first then MaybeAdcE. MBB.splice(Sta2->getIterator(), &MBB, Lda1->getIterator()); if (MaybeAdcE) { MBB.splice(Sta2->getIterator(), &MBB, MaybeAdcE->getIterator()); } Tax->eraseFromParent(); Txa->eraseFromParent(); ErasedH.insert(Lda1); Changed = true; } } // Pass 1z: re-run Pass 1's load-after-store elimination after Pass 1h. // Pass 1h's splice may have created newly-adjacent STAfi+LDAfi pairs // when the moved instructions exposed a producer (e.g., ADCfi) whose // result was already in A at the LDAfi point. Common in strcpy's // pointer-arithmetic: `ADCfi $a, lo_off ; STAfi $a, slot ; LDAfi $a, // slot ; STA_DP` collapses to `ADCfi ; STAfi ; STA_DP` after the LDA // is recognised as redundant. for (MachineBasicBlock &MBB : MF) { SmallVector Stores; for (MachineInstr &MI : MBB) { if (MI.getOpcode() == W65816::STAfi) { Stores.push_back(&MI); } } for (MachineInstr *StaMI : Stores) { if (tryEliminateLoadAfterStore(MBB, *StaMI, TRI)) { Changed = true; } } } // Pass 2: dead stores (STAfi to slot followed by another STAfi to // the same slot with no intervening read). This catches the // arg0_lo "preserve" spill that the regalloc emits even though the // value is consumed by the very next instruction. for (MachineBasicBlock &MBB : MF) { SmallVector Stores; for (MachineInstr &MI : MBB) if (MI.getOpcode() == W65816::STAfi) Stores.push_back(&MI); for (MachineInstr *StaMI : Stores) if (tryEliminateDeadStore(MBB, *StaMI)) Changed = true; } // Pass 3: zero-size unused local frame objects so the // PrologueEpilogue pass shrinks the prologue PHAs / TSC reservation. // Walk the MIR collecting which FIs are still referenced; any // *non-fixed* (local) FI with no remaining reference is dead. We // can't safely remove it (RemoveStackObject can shift indexes); we // just zero-size it via setObjectSize, which is enough for the // frame layout pass to skip it. MachineFrameInfo &MFI = MF.getFrameInfo(); if (MFI.getNumObjects() > 0) { BitVector Used(MFI.getObjectIndexEnd() - MFI.getObjectIndexBegin()); auto Mark = [&](int FI) { int Idx = FI - MFI.getObjectIndexBegin(); if (Idx >= 0 && Idx < (int)Used.size()) Used.set(Idx); }; for (MachineBasicBlock &MBB : MF) for (MachineInstr &MI : MBB) for (MachineOperand &MO : MI.operands()) if (MO.isFI()) Mark(MO.getIndex()); for (int FI = MFI.getObjectIndexBegin(); FI < MFI.getObjectIndexEnd(); ++FI) { // Skip fixed (arg) slots — those are "owned" by the caller. if (MFI.isFixedObjectIndex(FI)) continue; int Idx = FI - MFI.getObjectIndexBegin(); if (Idx < 0 || Idx >= (int)Used.size() || Used.test(Idx)) continue; // Already zero-sized? Skip. if (MFI.getObjectSize(FI) == 0) continue; // Don't touch dead-stripped objects either. if (MFI.isDeadObjectIndex(FI)) continue; MFI.setObjectSize(FI, 0); Changed = true; } } return Changed; }