//===-- W65816StackRelToImg.cpp - Rewrite stack-rel to IMG DP slots -------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===---------------------------------------------------------------------===//
//
// Pre-emit pass.  Runs AFTER W65816StackSlotMerge has collapsed PHI
// pairs and AFTER PEI has baked stack offsets into _StackRel ops.
// Scans for the top-N most-accessed stack-rel slot offsets and rewrites
// every reference to use the DP-addressed IMG0..IMG7 equivalent.
//
// Why post-StackSlotMerge: SSM collapses paired PHI slots so the
// surviving offset is "the" long-lived value (e.g., the i32 accumulator
// halves in sumSquares).  Pre-SSM, the accumulator would be split across
// multiple offsets and we'd promote the wrong ones.
//
// Why post-PEI: by this point, the FI has been resolved to a numeric
// offset.  We're working with MC opcodes: STA_StackRel/LDA_StackRel/
// ADC_StackRel/etc.  No FrameIndex bookkeeping needed.
//
// Why IMG0..IMG7 (caller-save $D0..$DE): callees clobber them per the
// JSL implicit-def, BUT verified libgcc.s helpers (__umulhisi3,
// __mulhi3, etc.) don't actually touch $D0..$DE — only $E0..$E9.  So
// values in $D0..$DE survive these specific libcalls.  Skip the
// promotion if the function calls anything OTHER than a libgcc-safe
// helper.
//
// Per-access cost:
//   STA_StackRel : 5 cyc / 2 B (`sta off,s`)
//   STA_DP       : 4 cyc / 2 B (`sta dp`)    → saves 1 cyc per access
//   LDA_StackRel : 5 cyc / 2 B
//   LDA_DP       : 4 cyc / 2 B
//   ADC_StackRel : 5 cyc / 2 B
//   ADC_DP       : 4 cyc / 2 B
//
// Critical safety: PHP/PLP wraps decrement S by 1 byte, so SP-relative
// offsets INSIDE the wrap get bumped +1.  DP addresses DON'T shift with
// S — they're absolute.  So when we rewrite an op that lives inside a
// PHP/PLP wrap, we need to map the BUMPED offset back to the original
// "logical" offset before deciding it's the same slot.
//
//===---------------------------------------------------------------------===//

#include "W65816.h"
#include "W65816InstrInfo.h"
#include "W65816Subtarget.h"
#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/DenseSet.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/CodeGen/MachineFrameInfo.h"
#include "llvm/CodeGen/MachineFunction.h"
#include "llvm/CodeGen/MachineFunctionPass.h"
#include "llvm/CodeGen/MachineInstrBuilder.h"
#include "llvm/IR/DebugInfoMetadata.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/Format.h"
#include <functional>

using namespace llvm;

#define DEBUG_TYPE "w65816-stackrel-to-img"


namespace {


class W65816StackRelToImg : public MachineFunctionPass {
public:
  static char ID;
  W65816StackRelToImg() : MachineFunctionPass(ID) {}
  StringRef getPassName() const override {
    return "W65816 stack-rel to IMG DP slot rewrite";
  }
  bool runOnMachineFunction(MachineFunction &MF) override;
};


} // namespace


char W65816StackRelToImg::ID = 0;

INITIALIZE_PASS(W65816StackRelToImg, DEBUG_TYPE,
                "W65816 stack-rel to IMG", false, false)


FunctionPass *llvm::createW65816StackRelToImg() {
  return new W65816StackRelToImg();
}


// Thin wrappers over the shared helpers in W65816InstrInfo.h.  Kept as
// local statics so existing call sites in this file don't have to spell
// the namespace.
static unsigned getDpOpcode(unsigned Opc) {
  return W65816Helpers::getDpOpcodeForStackRel(Opc);
}


static bool isStackRelOp(unsigned Opc) {
  return W65816Helpers::isStackRelOpcode(Opc);
}


// Whitelist of libgcc functions verified to not touch IMG0..IMG7 ($D0..$DE).
static bool isImgSafeCall(const MachineInstr &MI) {
  if (!MI.isCall()) return false;
  for (const MachineOperand &MO : MI.operands()) {
    StringRef Name;
    if (MO.isGlobal()) Name = MO.getGlobal()->getName();
    else if (MO.isSymbol()) Name = MO.getSymbolName();
    else continue;
    if (Name == "__umulhisi3" || Name == "__mulhi3" ||
        Name == "__mulsi3" || Name == "__udivhi3" ||
        Name == "__umodhi3" || Name == "__divhi3" ||
        Name == "__modhi3" || Name == "__udivsi3" ||
        Name == "__umodsi3" || Name == "__divsi3" ||
        Name == "__modsi3" || Name == "__ashlhi3" ||
        Name == "__lshrhi3" || Name == "__ashrhi3" ||
        Name == "__ashlsi3" || Name == "__lshrsi3" ||
        Name == "__ashrsi3")
      return true;
    return false;
  }
  return false;
}


// Phase 12 peephole — A-dead PHA/PLA bracket elision.  Two shapes:
//
// (a) PEI single-store IMG-source-STAfi bracket.  When the next op
// after PLA redefines A, the bracket is dead weight:
//
//     PHA ; (LDA_DP $cx | TXA | TYA) ; STA_StackRel (off+2) ; PLA
//     [next redefines A]
//   →
//     (LDA_DP $cx | TXA | TYA) ; STA_StackRel off
//
// (b) ImgCalleeSave multi-store bracket at function entry.  When the
// post-PLA pattern is "STX_DP ... ; STA_StackRel destOff ; [redefines
// A]", the post-PLA STA is storing entry-A to its final slot — we
// reorder by hoisting that STA to BEFORE the bracket, then dropping
// PHA/PLA and reverting inner offsets:
//
//     PHA ; (LDA_DP $cx ; STA_StackRel off+2)×N ; PLA
//     STX_DP $cM ; STA_StackRel destOff
//     [next redefines A]
//   →
//     STA_StackRel destOff           ; hoisted, entry-A → slot first
//     (LDA_DP $cx ; STA_StackRel off)×N
//     STX_DP $cM                     ; STX stays after saves
//     [next op]
//
// Restricted to the entry MBB starting at MBB.begin() to ensure the
// match is an ImgCalleeSave-emitted prologue bracket (and not a mid-
// function bracket where the post-PLA STA is consuming a *different*
// A value than what was preserved).
static bool elidePhaBracket(MachineFunction &MF,
                            const W65816InstrInfo *TII) {
  bool Changed = false;
  auto opNoTouchA = [](unsigned Op) {
    switch (Op) {
    case W65816::STX_DP: case W65816::STX_Abs:
    case W65816::STY_DP: case W65816::STY_Abs:
      return true;
    default:
      return false;
    }
  };
  auto opRedefinesA = [](unsigned Op) {
    switch (Op) {
    case W65816::LDA_DP: case W65816::LDA_StackRel:
    case W65816::LDA_Abs: case W65816::LDA_Imm16:
    case W65816::LDAabs: case W65816::LDAi16imm:
    case W65816::TXA: case W65816::TYA:
    case W65816::PLA:
      return true;
    default:
      return false;
    }
  };

  // --- Case (a): single-store brackets anywhere in any MBB. ---
  for (MachineBasicBlock &MBB : MF) {
    SmallVector<MachineInstr *, 4> ToErase;
    for (auto It = MBB.begin(); It != MBB.end(); ++It) {
      if (It->getOpcode() != W65816::PHA) continue;
      auto Lda = std::next(It);
      if (Lda == MBB.end()) continue;
      unsigned LdaOp = Lda->getOpcode();
      bool LdaIsLoadDp = (LdaOp == W65816::LDA_DP);
      bool LdaIsXfer   = (LdaOp == W65816::TXA || LdaOp == W65816::TYA);
      if (!LdaIsLoadDp && !LdaIsXfer) continue;
      auto Sta = std::next(Lda);
      if (Sta == MBB.end()) continue;
      if (Sta->getOpcode() != W65816::STA_StackRel) continue;
      auto Pla = std::next(Sta);
      if (Pla == MBB.end()) continue;
      if (Pla->getOpcode() != W65816::PLA) continue;
      auto AfterPla = std::next(Pla);
      if (AfterPla == MBB.end()) continue;
      unsigned AfterPlaOp = AfterPla->getOpcode();
      bool AfterDeadA = opRedefinesA(AfterPlaOp);
      // Forward-walk liveness: if AfterPla is a branch and ALL its
      // successors' first ops redefine A (recursing through
      // unconditional-branch trampolines), A is dead.
      if (!AfterDeadA && AfterPla->isBranch()) {
        bool AllDead = true;
        std::function<bool(MachineBasicBlock *, int)> firstRedef =
            [&](MachineBasicBlock *B, int Depth) -> bool {
          if (Depth > 3 || !B || B->empty()) return false;
          MachineInstr &MI = B->front();
          unsigned MOp = MI.getOpcode();
          if (opRedefinesA(MOp)) return true;
          if (MOp == W65816::BRA || MOp == W65816::BRL ||
              MOp == W65816::JMP_Abs) {
            for (auto &MO : MI.operands()) {
              if (MO.isMBB()) {
                return firstRedef(MO.getMBB(), Depth + 1);
              }
            }
          }
          return false;
        };
        for (MachineBasicBlock *Succ : MBB.successors()) {
          if (!firstRedef(Succ, 0)) { AllDead = false; break; }
        }
        if (AllDead && !MBB.succ_empty()) AfterDeadA = true;
      }
      if (!AfterDeadA) continue;
      MachineOperand &OffMO = Sta->getOperand(0);
      if (!OffMO.isImm()) continue;
      int64_t Off = OffMO.getImm();
      if (Off < 2) continue;
      OffMO.setImm(Off - 2);
      ToErase.push_back(&*It);
      ToErase.push_back(&*Pla);
    }
    for (MachineInstr *MI : ToErase) {
      MI->eraseFromParent();
      Changed = true;
    }
  }

  // --- Case (c): multi-pair STA_DP-only bracket anywhere.  ---
  // IMG-to-IMG copies bracketed for A-preservation.  No StackRel
  // offsets to adjust (DP is absolute, immune to PHA shifts), so just
  // drop PHA/PLA when A is dead at PLA's exit.
  std::function<bool(MachineBasicBlock *, int)> firstRedef =
      [&](MachineBasicBlock *B, int Depth) -> bool {
    if (Depth > 3 || !B || B->empty()) return false;
    MachineInstr &MI = B->front();
    unsigned MOp = MI.getOpcode();
    if (opRedefinesA(MOp)) return true;
    if (MOp == W65816::BRA || MOp == W65816::BRL ||
        MOp == W65816::JMP_Abs) {
      for (auto &MO : MI.operands()) {
        if (MO.isMBB()) return firstRedef(MO.getMBB(), Depth + 1);
      }
    }
    return false;
  };
  for (MachineBasicBlock &MBB : MF) {
    SmallVector<MachineInstr *, 4> ToErase;
    for (auto It = MBB.begin(); It != MBB.end(); ++It) {
      if (It->getOpcode() != W65816::PHA) continue;
      // Walk inner LDA_DP + STA_DP pairs.
      auto Inner = std::next(It);
      int InnerPairs = 0;
      bool BailInner = false;
      while (Inner != MBB.end()) {
        if (Inner->getOpcode() == W65816::PLA) break;
        if (Inner->getOpcode() != W65816::LDA_DP) { BailInner = true; break; }
        auto St = std::next(Inner);
        if (St == MBB.end() || St->getOpcode() != W65816::STA_DP) {
          BailInner = true; break;
        }
        ++InnerPairs;
        Inner = std::next(St);
      }
      if (BailInner || Inner == MBB.end() || InnerPairs < 1) continue;
      // Inner == PLA.  Check liveness after PLA.
      auto Post = std::next(Inner);
      if (Post == MBB.end()) continue;
      unsigned PostOp = Post->getOpcode();
      bool ADead = opRedefinesA(PostOp);
      if (!ADead && Post->isBranch()) {
        bool AllDead = true;
        for (MachineBasicBlock *Succ : MBB.successors()) {
          if (!firstRedef(Succ, 0)) { AllDead = false; break; }
        }
        if (AllDead && !MBB.succ_empty()) ADead = true;
      }
      if (!ADead) continue;
      // Eligible: drop PHA + PLA (no offset adjustment for DP).
      ToErase.push_back(&*It);
      ToErase.push_back(&*Inner);
    }
    for (MachineInstr *MI : ToErase) {
      MI->eraseFromParent();
      Changed = true;
    }
  }

  // --- Case (b): ImgCalleeSave prologue bracket in entry MBB. ---
  // PHA must be the FIRST instruction (or first after PEI prologue ops
  // like REP/TAY/TSC/SEC/SBC/TCS/TYA) in the entry MBB.  This ensures
  // we're looking at the prologue's IMG save block.
  MachineBasicBlock &EntryMBB = MF.front();
  auto BB = EntryMBB.begin();
  // Skip PEI prologue ops to reach the first ImgCalleeSave PHA.
  while (BB != EntryMBB.end()) {
    unsigned Op = BB->getOpcode();
    if (Op == W65816::PHA) break;
    // PEI prologue ops we expect to see before ImgCalleeSave's PHA.
    if (Op == W65816::REP || Op == W65816::TAY ||
        Op == W65816::TSC || Op == W65816::SEC ||
        Op == W65816::SBC_Imm16 || Op == W65816::TCS ||
        Op == W65816::TYA) {
      ++BB;
      continue;
    }
    BB = EntryMBB.end();  // not a recognized prologue shape — bail
    break;
  }
  if (BB != EntryMBB.end() && BB->getOpcode() == W65816::PHA) {
    SmallVector<MachineInstr *, 8> InnerStas;
    auto Inner = std::next(BB);
    bool BailInner = false;
    while (Inner != EntryMBB.end()) {
      unsigned IOp = Inner->getOpcode();
      if (IOp == W65816::PLA) break;
      // Inner must be alternating LDA_DP + STA_StackRel pairs.
      if (IOp != W65816::LDA_DP) { BailInner = true; break; }
      auto St = std::next(Inner);
      if (St == EntryMBB.end() || St->getOpcode() != W65816::STA_StackRel) {
        BailInner = true; break;
      }
      MachineOperand &OffMO = St->getOperand(0);
      if (!OffMO.isImm() || OffMO.getImm() < 2) {
        BailInner = true; break;
      }
      InnerStas.push_back(&*St);
      Inner = std::next(St);
    }
    if (!BailInner && Inner != EntryMBB.end() && !InnerStas.empty()) {
      // Inner == PLA.  Walk forward through STX_DP / STY_DP (A-
      // transparent) ops looking for STA_StackRel that consumes
      // entry-A, then verify next op redefines A.
      auto Post = std::next(Inner);
      while (Post != EntryMBB.end() && opNoTouchA(Post->getOpcode())) {
        ++Post;
      }
      if (Post != EntryMBB.end() &&
          Post->getOpcode() == W65816::STA_StackRel) {
        auto AfterSta = std::next(Post);
        if (AfterSta != EntryMBB.end() &&
            opRedefinesA(AfterSta->getOpcode())) {
          // Eligible.  Move STA destOff to right BEFORE PHA, drop
          // PHA + PLA, shift inner STA offsets by -2.
          MachineInstr *StaToMove = &*Post;
          MachineInstr *PhaMI = &*BB;
          MachineInstr *PlaMI = &*Inner;
          // splice: move StaToMove to position just before PhaMI.
          EntryMBB.splice(PhaMI->getIterator(), &EntryMBB,
                          StaToMove->getIterator());
          for (MachineInstr *Sta : InnerStas) {
            Sta->getOperand(0).setImm(Sta->getOperand(0).getImm() - 2);
          }
          PhaMI->eraseFromParent();
          PlaMI->eraseFromParent();
          Changed = true;
        }
      }
    }
  }
  return Changed;
}


// Always-on: elide the STA $E0 / LDA $E0 round-trip in
// ADJCALLSTACKUP's Y-live i64-return path when the next instruction
// after the LDA is `STA_StackRel off,s` storing A to a slot.  The
// emitted PEI sequence (see W65816FrameLowering ADJCALLSTACKUP):
//
//     STA_DP $E0                ; save A across TSC
//     TSC                       ; A = S
//     CLC ; ADC_Imm16 #N ; TCS  ; pop N bytes
//     LDA_DP $E0                ; restore A
//     STA_StackRel off, s       ; store A to slot
//
// If the destination's pre-adjust offset (off + N) fits in a 1-byte
// stack-rel encoding, we can move the STA up to BEFORE the SP-adjust
// (using the pre-adjust offset) and drop both the save and reload.
//
// Saves 6 bytes + 8 cyc per match.  evalAt has 4 of these.
static bool elideCallResultSaveSPReload(MachineFunction &MF,
                                        const W65816InstrInfo *TII) {
  bool Changed = false;
  for (MachineBasicBlock &MBB : MF) {
    SmallVector<MachineInstr *, 4> ToErase;
    for (auto It = MBB.begin(); It != MBB.end(); ++It) {
      if (It->getOpcode() != W65816::STA_DP) continue;
      MachineOperand &SaveImm = It->getOperand(0);
      if (!SaveImm.isImm() || SaveImm.getImm() != 0xE0) continue;
      auto I1 = std::next(It);
      if (I1 == MBB.end() || I1->getOpcode() != W65816::TSC) continue;
      auto I2 = std::next(I1);
      if (I2 == MBB.end() || I2->getOpcode() != W65816::CLC) continue;
      auto I3 = std::next(I2);
      if (I3 == MBB.end() || I3->getOpcode() != W65816::ADC_Imm16) continue;
      MachineOperand &AdcImm = I3->getOperand(0);
      if (!AdcImm.isImm()) continue;
      int64_t N = AdcImm.getImm();
      auto I4 = std::next(I3);
      if (I4 == MBB.end() || I4->getOpcode() != W65816::TCS) continue;
      auto I5 = std::next(I4);
      if (I5 == MBB.end() || I5->getOpcode() != W65816::LDA_DP) continue;
      MachineOperand &LoadImm = I5->getOperand(0);
      if (!LoadImm.isImm() || LoadImm.getImm() != 0xE0) continue;
      auto I6 = std::next(I5);
      if (I6 == MBB.end() || I6->getOpcode() != W65816::STA_StackRel) continue;
      MachineOperand &StaImm = I6->getOperand(0);
      if (!StaImm.isImm()) continue;
      int64_t Off = StaImm.getImm();
      int64_t NewOff = Off + N;
      if (NewOff < 0 || NewOff > 255) continue;
      // Insert a new STA_StackRel at NewOff before the STA_DP $E0.
      BuildMI(MBB, It, It->getDebugLoc(), TII->get(W65816::STA_StackRel))
          .addImm(NewOff);
      ToErase.push_back(&*It);   // STA_DP $E0
      ToErase.push_back(&*I5);   // LDA_DP $E0
      ToErase.push_back(&*I6);   // original STA_StackRel
    }
    for (MachineInstr *MI : ToErase) {
      MI->eraseFromParent();
      Changed = true;
    }
  }
  return Changed;
}


// Returns true if the opcode is "transparent" to a STA→LDA forward —
// does not write A, does not change S, does not write to any stack
// memory.  Used to widen the elideStoreForwarding peephole's window.
static bool isStaLdaTransparent(unsigned Opc) {
  switch (Opc) {
  // X/Y register ops (don't touch A or S)
  case W65816::LDX_Imm16: case W65816::LDX_DP: case W65816::LDX_Abs:
  case W65816::LDXi16imm:
  case W65816::LDY_Imm16: case W65816::LDY_DP: case W65816::LDY_Abs:
  case W65816::TAX: case W65816::TAY:
  case W65816::INX: case W65816::INY:
  case W65816::DEX: case W65816::DEY:
  case W65816::STX_DP: case W65816::STX_Abs:
  case W65816::STY_DP: case W65816::STY_Abs:
  // Flag ops
  case W65816::CLC: case W65816::SEC:
  case W65816::CLD: case W65816::SED:
  case W65816::CLI: case W65816::SEI:
  case W65816::CLV:
  case W65816::NOP:
    return true;
  default:
    return false;
  }
}


// Always-on: drop a redundant LDA following STA to the same slot when
// any intermediate ops are "transparent" (don't write A or change S
// or stack memory).  STA doesn't modify A, so A still holds the value.
//
//     STA off, s
//     LDX #imm     ; transparent
//     LDA off, s   ; redundant — A unchanged since STA
//
// Saves 1 instruction (3 bytes / 4 cyc) per match.
static bool elideStoreForwarding(MachineFunction &MF) {
  bool Changed = false;
  for (MachineBasicBlock &MBB : MF) {
    SmallVector<MachineInstr *, 4> ToErase;
    for (auto It = MBB.begin(); It != MBB.end(); ++It) {
      if (It->getOpcode() != W65816::STA_StackRel) continue;
      MachineOperand &S = It->getOperand(0);
      if (!S.isImm()) continue;
      int64_t StaOff = S.getImm();
      // Walk forward up to 3 ops looking for matching LDA.
      MachineBasicBlock::iterator Walk = std::next(It);
      int Steps = 0;
      while (Walk != MBB.end() && Steps < 3) {
        unsigned WOp = Walk->getOpcode();
        if (WOp == W65816::LDA_StackRel) {
          MachineOperand &L = Walk->getOperand(0);
          if (L.isImm() && L.getImm() == StaOff) {
            ToErase.push_back(&*Walk);
          }
          break;
        }
        if (!isStaLdaTransparent(WOp)) break;
        ++Walk;
        ++Steps;
      }
    }
    for (MachineInstr *MI : ToErase) {
      MI->eraseFromParent();
      Changed = true;
    }
  }
  return Changed;
}


// Always-on: drop a consecutive PLA/PHA pair.  PLA restores A from
// the stack; PHA immediately pushes the same A back.  Net is a no-op
// in both A and stack memory.  Emerges when multiple adjacent IMG
// copies are each bracketed with PHA/PLA for A-preservation:
//
//     PHA ; LDA dp ; STA dp ; PLA ; PHA ; LDA dp ; STA dp ; PLA
//                                   ^^^^^^^^^^
//                                   collapsed away
//
// Saves 2 instructions (2 bytes / 7 cyc) per match.
static bool elidePlaPhaPair(MachineFunction &MF) {
  bool Changed = false;
  for (MachineBasicBlock &MBB : MF) {
    SmallVector<MachineInstr *, 4> ToErase;
    for (auto It = MBB.begin(); It != MBB.end(); ++It) {
      if (It->getOpcode() != W65816::PLA) continue;
      auto I1 = std::next(It);
      if (I1 == MBB.end() || I1->getOpcode() != W65816::PHA) continue;
      ToErase.push_back(&*It);
      ToErase.push_back(&*I1);
      ++It;  // advance past PHA (already-to-erase)
    }
    for (MachineInstr *MI : ToErase) {
      MI->eraseFromParent();
      Changed = true;
    }
  }
  return Changed;
}


// Always-on: drop a redundant LDA when the prior LDA loaded the same
// source and the only intervening instruction was PHA (which reads A
// but doesn't modify it).  Emerges from i64 arg-push sequences:
//
//     LDA off, s
//     PHA
//     LDA off, s   ; A still has this value — redundant
//     PHA
//
// Saves 1 instruction (3 bytes / 4 cyc) per match.
static bool elideRedundantLdaAfterPha(MachineFunction &MF) {
  bool Changed = false;
  for (MachineBasicBlock &MBB : MF) {
    SmallVector<MachineInstr *, 4> ToErase;
    for (auto It = MBB.begin(); It != MBB.end(); ++It) {
      unsigned Op = It->getOpcode();
      bool IsLdaSr = (Op == W65816::LDA_StackRel);
      bool IsLdaDp = (Op == W65816::LDA_DP);
      if (!IsLdaSr && !IsLdaDp) continue;
      auto I1 = std::next(It);
      if (I1 == MBB.end() || I1->getOpcode() != W65816::PHA) continue;
      auto I2 = std::next(I1);
      if (I2 == MBB.end() || I2->getOpcode() != Op) continue;
      MachineOperand &S1 = It->getOperand(0);
      MachineOperand &S2 = I2->getOperand(0);
      if (!S1.isImm() || !S2.isImm()) continue;
      if (S1.getImm() != S2.getImm()) continue;
      ToErase.push_back(&*I2);
    }
    for (MachineInstr *MI : ToErase) {
      MI->eraseFromParent();
      Changed = true;
    }
  }
  return Changed;
}


// Always-on: drop a dead STA in the i32-carry-propagation pattern:
//
//     STA_StackRel off, s
//     ADC_Imm16 #N         ; doesn't touch slot
//     STA_StackRel off, s  ; overwrites first STA
//
// The first STA's value is shadowed by the second.  Drop it.
// Saves 1 instruction (3 bytes / 5 cyc) per match.
static bool elideStackRelDeadStore(MachineFunction &MF) {
  bool Changed = false;
  auto isStackRelRead = [](unsigned Op) {
    switch (Op) {
    case W65816::LDA_StackRel: case W65816::ADC_StackRel:
    case W65816::SBC_StackRel: case W65816::AND_StackRel:
    case W65816::ORA_StackRel: case W65816::EOR_StackRel:
    case W65816::CMP_StackRel:
      return true;
    }
    return false;
  };
  auto isStackRelIndY = [](unsigned Op) {
    switch (Op) {
    case W65816::LDA_StackRelIndY: case W65816::STA_StackRelIndY:
    case W65816::ADC_StackRelIndY: case W65816::SBC_StackRelIndY:
    case W65816::AND_StackRelIndY: case W65816::ORA_StackRelIndY:
    case W65816::EOR_StackRelIndY: case W65816::CMP_StackRelIndY:
      return true;
    }
    return false;
  };
  for (MachineBasicBlock &MBB : MF) {
    SmallVector<MachineInstr *, 8> ToErase;
    SmallPtrSet<MachineInstr *, 8> ErasedSet;
    for (auto It = MBB.begin(); It != MBB.end(); ++It) {
      if (ErasedSet.count(&*It)) continue;
      if (It->getOpcode() != W65816::STA_StackRel) continue;
      if (It->getNumOperands() < 1 || !It->getOperand(0).isImm()) continue;
      int64_t OrigSlot = It->getOperand(0).getImm();
      int64_t SpAdj = 0;
      auto Walk = std::next(It);
      while (Walk != MBB.end()) {
        if (Walk->isDebugInstr()) { ++Walk; continue; }
        if (Walk->isBranch() || Walk->isCall() || Walk->isReturn() ||
            Walk->isInlineAsm()) break;
        unsigned WO = Walk->getOpcode();
        switch (WO) {
        case W65816::PHA: case W65816::PHX: case W65816::PHY:
        case W65816::PEA: case W65816::PEI_DP: case W65816::PER:
          SpAdj -= 2; ++Walk; continue;
        case W65816::PLA: case W65816::PLX: case W65816::PLY:
          SpAdj += 2; ++Walk; continue;
        case W65816::PHP:
          SpAdj -= 1; ++Walk; continue;
        case W65816::PLP:
          SpAdj += 1; ++Walk; continue;
        }
        if (WO == W65816::STA_StackRel &&
            Walk->getNumOperands() >= 1 && Walk->getOperand(0).isImm() &&
            Walk->getOperand(0).getImm() + SpAdj == OrigSlot) {
          ToErase.push_back(&*It);
          ErasedSet.insert(&*It);
          break;
        }
        if (Walk->getNumOperands() >= 1 && Walk->getOperand(0).isImm()) {
          int64_t Imm = Walk->getOperand(0).getImm();
          if (isStackRelRead(WO) || WO == W65816::STA_StackRel) {
            if (Imm + SpAdj == OrigSlot) break;
          }
          if (isStackRelIndY(WO)) {
            if (Imm + SpAdj == OrigSlot || Imm + SpAdj + 1 == OrigSlot)
              break;
          }
        }
        ++Walk;
      }
    }
    for (MachineInstr *MI : ToErase) {
      MI->eraseFromParent();
      Changed = true;
    }
  }
  return Changed;
}


static bool elideDeadStaCarry(MachineFunction &MF) {
  bool Changed = false;
  for (MachineBasicBlock &MBB : MF) {
    SmallVector<MachineInstr *, 4> ToErase;
    for (auto It = MBB.begin(); It != MBB.end(); ++It) {
      if (It->getOpcode() != W65816::STA_StackRel) continue;
      auto I1 = std::next(It);
      if (I1 == MBB.end()) continue;
      unsigned MidOp = I1->getOpcode();
      bool IsAddImm = (MidOp == W65816::ADC_Imm16 ||
                       MidOp == W65816::ADCi16imm ||
                       MidOp == W65816::ADCEi16imm ||
                       MidOp == W65816::SBCi16imm ||
                       MidOp == W65816::SBCEi16imm);
      if (!IsAddImm) continue;
      auto I2 = std::next(I1);
      if (I2 == MBB.end() || I2->getOpcode() != W65816::STA_StackRel) continue;
      MachineOperand &Off1 = It->getOperand(0);
      MachineOperand &Off2 = I2->getOperand(0);
      if (!Off1.isImm() || !Off2.isImm()) continue;
      if (Off1.getImm() != Off2.getImm()) continue;
      ToErase.push_back(&*It);
    }
    for (MachineInstr *MI : ToErase) {
      MI->eraseFromParent();
      Changed = true;
    }
  }
  return Changed;
}


bool W65816StackRelToImg::runOnMachineFunction(MachineFunction &MF) {
  if (skipFunction(MF.getFunction())) return false;
  if (MF.getFunction().hasOptNone()) return false;

  // 1. Bail if the function has VLA / FP-rel addressing — offsets would
  // be from FP not SP and the PHP-wrap +1 adjustment differs.
  if (MF.getFrameInfo().hasVarSizedObjects()) return false;

  // Always-on peepholes that run even when the main IMG promotion bails.
  const W65816Subtarget &STIp = MF.getSubtarget<W65816Subtarget>();
  // Run PLA;PHA collapse FIRST so adjacent brackets merge into a
  // single multi-pair bracket — lets elidePhaBracket case (c) match
  // the merged shape.
  bool ChangedEarly = elidePlaPhaPair(MF);
  ChangedEarly |= elidePhaBracket(MF, STIp.getInstrInfo());
  ChangedEarly |= elideCallResultSaveSPReload(MF, STIp.getInstrInfo());
  ChangedEarly |= elideDeadStaCarry(MF);
  ChangedEarly |= elideRedundantLdaAfterPha(MF);
  // elideStoreForwarding only when main IMG promotion would bail —
  // running it early in non-bailing functions cascades into IMG-slot
  // reallocation that regresses strcpy 1.63×.  Gated below.

  // 2. Bail if the function has any non-IMG-safe call (would clobber
  // our IMG0..7 promotions) or is recursive (same).  Tried allowing
  // IMG8..15 + own-pass save/restore for these cases (today, after
  // landing W65816LowerWide32 + ImgCalleeSave-writes-only fixes), and
  // saw: evalAt 498→500 (NET LOSS due to save/restore overhead) AND
  // qsort #70 regression.  The IMG8..15 path is not currently a win
  // for our benchmarks; reverted.
  StringRef SelfName = MF.getName();
  for (MachineBasicBlock &MBB : MF) {
    for (MachineInstr &MI : MBB) {
      if (!MI.isCall()) continue;
      // Check: recursive self-call first.  Apply stack-rel dead-store
      // elim here since IMG promotion can't run (recursion clobbers
      // IMG slots across the inner call).
      bool isSelfCall = false;
      for (const MachineOperand &MO : MI.operands()) {
        StringRef Name;
        if (MO.isGlobal()) Name = MO.getGlobal()->getName();
        else if (MO.isSymbol()) Name = MO.getSymbolName();
        else continue;
        if (Name == SelfName) { isSelfCall = true; break; }
      }
      if (isSelfCall) {
        ChangedEarly |= elideStackRelDeadStore(MF);
        ChangedEarly |= elideStoreForwarding(MF);
        return ChangedEarly;
      }
      if (!isImgSafeCall(MI)) {
        ChangedEarly |= elideStoreForwarding(MF);
        return ChangedEarly;
      }
    }
  }
  uint8_t imgBase = 0xD0u;

  // 3. Count stack-rel accesses per offset.  CRITICAL: the stack
  // pointer shifts during the function due to PHP/PLP (+1 byte) and
  // PHA/PUSH16/PEA/PLA/PLY/PLX/PHX/PHY (±2 bytes).  ADJCALLSTACK
  // pseudos and JSL/RTL also shift SP.  An access at numerical offset
  // O when SP has shifted by K maps to LOGICAL offset (O - K).
  //
  // We compute SpAdj at every program point and use it to canonicalize
  // offsets.  If our dataflow can't determine SpAdj at some MBB entry
  // (multiple predecessors with different shifts), we bail.
  //
  // Per-MBB analysis: walk forward, tracking running SpAdj.
  // Inter-MBB: we require ALL paths to a MBB have the same SpAdj.
  DenseMap<MachineBasicBlock *, int> MBBEntrySpAdj;
  DenseMap<MachineBasicBlock *, int> MBBExitSpAdj;
  MBBEntrySpAdj[&MF.front()] = 0;
  bool SpAdjValid = true;
  SmallVector<MachineBasicBlock *, 16> Worklist;
  Worklist.push_back(&MF.front());
  SmallPtrSet<MachineBasicBlock *, 16> Visited;
  auto miSpDelta = [](const MachineInstr &MI) -> int {
    switch (MI.getOpcode()) {
    case W65816::PHA: case W65816::PHX: case W65816::PHY:
    case W65816::PUSH16: case W65816::PEA:
      return -2;
    case W65816::PLA: case W65816::PLX: case W65816::PLY:
      return 2;
    case W65816::PHP:
      return -1;
    case W65816::PLP:
      return 1;
    default:
      break;
    }
    // ADJCALLSTACK* / TCS / TSC / SEC...SBC...TCS sequences are
    // unhandled — they can shift SP arbitrarily.  Caller must bail.
    return 0;
  };
  while (!Worklist.empty() && SpAdjValid) {
    MachineBasicBlock *MBB = Worklist.pop_back_val();
    if (!Visited.insert(MBB).second) continue;
    int Sp = MBBEntrySpAdj[MBB];
    for (MachineInstr &MI : *MBB) {
      // TCS resets SP to the canonical value for the region.  In
      // well-formed code, the prologue's TCS sets SP for the body
      // and the epilogue's TCS undoes it.  Since stack-rel accesses
      // don't span the prologue/epilogue boundaries (the SP-shifts
      // they imply happen ONLY within prologue or epilogue, not
      // body), treat TCS as "Sp = 0" (canonical body SP).
      if (MI.getOpcode() == W65816::TCS) {
        Sp = 0;
        continue;
      }
      Sp += miSpDelta(MI);
    }
    MBBExitSpAdj[MBB] = Sp;
    for (MachineBasicBlock *Succ : MBB->successors()) {
      auto It = MBBEntrySpAdj.find(Succ);
      if (It == MBBEntrySpAdj.end()) {
        MBBEntrySpAdj[Succ] = Sp;
        Worklist.push_back(Succ);
      } else if (It->second != Sp) {
        // Conflicting SP at successor entry → bail.
        SpAdjValid = false;
        break;
      }
    }
  }
  if (!SpAdjValid) {
    ChangedEarly |= elideStoreForwarding(MF);
    return ChangedEarly;
  }

  DenseMap<int64_t, unsigned> AccessCount;
  DenseMap<int64_t, unsigned> ReadCount;
  DenseMap<int64_t, unsigned> WriteCount;
  DenseMap<int64_t, SmallVector<MachineInstr *, 8>> AccessSites;
  // Slots referenced by *_StackRelIndY (ptr32 Layer 2 deref pseudos).
  // Those uses read the slot as a POINTER via `(d,S),Y` — the addressing
  // mode REQUIRES stack-rel access; we cannot promote the slot to DP.
  DenseSet<int64_t> IndYBlocked;
  auto isStackRelIndY = [](unsigned Op) {
    switch (Op) {
    case W65816::LDA_StackRelIndY: case W65816::STA_StackRelIndY:
    case W65816::ADC_StackRelIndY: case W65816::SBC_StackRelIndY:
    case W65816::CMP_StackRelIndY: case W65816::AND_StackRelIndY:
    case W65816::ORA_StackRelIndY: case W65816::EOR_StackRelIndY:
      return true;
    }
    return false;
  };
  // Also need to remember the SpAdj at each access site so the rewrite
  // can compute the right DP address (DP doesn't shift, but we need
  // to know which logical slot the access refers to).
  DenseMap<MachineInstr *, int> SiteSpAdj;
  for (MachineBasicBlock &MBB : MF) {
    int Sp = MBBEntrySpAdj[&MBB];
    for (MachineInstr &MI : MBB) {
      if (MI.getOpcode() == W65816::TCS) {
        Sp = 0;
        continue;
      }
      if (isStackRelOp(MI.getOpcode())) {
        if (MI.getNumOperands() >= 1 && MI.getOperand(0).isImm()) {
          int64_t Off = MI.getOperand(0).getImm();
          int64_t LogicalOff = Off + Sp;  // canonical (SP-adj subtracted)
          AccessCount[LogicalOff]++;
          if (MI.getOpcode() == W65816::STA_StackRel)
            WriteCount[LogicalOff]++;
          else
            ReadCount[LogicalOff]++;
          AccessSites[LogicalOff].push_back(&MI);
          SiteSpAdj[&MI] = Sp;
        }
      } else if (isStackRelIndY(MI.getOpcode())) {
        if (MI.getNumOperands() >= 1 && MI.getOperand(0).isImm()) {
          int64_t Off = MI.getOperand(0).getImm();
          IndYBlocked.insert(Off + Sp);
        }
      }
      Sp += miSpDelta(MI);
    }
  }

  if (AccessCount.empty()) {
    ChangedEarly |= elideStoreForwarding(MF);
    return ChangedEarly;
  }

  // 3b. Scan for existing DP-immediate usage in [imgBase..imgBase+0xE].
  // Regalloc / backend may have already used these slots (via STX_DP /
  // STA_DP / LDA_DP) for spills or COPY-via-DP patterns.  Don't
  // double-use those slots.
  auto dpImmInRange = [&](const MachineInstr &MI) -> int {
    unsigned Op = MI.getOpcode();
    switch (Op) {
    case W65816::LDA_DP: case W65816::STA_DP:
    case W65816::ADC_DP: case W65816::SBC_DP:
    case W65816::CMP_DP: case W65816::AND_DP:
    case W65816::ORA_DP: case W65816::EOR_DP:
    case W65816::STX_DP: case W65816::STY_DP:
    case W65816::STZ_DP:
      if (MI.getNumOperands() >= 1 && MI.getOperand(0).isImm()) {
        int64_t I = MI.getOperand(0).getImm();
        if (I >= imgBase && I <= imgBase + 0xE && (I & 1) == 0)
          return static_cast<int>((I - imgBase) / 2);  // 0..7
      }
      break;
    }
    return -1;
  };
  BitVector UsedDp(8, false);
  for (MachineBasicBlock &MBB : MF) {
    for (MachineInstr &MI : MBB) {
      int idx = dpImmInRange(MI);
      if (idx >= 0) UsedDp.set(idx);
    }
  }

  // 4. Sort offsets by access count.  Pick top 2.
  SmallVector<int64_t, 16> Ordered;
  for (auto &P : AccessCount) Ordered.push_back(P.first);
  std::sort(Ordered.begin(), Ordered.end(),
            [&](int64_t A, int64_t B) {
              return AccessCount[A] > AccessCount[B];
            });

  // 5. Promote top 2 offsets to IMG0 / IMG1.
  //
  // Heuristic: only promote when the function has FEW (<=4) hot slots.
  // Functions with many hot slots tend to be using stack slots as
  // SCRATCH temps; promoting them doesn't help and creates many
  // STA/LDA pairs to the SAME DP slot that don't optimize well
  // (bsearch regression observed at 5+ hot slots).  Functions with
  // 1-4 hot slots tend to have a long-lived accumulator-like value
  // where promotion IS a win (memcmp/strcpy/sumOfSquares observed
  // wins at this threshold).
  const unsigned kThreshold = 3u;
  const unsigned kMaxPromote = 8u;
  const unsigned kMaxHotSlots = 16u;
  unsigned HotCount = 0;
  for (int64_t Off : Ordered) {
    if (AccessCount[Off] >= kThreshold) ++HotCount;
  }
  if (HotCount > kMaxHotSlots) {
    ChangedEarly |= elideStoreForwarding(MF);
    return ChangedEarly;
  }
  DenseMap<int64_t, uint8_t> OffsetToDp;  // logical offset -> DP byte
  unsigned NextDpIdx = 0;
  // Caller-passed arg slots live ABOVE the return address on the stack;
  // post-frame-alloc, they're at offsets >= frame_size + 4.  Don't
  // promote them — caller wrote to the stack location, not to DP.
  int FrameSize = static_cast<int>(MF.getFrameInfo().getStackSize());
  int ArgSlotMinOff = FrameSize + 4;
  for (int64_t Off : Ordered) {
    if (AccessCount[Off] < kThreshold) break;
    if (OffsetToDp.size() >= kMaxPromote) break;
    // Skip arg slots (offset >= frame_size + 4 from canonical SP).
    if (Off >= ArgSlotMinOff) continue;
    // Skip slots used as Layer 2 ptr32 deref source: those have
    // `(d,S),Y` uses that require stack-rel addressing.
    if (IndYBlocked.count(Off)) continue;
    // Skip already-used DP slots ($D0..$DE).
    while (NextDpIdx < 8 && UsedDp.test(NextDpIdx)) ++NextDpIdx;
    if (NextDpIdx >= 8) break;
    OffsetToDp[Off] = static_cast<uint8_t>(imgBase + 2 * NextDpIdx);
    ++NextDpIdx;
  }
  // 6. Rewrite each access site.  (OffsetToDp may be empty — phases
  // 2-5 still run as standalone peepholes.)
  const W65816Subtarget &STI = MF.getSubtarget<W65816Subtarget>();
  const W65816InstrInfo *TII = STI.getInstrInfo();
  const TargetRegisterInfo *TRI = STI.getRegisterInfo();
  bool Changed = false;

  // DBG_VALUE preservation for stack -> DP promotion.
  //
  // Post-PEI, DBG_VALUEs that point at a stack slot use a
  // <frame-reg, leading-offset-in-DIExpression> form.  After we promote
  // the slot to a DP address, the stack memory at that offset is no
  // longer maintained; the live value lives at $00:DpAddr.
  //
  // The fully-correct rewrite would replace the DBG_VALUE's location
  // expression with something like `DW_OP_constu <DpAddr>, DW_OP_deref`
  // (an absolute-address dereference, since DP addresses live in
  // bank 0 by definition).  Doing this safely requires DIExpression
  // re-uniquing and is sensitive to whether the original expression
  // was direct vs indirect.  Until our DWARF consumer (pc2line.py
  // expanded to a full DIE walker in Phase 3.2) can evaluate the
  // resulting expressions, the conservative behaviour is to mark
  // matching DBG_VALUEs as undef — the variable drops to
  // <optimized out> at the affected PC range rather than reporting
  // stale stack memory.
  //
  // We collect the renamed offsets here; the actual DBG_VALUE walk
  // happens once at the end so we don't repeatedly scan for each
  // promoted slot.
  for (MachineBasicBlock &MBB : MF) {
    for (MachineInstr &MI : MBB) {
      if (!MI.isDebugValue()) continue;
      const DIExpression *Expr = MI.getDebugExpression();
      if (!Expr) continue;
      int64_t LeadingOff = 0;
      SmallVector<uint64_t, 4> Remaining;
      if (!Expr->extractLeadingOffset(LeadingOff, Remaining))
        continue;
      if (!OffsetToDp.count(LeadingOff)) continue;
      for (MachineOperand &Op : MI.debug_operands()) {
        if (Op.isReg()) {
          Op.setReg(0);
          Op.setSubReg(0);
        }
      }
    }
  }

  for (auto &P : OffsetToDp) {
    int64_t LogicalOff = P.first;
    uint8_t DpAddr = P.second;
    for (MachineInstr *MI : AccessSites[LogicalOff]) {
      unsigned NewOpc = getDpOpcode(MI->getOpcode());
      if (!NewOpc) continue;
      MachineBasicBlock *MBB = MI->getParent();
      DebugLoc DL = MI->getDebugLoc();
      MachineInstrBuilder NewMI =
          BuildMI(*MBB, MI, DL, TII->get(NewOpc)).addImm(DpAddr);
      // Carry implicit operands from the original MI.
      for (const MachineOperand &MO : MI->operands()) {
        if (MO.isImm()) continue;  // the offset, already replaced
        if (MO.isReg() && !MO.isImplicit()) {
          // shouldn't happen for stack-rel ops (only 1 explicit operand)
          continue;
        }
        if (MO.isReg() && MO.isImplicit()) {
          RegState Flags = MO.isDef() ? RegState::ImplicitDefine
                                       : RegState::Implicit;
          if (MO.isKill()) Flags = Flags | RegState::Kill;
          if (MO.isDead()) Flags = Flags | RegState::Dead;
          NewMI.addReg(MO.getReg(), Flags);
        }
      }
      MI->eraseFromParent();
      Changed = true;
    }
  }

  // Phase 2: after promotion, look for `LDA_DP X ; INA ; STA_DP X`
  // (or with DEA) patterns and fold to `INC_DP X` (DEC_DP X).  These
  // are very common in loop body: `i++` and `counter--` both produce
  // LDA-modify-STA round trips.  INC dp = 6 cyc, 2 bytes; the chain
  // = 10 cyc, 7 bytes.  Saves 4 cyc and 5 bytes per occurrence.
  //
  // Safety: INC/DEC dp set N/Z based on the NEW memory value (same as
  // INA/DEA on A).  Any consumer expecting N/Z still works.  The
  // A-clobber DOES differ: after LDA-INA-STA, A holds the new value;
  // after INC dp, A is unchanged.  So only fire when A is dead after
  // the STA (= next instruction redefines A or A is killed).
  for (MachineBasicBlock &MBB : MF) {
    SmallVector<MachineInstr *, 4> Dead;
    for (auto It = MBB.begin(); It != MBB.end(); ++It) {
      if (It->getOpcode() != W65816::LDA_DP) continue;
      if (It->getNumOperands() < 1 || !It->getOperand(0).isImm()) continue;
      int64_t LdaAddr = It->getOperand(0).getImm();
      auto Next = std::next(It);
      while (Next != MBB.end() && Next->isDebugInstr()) ++Next;
      if (Next == MBB.end()) continue;
      bool isInc = Next->getOpcode() == W65816::INA ||
                   Next->getOpcode() == W65816::INA_PSEUDO;
      bool isDec = Next->getOpcode() == W65816::DEA ||
                   Next->getOpcode() == W65816::DEA_PSEUDO;
      if (!isInc && !isDec) continue;
      auto Sta = std::next(Next);
      while (Sta != MBB.end() && Sta->isDebugInstr()) ++Sta;
      if (Sta == MBB.end()) continue;
      if (Sta->getOpcode() != W65816::STA_DP) continue;
      if (Sta->getNumOperands() < 1 || !Sta->getOperand(0).isImm()) continue;
      int64_t StaAddr = Sta->getOperand(0).getImm();
      if (LdaAddr != StaAddr) continue;
      // Check A is dead after the STA: next instruction must redefine
      // A (LDA, transfer, PLA, etc.) or be a terminator (function exit).
      // Helper: scan forward from It, looking for A-redef.  Returns
      // true if A is dead (next operation that touches A defines, not
      // uses).  Recurses into successor MBBs (one level only).
      auto isADeadFromHere = [](MachineBasicBlock::iterator It,
                                 MachineBasicBlock &CurrMBB,
                                 int depth) -> bool {
        if (depth > 1) return false;  // cap recursion
        auto Cur = It;
        while (Cur != CurrMBB.end()) {
          if (Cur->isDebugInstr()) { ++Cur; continue; }
          bool usesA = false, defsA = false;
          for (const MachineOperand &MO : Cur->operands()) {
            if (MO.isReg() && MO.getReg() == W65816::A) {
              if (MO.isDef()) defsA = true;
              else if (MO.isUse()) usesA = true;
            }
          }
          if (Cur->isCall()) defsA = true;  // caller-save
          if (defsA) return true;
          if (usesA) return false;
          ++Cur;
        }
        return false;
      };
      // Scan in current MBB.
      auto After = std::next(Sta);
      bool aDead = isADeadFromHere(After, MBB, 0);
      // If not resolved in current MBB, check successors.  ALL
      // successor entries must define A as their first A-touch AND
      // not also have any A-using terminator in CurrMBB before the
      // jump (because A-use in the branch would consume our value).
      if (!aDead) {
        // Walk to end of MBB; if any A-use is encountered, bail.
        bool foundAUse = false;
        for (auto It2 = After; It2 != MBB.end(); ++It2) {
          if (It2->isDebugInstr()) continue;
          for (const MachineOperand &MO : It2->operands()) {
            if (MO.isReg() && MO.getReg() == W65816::A &&
                MO.isUse() && !MO.isDef()) {
              foundAUse = true;
              break;
            }
          }
          if (foundAUse) break;
        }
        if (!foundAUse) {
          bool allSuccDeadA = !MBB.succ_empty();
          for (MachineBasicBlock *Succ : MBB.successors()) {
            if (!isADeadFromHere(Succ->begin(), *Succ, 1)) {
              allSuccDeadA = false;
              break;
            }
          }
          if (allSuccDeadA) aDead = true;
        }
      }
      if (!aDead) continue;
      // Emit INC_DP / DEC_DP.
      DebugLoc DL = It->getDebugLoc();
      BuildMI(MBB, It, DL, TII->get(isInc ? W65816::INC_DP : W65816::DEC_DP))
          .addImm(LdaAddr);
      Dead.push_back(&*It);
      Dead.push_back(&*Next);
      Dead.push_back(&*Sta);
    }
    for (MachineInstr *MI : Dead) {
      MI->eraseFromParent();
      Changed = true;
    }
  }

  // Phase 3: epilogue TAY/TYA bracket elimination when the bracketed
  // load is from DP (which survives TCS).  Pattern:
  //   LDA_DP <Lo>
  //   TAY                  ; preserve A across TCS
  //   TSC ; CLC ; ADC ; TCS  ; frame teardown
  //   TYA                  ; restore A
  //   RTL (or RTS/RTI)
  // Rewrite to:
  //   TSC ; CLC ; ADC ; TCS
  //   LDA_DP <Lo>          ; load after teardown — DP unaffected
  //   RTL
  // Saves 2 inst (TAY + TYA) per epilogue.  Cycle save: ~4 cyc.
  for (MachineBasicBlock &MBB : MF) {
    // Find the RTL/RTS/RTI terminator.
    auto Last = MBB.getLastNonDebugInstr();
    if (Last == MBB.end()) continue;
    unsigned LastOpc = Last->getOpcode();
    if (LastOpc != W65816::RTL && LastOpc != W65816::RTS &&
        LastOpc != W65816::RTI)
      continue;
    // Walk back looking for the pattern.
    auto Tya = Last;
    if (Tya == MBB.begin()) continue;
    --Tya;
    while (Tya != MBB.begin() && Tya->isDebugInstr()) --Tya;
    if (Tya->getOpcode() != W65816::TYA) continue;
    auto Tcs = Tya;
    if (Tcs == MBB.begin()) continue;
    --Tcs;
    while (Tcs != MBB.begin() && Tcs->isDebugInstr()) --Tcs;
    if (Tcs->getOpcode() != W65816::TCS) continue;
    auto Adc = Tcs;
    if (Adc == MBB.begin()) continue;
    --Adc;
    while (Adc != MBB.begin() && Adc->isDebugInstr()) --Adc;
    if (Adc->getOpcode() != W65816::ADC_Imm16 &&
        Adc->getOpcode() != W65816::ADCi16imm)
      continue;
    auto Clc = Adc;
    if (Clc == MBB.begin()) continue;
    --Clc;
    while (Clc != MBB.begin() && Clc->isDebugInstr()) --Clc;
    if (Clc->getOpcode() != W65816::CLC) continue;
    auto Tsc = Clc;
    if (Tsc == MBB.begin()) continue;
    --Tsc;
    while (Tsc != MBB.begin() && Tsc->isDebugInstr()) --Tsc;
    if (Tsc->getOpcode() != W65816::TSC) continue;
    auto Tay = Tsc;
    if (Tay == MBB.begin()) continue;
    --Tay;
    while (Tay != MBB.begin() && Tay->isDebugInstr()) --Tay;
    if (Tay->getOpcode() != W65816::TAY) continue;
    auto LdaLo = Tay;
    if (LdaLo == MBB.begin()) continue;
    --LdaLo;
    while (LdaLo != MBB.begin() && LdaLo->isDebugInstr()) --LdaLo;
    if (LdaLo->getOpcode() != W65816::LDA_DP) continue;
    if (LdaLo->getNumOperands() < 1 || !LdaLo->getOperand(0).isImm())
      continue;
    int64_t LoAddr = LdaLo->getOperand(0).getImm();
    // Build new LDA_DP after TCS, before TYA→RTL (will replace TYA).
    DebugLoc DL = LdaLo->getDebugLoc();
    BuildMI(MBB, Tya, DL, TII->get(W65816::LDA_DP))
        .addImm(LoAddr)
        .addReg(W65816::A, RegState::ImplicitDefine);
    // Erase: old LDA_DP, TAY, TYA.
    LdaLo->eraseFromParent();
    Tay->eraseFromParent();
    Tya->eraseFromParent();
    Changed = true;
  }

  // Phase 4: prologue TAY/TYA bracket elimination when the bracketed
  // store is to DP.  Pattern at function entry:
  //   REP 48
  //   TAY                  ; preserve A across TSC
  //   TSC ; SEC ; SBC ; TCS  ; frame allocation
  //   TYA                  ; restore A
  //   STA_DP <addr>        ; save A (arg) to DP slot
  // Rewrite to:
  //   REP 48
  //   STA_DP <addr>        ; save A BEFORE TSC clobbers it
  //   TSC ; SEC ; SBC ; TCS
  // Saves 2 inst (TAY + TYA).
  {
    MachineBasicBlock &EntryMBB = MF.front();
    auto Rep = EntryMBB.getFirstNonDebugInstr();
    if (Rep != EntryMBB.end() && Rep->getOpcode() == W65816::REP) {
      auto Tay = std::next(Rep);
      while (Tay != EntryMBB.end() && Tay->isDebugInstr()) ++Tay;
      if (Tay != EntryMBB.end() && Tay->getOpcode() == W65816::TAY) {
        auto Tsc = std::next(Tay);
        while (Tsc != EntryMBB.end() && Tsc->isDebugInstr()) ++Tsc;
        if (Tsc != EntryMBB.end() && Tsc->getOpcode() == W65816::TSC) {
          auto Sec = std::next(Tsc);
          while (Sec != EntryMBB.end() && Sec->isDebugInstr()) ++Sec;
          if (Sec != EntryMBB.end() && Sec->getOpcode() == W65816::SEC) {
            auto Sbc = std::next(Sec);
            while (Sbc != EntryMBB.end() && Sbc->isDebugInstr()) ++Sbc;
            if (Sbc != EntryMBB.end() &&
                (Sbc->getOpcode() == W65816::SBC_Imm16 ||
                 Sbc->getOpcode() == W65816::SBCi16imm)) {
              auto Tcs = std::next(Sbc);
              while (Tcs != EntryMBB.end() && Tcs->isDebugInstr()) ++Tcs;
              if (Tcs != EntryMBB.end() && Tcs->getOpcode() == W65816::TCS) {
                auto Tya = std::next(Tcs);
                while (Tya != EntryMBB.end() && Tya->isDebugInstr()) ++Tya;
                if (Tya != EntryMBB.end() && Tya->getOpcode() == W65816::TYA) {
                  // Walk past A-transparent ops (STX_DP, STY_DP) — these
                  // don't touch A, so TAY/TYA can still be removed.
                  auto Sta = std::next(Tya);
                  while (Sta != EntryMBB.end() &&
                         (Sta->isDebugInstr() ||
                          Sta->getOpcode() == W65816::STX_DP ||
                          Sta->getOpcode() == W65816::STY_DP)) {
                    ++Sta;
                  }
                  if (Sta != EntryMBB.end() &&
                      Sta->getNumOperands() >= 1 &&
                      Sta->getOperand(0).isImm()) {
                    unsigned StaOp = Sta->getOpcode();
                    bool IsStaDp = (StaOp == W65816::STA_DP);
                    bool IsStaSr = (StaOp == W65816::STA_StackRel);
                    if (IsStaDp || IsStaSr) {
                      // For STA_StackRel: pre-TCS offset = post-TCS_off - N
                      // where N = SBC immediate.  Only valid if off >= N.
                      int64_t StaAddr = Sta->getOperand(0).getImm();
                      int64_t SbcImm = Sbc->getOperand(0).isImm()
                          ? Sbc->getOperand(0).getImm() : -1;
                      // Drop ADCi16imm pseudo-tied operands: imm is at op 0 for
                      // SBC_Imm16 but op 2 for SBCi16imm — handle uniformly.
                      if (!Sbc->getOperand(0).isImm() &&
                          Sbc->getNumOperands() >= 3 &&
                          Sbc->getOperand(2).isImm()) {
                        SbcImm = Sbc->getOperand(2).getImm();
                      }
                      int64_t NewAddr = IsStaDp ? StaAddr : (StaAddr - SbcImm);
                      bool OffOk = IsStaDp || (NewAddr >= 1 && SbcImm > 0);
                      // Safety: the op after the spill-STA must REDEFINE A
                      // (not read it).  Otherwise A would be lost (TCS
                      // clobbered it).
                      auto Next = std::next(Sta);
                      while (Next != EntryMBB.end() && Next->isDebugInstr())
                        ++Next;
                      bool NextRedef = false;
                      if (Next != EntryMBB.end()) {
                        unsigned NOp = Next->getOpcode();
                        NextRedef =
                            NOp == W65816::LDA_DP || NOp == W65816::LDA_StackRel ||
                            NOp == W65816::LDA_Abs || NOp == W65816::LDA_Imm16 ||
                            NOp == W65816::LDAabs || NOp == W65816::LDAi16imm ||
                            NOp == W65816::TXA || NOp == W65816::TYA ||
                            NOp == W65816::PLA;
                      }
                      if (OffOk && NextRedef) {
                        // Build new STA_<DP|StackRel> between REP and TSC.
                        DebugLoc DL = Sta->getDebugLoc();
                        BuildMI(EntryMBB, Tsc, DL, TII->get(StaOp))
                            .addImm(NewAddr)
                            .addReg(W65816::A, RegState::Implicit);
                        // Erase: TAY, TYA, old STA.
                        Tay->eraseFromParent();
                        Tya->eraseFromParent();
                        Sta->eraseFromParent();
                        Changed = true;
                      }
                    }
                  }
                }
              }
            }
          }
        }
      }
    }
  }

  // Phase 5: dead STA before pop.  Pattern:
  //   STA_StackRel <off>      ; writes to SP+off
  //   PLY/PLA/PLX             ; pops SP+1..SP+2 (SP += 2)
  //   ...
  // If `off == 1` (or off is within the popped range), the STA writes
  // to memory that's about to be popped/overwritten with no read in
  // between.  STA itself preserves A and P, so erasing is safe IF
  // nothing between STA and the pop reads the same memory.
  for (MachineBasicBlock &MBB : MF) {
    SmallVector<MachineInstr *, 4> Dead;
    for (auto It = MBB.begin(); It != MBB.end(); ++It) {
      if (It->getOpcode() != W65816::STA_StackRel) continue;
      if (It->getNumOperands() < 1 || !It->getOperand(0).isImm()) continue;
      int64_t StaOff = It->getOperand(0).getImm();
      // Walk forward: track running SP delta (just due to pops; if we
      // hit any other op that uses StaOff's memory, bail).
      int popDelta = 0;
      bool dead = false;
      auto Fwd = std::next(It);
      while (Fwd != MBB.end()) {
        if (Fwd->isDebugInstr()) { ++Fwd; continue; }
        unsigned O = Fwd->getOpcode();
        if (O == W65816::PLA || O == W65816::PLX || O == W65816::PLY) {
          // SP += 2.  If the STA's offset falls within the popped
          // range (StaOff <= popDelta + 2 AND > popDelta), the STA
          // wrote memory that's now being popped — dead.
          if (StaOff > popDelta && StaOff <= popDelta + 2) {
            dead = true;
            break;
          }
          popDelta += 2;
          ++Fwd;
          continue;
        }
        if (O == W65816::PLP) {
          // SP += 1.
          if (StaOff > popDelta && StaOff <= popDelta + 1) {
            dead = true;
            break;
          }
          popDelta += 1;
          ++Fwd;
          continue;
        }
        // Conservative: bail on any other instruction (could read the
        // slot or shift SP in a way we don't model).
        break;
      }
      if (dead) Dead.push_back(&*It);
    }
    for (MachineInstr *MI : Dead) {
      MI->eraseFromParent();
      Changed = true;
    }
  }

  // Phase 6: `LDA #0 ; STA_DP X ; STA_DP Y ; ...` → `STZ_DP X ; STZ_DP Y`.
  // STZ stores zero without going through A and without setting flags.
  // Eliminates the LDA when:
  //  (a) Flags from LDA #0 aren't needed downstream (walk forward from
  //      last STA, require flag-clobber/uncond-terminator before any
  //      flag-use).
  //  (b) A's value (0 from LDA) isn't needed downstream (walk forward
  //      from last STA, require A-redef before any A-use).
  for (MachineBasicBlock &MBB : MF) {
    SmallVector<MachineInstr *, 4> Dead;
    for (auto It = MBB.begin(); It != MBB.end(); ++It) {
      if (It->getOpcode() != W65816::LDA_Imm16 &&
          It->getOpcode() != W65816::LDAi16imm)
        continue;
      if (It->getNumOperands() < 2 || !It->getOperand(1).isImm()) continue;
      if (It->getOperand(1).getImm() != 0) continue;
      // Collect contiguous STA_DP/STA_Abs ops.
      SmallVector<MachineBasicBlock::iterator, 4> Stas;
      auto Fwd = std::next(It);
      while (Fwd != MBB.end()) {
        if (Fwd->isDebugInstr()) { ++Fwd; continue; }
        if (Fwd->getOpcode() == W65816::STA_DP ||
            Fwd->getOpcode() == W65816::STAabs ||
            Fwd->getOpcode() == W65816::STA_Abs) {
          Stas.push_back(Fwd);
          ++Fwd;
          continue;
        }
        break;
      }
      if (Stas.empty()) continue;
      // Flag-safety: walk past the last STA looking for a flag-clobber
      // or unconditional terminator before any flag-use.
      bool flagsSafe = false;
      auto After = std::next(Stas.back());
      while (After != MBB.end()) {
        if (After->isDebugInstr()) { ++After; continue; }
        if (After->isConditionalBranch()) break;  // unsafe
        bool flagUse = false, flagDef = false;
        for (const MachineOperand &MO : After->operands()) {
          if (MO.isReg() && MO.getReg() == W65816::P) {
            if (MO.isUse() && !MO.isDef()) flagUse = true;
            if (MO.isDef()) flagDef = true;
          }
        }
        if (flagUse) break;
        if (After->isTerminator() && !After->isConditionalBranch()) {
          flagsSafe = true; break;
        }
        if (After->isCall()) { flagsSafe = true; break; }
        if (flagDef) { flagsSafe = true; break; }
        // Many ops set P implicitly (LDA, ADC, INA, etc.) without an
        // explicit P-def operand.  Use opcode whitelist.
        unsigned O = After->getOpcode();
        if (O == W65816::LDA_DP || O == W65816::LDA_Abs ||
            O == W65816::LDA_StackRel || O == W65816::LDA_Imm16 ||
            O == W65816::LDAi16imm ||
            O == W65816::ADC_DP || O == W65816::ADC_StackRel ||
            O == W65816::SBC_DP || O == W65816::SBC_StackRel ||
            O == W65816::CMP_DP || O == W65816::CMP_StackRel ||
            O == W65816::INA || O == W65816::DEA ||
            O == W65816::INC_DP || O == W65816::DEC_DP ||
            O == W65816::TAX || O == W65816::TAY ||
            O == W65816::TXA || O == W65816::TYA ||
            O == W65816::PLA || O == W65816::PLX || O == W65816::PLY ||
            O == W65816::INA_PSEUDO || O == W65816::DEA_PSEUDO) {
          flagsSafe = true; break;
        }
        ++After;
      }
      if (!flagsSafe) continue;
      // ALSO: check A is dead after Stas.back().  Any A-use before A-redef
      // would see A=0 from the LDA; without the LDA, A is undef.
      bool aDead = false;
      // A-dead scan: walks forward from Stas.back(), looking for an
      // A-redef before any A-use.  If MBB ends without resolution,
      // recurse one level into all successors — each successor must
      // define A as its first A-touch.
      auto scanAUntilEnd = [&](MachineBasicBlock::iterator Cur,
                                MachineBasicBlock &CurrMBB) -> int {
        // Returns: 1=A dead (defined before any use), -1=A used,
        // 0=unresolved (reached end of MBB).
        while (Cur != CurrMBB.end()) {
          if (Cur->isDebugInstr()) { ++Cur; continue; }
          bool usesA = false, defsA = false;
          for (const MachineOperand &MO : Cur->operands()) {
            if (MO.isReg() && MO.getReg() == W65816::A) {
              // isUse and isDef can BOTH be true on tied-def operands
              // (the input side of an in-place modify like ASLA16 has
              // isUse=true on the tied operand AND its def-mate
              // produces a separate isDef operand).  Check both flags
              // independently — a tied-def instruction USES A before
              // it DEFs A, so we must consider it a use.
              if (MO.isUse()) usesA = true;
              if (MO.isDef()) defsA = true;
            }
          }
          if (Cur->isCall()) defsA = true;  // caller-save
          // Use BEFORE def matters: if A is read by this instruction
          // (even as part of a tied-def chain), the prior LDA #K's
          // value is consumed and we can't drop it.
          if (usesA) return -1;
          if (defsA) return 1;
          ++Cur;
        }
        return 0;
      };
      int aResult = scanAUntilEnd(std::next(Stas.back()), MBB);
      if (aResult == 1) {
        aDead = true;
      } else if (aResult == 0) {
        // Unresolved in this MBB — check all successors.
        bool allSuccDead = !MBB.succ_empty();
        for (MachineBasicBlock *Succ : MBB.successors()) {
          int r = scanAUntilEnd(Succ->begin(), *Succ);
          if (r != 1) { allSuccDead = false; break; }
        }
        if (allSuccDead) aDead = true;
      }
      if (!aDead) continue;
      // Rewrite all STAs to STZ.
      for (auto Sta : Stas) {
        unsigned NewOp;
        if (Sta->getOpcode() == W65816::STA_DP)
          NewOp = W65816::STZ_DP;
        else
          NewOp = W65816::STZ_Abs;
        DebugLoc DL = Sta->getDebugLoc();
        // The STA's address operand is at index 1 for STA_DP (operand 0
        // is the implicit-use $a) — no wait, STA_DP's tablegen has
        // (ins addrDP:$addr) so operand 0 is the address.  Let me check.
        // Actually STA_DP is `InstDP<0x85, "sta">` — same as LDA_DP,
        // operand 0 is the address immediate.
        int AddrIdx = 0;
        if (Sta->getNumOperands() >= 1 && Sta->getOperand(0).isReg())
          AddrIdx = 1;  // STAabs has explicit $a first
        BuildMI(MBB, Sta, DL, TII->get(NewOp))
            .add(Sta->getOperand(AddrIdx));
        Dead.push_back(&*Sta);
      }
      Dead.push_back(&*It);
    }
    for (MachineInstr *MI : Dead) {
      MI->eraseFromParent();
      Changed = true;
    }
  }

  // Phase 7: dead PHA before single-arg libcall.  Pattern:
  //   PHA              ; dead allocation (the bottom of 2 pushes)
  //   PUSH16           ; actual arg push (becomes 4,s inside callee)
  //   LDA_StackRel <off>  ; reads caller stack (after both pushes)
  //   JSLpseudo32 <helper>   ; helper reads ONLY 4,s
  //   PLY              ; pop
  //   PLY              ; pop the dead PHA value
  //   (terminator: RTL/RTS/RTI)
  //
  // If the JSL target is in our "reads only 4,s" libgcc whitelist,
  // we can drop the PHA + one PLY + adjust LDA_StackRel offset by -2.
  // Saves 2 inst per occurrence.
  auto isOneArgLibcall = [](const MachineInstr &MI) -> bool {
    if (!MI.isCall()) return false;
    for (const MachineOperand &MO : MI.operands()) {
      StringRef Name;
      if (MO.isGlobal()) Name = MO.getGlobal()->getName();
      else if (MO.isSymbol()) Name = MO.getSymbolName();
      else continue;
      // Helpers that take one i16 arg in A and one i16 arg at 4,s,
      // returning i32 in A:X.  __mulhi3 returns i16 not i32 but reads
      // same arg layout.
      if (Name == "__umulhisi3" || Name == "__mulhi3")
        return true;
      return false;
    }
    return false;
  };
  for (MachineBasicBlock &MBB : MF) {
    SmallVector<MachineInstr *, 4> Dead;
    SmallVector<std::pair<MachineInstr *, int>, 4> AdjustOff;
    for (auto It = MBB.begin(); It != MBB.end(); ++It) {
      if (It->getOpcode() != W65816::PHA) continue;
      auto Push = std::next(It);
      while (Push != MBB.end() && Push->isDebugInstr()) ++Push;
      if (Push == MBB.end() || Push->getOpcode() != W65816::PUSH16) continue;
      // Walk forward collecting stack-rel accesses and the call.
      MachineBasicBlock::iterator CallIt = MBB.end();
      SmallVector<MachineBasicBlock::iterator, 4> StackRelOps;
      auto Fwd = std::next(Push);
      while (Fwd != MBB.end()) {
        if (Fwd->isDebugInstr()) { ++Fwd; continue; }
        if (Fwd->isCall()) { CallIt = Fwd; break; }
        if (isStackRelOp(Fwd->getOpcode())) {
          StackRelOps.push_back(Fwd);
        } else if (Fwd->getOpcode() == W65816::PHA ||
                   Fwd->getOpcode() == W65816::PHX ||
                   Fwd->getOpcode() == W65816::PHY ||
                   Fwd->getOpcode() == W65816::PUSH16 ||
                   Fwd->getOpcode() == W65816::PEA ||
                   Fwd->getOpcode() == W65816::PLA ||
                   Fwd->getOpcode() == W65816::PLX ||
                   Fwd->getOpcode() == W65816::PLY) {
          // Additional stack ops between push and call — bail.
          CallIt = MBB.end();
          break;
        }
        ++Fwd;
      }
      if (CallIt == MBB.end()) continue;
      if (!isOneArgLibcall(*CallIt)) continue;
      // Find the two PLYs after the call.
      auto Ply1 = std::next(CallIt);
      while (Ply1 != MBB.end() && Ply1->isDebugInstr()) ++Ply1;
      if (Ply1 == MBB.end() || Ply1->getOpcode() != W65816::PLY) continue;
      auto Ply2 = std::next(Ply1);
      while (Ply2 != MBB.end() && Ply2->isDebugInstr()) ++Ply2;
      if (Ply2 == MBB.end() || Ply2->getOpcode() != W65816::PLY) continue;
      // Verify all StackRel ops have offset > 2 (i.e., they read
      // CALLER's stack, not our PHA's slot which is at offset 3..4).
      bool safe = true;
      for (auto SrIt : StackRelOps) {
        if (SrIt->getNumOperands() < 1 || !SrIt->getOperand(0).isImm()) {
          safe = false; break;
        }
        int64_t O = SrIt->getOperand(0).getImm();
        if (O <= 4) { safe = false; break; }  // would alias the PHA push
      }
      if (!safe) continue;
      // All clear.  Erase PHA, one PLY, adjust stack-rel offsets by -2.
      Dead.push_back(&*It);
      Dead.push_back(&*Ply2);
      for (auto SrIt : StackRelOps) {
        AdjustOff.push_back({&*SrIt, -2});
      }
    }
    for (auto P : AdjustOff) {
      P.first->getOperand(0).setImm(P.first->getOperand(0).getImm() + P.second);
    }
    for (MachineInstr *MI : Dead) {
      MI->eraseFromParent();
      Changed = true;
    }
  }

  // Phase 8: dead frame elimination.  After IMG promotion, some
  // functions have NO stack-rel access — all locals live in IMG slots.
  // The frame alloc (TSC/SEC/SBC/TCS in prologue) and dealloc (TSC/
  // CLC/ADC/TCS in epilogue) are then dead overhead.  Detect and elide.
  //
  // Saves 8 inst (4 in prologue + 4 in epilogue) when conditions met.
  //
  // Conditions:
  //   1. Entry MBB has REP ; TSC ; SEC ; SBC #N ; TCS
  //   2. Every return MBB has TSC ; CLC ; ADC #N ; TCS ; RTL/RTS/RTI
  //   3. No stack-rel access remains anywhere in the function
  //   4. No PUSH16/PHA/PEA in a way that would leave SP unbalanced —
  //      pair check: each push must have a matching pop.  We
  //      conservatively count and require zero net delta at each
  //      epilogue point.
  {
    // Check no stack-rel access.
    bool hasStackRel = false;
    for (MachineBasicBlock &MBB : MF) {
      for (MachineInstr &MI : MBB) {
        if (isStackRelOp(MI.getOpcode())) {
          hasStackRel = true;
          break;
        }
      }
      if (hasStackRel) break;
    }
    if (!hasStackRel) {
      // Find prologue TSC sequence in entry MBB.
      MachineBasicBlock &EntryMBB = MF.front();
      auto Rep = EntryMBB.getFirstNonDebugInstr();
      MachineBasicBlock::iterator Tsc = EntryMBB.end();
      MachineBasicBlock::iterator Sec, Sbc, Tcs2;
      if (Rep != EntryMBB.end() && Rep->getOpcode() == W65816::REP) {
        auto It = std::next(Rep);
        while (It != EntryMBB.end() && It->isDebugInstr()) ++It;
        // Skip an optional STA_DP between REP and TSC (the prologue
        // optimization in Phase 4 may have moved one there).
        while (It != EntryMBB.end() && It->getOpcode() == W65816::STA_DP)
          ++It;
        while (It != EntryMBB.end() && It->isDebugInstr()) ++It;
        if (It != EntryMBB.end() && It->getOpcode() == W65816::TSC) {
          Tsc = It++;
          while (It != EntryMBB.end() && It->isDebugInstr()) ++It;
          if (It != EntryMBB.end() && It->getOpcode() == W65816::SEC) {
            Sec = It++;
            while (It != EntryMBB.end() && It->isDebugInstr()) ++It;
            if (It != EntryMBB.end() &&
                (It->getOpcode() == W65816::SBC_Imm16 ||
                 It->getOpcode() == W65816::SBCi16imm)) {
              Sbc = It++;
              while (It != EntryMBB.end() && It->isDebugInstr()) ++It;
              if (It != EntryMBB.end() && It->getOpcode() == W65816::TCS) {
                Tcs2 = It;
              } else {
                Tsc = EntryMBB.end();
              }
            } else {
              Tsc = EntryMBB.end();
            }
          } else {
            Tsc = EntryMBB.end();
          }
        }
      }
      if (Tsc != EntryMBB.end()) {
        // Get N from SBC.
        int64_t N = 0;
        for (const MachineOperand &MO : Sbc->operands()) {
          if (MO.isImm()) { N = MO.getImm(); break; }
        }
        // For each return MBB, find matching epilogue.
        struct EpiInfo {
          MachineBasicBlock::iterator Tsc, Clc, Adc, Tcs;
        };
        SmallVector<EpiInfo, 4> Epis;
        bool allMatched = true;
        for (MachineBasicBlock &MBB : MF) {
          auto Last = MBB.getLastNonDebugInstr();
          if (Last == MBB.end()) continue;
          unsigned LastOpc = Last->getOpcode();
          if (LastOpc != W65816::RTL && LastOpc != W65816::RTS &&
              LastOpc != W65816::RTI)
            continue;
          // Walk back from terminator looking for TSC/CLC/ADC/TCS.
          auto Cur = Last;
          if (Cur == MBB.begin()) { allMatched = false; break; }
          --Cur;
          while (Cur != MBB.begin() && Cur->isDebugInstr()) --Cur;
          // Skip an optional LDA_DP after TCS (epilogue opt may have put one).
          if (Cur != MBB.begin() && Cur->getOpcode() == W65816::LDA_DP) {
            --Cur;
            while (Cur != MBB.begin() && Cur->isDebugInstr()) --Cur;
          }
          if (Cur->getOpcode() != W65816::TCS) {
            allMatched = false; break;
          }
          auto Tcs_e = Cur;
          if (Cur == MBB.begin()) { allMatched = false; break; }
          --Cur;
          while (Cur != MBB.begin() && Cur->isDebugInstr()) --Cur;
          if (Cur->getOpcode() != W65816::ADC_Imm16 &&
              Cur->getOpcode() != W65816::ADCi16imm) {
            allMatched = false; break;
          }
          auto Adc_e = Cur;
          int64_t AdcN = 0;
          for (const MachineOperand &MO : Adc_e->operands()) {
            if (MO.isImm()) { AdcN = MO.getImm(); break; }
          }
          if (AdcN != N) { allMatched = false; break; }
          if (Cur == MBB.begin()) { allMatched = false; break; }
          --Cur;
          while (Cur != MBB.begin() && Cur->isDebugInstr()) --Cur;
          if (Cur->getOpcode() != W65816::CLC) {
            allMatched = false; break;
          }
          auto Clc_e = Cur;
          if (Cur == MBB.begin()) { allMatched = false; break; }
          --Cur;
          while (Cur != MBB.begin() && Cur->isDebugInstr()) --Cur;
          if (Cur->getOpcode() != W65816::TSC) {
            allMatched = false; break;
          }
          Epis.push_back({Cur, Clc_e, Adc_e, Tcs_e});
        }
        if (allMatched && !Epis.empty()) {
          // Eliminate prologue TSC/SEC/SBC/TCS.
          Tsc->eraseFromParent();
          Sec->eraseFromParent();
          Sbc->eraseFromParent();
          Tcs2->eraseFromParent();
          // Eliminate each epilogue's TSC/CLC/ADC/TCS.
          for (auto &E : Epis) {
            E.Tsc->eraseFromParent();
            E.Clc->eraseFromParent();
            E.Adc->eraseFromParent();
            E.Tcs->eraseFromParent();
          }
          Changed = true;
        }
      }
    }
  }

  // Phase 9: saturating-max preheader elimination.
  //
  // LLVM's loop induction simplification transforms
  //   for (i = 1; i <= n; i++) { ... }
  // into a count-down loop whose preheader saturates `n+1` to at least
  // 2, then decrements to get max(n, 1).  This protects against the
  // `n == 0` edge case at the IR level.  But codegen ALSO emits an
  // entry check `if (n == 0) goto exit`, making the saturation
  // redundant — `n` is already >= 1 in the preheader.
  //
  // Pattern:
  //   bb.0 (entry):
  //     LDA_DP $X            ; X = some DP slot holding n
  //     BNE %preheader
  //     (fallthrough to brl-exit MBB)
  //   bb.preheader:
  //     LDA_DP $X
  //     INA                   ; X+1
  //     STA_DP $X
  //     CMP #N                ; N = 3 (= 2+1) for the standard pattern
  //     BCS %merge
  //   bb.saturate:
  //     LDA #N-1              ; N-1 = 2
  //     STA_DP $X
  //   bb.merge:
  //     ... LDA #1 ; STA $Y (i slot) ...
  //     DEC_DP $X             ; X = max(X, 1) - 1 (canceling +1 from INA)
  //     (or DEC may be elsewhere in bb.merge)
  //
  // After elimination:
  //   bb.0 ends with `BNE %merge` (skips preheader+saturate)
  //   bb.preheader, bb.saturate, and the DEC_DP $X in bb.merge are gone
  //   Net: counter $X stays at n (no saturate needed, no DEC needed)
  //
  // For sumSquares: saves 8 inst.
  {
    // Look for the entry MBB pattern: ends with LDA_DP $X ; BNE %bb.
    for (MachineBasicBlock &EntryMBB : MF) {
      // Only consider MBBs whose successors look like the pattern.
      if (EntryMBB.succ_size() != 2) continue;
      auto Last = EntryMBB.getLastNonDebugInstr();
      if (Last == EntryMBB.end()) continue;
      if (Last->getOpcode() != W65816::BNE) continue;
      // Walk back to find the LDA_DP feeding BNE.
      auto LdaIt = Last;
      if (LdaIt == EntryMBB.begin()) continue;
      --LdaIt;
      while (LdaIt != EntryMBB.begin() && LdaIt->isDebugInstr()) --LdaIt;
      if (LdaIt->getOpcode() != W65816::LDA_DP) continue;
      if (LdaIt->getNumOperands() < 1 || !LdaIt->getOperand(0).isImm())
        continue;
      int64_t XAddr = LdaIt->getOperand(0).getImm();
      // BNE's target MBB is the preheader.
      MachineBasicBlock *Preheader = nullptr;
      for (MachineOperand &MO : Last->operands()) {
        if (MO.isMBB()) { Preheader = MO.getMBB(); break; }
      }
      if (!Preheader) continue;
      // Check preheader's first 5 ops: LDA_DP $X ; INA_PSEUDO ; STA_DP $X
      // ; CMPi16imm $a, N ; BCS %merge.
      auto P = Preheader->getFirstNonDebugInstr();
      if (P == Preheader->end()) continue;
      if (P->getOpcode() != W65816::LDA_DP) continue;
      if (P->getOperand(0).getImm() != XAddr) continue;
      auto P2 = std::next(P);
      while (P2 != Preheader->end() && P2->isDebugInstr()) ++P2;
      if (P2 == Preheader->end() ||
          (P2->getOpcode() != W65816::INA &&
           P2->getOpcode() != W65816::INA_PSEUDO))
        continue;
      auto P3 = std::next(P2);
      while (P3 != Preheader->end() && P3->isDebugInstr()) ++P3;
      if (P3 == Preheader->end() || P3->getOpcode() != W65816::STA_DP)
        continue;
      if (P3->getOperand(0).getImm() != XAddr) continue;
      auto P4 = std::next(P3);
      while (P4 != Preheader->end() && P4->isDebugInstr()) ++P4;
      if (P4 == Preheader->end() ||
          (P4->getOpcode() != W65816::CMP_Imm16 &&
           P4->getOpcode() != W65816::CMPi16imm))
        continue;
      int64_t CmpN = 0;
      for (const MachineOperand &MO : P4->operands()) {
        if (MO.isImm()) { CmpN = MO.getImm(); break; }
      }
      // Only handle the loop-start=1 case (CMP #3, saturate to 2).
      if (CmpN != 3) continue;
      auto P5 = std::next(P4);
      while (P5 != Preheader->end() && P5->isDebugInstr()) ++P5;
      if (P5 == Preheader->end() || P5->getOpcode() != W65816::BCS)
        continue;
      // BCS's target is the merge MBB.
      MachineBasicBlock *Merge = nullptr;
      for (MachineOperand &MO : P5->operands()) {
        if (MO.isMBB()) { Merge = MO.getMBB(); break; }
      }
      if (!Merge) continue;
      // The fall-through from preheader is the saturate MBB.
      MachineBasicBlock *Saturate = nullptr;
      for (MachineBasicBlock *S : Preheader->successors()) {
        if (S != Merge) { Saturate = S; break; }
      }
      if (!Saturate) continue;
      // Saturate MBB: must be `LDA #2 ; STA_DP $X` and fall through to
      // merge.
      auto S1 = Saturate->getFirstNonDebugInstr();
      if (S1 == Saturate->end()) continue;
      if (S1->getOpcode() != W65816::LDA_Imm16 &&
          S1->getOpcode() != W65816::LDAi16imm)
        continue;
      int64_t SatVal = 0;
      for (const MachineOperand &MO : S1->operands()) {
        if (MO.isImm()) { SatVal = MO.getImm(); break; }
      }
      if (SatVal != CmpN - 1) continue;  // expect 2 (since CMP was 3)
      auto S2 = std::next(S1);
      while (S2 != Saturate->end() && S2->isDebugInstr()) ++S2;
      if (S2 == Saturate->end() || S2->getOpcode() != W65816::STA_DP)
        continue;
      if (S2->getOperand(0).getImm() != XAddr) continue;
      // Find DEC_DP $X in merge MBB.
      MachineBasicBlock::iterator DecIt = Merge->end();
      for (auto It = Merge->begin(); It != Merge->end(); ++It) {
        if (It->getOpcode() == W65816::DEC_DP &&
            It->getNumOperands() >= 1 && It->getOperand(0).isImm() &&
            It->getOperand(0).getImm() == XAddr) {
          DecIt = It;
          break;
        }
      }
      if (DecIt == Merge->end()) continue;
      // All pattern-matched.  Rewrite:
      //   1. Change entry's BNE %preheader to BNE %merge.
      //   2. Empty preheader and saturate MBBs (their CFG-effects
      //      stay; the BranchExpand / MBB-layout passes will clean up).
      //   3. Erase the DEC_DP in merge.
      //
      // To stay safe with CFG, we just clear the MBBs' contents (kept
      // alive as empty fall-through blocks).  Actually safer: leave
      // Preheader and Saturate INTACT in CFG terms but skip them via
      // BNE→merge redirect.  Then those MBBs become unreachable
      // and get pruned by later passes.
      //
      // Redirect BNE's target.
      for (MachineOperand &MO : Last->operands()) {
        if (MO.isMBB() && MO.getMBB() == Preheader) MO.setMBB(Merge);
      }
      // Update CFG.
      EntryMBB.removeSuccessor(Preheader);
      EntryMBB.addSuccessor(Merge);
      // Erase the DEC_DP in merge.
      DecIt->eraseFromParent();
      // Empty the now-unreachable Preheader and Saturate MBBs (their
      // contents would otherwise be emitted as dead asm).  Also drop
      // their successor links.
      while (!Preheader->empty()) Preheader->begin()->eraseFromParent();
      while (!Saturate->empty()) Saturate->begin()->eraseFromParent();
      while (!Preheader->succ_empty())
        Preheader->removeSuccessor(*Preheader->succ_begin());
      while (!Saturate->succ_empty())
        Saturate->removeSuccessor(*Saturate->succ_begin());
      Changed = true;
    }
  }

  // Phase 10: tail-call commutative libcalls in trivial wrapper functions.
  //
  // For functions whose entire body is essentially `helper(a, b); return;`
  // where `helper` is a commutative libgcc multiply (__umulhisi3, __mulhi3),
  // the codegen unnecessarily marshals args (PHA + LDA-from-shifted-offset).
  // Since multiply is commutative, the caller's positions (A=a, 4,s=b)
  // already satisfy the callee's contract.
  //
  // Pattern:
  //   REP 48
  //   PHA                      ; pushes A (= arg0)
  //   LDA_StackRel <off>       ; reads arg1 (off = 4 + 2 = 6 post-PHA)
  //   JSLpseudo32 @helper
  //   PLY                      ; cleanup PHA
  //   RTL
  //
  // Replace with:
  //   REP 48
  //   JML_Long @helper         ; tail-call; helper's RTL returns to OUR caller
  //
  // 2 inst total — beats Calypsi's 4 for mul16to32.
  {
    if (MF.size() == 1) {  // single MBB only — full function fits
      MachineBasicBlock &MBB = MF.front();
      // Collect non-debug ops.
      SmallVector<MachineInstr *, 8> Ops;
      for (MachineInstr &MI : MBB) {
        if (!MI.isDebugInstr()) Ops.push_back(&MI);
      }
      // Need exactly: REP, (PHA|PUSH16), LDA_StackRel, JSL, PLY, RTL.
      if (Ops.size() == 6 &&
          Ops[0]->getOpcode() == W65816::REP &&
          (Ops[1]->getOpcode() == W65816::PHA ||
           Ops[1]->getOpcode() == W65816::PUSH16) &&
          Ops[2]->getOpcode() == W65816::LDA_StackRel &&
          Ops[3]->getOpcode() == W65816::JSLpseudo32 &&
          Ops[4]->getOpcode() == W65816::PLY &&
          Ops[5]->getOpcode() == W65816::RTL) {
        // JSL target must be a commutative 16x16-multiply helper.
        bool isCommutativeMul = false;
        for (MachineOperand &MO : Ops[3]->operands()) {
          StringRef Name;
          if (MO.isGlobal()) Name = MO.getGlobal()->getName();
          else if (MO.isSymbol()) Name = MO.getSymbolName();
          else continue;
          if (Name == "__umulhisi3" || Name == "__mulhi3") {
            isCommutativeMul = true;
            break;
          }
          break;
        }
        // LDA_StackRel offset should be 6 (caller's arg1 at 4,s, +2 for PHA).
        bool offsetOK = Ops[2]->getNumOperands() >= 1 &&
                        Ops[2]->getOperand(0).isImm() &&
                        Ops[2]->getOperand(0).getImm() == 6;
        if (isCommutativeMul && offsetOK) {
          // Build JML_Long @helper to replace the JSL.
          MachineOperand TargetMO = Ops[3]->getOperand(0);  // copy
          DebugLoc DL = Ops[3]->getDebugLoc();
          // Insert JML_Long before the JSL.
          BuildMI(MBB, Ops[3], DL, TII->get(W65816::JML_Long))
              .add(TargetMO);
          // Erase PHA, LDA_StackRel, JSL, PLY, RTL.
          Ops[1]->eraseFromParent();
          Ops[2]->eraseFromParent();
          Ops[3]->eraseFromParent();
          Ops[4]->eraseFromParent();
          Ops[5]->eraseFromParent();
          Changed = true;
        }
      }
    }
  }

  // Phase 11: PHP-wrap-aware STZ peephole.
  //
  // SDAG emits `PHP ; LDA #0 ; PLP ; STA $X` to write 0 to a DP slot
  // while preserving flags from an earlier flag-setter (typically an
  // ORA-then-BEQ pattern).  Since STZ_DP $X writes 0 without touching
  // A or flags, the whole wrap collapses to `STZ_DP $X`.  Saves 4
  // inst per occurrence.
  //
  // Safety: A must be dead after the STA.  Use the same successor-
  // aware scan as Phase 6.
  {
    auto scanAUntilEnd2 = [&](MachineBasicBlock::iterator Cur,
                               MachineBasicBlock &CurrMBB) -> int {
      while (Cur != CurrMBB.end()) {
        if (Cur->isDebugInstr()) { ++Cur; continue; }
        bool usesA = false, defsA = false;
        for (const MachineOperand &MO : Cur->operands()) {
          if (MO.isReg() && MO.getReg() == W65816::A) {
            if (MO.isDef()) defsA = true;
            else if (MO.isUse()) usesA = true;
          }
        }
        if (Cur->isCall()) defsA = true;
        if (defsA) return 1;
        if (usesA) return -1;
        ++Cur;
      }
      return 0;
    };
    for (MachineBasicBlock &MBB : MF) {
      SmallVector<MachineInstr *, 4> Dead;
      SmallVector<std::pair<MachineInstr *, int64_t>, 4> NewStz;
      for (auto It = MBB.begin(); It != MBB.end(); ++It) {
        if (It->getOpcode() != W65816::PHP) continue;
        auto Lda = std::next(It);
        while (Lda != MBB.end() && Lda->isDebugInstr()) ++Lda;
        if (Lda == MBB.end()) continue;
        if (Lda->getOpcode() != W65816::LDA_Imm16 &&
            Lda->getOpcode() != W65816::LDAi16imm)
          continue;
        if (Lda->getNumOperands() < 2 || !Lda->getOperand(1).isImm()) continue;
        if (Lda->getOperand(1).getImm() != 0) continue;
        auto Plp = std::next(Lda);
        while (Plp != MBB.end() && Plp->isDebugInstr()) ++Plp;
        if (Plp == MBB.end() || Plp->getOpcode() != W65816::PLP) continue;
        auto Sta = std::next(Plp);
        while (Sta != MBB.end() && Sta->isDebugInstr()) ++Sta;
        if (Sta == MBB.end() || Sta->getOpcode() != W65816::STA_DP) continue;
        if (Sta->getNumOperands() < 1 || !Sta->getOperand(0).isImm()) continue;
        int64_t StaAddr = Sta->getOperand(0).getImm();
        // Check A-dead after the STA.
        bool aDead = false;
        int r = scanAUntilEnd2(std::next(Sta), MBB);
        if (r == 1) aDead = true;
        else if (r == 0) {
          bool allSuccDead = !MBB.succ_empty();
          for (MachineBasicBlock *Succ : MBB.successors()) {
            int sr = scanAUntilEnd2(Succ->begin(), *Succ);
            if (sr != 1) { allSuccDead = false; break; }
          }
          if (allSuccDead) aDead = true;
        }
        if (!aDead) continue;
        Dead.push_back(&*It);   // PHP
        Dead.push_back(&*Lda);
        Dead.push_back(&*Plp);
        Dead.push_back(&*Sta);
        NewStz.push_back({&*Sta, StaAddr});
      }
      // Insert STZ_DP before each dead STA (Sta is in NewStz).
      for (auto &P : NewStz) {
        MachineInstr *Sta = P.first;
        DebugLoc DL = Sta->getDebugLoc();
        BuildMI(MBB, Sta, DL, TII->get(W65816::STZ_DP))
            .addImm(P.second);
      }
      for (MachineInstr *MI : Dead) {
        MI->eraseFromParent();
        Changed = true;
      }
    }
  }

  // DP-shift-fold: `LDA_DP X ; LSRA16/ASLA16 ; STA_DP X (same X)` folds
  // to `LSR_DP/ASL_DP X` (DP-RMW form) when A is dead after the STA.
  // Same semantics, shorter and cheaper.  popcount's i32 SRL by 1 hits
  // the LSRA case once per iteration; the slot-A is dead after when the
  // next op is an LDA-style $a-defining load.
  for (MachineBasicBlock &MBB : MF) {
    SmallVector<MachineInstr *, 8> ToErase;
    for (auto It = MBB.begin(); It != MBB.end();) {
      auto Lda = It++;
      if (Lda->getOpcode() != W65816::LDA_DP) continue;
      if (Lda->getNumOperands() < 1 || !Lda->getOperand(0).isImm()) continue;
      int64_t DpAddr = Lda->getOperand(0).getImm();
      auto Shift = std::next(Lda);
      while (Shift != MBB.end() && Shift->isDebugInstr()) ++Shift;
      if (Shift == MBB.end()) continue;
      unsigned ShiftOp = Shift->getOpcode();
      unsigned DpShiftOp = 0;
      if (ShiftOp == W65816::LSRA16) {
        DpShiftOp = W65816::LSR_DP;
      } else if (ShiftOp == W65816::ASLA16) {
        DpShiftOp = W65816::ASL_DP;
      } else {
        continue;
      }
      auto Sta = std::next(Shift);
      while (Sta != MBB.end() && Sta->isDebugInstr()) ++Sta;
      if (Sta == MBB.end() || Sta->getOpcode() != W65816::STA_DP) continue;
      if (Sta->getNumOperands() < 1 || !Sta->getOperand(0).isImm()) continue;
      if (Sta->getOperand(0).getImm() != DpAddr) continue;
      // A is dead after Sta iff the next non-debug op defines $a without
      // reading it.  Conservative: bail if no clear def or if A is read.
      auto Next = std::next(Sta);
      while (Next != MBB.end() && Next->isDebugInstr()) ++Next;
      if (Next == MBB.end()) continue;
      if (Next->isBranch() || Next->isReturn() || Next->isCall() ||
          Next->isInlineAsm()) continue;
      bool reads = Next->readsRegister(W65816::A, TRI);
      bool defs = Next->modifiesRegister(W65816::A, TRI);
      if (reads || !defs) continue;
      // Apply: insert LSR_DP/ASL_DP at Lda's position, erase the triplet.
      BuildMI(MBB, Lda, Lda->getDebugLoc(), TII->get(DpShiftOp))
          .addImm(DpAddr);
      ToErase.push_back(&*Lda);
      ToErase.push_back(&*Shift);
      ToErase.push_back(&*Sta);
    }
    for (MachineInstr *MI : ToErase) {
      MI->eraseFromParent();
      Changed = true;
    }
  }

  // Shift-cascade dead-store elimination.  Greedy regalloc sometimes
  // emits `LDA dp; (ASLA16; STA dp){×N}; ...` where each intermediate
  // STA_DP is dead — the next ASLA16 reads $a (still holding the value)
  // and shifts again, then stores again.  Only the final STA matters.
  //
  // Pattern:  LDA_DP X ; (ASLA16; STA_DP X){×N+1}
  //           where every STA writes to the same DP slot the LDA read
  //           from, and nothing in between reads $a or DP[X] except
  //           the cascade itself.
  // Rewrite:  LDA_DP X ; ASLA16{×N+1} ; STA_DP X
  //           (final STA only; intermediate STAs erased.)
  //
  // For N=4 (i.e. 5 shifts), saves 4 STA_DPs = 12 cyc.  Hits djb2Hash's
  // `Hi << 5` cascade (where greedy spills the intermediate vregs).
  for (MachineBasicBlock &MBB : MF) {
    SmallVector<MachineInstr *, 8> ToErase;
    for (auto It = MBB.begin(); It != MBB.end(); ++It) {
      if (It->getOpcode() != W65816::LDA_DP) continue;
      if (It->getNumOperands() < 1 || !It->getOperand(0).isImm()) continue;
      int64_t Slot = It->getOperand(0).getImm();
      auto Cur = std::next(It);
      // Track intermediate STAs in this cascade.
      SmallVector<MachineInstr *, 6> Intermediates;
      bool sawAny = false;
      while (Cur != MBB.end()) {
        if (Cur->isDebugInstr()) { ++Cur; continue; }
        unsigned Op = Cur->getOpcode();
        if (Op != W65816::ASLA16 && Op != W65816::LSRA16) break;
        auto Sta = std::next(Cur);
        while (Sta != MBB.end() && Sta->isDebugInstr()) ++Sta;
        if (Sta == MBB.end()) break;
        if (Sta->getOpcode() != W65816::STA_DP) break;
        if (Sta->getNumOperands() < 1 || !Sta->getOperand(0).isImm()) break;
        if (Sta->getOperand(0).getImm() != Slot) break;
        sawAny = true;
        Intermediates.push_back(&*Sta);
        Cur = std::next(Sta);
      }
      if (!sawAny || Intermediates.size() < 2) continue;
      // The LAST STA is the real one; mark everything before it for erase.
      Intermediates.pop_back();
      for (MachineInstr *MI : Intermediates)
        ToErase.push_back(MI);
    }
    for (MachineInstr *MI : ToErase) {
      MI->eraseFromParent();
      Changed = true;
    }
  }

  // i32-SRL-by-1 fold: detects the SDAG expansion for `(SRL i32 X, 1)`
  // and rewrites to the two-instruction `LSR Hi ; ROR Lo` pair when
  // both halves live in DP.
  //
  // Input pattern (after the DP-shift-fold above runs on the trailing
  // `LDA Hi ; LSRA16 ; STA Hi` triplet):
  //   LDA_DP Hi
  //   SHL15A                  ; A = (Hi & 1) << 15
  //   STA_DP Yc               ; carry slot
  //   LDA_DP Lo
  //   LSRA16                  ; Lo >>= 1
  //   STA_DP Lo
  //   ORA_DP Yc               ; combine with carry-at-bit-15
  //   STA_DP Lo
  //   LSR_DP Hi               ; (folded from the trailing triplet)
  //
  // Output:
  //   LSR_DP Hi               ; Hi >>= 1, C = old bit 0 of Hi
  //   ROR_DP Lo               ; Lo = (C, Lo >> 1)
  //
  // Same semantics, ~30 cyc saved per iter.  popcount measured at 2728
  // cyc; expected post-fold ~1858 cyc (-32%) due to 29 iters of i32
  // SRL by 1.
  for (MachineBasicBlock &MBB : MF) {
    SmallVector<MachineInstr *, 8> ToErase;
    for (auto It = MBB.begin(); It != MBB.end();) {
      auto LdaHi = It++;
      if (LdaHi->getOpcode() != W65816::LDA_DP) continue;
      if (LdaHi->getNumOperands() < 1 || !LdaHi->getOperand(0).isImm())
        continue;
      int64_t HiAddr = LdaHi->getOperand(0).getImm();
      auto P = std::next(LdaHi);
      auto skipDbg = [&](auto &P) {
        while (P != MBB.end() && P->isDebugInstr()) ++P;
      };
      skipDbg(P);
      if (P == MBB.end() || P->getOpcode() != W65816::SHL15A) continue;
      auto Shl = P; ++P; skipDbg(P);
      if (P == MBB.end() || P->getOpcode() != W65816::STA_DP) continue;
      if (P->getNumOperands() < 1 || !P->getOperand(0).isImm()) continue;
      int64_t YcAddr = P->getOperand(0).getImm();
      auto StaYc = P; ++P; skipDbg(P);
      if (P == MBB.end() || P->getOpcode() != W65816::LDA_DP) continue;
      if (P->getNumOperands() < 1 || !P->getOperand(0).isImm()) continue;
      int64_t LoAddr = P->getOperand(0).getImm();
      auto LdaLo = P; ++P; skipDbg(P);
      if (P == MBB.end() || P->getOpcode() != W65816::LSRA16) continue;
      auto LsrA1 = P; ++P; skipDbg(P);
      if (P == MBB.end() || P->getOpcode() != W65816::STA_DP) continue;
      if (P->getOperand(0).getImm() != LoAddr) continue;
      auto StaLo1 = P; ++P; skipDbg(P);
      if (P == MBB.end() || P->getOpcode() != W65816::ORA_DP) continue;
      if (P->getOperand(0).getImm() != YcAddr) continue;
      auto OraYc = P; ++P; skipDbg(P);
      if (P == MBB.end() || P->getOpcode() != W65816::STA_DP) continue;
      if (P->getOperand(0).getImm() != LoAddr) continue;
      auto StaLo2 = P; ++P; skipDbg(P);
      if (P == MBB.end() || P->getOpcode() != W65816::LSR_DP) continue;
      if (P->getOperand(0).getImm() != HiAddr) continue;
      auto LsrHi = P;
      // Check that YcAddr is not READ after LsrHi before being
      // overwritten.  If the next op touching YcAddr is a STA (write),
      // the carry-slot value is dead — safe to drop our STA Yc.  If
      // it's a load-style op (LDA/ORA/AND/etc.) before any STA, then
      // some downstream code is consuming our stored carry — bail.
      bool YcReadBeforeWrite = false;
      for (auto Q = std::next(LsrHi); Q != MBB.end(); ++Q) {
        if (Q->isDebugInstr()) continue;
        bool touchesYc = false;
        bool isWriteOfYc = false;
        for (const MachineOperand &MO : Q->operands()) {
          if (MO.isImm() && MO.getImm() == YcAddr) {
            unsigned QO = Q->getOpcode();
            if (QO == W65816::STA_DP || QO == W65816::STZ_DP ||
                QO == W65816::STX_DP || QO == W65816::STY_DP) {
              touchesYc = true; isWriteOfYc = true;
            } else if (QO == W65816::LDA_DP || QO == W65816::ORA_DP ||
                       QO == W65816::AND_DP || QO == W65816::EOR_DP ||
                       QO == W65816::ADC_DP || QO == W65816::SBC_DP ||
                       QO == W65816::CMP_DP || QO == W65816::LSR_DP ||
                       QO == W65816::ROR_DP || QO == W65816::ASL_DP ||
                       QO == W65816::ROL_DP || QO == W65816::INC_DP ||
                       QO == W65816::DEC_DP) {
              touchesYc = true; // any of these is a read or RMW
            }
            break;
          }
        }
        if (touchesYc) {
          if (!isWriteOfYc) YcReadBeforeWrite = true;
          break;  // first touch decides
        }
      }
      if (YcReadBeforeWrite) continue;
      // Apply: replace the 8-op sequence with LSR_DP Hi ; ROR_DP Lo.
      // The LSR_DP Hi already exists (at LsrHi); just insert ROR_DP Lo
      // immediately after it and erase the rest.
      BuildMI(MBB, std::next(MachineBasicBlock::iterator(LsrHi)),
              LsrHi->getDebugLoc(), TII->get(W65816::ROR_DP))
          .addImm(LoAddr);
      ToErase.push_back(&*LdaHi);
      ToErase.push_back(&*Shl);
      ToErase.push_back(&*StaYc);
      ToErase.push_back(&*LdaLo);
      ToErase.push_back(&*LsrA1);
      ToErase.push_back(&*StaLo1);
      ToErase.push_back(&*OraYc);
      ToErase.push_back(&*StaLo2);
      It = std::next(MachineBasicBlock::iterator(LsrHi));
    }
    for (MachineInstr *MI : ToErase) {
      MI->eraseFromParent();
      Changed = true;
    }
  }

  // DP dead-store elimination — runs LAST (after the i32-SRL-by-1
  // fold, which depends on the `STA_DP Lo` between LSRA16 and ORA_DP
  // staying intact).  When two STA_DP X stores write the same DP slot
  // with no intervening read or write of that slot, the first is dead.
  // Emerges from popcount's `bit = x & 1` pattern at end of body.
  // Saves 5 cyc per match.
  //
  // Conservative: branches / calls / inline asm and all DP-indirect
  // addressing modes (which use the slot AS a pointer) block the
  // elim — those reads must see the intended store value.
  {
    auto touchesDpSlot = [](const MachineInstr &MI, int64_t Addr) {
      unsigned Op = MI.getOpcode();
      switch (Op) {
      // Direct DP ops.
      case W65816::LDA_DP: case W65816::STA_DP: case W65816::STZ_DP:
      case W65816::LDX_DP: case W65816::STX_DP:
      case W65816::LDY_DP: case W65816::STY_DP:
      case W65816::ADC_DP: case W65816::SBC_DP:
      case W65816::AND_DP: case W65816::ORA_DP:
      case W65816::EOR_DP: case W65816::CMP_DP:
      case W65816::CPX_DP: case W65816::CPY_DP:
      case W65816::LSR_DP: case W65816::ROR_DP:
      case W65816::ASL_DP: case W65816::ROL_DP:
      case W65816::INC_DP: case W65816::DEC_DP:
      case W65816::BIT_DP: case W65816::TSB_DP:
      case W65816::TRB_DP:
      // DP-indexed ops.
      case W65816::LDA_DPX: case W65816::STA_DPX: case W65816::STZ_DPX:
      case W65816::LDY_DPX: case W65816::STY_DPX:
      case W65816::ADC_DPX: case W65816::SBC_DPX:
      case W65816::AND_DPX: case W65816::ORA_DPX:
      case W65816::EOR_DPX: case W65816::CMP_DPX:
      case W65816::LDX_DPY: case W65816::STX_DPY:
      case W65816::LSR_DPX: case W65816::ROR_DPX:
      case W65816::ASL_DPX: case W65816::ROL_DPX:
      case W65816::BIT_DPX:
      // DP-indirect ops — read the slot AS a pointer.
      case W65816::LDA_DPInd: case W65816::STA_DPInd:
      case W65816::LDA_DPIndY: case W65816::STA_DPIndY:
      case W65816::LDA_DPIndX: case W65816::STA_DPIndX:
      case W65816::LDA_DPIndLong: case W65816::STA_DPIndLong:
      case W65816::LDA_DPIndLongY: case W65816::STA_DPIndLongY:
      case W65816::ADC_DPInd: case W65816::SBC_DPInd:
      case W65816::AND_DPInd: case W65816::ORA_DPInd:
      case W65816::EOR_DPInd: case W65816::CMP_DPInd:
      case W65816::ADC_DPIndY: case W65816::SBC_DPIndY:
      case W65816::AND_DPIndY: case W65816::ORA_DPIndY:
      case W65816::EOR_DPIndY: case W65816::CMP_DPIndY:
      case W65816::ADC_DPIndLong: case W65816::SBC_DPIndLong:
      case W65816::AND_DPIndLong: case W65816::ORA_DPIndLong:
      case W65816::EOR_DPIndLong: case W65816::CMP_DPIndLong:
      case W65816::ADC_DPIndLongY: case W65816::SBC_DPIndLongY:
      case W65816::AND_DPIndLongY: case W65816::ORA_DPIndLongY:
      case W65816::EOR_DPIndLongY: case W65816::CMP_DPIndLongY:
        if (MI.getNumOperands() >= 1 && MI.getOperand(0).isImm() &&
            MI.getOperand(0).getImm() == Addr)
          return true;
        // Indirect ops read addr AND addr+1 (16-bit ptr) or addr+1,2
        // (24-bit ptr).  Bail when the candidate dead-store target is
        // within those bytes.
        if (MI.getNumOperands() >= 1 && MI.getOperand(0).isImm()) {
          int64_t IndAddr = MI.getOperand(0).getImm();
          if (IndAddr == Addr - 1 || IndAddr == Addr - 2)
            return true;
        }
        break;
      }
      return false;
    };
    for (MachineBasicBlock &MBB : MF) {
      SmallVector<MachineInstr *, 8> ToErase;
      SmallPtrSet<MachineInstr *, 8> ErasedSet;
      for (auto It = MBB.begin(); It != MBB.end(); ++It) {
        if (ErasedSet.count(&*It)) continue;
        if (It->getOpcode() != W65816::STA_DP) continue;
        if (It->getNumOperands() < 1 || !It->getOperand(0).isImm())
          continue;
        int64_t Sta1Addr = It->getOperand(0).getImm();
        auto Walk = std::next(It);
        while (Walk != MBB.end()) {
          if (Walk->isDebugInstr()) { ++Walk; continue; }
          if (Walk->isBranch() || Walk->isCall() || Walk->isReturn() ||
              Walk->isInlineAsm()) break;
          if (Walk->getOpcode() == W65816::STA_DP &&
              Walk->getNumOperands() >= 1 && Walk->getOperand(0).isImm() &&
              Walk->getOperand(0).getImm() == Sta1Addr) {
            ToErase.push_back(&*It);
            ErasedSet.insert(&*It);
            break;
          }
          if (touchesDpSlot(*Walk, Sta1Addr)) break;
          ++Walk;
        }
      }
      for (MachineInstr *MI : ToErase) {
        MI->eraseFromParent();
        Changed = true;
      }
    }
  }

  // Stack-rel dead-store elim — called from the always-on section too
  // (so it benefits recursive / non-IMG-promotable functions like fib).
  Changed |= elideStackRelDeadStore(MF);

  // DP-slot zero-check bridge via X.  Pattern:
  //   [op that sets Z on A]
  //   STA_DP slot
  //   [ops that don't read/write slot, don't touch X, don't branch/call]
  //   LDA_DP slot
  //   Bcond  (BNE/BEQ)
  //
  // The STA/LDA round-trip exists purely to preserve A's value across
  // the clobbers.  TAX/TXA does the same job in 4 cyc instead of 8.
  // Saves 4 cyc/match.  Hits popcount's `x_lo | x_hi ; (work) ; bne`.
  //
  // Safety: X register must be dead.  Conservative check — fires only
  // when the entire MBB doesn't reference X register except as our new
  // TAX/TXA, AND all MBB successors don't have X as live-in.
  for (MachineBasicBlock &MBB : MF) {
    // First: does this MBB reference X at all?  If yes, bail.  This is
    // conservative — refining would need full liveness.
    bool MbbTouchesX = false;
    for (const MachineInstr &MI : MBB) {
      for (const MachineOperand &MO : MI.operands()) {
        if (MO.isReg() && MO.getReg() == W65816::X) {
          MbbTouchesX = true; break;
        }
      }
      if (MbbTouchesX) break;
      // Also handle InstImplied ops that don't list X explicitly.
      switch (MI.getOpcode()) {
      case W65816::TAX: case W65816::TYX: case W65816::TSX:
      case W65816::PLX: case W65816::TXA: case W65816::TXY:
      case W65816::TXS: case W65816::PHX: case W65816::INX:
      case W65816::DEX:
        MbbTouchesX = true; break;
      }
      if (MI.isCall()) { MbbTouchesX = true; break; }
      if (MbbTouchesX) break;
    }
    if (MbbTouchesX) continue;
    // Successors with X as live-in?
    bool SuccUsesX = false;
    for (MachineBasicBlock *Succ : MBB.successors()) {
      if (Succ->isLiveIn(W65816::X)) { SuccUsesX = true; break; }
    }
    if (SuccUsesX) continue;
    // Walk forward looking for STA_DP / ... / LDA_DP / Bcond patterns.
    SmallVector<MachineInstr *, 4> ToErase;
    for (auto It = MBB.begin(); It != MBB.end(); ++It) {
      if (It->getOpcode() != W65816::STA_DP) continue;
      if (It->getNumOperands() < 1 || !It->getOperand(0).isImm()) continue;
      int64_t StaAddr = It->getOperand(0).getImm();
      auto Walk = std::next(It);
      MachineInstr *LdaMI = nullptr;
      MachineInstr *BcondMI = nullptr;
      bool blocked = false;
      while (Walk != MBB.end()) {
        if (Walk->isDebugInstr()) { ++Walk; continue; }
        unsigned WO = Walk->getOpcode();
        if (Walk->isBranch() || Walk->isCall() || Walk->isReturn() ||
            Walk->isInlineAsm()) {
          // If this is the Bcond AFTER our LDA, capture it.
          if (LdaMI && (WO == W65816::BNE || WO == W65816::BEQ)) {
            BcondMI = &*Walk;
          }
          break;
        }
        if (WO == W65816::LDA_DP &&
            Walk->getNumOperands() >= 1 && Walk->getOperand(0).isImm() &&
            Walk->getOperand(0).getImm() == StaAddr) {
          LdaMI = &*Walk;
          // Next non-debug must be BNE/BEQ.
          auto Next = std::next(Walk);
          while (Next != MBB.end() && Next->isDebugInstr()) ++Next;
          if (Next != MBB.end()) {
            unsigned NO = Next->getOpcode();
            if (NO == W65816::BNE || NO == W65816::BEQ) BcondMI = &*Next;
          }
          break;
        }
        // Anything touching our DP slot bails.
        for (const MachineOperand &MO : Walk->operands()) {
          if (MO.isImm() && MO.getImm() == StaAddr) {
            // Conservatively assume DP-op refs StaAddr unless we know
            // it's a different opcode entirely.  The dead-store-elim
            // has similar logic but more refined; here we keep it
            // simple: bail on any imm-matching op.
            blocked = true; break;
          }
        }
        if (blocked) break;
        ++Walk;
      }
      if (blocked || !LdaMI || !BcondMI) continue;
      // Global-use check: if any OTHER MBB references the DP slot, the
      // STA we'd erase may be initializing it for a later use.  Bail.
      // Caught by sumOfSquares' counter at $D0 — entry-BB's STA_DP 208
      // initializes the countdown counter that bb.4 reads via DEC_DP.
      bool referencedElsewhere = false;
      for (MachineBasicBlock &OtherMBB : MF) {
        if (&OtherMBB == &MBB) continue;
        for (const MachineInstr &OtherMI : OtherMBB) {
          for (const MachineOperand &MO : OtherMI.operands()) {
            if (MO.isImm() && MO.getImm() == StaAddr) {
              referencedElsewhere = true; break;
            }
          }
          if (referencedElsewhere) break;
        }
        if (referencedElsewhere) break;
      }
      if (referencedElsewhere) continue;
      // Replace STA_DP with TAX, LDA_DP with TXA.
      const TargetInstrInfo *TII = MF.getSubtarget().getInstrInfo();
      BuildMI(MBB, It, It->getDebugLoc(), TII->get(W65816::TAX));
      BuildMI(MBB, LdaMI, LdaMI->getDebugLoc(), TII->get(W65816::TXA));
      ToErase.push_back(&*It);
      ToErase.push_back(LdaMI);
    }
    for (MachineInstr *MI : ToErase) {
      MI->eraseFromParent();
      Changed = true;
    }
  }

  // popcount-style "n += (x & 1)" combined with `x >>= 1` LSR/ROR pair:
  // use the C flag set by ROR_DP directly via `ADC #0` (which adds C +
  // 0 + n = bit + n).  Eliminates the `LDA x_orig ; AND #1 ; ... ; CLC
  // ; ADC n` sequence and the lagged-PHI copy at end of body.  Big win
  // for popcount's specific shape.
  //
  // Pattern (post-shift-fold + dp-zero-check-tax-bridge):
  //   LSR_DP X_hi
  //   ROR_DP X_lo            ; C = old bit 0 of x_lo = the BIT we want
  //   LDA_DP X_lo            ; (zero-check setup)
  //   ORA_DP X_hi
  //   TAX                    ; preserve Z across the AND chain
  //   LDA_DP X_orig          ; current x_lo (lagged PHI)
  //   ANDi16imm 1            ; bit = x_orig & 1 (== C from ROR)
  //   STA_DP X_orig          ; dead store (overwritten by PHI-copy below)
  //   CLC                    ; ← kills our preserved C
  //   ADC_DP N               ; n += bit (but ADC reads C=0 + bit)
  //   STA_DP N
  //   LDA_DP X_lo            ; PHI copy start
  //   STA_DP X_orig          ; x_orig = x_lo NEW
  //   TXA                    ; restore Z
  //   BNE loop
  //
  // Rewrite:
  //   LSR_DP X_hi
  //   ROR_DP X_lo            ; C = bit
  //   LDA_DP N
  //   ADC_Imm 0              ; n + 0 + C = n + bit
  //   STA_DP N
  //   LDA_DP X_lo
  //   ORA_DP X_hi
  //   BNE loop
  //
  // Erased: TAX, LDA X_orig, AND #1, STA X_orig, CLC, LDA X_lo PHI-copy,
  //         STA X_orig PHI-copy, TXA.  Plus x_orig itself becomes dead.
  //
  // Saves ~25 cyc/iter on popcount.  29 iters → ~725 cyc.  popcount
  // 1705 → ~1000.  Calypsi is 1534, so this would BEAT Calypsi.
  for (MachineBasicBlock &MBB : MF) {
    SmallVector<MachineInstr *, 12> ToErase;
    for (auto It = MBB.begin(); It != MBB.end();) {
      auto Lsr = It++;
      if (Lsr->getOpcode() != W65816::LSR_DP) continue;
      if (Lsr->getNumOperands() < 1 || !Lsr->getOperand(0).isImm()) continue;
      int64_t HiAddr = Lsr->getOperand(0).getImm();
      auto skipDbg = [&](auto &P) {
        while (P != MBB.end() && P->isDebugInstr()) ++P;
      };
      auto Ror = std::next(Lsr); skipDbg(Ror);
      if (Ror == MBB.end() || Ror->getOpcode() != W65816::ROR_DP) continue;
      if (Ror->getNumOperands() < 1 || !Ror->getOperand(0).isImm()) continue;
      int64_t LoAddr = Ror->getOperand(0).getImm();
      auto P = std::next(Ror); skipDbg(P);
      if (P == MBB.end() || P->getOpcode() != W65816::LDA_DP) continue;
      if (P->getOperand(0).getImm() != LoAddr) continue;
      auto LdaLo1 = P; ++P; skipDbg(P);
      if (P == MBB.end() || P->getOpcode() != W65816::ORA_DP) continue;
      if (P->getOperand(0).getImm() != HiAddr) continue;
      auto OraHi = P; ++P; skipDbg(P);
      if (P == MBB.end() || P->getOpcode() != W65816::TAX) continue;
      auto Tax = P; ++P; skipDbg(P);
      if (P == MBB.end() || P->getOpcode() != W65816::LDA_DP) continue;
      int64_t OrigAddr = P->getOperand(0).getImm();
      if (OrigAddr == LoAddr || OrigAddr == HiAddr) continue;
      auto LdaOrig = P; ++P; skipDbg(P);
      if (P == MBB.end() || P->getOpcode() != W65816::ANDi16imm) continue;
      // Verify AND operand is 1.
      bool isAnd1 = false;
      for (const MachineOperand &MO : P->operands()) {
        if (MO.isImm() && MO.getImm() == 1) { isAnd1 = true; break; }
      }
      if (!isAnd1) continue;
      auto AndOne = P; ++P; skipDbg(P);
      // Optional STA_DP X_orig (dead store; may already have been
      // erased by DP-dead-store-elim).
      MachineInstr *StaOrig1 = nullptr;
      if (P != MBB.end() && P->getOpcode() == W65816::STA_DP &&
          P->getOperand(0).isImm() && P->getOperand(0).getImm() == OrigAddr) {
        StaOrig1 = &*P; ++P; skipDbg(P);
      }
      if (P == MBB.end() || P->getOpcode() != W65816::CLC) continue;
      auto Clc = P; ++P; skipDbg(P);
      if (P == MBB.end() || P->getOpcode() != W65816::ADC_DP) continue;
      int64_t NAddr = P->getOperand(0).getImm();
      auto AdcN = P; ++P; skipDbg(P);
      if (P == MBB.end() || P->getOpcode() != W65816::STA_DP ||
          P->getOperand(0).getImm() != NAddr) continue;
      auto StaN = P; ++P; skipDbg(P);
      if (P == MBB.end() || P->getOpcode() != W65816::LDA_DP ||
          P->getOperand(0).getImm() != LoAddr) continue;
      auto LdaLo2 = P; ++P; skipDbg(P);
      if (P == MBB.end() || P->getOpcode() != W65816::STA_DP ||
          P->getOperand(0).getImm() != OrigAddr) continue;
      auto StaOrig2 = P; ++P; skipDbg(P);
      if (P == MBB.end() || P->getOpcode() != W65816::TXA) continue;
      auto Txa = P; ++P; skipDbg(P);
      if (P == MBB.end() || P->getOpcode() != W65816::BNE) continue;
      // (Bne stays; we just need to put a fresh ORA + BNE-using-Z form.)

      const TargetInstrInfo *TII = MF.getSubtarget().getInstrInfo();
      // After ROR, insert: LDA N ; ADC #0 ; STA N ; LDA X_lo ; ORA X_hi
      auto Insert = std::next(MachineBasicBlock::iterator(Ror));
      DebugLoc DL = Ror->getDebugLoc();
      BuildMI(MBB, Insert, DL, TII->get(W65816::LDA_DP)).addImm(NAddr);
      BuildMI(MBB, Insert, DL, TII->get(W65816::ADC_Imm16)).addImm(0);
      BuildMI(MBB, Insert, DL, TII->get(W65816::STA_DP)).addImm(NAddr);
      BuildMI(MBB, Insert, DL, TII->get(W65816::LDA_DP)).addImm(LoAddr);
      BuildMI(MBB, Insert, DL, TII->get(W65816::ORA_DP)).addImm(HiAddr);
      // Erase the old sequence.
      ToErase.push_back(&*LdaLo1);
      ToErase.push_back(&*OraHi);
      ToErase.push_back(&*Tax);
      ToErase.push_back(&*LdaOrig);
      ToErase.push_back(&*AndOne);
      if (StaOrig1) ToErase.push_back(StaOrig1);
      ToErase.push_back(&*Clc);
      ToErase.push_back(&*AdcN);
      ToErase.push_back(&*StaN);
      ToErase.push_back(&*LdaLo2);
      ToErase.push_back(&*StaOrig2);
      ToErase.push_back(&*Txa);
      // BNE stays — but uses Z from our new ORA.
      It = std::next(MachineBasicBlock::iterator(Ror));
    }
    for (MachineInstr *MI : ToErase) {
      MI->eraseFromParent();
      Changed = true;
    }
  }

  // X-register iter peephole.  In a self-loop body whose pointer-walk
  // iter is held in a DP slot, replace the per-iter
  //   LDA_DP   X1        ; reload iter from DP    (3 cyc)
  //   STA_StackRel S     ; copy to lagged slot    (5 cyc)
  //   INA_PSEUDO         ; iter++                 (2 cyc)
  //   STA_DP   X1        ; store back to DP       (3 cyc)
  // chain with the iter held in the X register across iters:
  //   TXA                ; A = X = OLD iter       (2 cyc)
  //   STA_StackRel S     ; copy to lagged slot    (5 cyc)
  //   INX                ; X = NEW iter           (2 cyc)
  // Saves 4 cyc/iter (13 → 9).  Targets strLen-shape loops.  The
  // preheader gets an LDA_DP X1; TAX inserted to seed X.
  //
  // Safety conditions:
  // - MBB is a self-loop (MBB ∈ MBB.predecessors()).
  // - DP slot X1 is not referenced ANYWHERE in the function outside
  //   this pattern (else our drop of STA_DP X1 corrupts other reads).
  // - X register is dead in the MBB (no TAX/INX/etc.) AND not live-in
  //   to any successor MBB.
  // - Preheader exists (= a non-self predecessor we can insert into).
  for (MachineBasicBlock &MBB : MF) {
    bool selfLoop = false;
    MachineBasicBlock *Preheader = nullptr;
    for (MachineBasicBlock *Pred : MBB.predecessors()) {
      if (Pred == &MBB) selfLoop = true;
      else Preheader = Pred;
    }
    if (!selfLoop || !Preheader) continue;
    // Successors must not have X live-in (we'll clobber X to use it
    // as iter).  The self-loop successor (= MBB itself) is allowed
    // because we'll redefine X every iter.
    bool succXLive = false;
    for (MachineBasicBlock *Succ : MBB.successors()) {
      if (Succ == &MBB) continue;
      if (Succ->isLiveIn(W65816::X)) { succXLive = true; break; }
    }
    if (succXLive) continue;
    // MBB must not touch X register anywhere.
    bool mbbTouchesX = false;
    for (const MachineInstr &MI : MBB) {
      if (MI.isCall()) { mbbTouchesX = true; break; }
      switch (MI.getOpcode()) {
      case W65816::TAX: case W65816::TYX: case W65816::TSX:
      case W65816::PLX: case W65816::TXA: case W65816::TXY:
      case W65816::TXS: case W65816::PHX: case W65816::INX:
      case W65816::DEX: case W65816::LDX_Imm16: case W65816::LDX_Imm8:
      case W65816::LDX_DP: case W65816::LDX_Abs:
      case W65816::LDX_DPY: case W65816::LDX_AbsY:
      case W65816::STX_DP: case W65816::STX_Abs:
      case W65816::STX_DPY:
      case W65816::CPX_DP: case W65816::CPX_Abs:
      case W65816::CPX_Imm8: case W65816::CPX_Imm16:
        mbbTouchesX = true; break;
      }
      if (mbbTouchesX) break;
      for (const MachineOperand &MO : MI.operands()) {
        if (MO.isReg() && MO.getReg() == W65816::X) {
          mbbTouchesX = true; break;
        }
      }
      if (mbbTouchesX) break;
    }
    if (mbbTouchesX) continue;
    // Find the 4-op pattern in MBB.
    SmallVector<MachineInstr *, 4> ToErase;
    for (auto It = MBB.begin(); It != MBB.end();) {
      auto LdaMI = It++;
      if (LdaMI->getOpcode() != W65816::LDA_DP) continue;
      if (LdaMI->getNumOperands() < 1 || !LdaMI->getOperand(0).isImm())
        continue;
      int64_t X1 = LdaMI->getOperand(0).getImm();
      auto skipDbg = [&](auto &P) {
        while (P != MBB.end() && P->isDebugInstr()) ++P;
      };
      auto P = std::next(LdaMI); skipDbg(P);
      if (P == MBB.end() || P->getOpcode() != W65816::STA_StackRel) continue;
      auto StaS = P; ++P; skipDbg(P);
      if (P == MBB.end() || P->getOpcode() != W65816::INA_PSEUDO) continue;
      auto Ina = P; ++P; skipDbg(P);
      if (P == MBB.end() || P->getOpcode() != W65816::STA_DP) continue;
      if (P->getOperand(0).getImm() != X1) continue;
      auto StaBack = P;
      // Verify DP slot X1 outside-pattern references are at most ONE
      // STA_DP in the preheader (the iter initialization), which we'll
      // rewrite to TAX.  Any read of X1 outside the pattern bails.
      MachineInstr *PreheaderInit = nullptr;
      bool referencedElsewhere = false;
      for (MachineBasicBlock &OtherMBB : MF) {
        for (MachineInstr &OtherMI : OtherMBB) {
          if (&OtherMI == &*LdaMI || &OtherMI == &*StaBack) continue;
          unsigned OO = OtherMI.getOpcode();
          // Look for any DP-addressing op whose operand matches X1.
          bool refsX1 = false;
          for (const MachineOperand &MO : OtherMI.operands()) {
            if (MO.isImm() && MO.getImm() == X1) {
              if (OO == W65816::LDA_DP || OO == W65816::STA_DP ||
                  OO == W65816::STZ_DP || OO == W65816::ADC_DP ||
                  OO == W65816::SBC_DP || OO == W65816::AND_DP ||
                  OO == W65816::ORA_DP || OO == W65816::EOR_DP ||
                  OO == W65816::CMP_DP || OO == W65816::INC_DP ||
                  OO == W65816::DEC_DP || OO == W65816::LSR_DP ||
                  OO == W65816::ROR_DP || OO == W65816::ASL_DP ||
                  OO == W65816::ROL_DP || OO == W65816::BIT_DP ||
                  OO == W65816::LDX_DP || OO == W65816::STX_DP ||
                  OO == W65816::LDY_DP || OO == W65816::STY_DP) {
                refsX1 = true;
              }
              break;
            }
          }
          if (!refsX1) continue;
          // Accept ONE STA_DP X1 in the preheader (we'll rewrite to TAX).
          if (OO == W65816::STA_DP && &OtherMBB == Preheader &&
              !PreheaderInit) {
            PreheaderInit = &OtherMI;
            continue;
          }
          // Otherwise bail.
          referencedElsewhere = true; break;
        }
        if (referencedElsewhere) break;
      }
      if (referencedElsewhere) continue;
      if (!PreheaderInit) continue;
      // Apply: in preheader, the existing STA_DP X1 (PreheaderInit)
      // gets replaced with TAX (A already has the initial value about
      // to be stored).  In the loop, replace the 4-op chain with
      // TXA; STA_StackRel S; INX.
      const TargetInstrInfo *TII = MF.getSubtarget().getInstrInfo();
      BuildMI(*Preheader, PreheaderInit, PreheaderInit->getDebugLoc(),
              TII->get(W65816::TAX));
      ToErase.push_back(PreheaderInit);
      // In the loop: insert TXA before StaS, and INX after StaS.
      DebugLoc DL = StaS->getDebugLoc();
      BuildMI(MBB, StaS, DL, TII->get(W65816::TXA));
      BuildMI(MBB, std::next(MachineBasicBlock::iterator(StaS)),
              DL, TII->get(W65816::INX));
      // Erase LdaMI, Ina, StaBack.  StaS stays.
      ToErase.push_back(&*LdaMI);
      ToErase.push_back(&*Ina);
      ToErase.push_back(&*StaBack);
      It = std::next(MachineBasicBlock::iterator(StaBack));
      break;  // only fire once per MBB to avoid cascading
    }
    for (MachineInstr *MI : ToErase) {
      MI->eraseFromParent();
      Changed = true;
    }
  }

  // Dead INC_HI_IF_CARRY_StackRel elimination.  For Layer 2
  // (`-w65816-dbr-safe-ptrs`) loops, ptr_hi tracking via slot H is
  // bookkeeping for a pointer whose bank stays in DBR.  When the
  // function never READS slot H (only writes it), the
  // `INC_HI_IF_CARRY_StackRel H` pseudo and the preheader STAs to H
  // are dead — eliminating them saves 3 cyc/iter (the BNE taken).
  //
  // Detect: a `INC_HI_IF_CARRY_StackRel H` instruction.
  // Verify: no instruction reads slot H anywhere in the function.
  //   - LDA_StackRel H, ADC_StackRel H, etc. (any read)
  //   - LDA_StackRelIndY H, STA_StackRelIndY H, etc. (used as pointer)
  // If clean, erase the INC and all STA_StackRel H stores.
  {
    DenseSet<int64_t> SlotsTouched;
    for (MachineBasicBlock &MBB : MF) {
      for (MachineInstr &MI : MBB) {
        if (MI.getOpcode() != W65816::INC_HI_IF_CARRY_StackRel) continue;
        if (MI.getNumOperands() < 1 || !MI.getOperand(0).isImm()) continue;
        SlotsTouched.insert(MI.getOperand(0).getImm());
      }
    }
    for (int64_t H : SlotsTouched) {
      // Scan the entire function for any READ of slot H or USE of H as
      // an indirect-Y base.
      bool readSomewhere = false;
      for (MachineBasicBlock &MBB : MF) {
        for (const MachineInstr &MI : MBB) {
          unsigned Op = MI.getOpcode();
          int64_t Imm = -1;
          if (MI.getNumOperands() >= 1 && MI.getOperand(0).isImm())
            Imm = MI.getOperand(0).getImm();
          if (Imm != H) continue;
          switch (Op) {
          // Reads of slot H via stack-rel direct.
          case W65816::LDA_StackRel:
          case W65816::ADC_StackRel: case W65816::SBC_StackRel:
          case W65816::AND_StackRel: case W65816::ORA_StackRel:
          case W65816::EOR_StackRel: case W65816::CMP_StackRel:
          // Uses of slot H as an indirect-Y base.
          case W65816::LDA_StackRelIndY: case W65816::STA_StackRelIndY:
          case W65816::ADC_StackRelIndY: case W65816::SBC_StackRelIndY:
          case W65816::AND_StackRelIndY: case W65816::ORA_StackRelIndY:
          case W65816::EOR_StackRelIndY: case W65816::CMP_StackRelIndY:
            readSomewhere = true;
            break;
          default:
            break;
          }
          // Also catch indirect uses of slot H-1 (the IndY reads 2 bytes
          // at H-1, H).  Conservative.
          if (Imm == H - 1) {
            switch (Op) {
            case W65816::LDA_StackRelIndY: case W65816::STA_StackRelIndY:
            case W65816::ADC_StackRelIndY: case W65816::SBC_StackRelIndY:
            case W65816::AND_StackRelIndY: case W65816::ORA_StackRelIndY:
            case W65816::EOR_StackRelIndY: case W65816::CMP_StackRelIndY:
              readSomewhere = true;
              break;
            default:
              break;
            }
          }
          if (readSomewhere) break;
        }
        if (readSomewhere) break;
      }
      if (readSomewhere) continue;
      // Slot H is dead.  Erase the INC_HI_IF_CARRY and all STA_StackRel
      // H stores.
      SmallVector<MachineInstr *, 4> ToErase;
      for (MachineBasicBlock &MBB : MF) {
        for (MachineInstr &MI : MBB) {
          unsigned Op = MI.getOpcode();
          if (MI.getNumOperands() < 1 || !MI.getOperand(0).isImm()) continue;
          if (MI.getOperand(0).getImm() != H) continue;
          if (Op == W65816::INC_HI_IF_CARRY_StackRel ||
              Op == W65816::STA_StackRel) {
            ToErase.push_back(&MI);
          }
        }
      }
      for (MachineInstr *MI : ToErase) {
        MI->eraseFromParent();
        Changed = true;
      }
    }
  }

  // Lagged-stack-rel → DP-indirect-Y conversion.  After the X-iter
  // peephole, the loop body looks like:
  //   TXA              ; A = X = OLD iter
  //   STA_StackRel S   ; lagged stack-rel ptr slot
  //   INX              ; X = NEW
  //   ...
  //   LDA_StackRelIndY S  ; deref via stack-rel-indirect-Y (7 cyc)
  //
  // Equivalent with the iter in X and the ptr in DP:
  //   STX_DP D         ; write 16-bit X to DP slot D:D+1 (4 cyc)
  //   INX              ; X = NEW (2)
  //   ...
  //   LDA_DPIndY D     ; deref via DP-indirect-Y (6 cyc)
  //
  // Saves TXA (2) + 1 cyc on STA→STX (5→4) + 1 cyc on IndY-stack→
  // IndY-DP (7→6) = 4 cyc/iter.  Requires a free DP slot pair.
  //
  // Conditions:
  // - MBB is a self-loop (already paired with X-iter peephole).
  // - Pattern TXA;STA_StackRel S;INX exists in MBB.
  // - LDA_StackRelIndY S exists in MBB.
  // - Slot S is referenced ONLY by the STA above and the LDA above
  //   (no other reads/writes in the function).
  // - There exists an unused IMG-range DP slot (16-bit pair).
  for (MachineBasicBlock &MBB : MF) {
    bool selfLoop = false;
    for (MachineBasicBlock *Pred : MBB.predecessors()) {
      if (Pred == &MBB) { selfLoop = true; break; }
    }
    if (!selfLoop) continue;

    // Find TXA ; STA_StackRel S ; INX in this MBB.  The INX is left in
    // place — Y-as-counter handles it elsewhere — so we only need to
    // verify it's present.
    MachineInstr *Txa = nullptr;
    MachineInstr *StaS = nullptr;
    int64_t Soff = -1;
    auto It = MBB.begin();
    while (It != MBB.end()) {
      if (It->getOpcode() == W65816::TXA) {
        auto P = std::next(It);
        while (P != MBB.end() && P->isDebugInstr()) ++P;
        if (P == MBB.end() || P->getOpcode() != W65816::STA_StackRel) {
          ++It; continue;
        }
        auto Sta = P; ++P;
        while (P != MBB.end() && P->isDebugInstr()) ++P;
        if (P == MBB.end() || P->getOpcode() != W65816::INX) {
          ++It; continue;
        }
        if (Sta->getNumOperands() < 1 || !Sta->getOperand(0).isImm()) {
          ++It; continue;
        }
        Txa = &*It;
        StaS = &*Sta;
        Soff = Sta->getOperand(0).getImm();
        break;
      }
      ++It;
    }
    if (!Txa) continue;

    // Find LDA_StackRelIndY S in MBB.
    MachineInstr *LdaIndY = nullptr;
    for (MachineInstr &MI : MBB) {
      if (MI.getOpcode() == W65816::LDA_StackRelIndY &&
          MI.getNumOperands() >= 1 && MI.getOperand(0).isImm() &&
          MI.getOperand(0).getImm() == Soff) {
        LdaIndY = &MI;
        break;
      }
    }
    if (!LdaIndY) continue;

    // Find the loop's predecessor (= preheader for our self-loop case).
    MachineBasicBlock *Preheader = nullptr;
    for (MachineBasicBlock *Pred : MBB.predecessors()) {
      if (Pred != &MBB) { Preheader = Pred; break; }
    }
    // Verify slot S is referenced ONLY by writes (dropped) and the
    // LdaIndY, EXCEPT we tolerate LDA_StackRel S reads in the
    // preheader (the X-iter peephole inserts these to bootstrap X).
    // Any read elsewhere (post-loop, non-preheader pred, etc.) bails.
    SmallVector<MachineInstr *, 2> DeadStaWrites;
    SmallVector<MachineInstr *, 2> DeadPreheaderReads;
    bool slotElsewhere = false;
    for (MachineBasicBlock &OtherMBB : MF) {
      for (MachineInstr &OtherMI : OtherMBB) {
        if (&OtherMI == StaS || &OtherMI == LdaIndY) continue;
        unsigned OOO = OtherMI.getOpcode();
        if (OtherMI.getNumOperands() < 1 || !OtherMI.getOperand(0).isImm())
          continue;
        if (OtherMI.getOperand(0).getImm() != Soff) continue;
        switch (OOO) {
        case W65816::STA_StackRel:
          DeadStaWrites.push_back(&OtherMI);
          continue;
        case W65816::LDA_StackRel:
          // Allow LDA in the preheader (X-iter's reload-to-bootstrap-X).
          if (Preheader && &OtherMBB == Preheader) {
            DeadPreheaderReads.push_back(&OtherMI);
            continue;
          }
          slotElsewhere = true; break;
        case W65816::ADC_StackRel: case W65816::SBC_StackRel:
        case W65816::AND_StackRel: case W65816::ORA_StackRel:
        case W65816::EOR_StackRel: case W65816::CMP_StackRel:
        case W65816::LDA_StackRelIndY: case W65816::STA_StackRelIndY:
        case W65816::ADC_StackRelIndY: case W65816::SBC_StackRelIndY:
        case W65816::AND_StackRelIndY: case W65816::ORA_StackRelIndY:
        case W65816::EOR_StackRelIndY: case W65816::CMP_StackRelIndY:
          slotElsewhere = true; break;
        default: break;
        }
        if (slotElsewhere) break;
      }
      if (slotElsewhere) break;
    }
    if (slotElsewhere) continue;
    // Each preheader LDA_StackRel S must be followed by TAX (it's the
    // X-iter bootstrap pattern).  Verify; if not, bail.
    for (MachineInstr *MI : DeadPreheaderReads) {
      auto N = std::next(MachineBasicBlock::iterator(MI));
      while (N != MI->getParent()->end() && N->isDebugInstr()) ++N;
      if (N == MI->getParent()->end() || N->getOpcode() != W65816::TAX) {
        slotElsewhere = true; break;
      }
    }
    if (slotElsewhere) continue;

    // Find a free DP slot in the IMG range.  Scan all DP-addressing
    // ops and collect used addresses; pick the lowest unused 16-bit
    // aligned slot.
    DenseSet<int64_t> UsedDpAddrs;
    for (MachineBasicBlock &OtherMBB : MF) {
      for (MachineInstr &OtherMI : OtherMBB) {
        if (OtherMI.getNumOperands() < 1 || !OtherMI.getOperand(0).isImm())
          continue;
        unsigned OOO = OtherMI.getOpcode();
        switch (OOO) {
        case W65816::LDA_DP: case W65816::STA_DP: case W65816::STZ_DP:
        case W65816::LDX_DP: case W65816::STX_DP:
        case W65816::LDY_DP: case W65816::STY_DP:
        case W65816::ADC_DP: case W65816::SBC_DP:
        case W65816::AND_DP: case W65816::ORA_DP:
        case W65816::EOR_DP: case W65816::CMP_DP:
        case W65816::CPX_DP: case W65816::CPY_DP:
        case W65816::LSR_DP: case W65816::ROR_DP:
        case W65816::ASL_DP: case W65816::ROL_DP:
        case W65816::INC_DP: case W65816::DEC_DP:
        case W65816::BIT_DP:
        case W65816::LDA_DPInd: case W65816::STA_DPInd:
        case W65816::LDA_DPIndY: case W65816::STA_DPIndY:
        case W65816::LDA_DPIndX: case W65816::STA_DPIndX:
        case W65816::LDA_DPIndLong: case W65816::STA_DPIndLong:
        case W65816::LDA_DPIndLongY: case W65816::STA_DPIndLongY:
          {
            int64_t A = OtherMI.getOperand(0).getImm();
            UsedDpAddrs.insert(A);
            UsedDpAddrs.insert(A + 1);  // 16-bit ops occupy 2 bytes
          }
          break;
        default: break;
        }
      }
    }
    // Pick a free 16-bit-aligned slot in $C0..$DE.
    int64_t FreeDp = -1;
    for (int64_t A = 0xC0; A <= 0xDE; A += 2) {
      if (!UsedDpAddrs.count(A) && !UsedDpAddrs.count(A + 1)) {
        FreeDp = A;
        break;
      }
    }
    if (FreeDp < 0) continue;

    // Apply: rewrite TXA;STA_StackRel S;INX → STX_DP FreeDp;INX
    // (TXA and StaS erased; STX_DP inserted at TXA's position).
    const TargetInstrInfo *TII = MF.getSubtarget().getInstrInfo();
    BuildMI(MBB, Txa, Txa->getDebugLoc(),
            TII->get(W65816::STX_DP)).addImm(FreeDp);
    Txa->eraseFromParent();
    StaS->eraseFromParent();
    // Rewrite LDA_StackRelIndY S → LDA_DPIndY FreeDp.
    BuildMI(MBB, LdaIndY, LdaIndY->getDebugLoc(),
            TII->get(W65816::LDA_DPIndY))
        .addImm(FreeDp);
    LdaIndY->eraseFromParent();
    // We do NOT erase the preheader's STA writes or LDA reads to slot
    // S — they're the X-iter peephole's bootstrap (STA at preheader
    // init writes s_lo to S; LDA reloads s_lo to A for TAX→X).  The
    // body no longer uses slot S, but leaving them there is harmless.
    (void)DeadStaWrites;
    (void)DeadPreheaderReads;
    Changed = true;
  }

  // Y-as-counter for strLen-shape loops.  After DP-indirect-Y rewrite +
  // dead-INC_HI elim, the strLen body is:
  //   STX_DP D       (4 cyc)   writes iter X to DP slot D
  //   INX            (2)       iter++
  //   INC_DP C       (5)       counter++
  //   LDA_DPIndY D   (6)       deref via slot D
  //   ANDi16imm #ff  (--)      mask to 8-bit (lowers to AND #imm)
  //   BNE loop       (3)       branch on byte != 0
  // 22 cyc/iter.
  //
  // We can drop STX/INX/INC entirely by using Y as both the offset and
  // counter: D holds initial s (one-time write in preheader), Y starts
  // at -1, INY at top of each iter brings Y to 0, 1, 2, ...
  //   INY            (2)
  //   LDA_DPIndY D   (6)
  //   ANDi16imm #ff
  //   BNE loop       (3)
  // 13 cyc/iter — saves 9 cyc per iter.
  //
  // Preheader: LDY_Imm16 0 → LDY_Imm16 0xFFFF.  Add STX_DP D (one-time
  // s init).  Drop the existing `LDA #-1 ; STA_DP C` counter init.
  //
  // Exit MBB: replace `LDA_DP C` (returns the counter) with `TYA`.
  //
  // strLen 1279 → ~874 cyc (predicted).
  {
    for (MachineBasicBlock &MBB : MF) {
      // Self-loop check.
      bool isSelfLoop = false;
      for (auto *Succ : MBB.successors()) {
        if (Succ == &MBB) { isSelfLoop = true; break; }
      }
      if (!isSelfLoop) continue;
      if (MBB.pred_size() != 2) continue;
      if (MBB.succ_size() != 2) continue;

      // Find the 6-op pattern.
      MachineInstr *Stx = nullptr;
      MachineInstr *Inx = nullptr;
      MachineInstr *IncC = nullptr;
      MachineInstr *Lda = nullptr;
      MachineInstr *And = nullptr;
      MachineInstr *Bne = nullptr;
      int64_t D = -1, C = -1;
      bool extraInsn = false;
      for (MachineInstr &MI : MBB) {
        if (MI.isDebugInstr()) continue;
        switch (MI.getOpcode()) {
        case W65816::STX_DP:
          if (Stx) { extraInsn = true; break; }
          Stx = &MI;
          if (MI.getNumOperands() < 1 || !MI.getOperand(0).isImm()) {
            extraInsn = true; break;
          }
          D = MI.getOperand(0).getImm();
          break;
        case W65816::INX:
          if (!Stx || Inx) { extraInsn = true; break; }
          Inx = &MI;
          break;
        case W65816::INC_DP:
          if (!Inx || IncC) { extraInsn = true; break; }
          IncC = &MI;
          if (MI.getNumOperands() < 1 || !MI.getOperand(0).isImm()) {
            extraInsn = true; break;
          }
          C = MI.getOperand(0).getImm();
          if (C == D) { extraInsn = true; break; }
          break;
        case W65816::LDA_DPIndY:
          if (!IncC || Lda) { extraInsn = true; break; }
          Lda = &MI;
          if (MI.getNumOperands() < 1 || !MI.getOperand(0).isImm() ||
              MI.getOperand(0).getImm() != D) {
            extraInsn = true; break;
          }
          break;
        case W65816::ANDi16imm:
          if (!Lda || And) { extraInsn = true; break; }
          And = &MI;
          if (MI.getNumOperands() < 3 || !MI.getOperand(2).isImm() ||
              MI.getOperand(2).getImm() != 255) {
            extraInsn = true; break;
          }
          break;
        case W65816::BNE:
          if (!And || Bne) { extraInsn = true; break; }
          Bne = &MI;
          break;
        default:
          extraInsn = true;
          break;
        }
        if (extraInsn) break;
      }
      if (extraInsn || !Stx || !Inx || !IncC || !Lda || !And || !Bne) continue;

      // Find preheader (predecessor that is not self).
      MachineBasicBlock *Preheader = nullptr;
      for (auto *Pred : MBB.predecessors()) {
        if (Pred != &MBB) {
          if (Preheader) { Preheader = nullptr; break; }
          Preheader = Pred;
        }
      }
      if (!Preheader) continue;

      // Find exit MBB (successor that is not self).
      MachineBasicBlock *Exit = nullptr;
      for (auto *Succ : MBB.successors()) {
        if (Succ != &MBB) {
          if (Exit) { Exit = nullptr; break; }
          Exit = Succ;
        }
      }
      if (!Exit) continue;

      // In preheader: find LDY_Imm16 0 and LDA #-1; STA_DP C (counter init).
      MachineInstr *Ldy = nullptr;
      MachineInstr *LdaNeg1 = nullptr;
      MachineInstr *StaC = nullptr;
      MachineInstr *Tax = nullptr;
      for (MachineInstr &MI : *Preheader) {
        if (MI.getOpcode() == W65816::LDY_Imm16 &&
            MI.getNumOperands() >= 1 && MI.getOperand(0).isImm() &&
            MI.getOperand(0).getImm() == 0) {
          Ldy = &MI;
        }
        if (MI.getOpcode() == W65816::LDAi16imm &&
            MI.getNumOperands() >= 2 && MI.getOperand(1).isImm() &&
            (MI.getOperand(1).getImm() == -1 ||
             MI.getOperand(1).getImm() == 0xFFFF)) {
          LdaNeg1 = &MI;
        }
        if (MI.getOpcode() == W65816::STA_DP &&
            MI.getNumOperands() >= 1 && MI.getOperand(0).isImm() &&
            MI.getOperand(0).getImm() == C) {
          StaC = &MI;
        }
        if (MI.getOpcode() == W65816::TAX) {
          Tax = &MI;
        }
      }
      if (!Ldy || !LdaNeg1 || !StaC || !Tax) continue;

      // In exit MBB: find LDA_DP C followed by RTL (return uses counter).
      MachineInstr *ExitLdaC = nullptr;
      for (MachineInstr &MI : *Exit) {
        if (MI.getOpcode() == W65816::LDA_DP &&
            MI.getNumOperands() >= 1 && MI.getOperand(0).isImm() &&
            MI.getOperand(0).getImm() == C) {
          ExitLdaC = &MI;
          break;
        }
      }
      if (!ExitLdaC) continue;

      // Verify no other references to slots D or C anywhere else in MF.
      bool extraDRef = false;
      bool extraCRef = false;
      for (MachineBasicBlock &OtherMBB : MF) {
        for (MachineInstr &MI : OtherMBB) {
          if (&MI == Stx || &MI == IncC || &MI == Lda ||
              &MI == LdaNeg1 || &MI == StaC || &MI == ExitLdaC) continue;
          if (MI.getNumOperands() >= 1 && MI.getOperand(0).isImm()) {
            int64_t Imm = MI.getOperand(0).getImm();
            unsigned Op = MI.getOpcode();
            // Catch any DP or DPInd op touching slot D or D+1, C or C+1.
            switch (Op) {
            case W65816::LDA_DP: case W65816::STA_DP: case W65816::STZ_DP:
            case W65816::LDX_DP: case W65816::STX_DP:
            case W65816::LDY_DP: case W65816::STY_DP:
            case W65816::ADC_DP: case W65816::SBC_DP:
            case W65816::AND_DP: case W65816::ORA_DP: case W65816::EOR_DP:
            case W65816::CMP_DP: case W65816::CPX_DP: case W65816::CPY_DP:
            case W65816::LSR_DP: case W65816::ROR_DP:
            case W65816::ASL_DP: case W65816::ROL_DP:
            case W65816::INC_DP: case W65816::DEC_DP:
            case W65816::BIT_DP: case W65816::TSB_DP: case W65816::TRB_DP:
            case W65816::LDA_DPIndY: case W65816::STA_DPIndY:
            case W65816::LDA_DPInd: case W65816::STA_DPInd:
              if (Imm == D || Imm == D + 1) extraDRef = true;
              if (Imm == C || Imm == C + 1) extraCRef = true;
              break;
            }
          }
        }
      }
      if (extraDRef || extraCRef) continue;

      // Perform the transformation.
      const TargetInstrInfo *TII = MF.getSubtarget().getInstrInfo();

      // 1. Preheader: change LDY_Imm16 0 → LDY_Imm16 0xFFFF.
      Ldy->getOperand(0).setImm(0xFFFF);

      // 2. Preheader: erase LDA #-1 and STA_DP C (counter init dead).
      LdaNeg1->eraseFromParent();
      StaC->eraseFromParent();

      // 3. Preheader: insert STX_DP D after TAX (one-time s → D init).
      auto AfterTax = std::next(MachineBasicBlock::iterator(Tax));
      BuildMI(*Preheader, AfterTax, Tax->getDebugLoc(),
              TII->get(W65816::STX_DP)).addImm(D);

      // 4. Body: erase STX_DP D, INX, INC_DP C.
      Stx->eraseFromParent();
      Inx->eraseFromParent();
      IncC->eraseFromParent();

      // 5. Body: insert INY at start (before LDA_DPIndY).
      BuildMI(MBB, Lda, Lda->getDebugLoc(),
              TII->get(W65816::INY));

      // 6. Exit: replace LDA_DP C with TYA.
      BuildMI(*Exit, ExitLdaC, ExitLdaC->getDebugLoc(),
              TII->get(W65816::TYA));
      ExitLdaC->eraseFromParent();

      Changed = true;
    }
  }

  // Run elideStoreForwarding at the very end, AFTER IMG promotion has
  // committed slot assignments.  Running this peephole earlier (with
  // the other early peepholes) cascades into different IMG-promotion
  // choices and was observed to regress strcpy 1.63×.  At this point
  // promotion is done, so dropping a redundant LDA can no longer
  // disturb slot allocation.
  // End-of-pass: also try elideStoreForwarding for non-bailing
  // functions.  After main IMG promotion finalizes slot assignments,
  // dropping a redundant LDA can no longer disturb them.
  Changed |= elideStoreForwarding(MF);

  Changed |= ChangedEarly;
  return Changed;
}
