//===-- W65816UnLSR.cpp - Undo LSR for global-array pointer-walks --------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===---------------------------------------------------------------------===//
//
// Post-LSR IR pass.  LSR converts `for (i; i<n; i++) arr[i]` loops into
// pointer-walking:
//
//   %lsr.iv1 = phi ptr [@arr, ...], [%scevgep, ...]
//   %lsr.iv  = phi i16 [%n, ...], [%lsr.iv.next, ...]
//   %v = load i16, ptr %lsr.iv1
//   %scevgep = getelementptr i8, ptr %lsr.iv1, i32 stride
//   %lsr.iv.next = add i16 %lsr.iv, -1
//
// On the W65816 this is a regression: pointer-deref needs the 24-bit
// `[dp],Y` addressing (sets up $E0/$E2 each iter), whereas indexed
// access via `lda <global>, X` is 3 bytes / 5 cyc DBR-relative.  The
// LDA_AbsX SDAG combine recognizes `(load (add Wrapper, idx))` but
// LSR has destroyed that shape.
//
// This pass detects the shape and converts back: introduce a forward
// counter PHI (0, 1, 2, ...), replace loads/stores through the pointer
// PHI with indexed GEPs `getelementptr T, @global, %i`.  The old pointer
// PHI becomes dead and is DCE'd.  The existing count-down counter PHI
// stays as the loop control.
//
// Restrictions:
//  * Pointer PHI's initial value must be a `GlobalAddress` (otherwise
//    it might point at a non-DBR-bank object, and our combine doesn't
//    fire / produces wrong code).
//  * Pointer PHI must have exactly two incoming values (header + latch).
//  * Pointer PHI's only uses must be the stride GEP and loads/stores.
//  * Stride must be a positive constant.
//  * The loop must have an existing trip-count counter so we don't
//    introduce extra register pressure.
//
//===---------------------------------------------------------------------===//

#include "W65816.h"
#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/SmallPtrSet.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/Analysis/LoopInfo.h"
#include "llvm/Analysis/ScalarEvolution.h"
#include "llvm/InitializePasses.h"
#include "llvm/IR/Constants.h"
#include "llvm/IR/Function.h"
#include "llvm/IR/IRBuilder.h"
#include "llvm/IR/InstIterator.h"
#include "llvm/IR/Instructions.h"
#include "llvm/IR/Module.h"
#include "llvm/Pass.h"
#include "llvm/Support/Debug.h"

using namespace llvm;

#define DEBUG_TYPE "w65816-un-lsr"

namespace llvm {
void initializeW65816UnLSRPass(PassRegistry &);
}

namespace {

class W65816UnLSR : public FunctionPass {
public:
  static char ID;
  W65816UnLSR() : FunctionPass(ID) {}

  StringRef getPassName() const override {
    return "W65816 undo LSR pointer-walk for global-array access";
  }

  void getAnalysisUsage(AnalysisUsage &AU) const override {
    AU.addRequired<LoopInfoWrapperPass>();
    AU.setPreservesCFG();
  }

  bool runOnFunction(Function &F) override;

private:
  bool processLoop(Loop *L);
  bool processCounterToPtrPHIs(Loop *L);
  bool processReturnedCounter(Loop *L);
};

} // namespace

char W65816UnLSR::ID = 0;

INITIALIZE_PASS_BEGIN(W65816UnLSR, DEBUG_TYPE,
                      "W65816 undo LSR for global-array", false, false)
INITIALIZE_PASS_DEPENDENCY(LoopInfoWrapperPass)
INITIALIZE_PASS_END(W65816UnLSR, DEBUG_TYPE,
                    "W65816 undo LSR for global-array", false, false)

FunctionPass *llvm::createW65816UnLSR() { return new W65816UnLSR(); }

bool W65816UnLSR::runOnFunction(Function &F) {
  if (F.hasOptNone())
    return false;
  LoopInfo &LI = getAnalysis<LoopInfoWrapperPass>().getLoopInfo();
  bool Changed = false;
  for (Loop *L : LI) {
    Changed |= processLoop(L);
    Changed |= processCounterToPtrPHIs(L);
    // processReturnedCounter remains disabled — see note above.
    SmallVector<Loop *, 4> Worklist(L->begin(), L->end());
    while (!Worklist.empty()) {
      Loop *Sub = Worklist.pop_back_val();
      Changed |= processLoop(Sub);
      Changed |= processCounterToPtrPHIs(Sub);
      Worklist.append(Sub->begin(), Sub->end());
    }
  }
  return Changed;
}


// strLen-style undo: LSR converts `return p - s` into a counter PHI
// `%lsr.iv` that increments per iter and is returned directly:
//   %lsr.iv = phi i16 [-1, %entry], [%lsr.iv.next, %latch]
//   %p.0    = phi ptr [%s, %entry], [%incdec.ptr, %latch]
//   %incdec.ptr = getelementptr i8, %p.0, i32 1
//   %lsr.iv.next = add i16 %lsr.iv, 1
//   br ..., %exit, %loop
// %exit:
//   ret i16 %lsr.iv.next
//
// LSR's reasoning: cheaper to maintain a counter than compute (p - s)
// at exit.  On W65816 the opposite is true: counter inc per iter costs
// 5 cyc/iter * N iters; one-time sub at exit costs ~10 cyc total.
//
// This undo finds the counter PHI, verifies its only out-of-loop use
// is via LCSSA → return, finds the sibling pointer PHI with the same
// stride, and replaces the return value with
// `(i16)(p_lcssa - base) + (K_init + 1)`.  Erases the counter PHI.
//
// Saves ~5 cyc/iter on strLen-shape loops with a returned counter.
bool W65816UnLSR::processReturnedCounter(Loop *L) {
  BasicBlock *Header = L->getHeader();
  BasicBlock *Latch = L->getLoopLatch();
  BasicBlock *Preheader = L->getLoopPreheader();
  if (!Latch || !Preheader) return false;

  // Single-exit loop.
  SmallVector<BasicBlock *, 2> ExitBlocks;
  L->getExitBlocks(ExitBlocks);
  if (ExitBlocks.size() != 1) return false;
  BasicBlock *Exit = ExitBlocks[0];

  // Find a candidate counter PHI: integer, init=ConstantInt, step=+1.
  PHINode *CounterPHI = nullptr;
  ConstantInt *KInit = nullptr;
  BinaryOperator *CounterStep = nullptr;
  for (PHINode &PN : Header->phis()) {
    if (!PN.getType()->isIntegerTy()) continue;
    if (PN.getNumIncomingValues() != 2) continue;
    Value *Init = nullptr, *Step = nullptr;
    for (unsigned i = 0; i < PN.getNumIncomingValues(); ++i) {
      BasicBlock *Pred = PN.getIncomingBlock(i);
      if (L->contains(Pred)) Step = PN.getIncomingValue(i);
      else                    Init = PN.getIncomingValue(i);
    }
    if (!Init || !Step) continue;
    auto *InitC = dyn_cast<ConstantInt>(Init);
    if (!InitC) continue;
    auto *StepBO = dyn_cast<BinaryOperator>(Step);
    if (!StepBO || StepBO->getOpcode() != Instruction::Add) continue;
    Value *Other = nullptr;
    if (StepBO->getOperand(0) == &PN) Other = StepBO->getOperand(1);
    else if (StepBO->getOperand(1) == &PN) Other = StepBO->getOperand(0);
    if (!Other) continue;
    auto *StepCI = dyn_cast<ConstantInt>(Other);
    if (!StepCI || !StepCI->isOne()) continue;
    CounterPHI = &PN;
    KInit = InitC;
    CounterStep = StepBO;
    break;
  }
  if (!CounterPHI) return false;

  // The counter PHI must be used INSIDE the loop only by its increment
  // and OUTSIDE the loop only via an LCSSA PHI in the exit block that
  // feeds a return.  Same for the increment.
  auto isOnlyInLoopUseTheStep = [&](Value *V) {
    for (User *U : V->users()) {
      auto *UI = dyn_cast<Instruction>(U);
      if (!UI) return false;
      if (!L->contains(UI)) continue;  // out-of-loop is handled separately
      if (UI == CounterStep) continue;
      // The PHI itself is allowed (V might be CounterStep, used by
      // CounterPHI's back-edge incoming).
      if (UI == CounterPHI) continue;
      return false;
    }
    return true;
  };
  if (!isOnlyInLoopUseTheStep(CounterPHI)) return false;
  if (!isOnlyInLoopUseTheStep(CounterStep)) return false;

  // Find a use of CounterPHI or CounterStep that's a ReturnInst.
  // The use might be DIRECT (no LCSSA — common after LCSSA cleanup)
  // or via an LCSSA PHI in the exit block.
  ReturnInst *Ret = nullptr;
  Value *RetSource = nullptr;          // the value the ret reads
  PHINode *ExitLCSSA = nullptr;        // optional LCSSA PHI to erase
  bool fromNext = false;                // true if return source is CounterStep
  auto findRet = [&](Value *V, bool isNext) -> bool {
    for (User *U : V->users()) {
      auto *UI = dyn_cast<Instruction>(U);
      if (!UI) continue;
      // Skip in-loop uses (those are the counter increment chain).
      if (L->contains(UI->getParent())) continue;
      if (auto *R = dyn_cast<ReturnInst>(UI)) {
        if (R->getReturnValue() != V) continue;
        Ret = R; RetSource = V; fromNext = isNext; return true;
      }
      // LCSSA PHI in the exit block?
      if (auto *PN = dyn_cast<PHINode>(UI)) {
        if (PN->getParent() != Exit) continue;
        if (PN->getNumIncomingValues() != 1) continue;
        if (PN->getIncomingValue(0) != V) continue;
        if (!PN->hasOneUse()) continue;
        auto *R = dyn_cast<ReturnInst>(PN->user_back());
        if (!R || R->getReturnValue() != PN) continue;
        Ret = R; RetSource = V; fromNext = isNext; ExitLCSSA = PN;
        return true;
      }
    }
    return false;
  };
  if (!findRet(CounterStep, true) && !findRet(CounterPHI, false))
    return false;

  // Find a sibling pointer PHI: init=Base, latch incoming is a
  // `getelementptr i8, %ptr, 1` of itself.
  PHINode *PtrPHI = nullptr;
  Value *Base = nullptr;
  GetElementPtrInst *PtrStep = nullptr;
  for (PHINode &PN : Header->phis()) {
    if (!PN.getType()->isPointerTy()) continue;
    if (PN.getNumIncomingValues() != 2) continue;
    Value *Init = nullptr, *Step = nullptr;
    for (unsigned i = 0; i < PN.getNumIncomingValues(); ++i) {
      BasicBlock *Pred = PN.getIncomingBlock(i);
      if (L->contains(Pred)) Step = PN.getIncomingValue(i);
      else                    Init = PN.getIncomingValue(i);
    }
    if (!Init || !Step) continue;
    auto *StepGEP = dyn_cast<GetElementPtrInst>(Step);
    if (!StepGEP) continue;
    if (StepGEP->getPointerOperand() != &PN) continue;
    if (StepGEP->getNumIndices() != 1) continue;
    if (!StepGEP->getSourceElementType()->isIntegerTy(8)) continue;
    auto *StrideCI = dyn_cast<ConstantInt>(StepGEP->getOperand(1));
    if (!StrideCI || !StrideCI->isOne()) continue;
    PtrPHI = &PN;
    Base = Init;
    PtrStep = StepGEP;
    break;
  }
  if (!PtrPHI) return false;

  // The pointer-PHI must have an LCSSA in the exit (so we can compute
  // p_lcssa - base).  Find it or create one.
  PHINode *PtrLCSSA = nullptr;
  for (PHINode &EPN : Exit->phis()) {
    if (EPN.getNumIncomingValues() != 1) continue;
    if (EPN.getIncomingValue(0) == PtrPHI) {
      PtrLCSSA = &EPN; break;
    }
  }
  if (!PtrLCSSA) {
    // Create LCSSA for PtrPHI.
    IRBuilder<> B(&Exit->front());
    PtrLCSSA = B.CreatePHI(PtrPHI->getType(), 1, "unlsr.p.lcssa");
    PtrLCSSA->addIncoming(PtrPHI, Latch);
  }

  // Build replacement value: (i16)(p_lcssa - base) + (K_init + (fromNext ? 1 : 0))
  // For fromNext=true (returning %counter.next): value = K_init + iters
  //   p_lcssa - base = iters (in bytes, stride 1) → value = K_init + (p_lcssa - base)
  //   But we want: counter.next at exit = K_init + iters; and p_lcssa - base = iters.
  //   So replacement = (i16)(p_lcssa - base) + K_init.
  //   For strLen: K_init = -1; iters at exit = K (where ret = K - 1 + 1 = K)
  //     Wait let me re-derive. counter init = -1. iter 1 entry: counter = -1.
  //     iter 1 exit: counter.next = 0. Suppose exit-iter is iter K. Then at
  //     iter K's icmp-true, counter.next = -1 + K.
  //   And p_lcssa = base + (K - 1) (since iter K had p.0 = base + K-1).
  //   So p_lcssa - base = K - 1.
  //   We want counter.next = K - 1 (because exit-iter is iter K, but counter.next
  //     was computed before icmp tested 0 - so it's K - 1 (with K iters = K decisions))
  //   Hmm, off-by-one is tricky.  Let me just test empirically.

  // The "return value type" we'll cast to.
  Type *RetTy = Ret->getReturnValue()->getType();
  if (!RetTy->isIntegerTy()) return false;
  Instruction *InsertPt = ExitLCSSA ? ExitLCSSA->getNextNode() : Ret;
  IRBuilder<> B(InsertPt);
  // (p_lcssa - base) as integer.
  Value *PLcssaInt = B.CreatePtrToInt(PtrLCSSA, Type::getInt32Ty(Header->getContext()), "unlsr.plcssa.i");
  Value *BaseInt = B.CreatePtrToInt(Base, Type::getInt32Ty(Header->getContext()), "unlsr.base.i");
  Value *Diff = B.CreateSub(PLcssaInt, BaseInt, "unlsr.diff");
  // Truncate to counter type.
  Value *DiffI = B.CreateTrunc(Diff, CounterPHI->getType(), "unlsr.diff.trunc");
  // For fromNext (returning %counter.next): replacement = diff + (K_init + 1).
  //   At exit, counter.next = K_init + iters.
  //   p_lcssa - base = iters (in bytes; stride 1).  Wait but iters is the iter count.
  //   Let me re-check with concrete example.
  //   strLen("a\0"): iter 1: p.0 = s, *p='a'!=0, p++, counter=-1, counter.next=0.
  //     iter 2: p.0 = s+1, *p=0, exit. counter=0, counter.next=1.
  //   At exit: counter.next = 1. iters before exit-iter's icmp-true = 2.
  //   p_lcssa = s+1 (the iter-2 entry value).  p_lcssa - base = 1.
  //   counter.next = 1 = K_init + 2 = -1 + 2 = 1. ✓
  //   p_lcssa - base = 1.  So counter.next = p_lcssa - base + 0.
  //   (K_init + iters - (iters - (p_lcssa - base))) = K_init + (p_lcssa - base) = K_init + 1.
  //   Wait: counter.next = K_init + iters; p_lcssa - base = iters - 1.
  //   So counter.next = K_init + (p_lcssa - base) + 1.
  //   For K_init = -1: counter.next = -1 + 1 + 1 = 1 if iters=2. ✓
  //   So replacement = diff + (K_init + 1).
  int64_t Adjust = KInit->getSExtValue() + (fromNext ? 1 : 0);
  Value *Result = DiffI;
  if (Adjust != 0) {
    Result = B.CreateAdd(DiffI,
                          ConstantInt::get(CounterPHI->getType(), Adjust),
                          "unlsr.result");
  }
  // Cast to return type if different.
  if (Result->getType() != RetTy) {
    if (CounterPHI->getType()->getIntegerBitWidth() <
        RetTy->getIntegerBitWidth())
      Result = B.CreateZExt(Result, RetTy);
    else
      Result = B.CreateTrunc(Result, RetTy);
  }
  // Replace the return.  If there's an LCSSA PHI, replace it.  Otherwise
  // replace the direct use in `ret`.
  if (ExitLCSSA) {
    ExitLCSSA->replaceAllUsesWith(Result);
    ExitLCSSA->eraseFromParent();
  } else {
    Ret->setOperand(0, Result);
  }

  // Erase the counter PHI and its increment.
  CounterStep->replaceAllUsesWith(UndefValue::get(CounterPHI->getType()));
  CounterPHI->replaceAllUsesWith(UndefValue::get(CounterPHI->getType()));
  CounterStep->eraseFromParent();
  CounterPHI->eraseFromParent();
  return true;
}


// strcpy-style undo: LSR converts two pointer PHIs (`src.addr.0` and
// `d.0` each stepping by 1) into a single counter PHI (`lsr.iv`) plus
// GEPs `(base, counter)` per iter.  On 65816 the counter+GEP form
// each iter does i32 (base + counter) on each pointer — much more
// expensive than just incrementing two i16 pointer PHIs.
//
// Pattern (post-LSR):
//   %lsr.iv = phi i32 [0, %entry], [%lsr.iv.next, %latch]
//   %scevgep_i = getelementptr i8, ptr %base_i, i32 %lsr.iv  (for each base_i)
//   ... loads/stores via %scevgep_i ...
//   %lsr.iv.next = add i32 %lsr.iv, 1
//
// Where each %base_i is loop-invariant (typically a function arg).
//
// Rewrite: for each base_i, introduce a pointer PHI that strides by 1
// per iter.  Replace %scevgep_i with the new pointer PHI.  If counter
// has no other uses, eliminate it.
bool W65816UnLSR::processCounterToPtrPHIs(Loop *L) {
  BasicBlock *Header = L->getHeader();
  BasicBlock *Latch = L->getLoopLatch();
  BasicBlock *Preheader = L->getLoopPreheader();
  if (!Latch || !Preheader) return false;

  // Find an integer counter PHI starting at 0 with step +1.
  PHINode *Counter = nullptr;
  Value *CounterNext = nullptr;
  for (PHINode &PN : Header->phis()) {
    if (!PN.getType()->isIntegerTy()) continue;
    if (PN.getNumIncomingValues() != 2) continue;
    Value *Init = nullptr, *Step = nullptr;
    for (unsigned i = 0; i < PN.getNumIncomingValues(); ++i) {
      BasicBlock *Pred = PN.getIncomingBlock(i);
      if (L->contains(Pred)) Step = PN.getIncomingValue(i);
      else                    Init = PN.getIncomingValue(i);
    }
    if (!Init || !Step) continue;
    auto *InitCI = dyn_cast<ConstantInt>(Init);
    if (!InitCI || !InitCI->isZero()) continue;
    auto *StepBO = dyn_cast<BinaryOperator>(Step);
    if (!StepBO || StepBO->getOpcode() != Instruction::Add) continue;
    Value *Other = nullptr;
    if (StepBO->getOperand(0) == &PN) Other = StepBO->getOperand(1);
    else if (StepBO->getOperand(1) == &PN) Other = StepBO->getOperand(0);
    if (!Other) continue;
    auto *StepCI = dyn_cast<ConstantInt>(Other);
    if (!StepCI || !StepCI->isOne()) continue;
    Counter = &PN;
    CounterNext = StepBO;
    break;
  }
  if (!Counter) return false;

  // Find GEPs `getelementptr i8, %base, %counter` (or %counter.next)
  // where base is loop-invariant.  Collect them and verify the counter
  // has no OTHER uses outside this pattern.
  SmallVector<GetElementPtrInst *, 4> GEPs;
  for (User *U : Counter->users()) {
    if (U == CounterNext) continue;
    auto *GEP = dyn_cast<GetElementPtrInst>(U);
    if (!GEP) return false;
    if (GEP->getNumIndices() != 1) return false;
    if (GEP->getOperand(1) != Counter) return false;
    Value *Base = GEP->getPointerOperand();
    // base must be loop-invariant.  Instructions inside the loop fail;
    // arguments and globals are always invariant.
    if (auto *BaseI = dyn_cast<Instruction>(Base))
      if (L->contains(BaseI)) return false;
    if (!Base->getType()->isPointerTy()) return false;
    // Only handle the i8 element type (byte stride).  Other strides
    // would need different ptr-PHI step values.
    if (!GEP->getSourceElementType()->isIntegerTy(8)) return false;
    GEPs.push_back(GEP);
  }
  // Also accept if CounterNext is used as a GEP index (sometimes LSR
  // uses the post-increment value).  Walk those too.
  for (User *U : CounterNext->users()) {
    if (U == Counter) continue;
    auto *GEP = dyn_cast<GetElementPtrInst>(U);
    if (GEP) {
      // Bail if CounterNext is used as a GEP index — we'd need to add
      // a +1 offset to the new pointer PHI to match.  Keep this simple
      // for now: only handle uses of Counter, not CounterNext.
      if (GEP->getNumIndices() == 1 && GEP->getOperand(1) == CounterNext)
        return false;
    }
    // Allow icmp / branch / other non-GEP uses of CounterNext — those
    // are the loop's exit test, fine to leave alone.
  }
  if (GEPs.empty()) return false;

  // For each unique base, build a pointer PHI.
  LLVMContext &Ctx = Header->getContext();
  Type *I8 = Type::getInt8Ty(Ctx);
  DenseMap<Value *, PHINode *> BasePhis;
  for (GetElementPtrInst *GEP : GEPs) {
    Value *Base = GEP->getPointerOperand();
    if (BasePhis.count(Base)) continue;
    IRBuilder<> B(&Header->front());
    PHINode *PtrPHI = B.CreatePHI(Base->getType(), 2, "unlsr.ptr");
    PtrPHI->addIncoming(Base, Preheader);
    // Build the step GEP in the latch (just before terminator).
    IRBuilder<> BL(Latch->getTerminator());
    Value *PtrNext = BL.CreateGEP(I8, PtrPHI,
                                   ConstantInt::get(Type::getInt16Ty(Ctx), 1),
                                   "unlsr.ptr.next");
    PtrPHI->addIncoming(PtrNext, Latch);
    BasePhis[Base] = PtrPHI;
  }

  // Replace each GEP's uses with the corresponding pointer PHI.
  for (GetElementPtrInst *GEP : GEPs) {
    GEP->replaceAllUsesWith(BasePhis[GEP->getPointerOperand()]);
  }
  // Erase the now-dead GEPs.
  for (GetElementPtrInst *GEP : GEPs) {
    if (GEP->use_empty()) GEP->eraseFromParent();
  }

  // If counter has no other uses (besides CounterNext and the latch
  // incoming), eliminate it.  CounterNext might still be used by the
  // exit test — leave that alone.
  bool counterDead = true;
  for (User *U : Counter->users()) {
    if (U == CounterNext) continue;
    counterDead = false;
    break;
  }
  if (counterDead) {
    // CounterNext might be used by other PHIs / icmp.  Don't erase if so.
    bool counterNextHasOtherUses = false;
    for (User *U : CounterNext->users()) {
      if (U == Counter) continue;
      counterNextHasOtherUses = true;
      break;
    }
    if (!counterNextHasOtherUses) {
      Type *IntT = Counter->getType();
      cast<Instruction>(CounterNext)->replaceAllUsesWith(
          UndefValue::get(IntT));
      Counter->replaceAllUsesWith(UndefValue::get(IntT));
      cast<Instruction>(CounterNext)->eraseFromParent();
      Counter->eraseFromParent();
    }
  }
  return true;
}

bool W65816UnLSR::processLoop(Loop *L) {
  BasicBlock *Header = L->getHeader();
  BasicBlock *Latch = L->getLoopLatch();
  if (!Latch)
    return false;
  // Single-block loops are fine (header == latch).

  // Find a pointer PHI whose initial value is a GlobalAddress and whose
  // latch-incoming is a constant-stride GEP off itself.
  SmallVector<PHINode *, 4> Candidates;
  for (PHINode &PN : Header->phis()) {
    if (!PN.getType()->isPointerTy()) continue;
    if (PN.getNumIncomingValues() != 2) continue;
    Candidates.push_back(&PN);
  }

  bool Changed = false;
  for (PHINode *PtrPHI : Candidates) {
    // Identify which incoming is from the preheader vs the latch.
    Value *InitVal = nullptr;
    Value *StepVal = nullptr;
    for (unsigned i = 0; i < PtrPHI->getNumIncomingValues(); ++i) {
      BasicBlock *Pred = PtrPHI->getIncomingBlock(i);
      Value *Inc = PtrPHI->getIncomingValue(i);
      if (L->contains(Pred))
        StepVal = Inc;
      else
        InitVal = Inc;
    }
    if (!InitVal || !StepVal) continue;

    // InitVal must be a GlobalAddress (GlobalVariable).  Could relax to
    // any constant base, but globals are the only case our combine
    // handles correctly.
    auto *InitGV = dyn_cast<GlobalVariable>(InitVal);
    if (!InitGV) continue;

    // StepVal must be a `getelementptr i8, %PtrPHI, ConstantInt`.
    auto *StepGEP = dyn_cast<GetElementPtrInst>(StepVal);
    if (!StepGEP) continue;
    if (StepGEP->getPointerOperand() != PtrPHI) continue;
    if (StepGEP->getNumIndices() != 1) continue;
    auto *StrideCI = dyn_cast<ConstantInt>(StepGEP->getOperand(1));
    if (!StrideCI) continue;
    int64_t StrideBytes = StrideCI->getSExtValue();
    if (StrideBytes <= 0) continue;
    // Determine element type for the indexed GEP.  Walk all loads/
    // stores through PtrPHI; require they all access the same primitive
    // type whose size equals StrideBytes (typical: i16 array → stride 2).
    Type *ElemTy = nullptr;
    SmallVector<Instruction *, 4> Accesses;
    bool ok = true;
    for (User *U : PtrPHI->users()) {
      if (U == StepGEP) continue;
      if (auto *LD = dyn_cast<LoadInst>(U)) {
        if (LD->getPointerOperand() != PtrPHI) { ok = false; break; }
        Type *T = LD->getType();
        if (!T->isIntegerTy()) { ok = false; break; }
        if (!ElemTy) ElemTy = T;
        else if (ElemTy != T) { ok = false; break; }
        Accesses.push_back(LD);
        continue;
      }
      if (auto *ST = dyn_cast<StoreInst>(U)) {
        if (ST->getPointerOperand() != PtrPHI) { ok = false; break; }
        Type *T = ST->getValueOperand()->getType();
        if (!T->isIntegerTy()) { ok = false; break; }
        if (!ElemTy) ElemTy = T;
        else if (ElemTy != T) { ok = false; break; }
        Accesses.push_back(ST);
        continue;
      }
      // Any other use → bail.
      ok = false;
      break;
    }
    if (!ok || !ElemTy || Accesses.empty()) continue;
    Module &M = *Header->getModule();
    const DataLayout &DL = M.getDataLayout();
    uint64_t ElemBytes = DL.getTypeAllocSize(ElemTy);
    if ((int64_t)ElemBytes != StrideBytes) continue;

    // We have a clean global-array pointer-walk.  Build a forward
    // counter PHI starting at 0, incrementing by 1.  Replace the
    // pointer-PHI's loads/stores with `GEP ElemTy, @global, %i`.
    LLVMContext &Ctx = Header->getContext();
    IntegerType *I16 = Type::getInt16Ty(Ctx);
    IRBuilder<> B(&Header->front());
    PHINode *Counter = B.CreatePHI(I16, 2, "unlsr.i");
    BasicBlock *Preheader = L->getLoopPreheader();
    if (!Preheader) continue;
    Counter->addIncoming(ConstantInt::get(I16, 0), Preheader);
    // Build the counter increment in the latch, just before the
    // terminator.
    IRBuilder<> BL(Latch->getTerminator());
    Value *CounterNext = BL.CreateAdd(Counter, ConstantInt::get(I16, 1),
                                       "unlsr.i.next");
    Counter->addIncoming(CounterNext, Latch);

    // Replace each load/store with a GEP-then-load/store.
    for (Instruction *Acc : Accesses) {
      IRBuilder<> BA(Acc);
      Value *Idx = Counter;
      // GEP type is i16 elem, idx is i16.
      Value *Addr = BA.CreateGEP(ElemTy, InitGV, Idx, "unlsr.addr");
      if (auto *LD = dyn_cast<LoadInst>(Acc)) {
        LD->setOperand(LoadInst::getPointerOperandIndex(), Addr);
      } else if (auto *ST = dyn_cast<StoreInst>(Acc)) {
        ST->setOperand(StoreInst::getPointerOperandIndex(), Addr);
      }
    }
    // Erase the now-unused step GEP and pointer PHI.
    if (StepGEP->use_empty()) StepGEP->eraseFromParent();
    if (PtrPHI->use_empty()) PtrPHI->eraseFromParent();

    // Try to eliminate the count-down LSR counter (`%lsr.iv = phi i16
    // [%n, preheader], [%lsr.iv.next, latch]; %lsr.iv.next = add %lsr.iv,
    // -1; %exitcond = icmp eq %lsr.iv.next, 0`).  Replace the exit
    // comparison with `icmp eq %unlsr.i.next, %n`, then erase the
    // count-down chain.  Saves one i16 PHI + dec + cmp per loop.
    for (PHINode &LSR : Header->phis()) {
      if (&LSR == Counter) continue;
      if (!LSR.getType()->isIntegerTy(16)) continue;
      if (LSR.getNumIncomingValues() != 2) continue;
      // Identify preheader vs latch incomings.
      Value *LsrInit = nullptr;
      Value *LsrStep = nullptr;
      for (unsigned i = 0; i < LSR.getNumIncomingValues(); ++i) {
        BasicBlock *Pred = LSR.getIncomingBlock(i);
        if (L->contains(Pred)) LsrStep = LSR.getIncomingValue(i);
        else                    LsrInit = LSR.getIncomingValue(i);
      }
      if (!LsrInit || !LsrStep) continue;
      // %lsr.iv.next must be `add %lsr.iv, -1`.
      auto *LsrStepBO = dyn_cast<BinaryOperator>(LsrStep);
      if (!LsrStepBO || LsrStepBO->getOpcode() != Instruction::Add) continue;
      Value *OtherOp = nullptr;
      if (LsrStepBO->getOperand(0) == &LSR) OtherOp = LsrStepBO->getOperand(1);
      else if (LsrStepBO->getOperand(1) == &LSR) OtherOp = LsrStepBO->getOperand(0);
      if (!OtherOp) continue;
      auto *DecCI = dyn_cast<ConstantInt>(OtherOp);
      if (!DecCI || !DecCI->isMinusOne()) continue;
      // %lsr.iv.next typically has 2 uses: the icmp exit comparison and
      // the PHI back-edge (LSR.users includes LsrStepBO as the latch
      // incoming).  Allow that; find the icmp among the uses.
      ICmpInst *ExitCmp = nullptr;
      bool extraUse = false;
      for (User *U : LsrStepBO->users()) {
        if (U == &LSR) continue;
        if (auto *IC = dyn_cast<ICmpInst>(U)) {
          if (ExitCmp) { extraUse = true; break; }
          ExitCmp = IC;
          continue;
        }
        extraUse = true;
        break;
      }
      if (extraUse || !ExitCmp) continue;
      if (ExitCmp->getPredicate() != ICmpInst::ICMP_EQ &&
          ExitCmp->getPredicate() != ICmpInst::ICMP_NE) continue;
      Value *CmpOther = nullptr;
      if (ExitCmp->getOperand(0) == LsrStepBO)
        CmpOther = ExitCmp->getOperand(1);
      else if (ExitCmp->getOperand(1) == LsrStepBO)
        CmpOther = ExitCmp->getOperand(0);
      if (!CmpOther) continue;
      auto *CmpZero = dyn_cast<ConstantInt>(CmpOther);
      if (!CmpZero || !CmpZero->isZero()) continue;
      bool lsrOnlyIncrUse = true;
      for (User *U : LSR.users()) {
        if (U == LsrStepBO) continue;
        lsrOnlyIncrUse = false; break;
      }
      if (!lsrOnlyIncrUse) continue;
      // CounterNext was inserted just before the latch terminator, but
      // ExitCmp may live higher in the block (originally placed there
      // by LSR).  Move CounterNext to just before ExitCmp so it
      // dominates the cmp's use.
      if (auto *CounterNextI = dyn_cast<Instruction>(CounterNext))
        CounterNextI->moveBefore(ExitCmp->getIterator());
      // Replace exit comparison: icmp eq %unlsr.i.next, %lsrInit.
      ExitCmp->setOperand(0, CounterNext);
      ExitCmp->setOperand(1, LsrInit);
      // RAUW each value with undef before erasing — they have a mutual
      // reference (PHI → BO via latch-incoming, BO → PHI via add operand).
      Type *I16T = LSR.getType();
      LSR.replaceAllUsesWith(UndefValue::get(I16T));
      LsrStepBO->replaceAllUsesWith(UndefValue::get(I16T));
      LsrStepBO->eraseFromParent();
      LSR.eraseFromParent();
      break;
    }

    Changed = true;
  }
  return Changed;
}
