Several miscompile bugs fixed. libc can now open more than 5 files. Derp.

runtime/src/libc.c

@@ -1155,7 +1155,13 @@ typedef struct __sFILE {
     unsigned short refNum;     // GS/OS file reference (kind=GSOS only)
 } FILE;
 
-#define MFS_MAX_FILES 8
+// Slots 0-2 are stdin/stdout/stderr; 3..MFS_MAX_FILES-1 are usable FILE*
+// slots (both GS/OS and memory files draw from this table -- see fopen).
+// At 8 that left only 5 concurrent opens, which starved apps that keep
+// several files open at once (e.g. AGI holds one FILE* per game VOL.x
+// archive). Each slot is a ~30-byte FILE plus a 24-byte tmpfile-name row,
+// so 24 slots (21 usable) costs well under 1.5 KB of BSS.
+#define MFS_MAX_FILES 24
 
 // Per-FILE-slot tmpfile name storage.  Parallel to __mfs[] so an
 // auto-delete FILE can own its name without the caller having to pass

src/llvm/lib/Target/W65816/W65816FrameLowering.cpp

@@ -101,6 +101,26 @@ bool W65816FrameLowering::hasReservedCallFrame(const MachineFunction &MF) const
   return false;
 }
 
+void W65816FrameLowering::determineCalleeSaves(MachineFunction &MF,
+                                               BitVector &SavedRegs,
+                                               RegScavenger *RS) const {
+  TargetFrameLowering::determineCalleeSaves(MF, SavedRegs, RS);
+  // A VLA function caches its entry SP in DP $F4/$F5, which the epilogue
+  // uses to unwind the (statically-unknown) VLA allocation and which the
+  // post-call $F6 recompute derives from.  $F4 is a single global DP
+  // location: a nested VLA callee overwrites it in its own prologue and
+  // never restores it, so without a callee-side save/restore the caller's
+  // $F4 is stale after the call -> wrong SP unwind + wrong $F6.  Reserve a
+  // 2-byte frame slot here (before frame-object offsets are finalized) so
+  // PEI includes it in the frame; emitPrologue/emitEpilogue fill and
+  // restore it.  Only VLA functions pay this.
+  if (MF.getFrameInfo().hasVarSizedObjects()) {
+    int FI = MF.getFrameInfo().CreateStackObject(2, Align(2),
+                                                 /*isSpillSlot=*/false);
+    MF.getInfo<W65816MachineFunctionInfo>()->setF4SaveFrameIndex(FI);
+  }
+}
+
 void W65816FrameLowering::emitPrologue(MachineFunction &MF,
                                        MachineBasicBlock &MBB) const {
   // Pick canonical mode based on whether the function uses any 8-bit
@@ -166,10 +186,17 @@ void W65816FrameLowering::emitPrologue(MachineFunction &MF,
   // $E0..$EF is libcall scratch.  TAY around the TSC preserves A
   // (which holds arg0).
   if (HasVLA) {
-    BuildMI(MBB, MBBI, DL, TII.get(W65816::TAY));    // save A
+    BuildMI(MBB, MBBI, DL, TII.get(W65816::TAY));    // save A (arg0) in Y
+    // Callee-save the caller's $F4 BEFORE we overwrite it below: stash it
+    // in $E0 (libcall-scratch DP, free this early -- no calls yet) and
+    // flush it to the reserved frame slot once $F6 is set (end of prologue).
+    if (MF.getInfo<W65816MachineFunctionInfo>()->getF4SaveFrameIndex() >= 0) {
+      BuildMI(MBB, MBBI, DL, TII.get(W65816::LDA_DP)).addImm(0xF4);
+      BuildMI(MBB, MBBI, DL, TII.get(W65816::STA_DP)).addImm(0xE0);
+    }
     BuildMI(MBB, MBBI, DL, TII.get(W65816::TSC));    // A = SP
     BuildMI(MBB, MBBI, DL, TII.get(W65816::STA_DP)).addImm(0xF4);
-    BuildMI(MBB, MBBI, DL, TII.get(W65816::TYA));    // restore A
+    BuildMI(MBB, MBBI, DL, TII.get(W65816::TYA));    // restore A (arg0)
   }
 
   if (StackSize > 0) {
@@ -229,6 +256,35 @@ void W65816FrameLowering::emitPrologue(MachineFunction &MF,
     BuildMI(MBB, MBBI, DL, TII.get(W65816::STZ_DP)).addImm(0xF8);
     BuildMI(MBB, MBBI, DL, TII.get(W65816::TYA));
   }
+
+  // Flush the caller's $F4 (stashed in $E0 above) to its reserved frame
+  // slot, now that $F6 (static base) is established.  Address the slot via
+  // ($F6),Y exactly like eliminateFrameIndex's far path so the disp is
+  // correct for any frame size.  ($F6),Y is SP-independent, so the PHA that
+  // preserves arg0 does not perturb it.
+  {
+    int F4SaveFI =
+        MF.getInfo<W65816MachineFunctionInfo>()->getF4SaveFrameIndex();
+    if (HasVLA && F4SaveFI >= 0) {
+      const MachineFrameInfo &MFI = MF.getFrameInfo();
+      int FrameOffset = MFI.getObjectOffset(F4SaveFI);
+      int FPOff = FrameOffset + (int)MFI.getStackSize();
+      if (FrameOffset < 0)
+        FPOff += 1;
+      BuildMI(MBB, MBBI, DL, TII.get(W65816::PHA));                  // save arg0
+      BuildMI(MBB, MBBI, DL, TII.get(W65816::LDA_DP)).addImm(0xE0);  // A = caller $F4
+      BuildMI(MBB, MBBI, DL, TII.get(W65816::STY_DP)).addImm(0xFA)
+          .addReg(W65816::Y, RegState::Implicit | RegState::Undef);
+      BuildMI(MBB, MBBI, DL, TII.get(W65816::LDY_Imm16)).addImm(FPOff);
+      BuildMI(MBB, MBBI, DL, TII.get(W65816::STA_DPIndLongY))
+          .addImm(0xF6)
+          .addReg(W65816::A, RegState::Implicit)
+          .addReg(W65816::Y, RegState::Implicit);
+      BuildMI(MBB, MBBI, DL, TII.get(W65816::LDY_DP)).addImm(0xFA);
+      BuildMI(MBB, MBBI, DL, TII.get(W65816::PLA))
+          .addReg(W65816::A, RegState::ImplicitDefine);              // restore arg0
+    }
+  }
 }
 
 void W65816FrameLowering::emitEpilogue(MachineFunction &MF,
@@ -269,6 +325,40 @@ void W65816FrameLowering::emitEpilogue(MachineFunction &MF,
   // returns where Y is also live, route the save through DP $E0
   // ($E0..$EF is libcall scratch — guaranteed dead by epilogue time).
   if (HasVLA) {
+    int F4SaveFI =
+        MF.getInfo<W65816MachineFunctionInfo>()->getF4SaveFrameIndex();
+    if (F4SaveFI >= 0) {
+      // Restore the caller's $F4 (a nested VLA call clobbered it), then
+      // return.  Sequence:
+      //   1) stash the return value A in $E2 -- X and Y are left untouched
+      //      so i64/double returns survive without a YLive special case;
+      //   2) unwind SP from THIS function's $F4 (its entry SP) -- must run
+      //      BEFORE $F4 is overwritten;
+      //   3) read the caller's saved $F4 from its slot via ($F6),Y ($F6 is
+      //      valid here -- recomputed after every call, untouched by TCS,
+      //      and the slot bytes below SP are intact until the RTL) and
+      //      write it back to $F4;
+      //   4) reload the return value.
+      const MachineFrameInfo &MFI = MF.getFrameInfo();
+      int FrameOffset = MFI.getObjectOffset(F4SaveFI);
+      int FPOff = FrameOffset + (int)StackSize;
+      if (FrameOffset < 0)
+        FPOff += 1;
+      BuildMI(MBB, MBBI, DL, TII.get(W65816::STA_DP)).addImm(0xE2);  // $E2 = ret A
+      BuildMI(MBB, MBBI, DL, TII.get(W65816::LDA_DP)).addImm(0xF4);  // A = entry SP
+      BuildMI(MBB, MBBI, DL, TII.get(W65816::TCS));                  // SP = entry SP
+      BuildMI(MBB, MBBI, DL, TII.get(W65816::STY_DP)).addImm(0xFA)
+          .addReg(W65816::Y, RegState::Implicit | RegState::Undef);
+      BuildMI(MBB, MBBI, DL, TII.get(W65816::LDY_Imm16)).addImm(FPOff);
+      BuildMI(MBB, MBBI, DL, TII.get(W65816::LDA_DPIndLongY)).addImm(0xF6)
+          .addReg(W65816::A, RegState::ImplicitDefine)
+          .addReg(W65816::Y, RegState::Implicit);
+      BuildMI(MBB, MBBI, DL, TII.get(W65816::LDY_DP)).addImm(0xFA);
+      BuildMI(MBB, MBBI, DL, TII.get(W65816::STA_DP)).addImm(0xF4);  // $F4 = caller val
+      BuildMI(MBB, MBBI, DL, TII.get(W65816::LDA_DP)).addImm(0xE2);  // A = ret
+      return;
+    }
+    // Leaf VLA (no reserved slot): $F4 still holds this function's entry SP.
     if (YLive) {
       BuildMI(MBB, MBBI, DL, TII.get(W65816::STA_DP)).addImm(0xE0);
       BuildMI(MBB, MBBI, DL, TII.get(W65816::LDA_DP)).addImm(0xF4);
@@ -342,8 +432,42 @@ MachineBasicBlock::iterator W65816FrameLowering::eliminateCallFramePseudoInstr(
   // sum's mid-high.
   if (I->getOpcode() == W65816::ADJCALLSTACKUP) {
     int N = I->getOperand(0).getImm();
-    if (N > 0) {
+    // A function with a large (>200B) frame or VLAs caches its own SP in
+    // the DP frame pointer $F6/$F7 (see emitPrologue) for far ($F6],Y)
+    // stack access.  $F6 is NOT preserved across calls: any callee with
+    // its own large frame overwrites $F6 in its prologue and never
+    // restores it.  Because $F6 is merely a cache of SP, we recompute it
+    // rather than save/restore it: after every call, re-store SP into
+    // $F6 before the next FP-relative access reads a stale base.  Without
+    // this, a far-loaded pointer (e.g. jlDataOpen's `mode` param, read
+    // FP-relative after an snprintf whose vsnprintf clobbered $F6) arrives
+    // as nondeterministic garbage.  This is needed even when N == 0 (a
+    // call with no stack args still clobbers $F6).
+    bool usesDpFP = MF.getFrameInfo().getStackSize() > 200 ||
+                    MF.getFrameInfo().hasVarSizedObjects();
+    if (N > 0 || usesDpFP) {
       DebugLoc DL = I->getDebugLoc();
+      // Re-store the DP frame pointer $F6 for DpFP functions.  For a
+      // plain large frame, $F6 == SP, and A already holds SP at each
+      // call site below, so a single STA suffices.  For a VLA function
+      // $F6 is the *static* frame base ($F4 entry-SP minus the static
+      // frame size) and does NOT track SP once a VLA alloc has lowered
+      // it -- recompute it from $F4 instead (clobbers A, which every
+      // caller has already stashed in Y or $E0 and restores afterward).
+      bool hasVLA = MF.getFrameInfo().hasVarSizedObjects();
+      uint64_t frameSize = MF.getFrameInfo().getStackSize();
+      auto reestablishFP = [&]() {
+        if (!usesDpFP)
+          return;
+        if (hasVLA) {
+          BuildMI(MBB, I, DL, TII.get(W65816::LDA_DP)).addImm(0xF4);
+          BuildMI(MBB, I, DL, TII.get(W65816::SEC));
+          BuildMI(MBB, I, DL, TII.get(W65816::SBC_Imm16)).addImm(frameSize);
+          BuildMI(MBB, I, DL, TII.get(W65816::STA_DP)).addImm(0xF6);
+        } else {
+          BuildMI(MBB, I, DL, TII.get(W65816::STA_DP)).addImm(0xF6);
+        }
+      };
       bool YLive = false;
       bool XLive = false;
       // Walk forward looking for COPY %vreg = $x / $y — LowerCall's
@@ -375,20 +499,30 @@ MachineBasicBlock::iterator W65816FrameLowering::eliminateCallFramePseudoInstr(
         // change anything here — track it for symmetry.
         BuildMI(MBB, I, DL, TII.get(W65816::STA_DP)).addImm(0xE0);
         BuildMI(MBB, I, DL, TII.get(W65816::TSC));
-        BuildMI(MBB, I, DL, TII.get(W65816::CLC));
+        if (N > 0) {
-        BuildMI(MBB, I, DL, TII.get(W65816::ADC_Imm16)).addImm(N);
+          BuildMI(MBB, I, DL, TII.get(W65816::CLC));
-        BuildMI(MBB, I, DL, TII.get(W65816::TCS));
+          BuildMI(MBB, I, DL, TII.get(W65816::ADC_Imm16)).addImm(N);
+          BuildMI(MBB, I, DL, TII.get(W65816::TCS));
+        }
+        // A holds SP here (frame SP after any arg release): cache it in $F6
+        // (or recompute from $F4 for VLA functions).
+        reestablishFP();
         BuildMI(MBB, I, DL, TII.get(W65816::LDA_DP)).addImm(0xE0);
         (void)XLive;
-      } else if (N <= 14 && (N % 2) == 0) {
+      } else if (N > 0 && N <= 14 && (N % 2) == 0 && !usesDpFP) {
         for (int i = 0; i < N / 2; ++i)
           BuildMI(MBB, I, DL, TII.get(W65816::PLY));
       } else {
         BuildMI(MBB, I, DL, TII.get(W65816::TAY));
         BuildMI(MBB, I, DL, TII.get(W65816::TSC));
-        BuildMI(MBB, I, DL, TII.get(W65816::CLC));
+        if (N > 0) {
-        BuildMI(MBB, I, DL, TII.get(W65816::ADC_Imm16)).addImm(N);
+          BuildMI(MBB, I, DL, TII.get(W65816::CLC));
-        BuildMI(MBB, I, DL, TII.get(W65816::TCS));
+          BuildMI(MBB, I, DL, TII.get(W65816::ADC_Imm16)).addImm(N);
+          BuildMI(MBB, I, DL, TII.get(W65816::TCS));
+        }
+        // A holds SP here (frame SP after any arg release): cache it in $F6
+        // (or recompute from $F4 for VLA functions).
+        reestablishFP();
         BuildMI(MBB, I, DL, TII.get(W65816::TYA));
       }
     }

src/llvm/lib/Target/W65816/W65816FrameLowering.h

@@ -31,6 +31,9 @@ public:
   eliminateCallFramePseudoInstr(MachineFunction &MF, MachineBasicBlock &MBB,
                                 MachineBasicBlock::iterator I) const override;
 
+  void determineCalleeSaves(MachineFunction &MF, BitVector &SavedRegs,
+                            RegScavenger *RS) const override;
+
   bool hasReservedCallFrame(const MachineFunction &MF) const override;
 };
 

src/llvm/lib/Target/W65816/W65816MachineFunctionInfo.h

@@ -48,6 +48,13 @@ class W65816MachineFunctionInfo : public MachineFunctionInfo {
   /// exceeds 0xFF through `(F6),Y` indirect-indexed addressing.
   bool UsesDpFP = false;
 
+  /// FrameIndex of a 2-byte slot reserved (VLA functions only) to
+  /// callee-save the caller's DP $F4/$F5 (the VLA entry-SP pointer).
+  /// A nested VLA callee overwrites $F4 in its own prologue; without
+  /// this save/restore the caller's epilogue SP-restore and $F6
+  /// recompute would read a stale $F4.  -1 when unused.
+  int F4SaveFrameIndex = -1;
+
 
 public:
   W65816MachineFunctionInfo() = default;
@@ -77,6 +84,9 @@ public:
 
   bool getUsesDpFP() const { return UsesDpFP; }
   void setUsesDpFP(bool V) { UsesDpFP = V; }
+
+  int getF4SaveFrameIndex() const { return F4SaveFrameIndex; }
+  void setF4SaveFrameIndex(int Index) { F4SaveFrameIndex = Index; }
 };
 
 } // namespace llvm