Yet more fixes!

runtime/include/c++/cmath

@@ -26,11 +26,23 @@
 #undef isinf
 #undef isfinite
 #undef signbit
+#undef isgreater
+#undef isgreaterequal
+#undef isless
+#undef islessequal
+#undef islessgreater
+#undef isunordered
 
 inline bool isnan(double x) { return ::__isnan_d(x) != 0; }
 inline bool isinf(double x) { return ::__isinf_d(x) != 0; }
 inline bool isfinite(double x) { return ::__isfinite_d(x) != 0; }
 inline bool signbit(double x) { return ::__signbit_d(x) != 0; }
+inline bool isgreater(double x, double y) { return __builtin_isgreater(x, y); }
+inline bool isgreaterequal(double x, double y) { return __builtin_isgreaterequal(x, y); }
+inline bool isless(double x, double y) { return __builtin_isless(x, y); }
+inline bool islessequal(double x, double y) { return __builtin_islessequal(x, y); }
+inline bool islessgreater(double x, double y) { return __builtin_islessgreater(x, y); }
+inline bool isunordered(double x, double y) { return __builtin_isunordered(x, y); }
 
 namespace std {
 
@@ -166,6 +178,14 @@ using ::scalblnf;
 // ---- Classification (function form) ---------------------------------
 using ::fpclassify;
 
+// ---- Comparison (function form per C++ <cmath>) ---------------------
+using ::isgreater;
+using ::isgreaterequal;
+using ::isless;
+using ::islessequal;
+using ::islessgreater;
+using ::isunordered;
+
 }  // namespace std
 
 #endif  // _W65816_CXX_CMATH

runtime/include/math.h

@@ -25,6 +25,18 @@ int __signbit_d (double x);
 #define isfinite(x) __isfinite_d((double)(x))
 #define signbit(x)  __signbit_d((double)(x))
 
+// ---- C99 comparison macros ------------------------------------------
+// False, never an error, when either operand is a NaN.  The soft-float
+// compares raise nothing, so clang's builtins (which LLVM lowers to the
+// ordinary compare helpers, plus __unordsf2 / __unorddf2 for the
+// unordered tests) are all these need to be.
+#define isgreater(x, y)      __builtin_isgreater((x), (y))
+#define isgreaterequal(x, y) __builtin_isgreaterequal((x), (y))
+#define isless(x, y)         __builtin_isless((x), (y))
+#define islessequal(x, y)    __builtin_islessequal((x), (y))
+#define islessgreater(x, y)  __builtin_islessgreater((x), (y))
+#define isunordered(x, y)    __builtin_isunordered((x), (y))
+
 // ---- Absolute / sign ------------------------------------------------
 double fabs    (double x);
 float  fabsf   (float  x);

runtime/src/math.c

@@ -46,6 +46,73 @@ static float fFromBits(uint32_t b) {
 }
 
 
+#define D_SIGN   0x8000000000000000ULL
+#define D_INF    0x7FF0000000000000ULL
+#define D_ONE    0x3FF0000000000000ULL
+
+
+// Used before their definitions (this file does not include math.h).
+double exp(double x);
+double log(double x);
+double rint(double x);
+
+
+// clang's library-call simplifier rewrites (float)floor((double)x) into
+// floorf(x), which is exact for the rounding and min/max functions; inside
+// floorf itself that is a call to itself, so every floorf, ceilf, truncf,
+// roundf, fmaxf and fminf recursed until the stack overran the program.
+// no_builtin("f") stops the rewrite inside the one wrapper that needs it.
+#define NO_SHRINK(f) __attribute__((no_builtin(#f)))
+
+
+// Special-value paths kept out of line: inlined, they grew the ordinary
+// path's frame and register traffic by several percent.
+#define COLD __attribute__((noinline))
+
+
+// The top word of a double's bits: sign, exponent and the mantissa's top
+// four bits.  Free to extract, so exponent tests go through it instead of
+// 64-bit compares.
+static unsigned topWord(uint64_t b) {
+    return (unsigned)(b >> 48);
+}
+
+
+// 1 when the double with bits b is an infinity or a NaN.  The top word
+// holds the whole exponent, so this is one masked compare.
+static int isNonFinite(uint64_t b) {
+    return (topWord(b) & 0x7FF0u) == 0x7FF0u;
+}
+
+
+// 1 when the double with bits b is a NaN (its magnitude is above
+// infinity's).  A 64-bit compare, so callers screen with isNonFinite.
+static int isNanBits(uint64_t b) {
+    return (b & ~D_SIGN) > D_INF;
+}
+
+
+// 1 when the double with bits b is +0 or -0.  Spelled word by word: the
+// optimizer turns a 64-bit `(b & ~sign) == 0` back into `x == 0.0`, which
+// on this target is a soft-float libcall compare (~500 cycles).
+static int isZeroBits(uint64_t b) {
+    return (((unsigned)(b >> 48) & 0x7FFFu) | (unsigned)(b >> 32) |
+            (unsigned)(b >> 16) | (unsigned)b) == 0u;
+}
+
+
+// sqrt and log of an infinity or a NaN: +inf maps to itself, a NaN to
+// itself quieted (x + x does both), -inf is a domain error like any other
+// negative argument.  Without this the series below turned +inf into a
+// finite value and a NaN into a plausible-looking number.
+static double nonFiniteArg(double x, uint64_t b) {
+    if (b == 0xFFF0000000000000ULL) {
+        return __builtin_nan("");
+    }
+    return x + x;
+}
+
+
 double fabs(double x) {
     return dFromBits(dToBits(x) & ~((uint64_t)1 << 63));
 }
@@ -75,7 +142,12 @@ double floor(double x) {
     uint64_t b = dToBits(x);
     int      e = (int)((b >> 52) & 0x7FF) - 1023;
     if (e < 0) {
-        // |x| < 1: floor(positive) = 0, floor(negative) = -1.
+        // |x| < 1: floor(positive) = 0, floor(negative) = -1, and a zero
+        // keeps its sign.  -1 for -0 made ceil(+0), which is -floor(-0),
+        // come out as 1.
+        if (isZeroBits(b)) {
+            return x;
+        }
         return (b >> 63) ? -1.0 : 0.0;
     }
     if (e >= 52) {
@@ -100,30 +172,54 @@ double ceil(double x) {
 }
 
 
-float floorf(float x) {
+NO_SHRINK(floor) float floorf(float x) {
     return (float)floor((double)x);
 }
 
 
-float ceilf(float x) {
+NO_SHRINK(ceil) float ceilf(float x) {
     return (float)ceil((double)x);
 }
 
 
-// fmod(x, y) = x - trunc(x / y) * y.  trunc(q) = round-toward-zero.
+// fmod(x, y) = x - trunc(x / y) * y, which takes the sign of x.  The
+// special cases follow C99 Annex F: a NaN operand, an infinite x or a zero
+// y give NaN; an infinite y or a zero x give x back.  They are screened on
+// the bits, which also replaced a `y == 0.0` libcall compare.
+static COLD double fmodSpecial(double x, double y) {
+    uint64_t xb = dToBits(x);
+    uint64_t yb = dToBits(y);
+    if (isNanBits(xb) || isNanBits(yb)) {
+        return x + y;                       // the NaN, quieted
+    }
+    if (isNonFinite(xb) || isZeroBits(yb)) {
+        return __builtin_nan("");           // domain error
+    }
+    return x;                               // infinite y, or a zero x
+}
+
+
 double fmod(double x, double y) {
-    if (y == 0.0) {
+    uint64_t xb = dToBits(x);
-        return 0.0;  // implementation-defined; we return 0
+    uint64_t yb = dToBits(y);
+    if (isNonFinite(xb) || isNonFinite(yb) || isZeroBits(yb) || isZeroBits(xb)) {
+        return fmodSpecial(x, y);
     }
     double q = x / y;
-    // trunc: floor for positives, ceil for negatives.
+    // trunc: floor for positives, ceil for negatives.  The sign test reads
+    // the bits, which is cheaper than a `<` libcall compare.
     double t;
-    if (q < 0.0) {
+    if (dToBits(q) >> 63) {
         t = -floor(-q);
     } else {
         t = floor(q);
     }
-    return x - t * y;
+    double r = x - t * y;
+    // An exact multiple leaves x + -x, which is +0; the result takes x's sign.
+    if (isZeroBits(dToBits(r))) {
+        return (xb >> 63) ? -0.0 : 0.0;
+    }
+    return r;
 }
 
 
@@ -146,6 +242,9 @@ double sqrt(double x) {
     if ((b & ~((uint64_t)1 << 63)) == 0) {
         return x;
     }
+    if (isNonFinite(b)) {
+        return nonFiniteArg(x, b);
+    }
     if (b & ((uint64_t)1 << 63)) {
         return 0.0 / 0.0;  // NaN for negatives
     }
@@ -168,38 +267,60 @@ float sqrtf(float x) {
 }
 
 
-// pow(x, y) supports two restricted regimes:
+// 0 when the finite, nonzero y is not an integer, 1 when it is an odd one
-//   1. integer y (positive, negative, or zero) — repeated multiply.
+// and 2 when it is an even one.  e is y's unbiased exponent; only pow's
-//   2. y == 0.5 — returns sqrt(x).
+// rare paths need this, the |y| < 2^15 integer path tests (int)y instead.
-// Anything else (general real exponent) needs exp/log; not yet
+static int intClass(double y, int e) {
-// landed.  Returning 0 is the documented behaviour for now.
+    if (e < 0) {
-double pow(double x, double y) {
+        return 0;                       // 0 < |y| < 1
-    if (y == 0.0) {
-        return 1.0;
     }
-    if (y == 0.5) {
+    if (e >= 53) {
-        return sqrt(x);
+        return 2;                       // every double from 2^53 up is even
     }
-    // Test for integer y by floor-equal trick (no soft-fp comparison).
+    if (dToBits(floor(y)) != dToBits(y)) {
-    double yi = floor(y);
+        return 0;
-    uint64_t yi_b, y_b;
-    __builtin_memcpy(&yi_b, &yi, sizeof(yi_b));
-    __builtin_memcpy(&y_b,  &y,  sizeof(y_b));
-    if (yi_b != y_b) {
-        return 0.0;  // non-integer, non-0.5 y not supported yet
     }
-    // y is a whole number; convert via __fixdfsi.  Range -32768..32767
+    double h = y * 0.5;
-    // covers any practical exponent.  Use unsigned for the magnitude
+    return (dToBits(floor(h)) == dToBits(h)) ? 2 : 1;
-    // to avoid signed-overflow UB on INT_MIN.
-    int sn = (int)yi;
-    int neg = 0;
-    unsigned int n;
-    if (sn < 0) {
-        neg = 1;
-        n = 0u - (unsigned int)sn;
-    } else {
-        n = (unsigned int)sn;
 }
+
+
+// The high word's mantissa bits below the unit of a double whose unbiased
+// exponent is 0..14: nonzero ones make it a non-integer.
+static const uint32_t kPowFracHi[15] = {
+    0xFFFFFUL, 0x7FFFFUL, 0x3FFFFUL, 0x1FFFFUL, 0xFFFFUL, 0x7FFFUL, 0x3FFFUL,
+    0x1FFFUL, 0xFFFUL, 0x7FFUL, 0x3FFUL, 0x1FFUL, 0xFFUL, 0x7FUL, 0x3FUL
+};
+
+
+// pow when x is a zero or an infinity or either operand is a NaN or y is
+// an infinity (y is not a zero, x is not +1).  C99 F.9.4.4: a NaN operand
+// gives NaN; pow(-1, +-inf) is 1, otherwise an infinite y gives +inf when
+// |x| > 1 agrees with y > 0 and +0 when not; a zero or infinite x gives
+// +inf or +0 by y's sign (inverted for a zero x), negated for a negative x
+// raised to an odd integer.
+static COLD double powSpecial(double x, double y, uint64_t xb, uint64_t yb) {
+    if (isNanBits(xb) || isNanBits(yb)) {
+        return x + y;                   // the NaN, quieted
+    }
+    uint64_t xa = xb & ~D_SIGN;
+    int yNeg = (int)(yb >> 63);
+    if ((yb & ~D_SIGN) == D_INF) {
+        if (xa == D_ONE) {
+            return 1.0;                 // pow(-1, +-inf)
+        }
+        return ((xa > D_ONE) != yNeg) ? __builtin_inf() : 0.0;
+    }
+    int e = (int)((topWord(yb) >> 4) & 0x7FFu) - 1023;
+    int huge = isZeroBits(xb) ? yNeg : !yNeg;
+    double r = huge ? __builtin_inf() : 0.0;
+    return ((xb >> 63) && intClass(y, e) == 1) ? -r : r;
+}
+
+
+// x raised to the integer n (1 / x^n when neg) by repeated squaring,
+// exact whenever the result is representable.
+static double powInt(double x, unsigned int n, int neg) {
     double r = 1.0;
     double base = x;
     while (n > 0) {
@@ -216,6 +337,53 @@ double pow(double x, double y) {
 }
 
 
+// pow(x, y).  C99 F.9.4.4's special cases first: pow(x, +-0) and pow(+1, y)
+// are 1 even when the other operand is a NaN; zeros, infinities and NaNs go
+// to powSpecial.  An integer y below 2^15 in magnitude is repeated
+// multiplication (exact whenever the result is representable); any other y
+// is exp(y * log|x|), as accurate as those (~1e-6), and a negative x
+// raised to a non-integer is a domain error.  Every non-integer y except
+// 0.5 used to return 0.
+double pow(double x, double y) {
+    uint64_t xb = dToBits(x);
+    uint64_t yb = dToBits(y);
+    if (isZeroBits(yb) || xb == D_ONE) {
+        return 1.0;
+    }
+    if (isNonFinite(xb) || isNonFinite(yb) || isZeroBits(xb)) {
+        return powSpecial(x, y, xb, yb);
+    }
+    int e = (int)((topWord(yb) >> 4) & 0x7FFu) - 1023;
+    int xNeg = (int)(xb >> 63);
+    if (e < 15) {
+        // 1 <= |y| < 2^15: y is an integer when every mantissa bit below
+        // its unit is clear.  From the bits: (int)y and back through
+        // __fixdfsi / __floatsidf normalize bit by bit and cost more than
+        // the multiplications of pow(2, 10).  The high word's bit 20 is the
+        // implicit leading 1; bits 5..20 of it, as 16 bits, hold every bit
+        // of an integer below 2^15.
+        uint32_t hi = (uint32_t)(yb >> 32);
+        if (e >= 0 && (uint32_t)yb == 0 && (hi & kPowFracHi[e]) == 0) {
+            unsigned int n = (((unsigned int)(hi >> 5) & 0x7FFFu) | 0x8000u) >> (15 - e);
+            return powInt(x, n, (int)(yb >> 63));
+        }
+        if (xNeg) {
+            return __builtin_nan("");   // negative x, non-integer y
+        }
+        if (yb == 0x3FE0000000000000ULL) {
+            return sqrt(x);             // y == 0.5
+        }
+        return exp(y * log(x));
+    }
+    int yc = intClass(y, e);
+    if (xNeg && yc == 0) {
+        return __builtin_nan("");
+    }
+    double r = exp(y * log(fabs(x)));
+    return (xNeg && yc == 1) ? -r : r;
+}
+
+
 float powf(float x, float y) {
     return (float)pow((double)x, (double)y);
 }
@@ -300,15 +468,37 @@ float tanf(float x) {
 // exp via 2^k * e^r where x = k*ln2 + r, |r| < ln2/2.  Then Taylor
 // series for e^r converges in ~10 terms.  k * 2 multiplication uses
 // the IEEE-754 layout (add k to exponent field).
+// 1/n for the Taylor terms: a table of the correctly rounded constants,
+// which the run-time 1.0 / n computed identically at ~60000 cycles each.
+static const double kInvN[13] = {
+    0.0, 1.0, 1.0 / 2.0, 1.0 / 3.0, 1.0 / 4.0, 1.0 / 5.0, 1.0 / 6.0,
+    1.0 / 7.0, 1.0 / 8.0, 1.0 / 9.0, 1.0 / 10.0, 1.0 / 11.0, 1.0 / 12.0
+};
+
+
 double exp(double x) {
-    double k_d = floor(x / M_LN2 + 0.5);
+    uint64_t xb = dToBits(x);
+    // |x| >= 1024, the infinities and the NaNs: the result is surely past
+    // the double range (and k below would not fit an int), so exp(+inf)
+    // and every large x overflow to +inf and exp(-inf) and every large
+    // negative x underflow to +0.  A NaN comes back quieted.
+    if ((topWord(xb) & 0x7FF0u) >= 0x4090u) {
+        if (isNanBits(xb)) {
+            return x + x;
+        }
+        return (xb >> 63) ? 0.0 : __builtin_inf();
+    }
+    // k = round(x / ln2); multiplying by 1/ln2 instead of dividing saves a
+    // soft division, and any k within one of the exact one still leaves r
+    // well inside the series' fast-converging range.
+    double k_d = floor(x * 1.44269504088896340736 + 0.5);
     double r   = x - k_d * M_LN2;
     int    k   = (int)k_d;
-    // Taylor: 1 + r + r^2/2 + r^3/6 + ... + r^10/3628800
+    // Taylor: 1 + r + r^2/2 + r^3/6 + ... + r^12/479001600
     double term = 1.0;
     double s    = 1.0;
     for (int n = 1; n <= 12; n++) {
-        term = term * r * (1.0 / (double)n);
+        term = term * r * kInvN[n];
         s    = s + term;
     }
     // Multiply by 2^k by adding k to the IEEE-754 exponent field.
@@ -345,6 +535,9 @@ double log(double x) {
     if ((b & ~((uint64_t)1 << 63)) == 0) {
         return -1.0 / 0.0;
     }
+    if (isNonFinite(b)) {
+        return nonFiniteArg(x, b);
+    }
     if (b & ((uint64_t)1 << 63)) {
         return 0.0 / 0.0;  // log(negative) = NaN (domain error)
     }
@@ -353,13 +546,19 @@ double log(double x) {
     b = (b & ~((uint64_t)0x7FF << 52)) | ((uint64_t)1023 << 52);
     double m;
     __builtin_memcpy(&m, &b, sizeof(m));
+    // 1/(2n+1) for the series: a table of the correctly rounded constants
+    // (the run-time division gave the same values at ~60000 cycles each).
+    static const double kInvOdd[9] = {
+        1.0, 1.0 / 3.0, 1.0 / 5.0, 1.0 / 7.0, 1.0 / 9.0, 1.0 / 11.0,
+        1.0 / 13.0, 1.0 / 15.0, 1.0 / 17.0
+    };
     double u  = (m - 1.0) / (m + 1.0);
     double u2 = u * u;
     double term = u;
     double s    = term;
     for (int n = 1; n <= 8; n++) {
         term = term * u2;
-        s    = s + term * (1.0 / (double)(2 * n + 1));
+        s    = s + term * kInvOdd[n];
     }
     return 2.0 * s + (double)e * M_LN2;
 }
@@ -447,7 +646,10 @@ float atan2f(float y, float x) {
 double asin(double x) {
     double r = 1.0 - x * x;
     if (r <= 0.0) {
-        // |x| >= 1; clamp to ±pi/2.
+        // |x| > 1 is a domain error; |x| == 1 is exactly +-pi/2.
+        if (r < 0.0) {
+            return __builtin_nan("");
+        }
         uint64_t b;
         __builtin_memcpy(&b, &x, sizeof(b));
         return (b >> 63) ? -M_HALF_PI : M_HALF_PI;
@@ -471,7 +673,12 @@ float acosf(float x) {
 }
 
 
+// sinh(+-0) is +-0 (the formula gives +0 for both); infinities and NaNs
+// come through exp: sinh(+-inf) = +-inf.
 double sinh(double x) {
+    if (isZeroBits(dToBits(x))) {
+        return x;
+    }
     double e = exp(x);
     return (e - 1.0 / e) * 0.5;
 }
@@ -493,7 +700,20 @@ float coshf(float x) {
 }
 
 
+// tanh is +-1 to double precision from |x| = 22 on; from 32 (a top-word
+// test) the formula would reach inf / inf = NaN once exp(2x) overflows, so
+// return +-1 there, which also covers tanh(+-inf).  tanh(+-0) is +-0.
 double tanh(double x) {
+    uint64_t b = dToBits(x);
+    if ((topWord(b) & 0x7FF0u) >= 0x4040u) {
+        if (isNanBits(b)) {
+            return x + x;
+        }
+        return (b >> 63) ? -1.0 : 1.0;
+    }
+    if (isZeroBits(b)) {
+        return x;
+    }
     double e2 = exp(x + x);
     return (e2 - 1.0) / (e2 + 1.0);
 }
@@ -540,18 +760,29 @@ double trunc(double x) {
 }
 
 
-float truncf(float x) {
+NO_SHRINK(trunc) float truncf(float x) {
     return (float)trunc((double)x);
 }
 
 
-// round: round half away from zero.
+// round: round half away from zero.  floor(|x| + 0.5) is exact only in
+// between: below 0.5 the addition can round up to 1 (0.49999999999999994
+// rounded to 1), and from 2^52 on, where every double is an integer, it
+// can round an odd x up to the next even one.
 double round(double x) {
-    return (dToBits(x) >> 63) ? -floor(-x + 0.5) : floor(x + 0.5);
+    uint64_t b = dToBits(x);
+    unsigned e = (topWord(b) >> 4) & 0x7FFu;
+    if (e < 1022u) {
+        return (b >> 63) ? -0.0 : 0.0;  // |x| < 0.5, zeros keep their sign
+    }
+    if (e >= 1075u) {
+        return x + 0.0;                 // integer, infinity or NaN (quieted)
+    }
+    return (b >> 63) ? -floor(-x + 0.5) : floor(x + 0.5);
 }
 
 
-float roundf(float x) {
+NO_SHRINK(round) float roundf(float x) {
     return (float)round((double)x);
 }
 
@@ -573,12 +804,19 @@ double fmin(double x, double y) {
 
 
 double fdim(double x, double y) {
-    return x > y ? x - y : 0.0;
+    if (x > y) {
+        return x - y;
+    }
+    // Unordered compares false too: a NaN operand gives NaN, not +0.
+    if (isNanBits(dToBits(x)) || isNanBits(dToBits(y))) {
+        return x + y;
+    }
+    return 0.0;
 }
 
 
-float fmaxf(float x, float y) { return (float)fmax((double)x, (double)y); }
+NO_SHRINK(fmax) float fmaxf(float x, float y) { return (float)fmax((double)x, (double)y); }
-float fminf(float x, float y) { return (float)fmin((double)x, (double)y); }
+NO_SHRINK(fmin) float fminf(float x, float y) { return (float)fmin((double)x, (double)y); }
 float fdimf(float x, float y) { return (float)fdim((double)x, (double)y); }
 
 
@@ -692,12 +930,19 @@ float exp2f(float x)  { return (float)exp2((double)x); }
 // log1p(x) = log(1 + x).  For |x| small the naive form loses precision;
 // we accept that for now since this is a standalone runtime, not a
 // numerics library.
-double log1p(double x) { return log(1.0 + x); }
+double log1p(double x) {
+    // log1p(+-0) is +-0; log(1 + -0) would be +0.
+    return isZeroBits(dToBits(x)) ? x : log(1.0 + x);
+}
 float  log1pf(float x) { return (float)log1p((double)x); }
 
 
 // expm1(x) = exp(x) - 1.  Same loss-of-precision caveat near 0.
-double expm1(double x) { return exp(x) - 1.0; }
+double expm1(double x) {
+    // expm1(+-0) is +-0; exp(-0) - 1 would be +0.  expm1(-inf) = -1 and
+    // expm1(+inf) = +inf come through exp.
+    return isZeroBits(dToBits(x)) ? x : exp(x) - 1.0;
+}
 float  expm1f(float x) { return (float)expm1((double)x); }
 
 
@@ -708,6 +953,13 @@ float  expm1f(float x) { return (float)expm1((double)x); }
 // past that you'd want the standard scale-by-max trick.
 
 double hypot(double x, double y) {
+    uint64_t xb = dToBits(x);
+    uint64_t yb = dToBits(y);
+    // C99 F.9.4.3: an infinite operand gives +inf even when the other is a
+    // NaN; otherwise a NaN operand gives NaN (through the arithmetic).
+    if ((xb & ~D_SIGN) == D_INF || (yb & ~D_SIGN) == D_INF) {
+        return __builtin_inf();
+    }
     double xx = x * x;
     double yy = y * y;
     double s  = xx + yy;
@@ -728,7 +980,15 @@ float hypotf(float x, float y) {
 // known builtin and either inlines it (with bad fold of pow(x,1/3))
 // or DCEs the call entirely (cbrt body collapses to "return 0").
 double cbrt(double x) {
-    if (x == 0.0) return x;
+    uint64_t xb = dToBits(x);
+    // cbrt(+-inf) = +-inf and a NaN comes back quieted (x + x does both;
+    // the iteration below turned inf into inf / inf = NaN); cbrt(+-0) = +-0.
+    if (isNonFinite(xb)) {
+        return x + x;
+    }
+    if (isZeroBits(xb)) {
+        return x;
+    }
     int neg = (int)(dToBits(x) >> 63) & 1;
     double a = neg ? -x : x;
     double r = a;                        // crude initial guess
@@ -768,8 +1028,8 @@ double acosh(double x) {
 double atanh(double x) {
     // atanh(x) = 0.5 * log((1+x)/(1-x)) for |x| < 1.  Returns +/-inf
     // at x = +/-1 and NaN outside [-1, 1].
-    if (x >= 1.0)  return  __builtin_inff();
+    if (x >= 1.0)  return (x == 1.0)  ?  __builtin_inff() : __builtin_nan("");
-    if (x <= -1.0) return -__builtin_inff();
+    if (x <= -1.0) return (x == -1.0) ? -__builtin_inff() : __builtin_nan("");
     return 0.5 * log((1.0 + x) / (1.0 - x));
 }
 
@@ -800,10 +1060,16 @@ float nanf(const char *tagp) { (void)tagp; return __builtin_nanf(""); }
 
 
 double remainder(double x, double y) {
-    // IEEE 754 remainder: x - n*y where n = round-to-nearest-even(x/y).
+    // IEEE 754 remainder: x - n*y where n = round-to-nearest-even(x/y),
-    // Falls back to fmod for non-finite cases via the existing primitives.
+    // which is rint (round() rounded ties away: remainder(5, 2) was -1).
+    // remainder(x, +-inf) is x for a finite x (C99 F.9.7.2); x / y would
+    // make it 0 * inf = NaN.
     if (y == 0.0)                return __builtin_nanf("");
-    double quotient = round(x / y);
+    uint64_t yb = dToBits(y);
+    if ((yb & ~D_SIGN) == D_INF && !isNonFinite(dToBits(x))) {
+        return x;
+    }
+    double quotient = rint(x / y);
     return x - quotient * y;
 }
 
@@ -811,10 +1077,39 @@ double remainder(double x, double y) {
 float remainderf(float x, float y) { return (float)remainder((double)x, (double)y); }
 
 
-double rint(double x)        { return round(x); }
+// rint / nearbyint round in the current rounding mode, which is always
-float  rintf(float  x)       { return roundf(x); }
+// to nearest, ties to even, here (the soft-float ignores fesetround).  They
-double nearbyint(double x)   { return round(x); }
+// used to be round(), which rounds ties away from zero (rint(2.5) was 3).
-float  nearbyintf(float  x)  { return roundf(x); }
+// Done on the bits, like floor: the fraction bits below the unit are
+// dropped, and one unit is added when they exceed half a unit or equal it
+// with an odd integer part.  A carry out of the mantissa steps the
+// exponent, which is exactly the next power of two.
+double rint(double x) {
+    uint64_t b = dToBits(x);
+    int e = (int)((topWord(b) >> 4) & 0x7FFu) - 1023;
+    if (e >= 52) {
+        return x + 0.0;                 // integer, infinity or NaN (quieted)
+    }
+    uint64_t sign = b & D_SIGN;
+    if (e < 0) {
+        // |x| < 1: +-0 up to a half (ties go to the even 0), +-1 above it.
+        uint64_t mag = b & ~D_SIGN;
+        return dFromBits(sign | ((mag > 0x3FE0000000000000ULL) ? D_ONE : 0));
+    }
+    uint64_t unit = (uint64_t)1 << (52 - e);
+    uint64_t half = unit >> 1;
+    uint64_t frac = b & (unit - 1);
+    b &= ~(unit - 1);
+    // For e == 0 the unit bit is the exponent's lowest bit, which is 1 for
+    // 1023: the integer part 1 is odd, as it should be.
+    if (frac > half || (frac == half && (b & unit))) {
+        b += unit;
+    }
+    return dFromBits(b);
+}
+NO_SHRINK(rint) float rintf(float x) { return (float)rint((double)x); }
+double nearbyint(double x)   { return rint(x); }
+NO_SHRINK(rint) float nearbyintf(float x) { return (float)rint((double)x); }
 
 
 long lround(double x) {
@@ -828,12 +1123,12 @@ long lroundf(float x) {
 
 
 long lrint(double x) {
-    return (long)round(x);
+    return (long)rint(x);
 }
 
 
 long lrintf(float x) {
-    return (long)roundf(x);
+    return (long)rintf(x);
 }
 
 

runtime/src/softDouble.c

@@ -5,6 +5,13 @@
 // Earlier attempts crashed the Register Coalescer; the greedy
 // regalloc landing fixed the underlying register pressure problem.
 // Each routine is broken into small helpers to keep frames shallow.
+//
+// NaN and infinity follow IEEE 754 and compiler-rt's libcall contract,
+// as softFloat.c does: a NaN operand comes back quieted with its payload
+// kept, invalid operations return the default NaN, and comparisons with a
+// NaN are unordered.  Double to integer saturates: NaN converts to 0,
+// +inf and too-large values to the maximum, -inf and too-negative values
+// to the minimum (0 for unsigned).
 
 // Local typedefs (no stdint.h — clang's host stdint pulls glibc).
 typedef unsigned long long u64;
@@ -21,6 +28,19 @@ typedef unsigned char      u8;
 #define DMANT_LEAD 0x0010000000000000ULL
 #define DEXP_SHIFT 52
 #define DEXP_BIAS  1023
+#define DQUIET_BIT  0x0008000000000000ULL  // mantissa MSB: set = quiet NaN
+#define DDEFAULT_NAN 0x7FF8000000000000ULL // what an invalid op returns
+#define DPC_ZERO    0                      // dclass results
+#define DPC_NORMAL  1
+#define DPC_INF     2
+#define DPC_NAN     3
+#define DPC_SPECIAL 2                      // set for inf and NaN only
+// The special-value paths are kept out of line: inlined, they grew the
+// hot functions' frames and the compiler speculated their tests into the
+// normal path.
+#define COLD __attribute__((noinline))
+
+COLD s16 __unorddf2(u64 a, u64 b);
 
 // Pack sign / unbiased-exp / mantissa-with-leading-bit into IEEE-754
 // double.  Returns sign for zero or underflow; sign|inf for overflow.
@@ -45,16 +65,57 @@ static u16 dclass(u64 x, u64 *out_sign, s16 *out_exp, u64 *out_mant) {
     if (e == 0) {
         *out_exp = 0;
         *out_mant = 0;
-        return 0;
+        return DPC_ZERO;
     }
     if (e == 0x7FF) {
         *out_exp = 0x7FF;
         *out_mant = m;
-        return (m == 0) ? 2 : 3;
+        return (m == 0) ? DPC_INF : DPC_NAN;
     }
     *out_exp = e - DEXP_BIAS;
     *out_mant = m | DMANT_LEAD;
-    return 1;
+    return DPC_NORMAL;
+}
+
+// a op b when an operand is a zero, an infinity or a NaN, apart from the
+// zero cases the hot routines answer themselves (op: DSP_ADD / DSP_MUL /
+// DSP_DIV).  One out-of-line routine for all three keeps the rarely used
+// code small, and it reclassifies the operands itself so the call sites
+// stay small: bigger ones perturbed the hot routines' register allocation
+// enough to cost the ordinary path 2-4%.
+#define DSP_ADD 0
+#define DSP_MUL 1
+#define DSP_DIV 2
+static COLD u64 dpSpecial(u16 op, u64 a, u64 b) {
+    u64 sa, sb, ma, mb;
+    s16 ea, eb;
+    u16 ca = dclass(a, &sa, &ea, &ma);
+    u16 cb = dclass(b, &sb, &eb, &mb);
+    // The first NaN operand wins, quieted; keeping its payload lets a
+    // NaN's origin survive arithmetic.
+    if (ca == DPC_NAN) return a | DQUIET_BIT;
+    if (cb == DPC_NAN) return b | DQUIET_BIT;
+    u64 sign = (a ^ b) & DSIGN_BIT;
+    if (op == DSP_ADD) {
+        // Opposite infinities cancel to nothing representable: invalid.
+        if (ca == DPC_INF) return (cb == DPC_INF && sign) ? DDEFAULT_NAN : a;
+        if (cb == DPC_INF) return b;
+        // 0 + 0 is -0 only when both are -0 (round to nearest).
+        if (ca == DPC_ZERO) return (cb == DPC_ZERO) ? (a & b & DSIGN_BIT) : b;
+        return a;  // x + 0
+    }
+    if (op == DSP_MUL) {
+        // 0 * inf has no meaningful magnitude: invalid.
+        if (ca == DPC_INF || cb == DPC_INF) {
+            return (ca == DPC_ZERO || cb == DPC_ZERO) ? DDEFAULT_NAN : (sign | DEXP_MASK);
+        }
+        return sign;  // a zero operand
+    }
+    if (ca == DPC_INF) return (cb == DPC_INF) ? DDEFAULT_NAN : (sign | DEXP_MASK);
+    if (cb == DPC_INF) return sign;  // finite / inf = signed zero
+    // x / 0 is a signed infinity, but 0 / 0 is invalid.
+    if (cb == DPC_ZERO) return (ca == DPC_ZERO) ? DDEFAULT_NAN : (sign | DEXP_MASK);
+    return sign;  // 0 / x
 }
 
 u64 __adddf3(u64 a, u64 b) {
@@ -62,8 +123,11 @@ u64 __adddf3(u64 a, u64 b) {
     s16 ea, eb;
     u16 ca = dclass(a, &sa, &ea, &ma);
     u16 cb = dclass(b, &sb, &eb, &mb);
-    if (ca == 0) return b;
+    // x + 0 and 0 + x stay fast; every other zero, infinity or NaN case
-    if (cb == 0) return a;
+    // goes out of line.
+    if (ca == DPC_ZERO) return (cb == DPC_NORMAL) ? b : dpSpecial(DSP_ADD, a, b);
+    if (cb == DPC_ZERO) return (ca == DPC_NORMAL) ? a : dpSpecial(DSP_ADD, a, b);
+    if (__builtin_expect((ca | cb) & DPC_SPECIAL, 0)) return dpSpecial(DSP_ADD, a, b);
     // Shift mantissas left by 3 to reserve guard / round / sticky bits
     // below position 0.  Lead bit is now at position 55 instead of 52.
     // The sticky bit is preserved by ORing it into the LSB whenever a
@@ -215,7 +279,12 @@ u64 __muldf3(u64 a, u64 b) {
     u16 ca = dclass(a, &sa, &ea, &ma);
     u16 cb = dclass(b, &sb, &eb, &mb);
     u64 sr = sa ^ sb;
-    if (ca == 0 || cb == 0) return sr;
+    // x * 0 and 0 * x stay fast; every other zero, infinity or NaN case
+    // goes out of line.
+    if ((ca == DPC_ZERO && cb == DPC_NORMAL) || (cb == DPC_ZERO && ca == DPC_NORMAL)) {
+        return sr;
+    }
+    if (__builtin_expect(ca != DPC_NORMAL || cb != DPC_NORMAL, 0)) return dpSpecial(DSP_MUL, a, b);
     u16 carry;
     u16 round_bits;
     u64 mr = mulhi64Aligned(ma, mb, &carry, &round_bits);
@@ -239,8 +308,10 @@ u64 __divdf3(u64 a, u64 b) {
     u16 ca = dclass(a, &sa, &ea, &ma);
     u16 cb = dclass(b, &sb, &eb, &mb);
     u64 sr = sa ^ sb;
-    if (ca == 0) return sr;
+    // 0 / x stays fast; every other zero, infinity or NaN case goes out of
-    if (cb == 0) return sr | DEXP_MASK;  // div-by-zero → inf
+    // line.
+    if (ca == DPC_ZERO && cb == DPC_NORMAL) return sr;
+    if (__builtin_expect(ca != DPC_NORMAL || cb != DPC_NORMAL, 0)) return dpSpecial(DSP_DIV, a, b);
     // Long division: handle the leading quotient bit explicitly (since
     // we need to "consume" the dividend's leading 1 by subtracting),
     // then generate 52 more fractional bits by shifting r left and
@@ -283,7 +354,10 @@ u64 __divdf3(u64 a, u64 b) {
     return dpack(sr, er, q);
 }
 
-s16 __cmpdf2(u64 a, u64 b) {
+// Order of a against b: -1, 0 or 1.  Neither may be a NaN.  Infinities
+// need no case of their own: their bit patterns are the largest
+// magnitudes.
+static s16 dorder(u64 a, u64 b) {
     u64 sa = a & DSIGN_BIT;
     u64 sb = b & DSIGN_BIT;
     if (sa != sb) {
@@ -296,6 +370,45 @@ s16 __cmpdf2(u64 a, u64 b) {
     return a < b ? -1 : 1;
 }
 
+// 1 when a or b might be a NaN: only an all-ones exponent can be one.
+// With the sign forced on, a top word of 0xFFF0 or more has all exponent
+// bits set.  Spelled as a 16-bit compare because the equivalent mask test
+// is widened by the optimizer into a 64-bit one, several times dearer.
+static u16 dmaybeNan(u64 a, u64 b) {
+    return ((u16)(a >> 48) | 0x8000) >= 0xFFF0 ||
+           ((u16)(b >> 48) | 0x8000) >= 0xFFF0;
+}
+
+// 1 when x is a NaN: its magnitude is above infinity's.
+static u16 disNan(u64 x) {
+    return (x & ~DSIGN_BIT) > DEXP_MASK;
+}
+
+// Order of a against b, or `unord` when either is a NaN.  Forced inline
+// into each helper, and the NaN test kept inline behind the cheap screen:
+// a call anywhere in a helper stops the stack-slot promotion that keeps
+// the ordinary compare fast.
+static inline __attribute__((always_inline)) s16 dcmp(u64 a, u64 b, s16 unord) {
+    if (__builtin_expect(dmaybeNan(a, b), 0) && (disNan(a) || disNan(b))) {
+        return unord;
+    }
+    return dorder(a, b);
+}
+
+// 0 when a == b, else 1 (unordered included).  eq / ne only need that,
+// and it is less than half the code of the three-way order.
+static inline __attribute__((always_inline)) s16 dne(u64 a, u64 b) {
+    if (__builtin_expect(dmaybeNan(a, b), 0) && (disNan(a) || disNan(b))) {
+        return 1;
+    }
+    if (a == b) return 0;
+    return ((a | b) << 1) != 0;  // +0 == -0
+}
+
+s16 __cmpdf2(u64 a, u64 b) {
+    return dcmp(a, b, 1);
+}
+
 s16 __unorddf2(u64 a, u64 b) {
     // Returns nonzero if either is NaN.
     u64 ea = (a >> DEXP_SHIFT) & 0x7FF;
@@ -311,14 +424,19 @@ s16 __unorddf2(u64 a, u64 b) {
 // return the three-way order, as softFloat.c's do.  They used to return C
 // booleans, which made every double `<` false, `>=` always true and `<=`
 // inverted (CoreMark's "must run 10 s" check, fmod of a negative quotient
-// and asin all went wrong).  eq/ne are tested with == 0 / != 0, for which
+// and asin all went wrong).  For a NaN operand they follow compiler-rt:
-// "nonzero when different" is enough.
+// the helpers tested with `< 0` / `<= 0` / `== 0` / `!= 0` (lt, le, eq, ne)
-s16 __eqdf2(u64 a, u64 b) { return __cmpdf2(a, b) != 0; }
+// answer 1, the ones tested with `> 0` / `>= 0` (gt, ge) answer -1, so
-s16 __nedf2(u64 a, u64 b) { return __cmpdf2(a, b) != 0; }
+// every ordered predicate is false and its negation true.  The unordered
-s16 __ltdf2(u64 a, u64 b) { return __cmpdf2(a, b); }
+// predicates (ult is !(a >= b), ugt is !(a <= b), ...) are lowered to
-s16 __ledf2(u64 a, u64 b) { return __cmpdf2(a, b); }
+// these same helpers with the test inverted, and ueq / one add a
-s16 __gtdf2(u64 a, u64 b) { return __cmpdf2(a, b); }
+// __unorddf2 call.
-s16 __gedf2(u64 a, u64 b) { return __cmpdf2(a, b); }
+s16 __eqdf2(u64 a, u64 b) { return dne(a, b); }
+s16 __nedf2(u64 a, u64 b) { return dne(a, b); }
+s16 __ltdf2(u64 a, u64 b) { return dcmp(a, b, 1); }
+s16 __ledf2(u64 a, u64 b) { return dcmp(a, b, 1); }
+s16 __gtdf2(u64 a, u64 b) { return dcmp(a, b, -1); }
+s16 __gedf2(u64 a, u64 b) { return dcmp(a, b, -1); }
 
 // double <-> float conversions.
 u64 __extendsfdf2(u32 x) {
@@ -327,7 +445,12 @@ u64 __extendsfdf2(u32 x) {
     u32 m = x & 0x7FFFFFUL;
     if (e == 0) return sign;
     if (e == 0xFF) {
-        return sign | DEXP_MASK | ((u64)m << 29);
+        // Infinity, or a NaN quieted with its payload moved up to the
+        // top of the wider mantissa (so a float NaN's payload survives a
+        // round trip through double).
+        u64 r = sign | DEXP_MASK | ((u64)m << 29);
+        if (m != 0) r |= DQUIET_BIT;
+        return r;
     }
     s16 unbiased = e - 127;
     return dpack(sign, unbiased, ((u64)m << 29) | DMANT_LEAD);
@@ -339,7 +462,10 @@ u32 __truncdfsf2(u64 x) {
     u64 m = x & DMANT_MASK;
     if (e == 0) return (u32)sign;
     if (e == 0x7FF) {
-        return (u32)sign | 0x7F800000UL | (u32)(m >> 29);
+        // A NaN whose payload lives only in the low 29 bits would lose it
+        // all and become infinity; setting the quiet bit keeps it a NaN.
+        if (m == 0) return (u32)sign | 0x7F800000UL;
+        return (u32)sign | 0x7FC00000UL | (u32)(m >> 29);
     }
     s16 unbiased = e - DEXP_BIAS;
     s16 fexp = unbiased + 127;
@@ -373,7 +499,11 @@ s32 __fixdfsi(u64 x) {
     u64 sign = x & DSIGN_BIT;
     s16 e = (s16)((x >> DEXP_SHIFT) & 0x7FF);
     if (e == 0) return 0;
-    if (e == 0x7FF) return sign ? (s32)0x80000000 : 0x7FFFFFFF;
+    if (e == 0x7FF) {
+        // NaN converts to 0; infinities saturate like overflow.
+        if (x & DMANT_MASK) return 0;
+        return sign ? (s32)0x80000000 : 0x7FFFFFFF;
+    }
     s16 unbiased = e - DEXP_BIAS;
     if (unbiased < 0) return 0;
     if (unbiased > 30) return sign ? (s32)0x80000000 : 0x7FFFFFFF;
@@ -385,13 +515,13 @@ s32 __fixdfsi(u64 x) {
 
 
 // __fixunsdfsi: unsigned double → uint32. Saturates to 0 for negative
-// inputs, to 0xFFFFFFFF for inputs >= 2^32. Used by clang when casting
+// inputs and NaN, to 0xFFFFFFFF for inputs >= 2^32 and +inf. Used by clang
-// double values to unsigned integer types.
+// when casting double values to unsigned integer types.
 u32 __fixunsdfsi(u64 x) {
     if (x & DSIGN_BIT) return 0;                 // negative → 0
     u16 e = (u16)((x >> DEXP_SHIFT) & 0x7FF);
     if (e == 0) return 0;
-    if (e == 0x7FF) return 0xFFFFFFFF;
+    if (e == 0x7FF) return (x & DMANT_MASK) ? 0 : 0xFFFFFFFF;
     s16 unbiased = (s16)e - DEXP_BIAS;
     if (unbiased < 0) return 0;
     if (unbiased > 31) return 0xFFFFFFFF;

runtime/src/softFloat.c

@@ -16,14 +16,18 @@
 //
 // Limitations (V1):
 //   - No subnormal / denormal handling — values flush to zero.
-//   - No NaN / Inf handling — operations on these give garbage but
-//     don't crash.
 //   - Round-to-zero (truncation) only; no banker's rounding.
 //   - Add/sub use a 24-bit mantissa; underflow rounding is crude.
 //
-// These are correct enough for end-to-end test programs that do
+// NaN and infinity follow IEEE 754 (and compiler-rt, whose libcall
-// "normal" arithmetic in the representable range.  Production-grade
+// contract LLVM lowers to): a NaN operand comes back quieted with its
-// IEEE compliance is a significantly bigger project.
+// payload kept, an invalid operation (inf - inf, 0 * inf, 0 / 0,
+// inf / inf) returns the default NaN, infinities add, multiply and divide
+// as IEEE says, and every comparison with a NaN is unordered.  The checks
+// hang off the class fpClass already computes, so ordinary operands pay
+// one branch.  Float to integer saturates: NaN converts to 0, +inf and
+// too-large values to the type's maximum, -inf and too-negative values to
+// its minimum (0 for unsigned).
 
 typedef unsigned long u32;
 typedef long          s32;
@@ -37,6 +41,19 @@ typedef int           s16;
 #define EXP_BIAS   127
 #define MANT_MASK  0x007FFFFFUL
 #define MANT_LEAD  0x00800000UL  // implicit leading 1
+#define QUIET_BIT  0x00400000UL  // mantissa MSB: set = quiet NaN
+#define DEFAULT_NAN 0x7FC00000UL // what an invalid operation returns
+#define FPC_ZERO   0             // fpClass results
+#define FPC_NORMAL 1
+#define FPC_INF    2
+#define FPC_NAN    3
+#define FPC_SPECIAL 2            // set in the class of an inf or NaN only
+// The special-value paths are kept out of line: inlined, they grew the
+// hot functions' frames and the compiler speculated their tests into the
+// normal path.
+#define COLD __attribute__((noinline))
+
+COLD s16 __unordsf2(u32 a, u32 b);
 
 static u16 fpClass(u32 x, u32 *out_sign, s16 *out_exp, u32 *out_mant) {
     *out_sign = x & SIGN_BIT;
@@ -46,18 +63,58 @@ static u16 fpClass(u32 x, u32 *out_sign, s16 *out_exp, u32 *out_mant) {
         // Zero or subnormal — treat as zero (flush).
         *out_exp = 0;
         *out_mant = 0;
-        return 0;  // zero
+        return FPC_ZERO;
     }
     if (e == 0xFF) {
         // Inf or NaN — return as-is, caller decides.
         *out_exp = 0xFF;
         *out_mant = m;
-        return (m == 0) ? 2 : 3;  // 2=inf, 3=nan
+        return (m == 0) ? FPC_INF : FPC_NAN;
     }
     // Normal — restore implicit leading 1.
     *out_exp = e - EXP_BIAS;
     *out_mant = m | MANT_LEAD;
-    return 1;  // normal
+    return FPC_NORMAL;
+}
+
+// a op b when an operand is a zero, an infinity or a NaN, apart from the
+// zero cases the hot routines answer themselves (op: SP_ADD / SP_MUL /
+// SP_DIV).  One out-of-line routine for all three keeps the rarely used
+// code small, and it reclassifies the operands itself so the call sites
+// stay small (see softDouble.c's dpSpecial).
+#define SP_ADD 0
+#define SP_MUL 1
+#define SP_DIV 2
+static COLD u32 fpSpecial(u16 op, u32 a, u32 b) {
+    u32 sa, sb, ma, mb;
+    s16 ea, eb;
+    u16 ca = fpClass(a, &sa, &ea, &ma);
+    u16 cb = fpClass(b, &sb, &eb, &mb);
+    // The first NaN operand wins, quieted; keeping its payload lets a
+    // NaN's origin survive arithmetic.
+    if (ca == FPC_NAN) return a | QUIET_BIT;
+    if (cb == FPC_NAN) return b | QUIET_BIT;
+    u32 sign = (a ^ b) & SIGN_BIT;
+    if (op == SP_ADD) {
+        // Opposite infinities cancel to nothing representable: invalid.
+        if (ca == FPC_INF) return (cb == FPC_INF && sign) ? DEFAULT_NAN : a;
+        if (cb == FPC_INF) return b;
+        // 0 + 0 is -0 only when both are -0 (round to nearest).
+        if (ca == FPC_ZERO) return (cb == FPC_ZERO) ? (a & b & SIGN_BIT) : b;
+        return a;  // x + 0
+    }
+    if (op == SP_MUL) {
+        // 0 * inf has no meaningful magnitude: invalid.
+        if (ca == FPC_INF || cb == FPC_INF) {
+            return (ca == FPC_ZERO || cb == FPC_ZERO) ? DEFAULT_NAN : (sign | EXP_MASK);
+        }
+        return sign;  // a zero operand
+    }
+    if (ca == FPC_INF) return (cb == FPC_INF) ? DEFAULT_NAN : (sign | EXP_MASK);
+    if (cb == FPC_INF) return sign;  // finite / inf = signed zero
+    // x / 0 is a signed infinity, but 0 / 0 is invalid.
+    if (cb == FPC_ZERO) return (ca == FPC_ZERO) ? DEFAULT_NAN : (sign | EXP_MASK);
+    return sign;  // 0 / x
 }
 
 static u32 fpPack(u32 sign, s16 exp, u32 mant) {
@@ -82,8 +139,11 @@ u32 __addsf3(u32 a, u32 b) {
     s16 ea, eb;
     u16 ca = fpClass(a, &sa, &ea, &ma);
     u16 cb = fpClass(b, &sb, &eb, &mb);
-    if (ca == 0) return b;
+    // x + 0 and 0 + x stay fast; every other zero, infinity or NaN case
-    if (cb == 0) return a;
+    // goes out of line.
+    if (ca == FPC_ZERO) return (cb == FPC_NORMAL) ? b : fpSpecial(SP_ADD, a, b);
+    if (cb == FPC_ZERO) return (ca == FPC_NORMAL) ? a : fpSpecial(SP_ADD, a, b);
+    if (__builtin_expect((ca | cb) & FPC_SPECIAL, 0)) return fpSpecial(SP_ADD, a, b);
 
     // Align: shift smaller-exp mantissa right.
     if (ea > eb) {
@@ -103,12 +163,15 @@ u32 __addsf3(u32 a, u32 b) {
         u32 m = ma + mb;
         return fpPack(sa, ea, m);
     } else {
-        // Different signs — subtract the smaller magnitude.
+        // Different signs -- subtract the smaller magnitude.  Exact
-        if (ma >= mb) {
+        // cancellation is +0 whichever operand was negative.
+        if (ma > mb) {
             return fpPack(sa, ea, ma - mb);
-        } else {
+        }
+        if (mb > ma) {
             return fpPack(sb, eb, mb - ma);
         }
+        return 0;
     }
 }
 
@@ -126,7 +189,13 @@ u32 __mulsf3(u32 a, u32 b) {
     u16 ca = fpClass(a, &sa, &ea, &ma);
     u16 cb = fpClass(b, &sb, &eb, &mb);
     u32 sign = sa ^ sb;
-    if (ca == 0 || cb == 0) return sign;  // zero
+    // x * 0 and 0 * x stay fast; every other zero, infinity or NaN case
+    // goes out of line.  (This spelling measured faster than softDouble.c's
+    // for this routine: the difference is register allocation, not work.)
+    if (ca == FPC_ZERO || cb == FPC_ZERO) {
+        return ((ca | cb) & FPC_SPECIAL) ? fpSpecial(SP_MUL, a, b) : sign;
+    }
+    if (__builtin_expect((ca | cb) & FPC_SPECIAL, 0)) return fpSpecial(SP_MUL, a, b);
 
     // 24-bit x 24-bit -> 48-bit product.  Take top 24 bits.
     // We approximate by multiplying the 16-bit halves and combining.
@@ -171,8 +240,10 @@ u32 __divsf3(u32 a, u32 b) {
     u16 ca = fpClass(a, &sa, &ea, &ma);
     u16 cb = fpClass(b, &sb, &eb, &mb);
     u32 sign = sa ^ sb;
-    if (cb == 0) return sign | EXP_MASK;  // div-by-zero -> inf
+    // 0 / x stays fast; every other zero, infinity or NaN case goes out of
-    if (ca == 0) return sign;
+    // line.
+    if (ca == FPC_ZERO && cb == FPC_NORMAL) return sign;
+    if (__builtin_expect(ca != FPC_NORMAL || cb != FPC_NORMAL, 0)) return fpSpecial(SP_DIV, a, b);
 
     // Long division: quotient = ma/mb, in 24+1 bits.  We shift ma left
     // until larger than mb, accumulating quotient bits.  Use a 32-bit
@@ -192,10 +263,12 @@ u32 __divsf3(u32 a, u32 b) {
     return fpPack(sign, new_exp, q);
 }
 
-s16 __cmpsf2(u32 a, u32 b) {
+// Order of a against b: -1, 0 or 1.  Neither may be a NaN.  Kept free of
-    // Returns -1 if a<b, 0 if a==b, 1 if a>b.
+// calls: one would push its values into callee-saved slots and, on this
-    // For NaN, libgcc returns 1 from cmpsf2 (no-NaN convention).  We
+// target, stop the stack-slot promotion for the whole function.
-    // skip NaN handling.
+// Infinities need no case of their own: their bit patterns are the
+// largest magnitudes.
+static s16 forder(u32 a, u32 b) {
     if (a == b) return 0;
     u32 sa = a & SIGN_BIT;
     u32 sb = b & SIGN_BIT;
@@ -211,12 +284,47 @@ s16 __cmpsf2(u32 a, u32 b) {
     return sa ? -r : r;
 }
 
-s16 __eqsf2(u32 a, u32 b) { return __cmpsf2(a, b) != 0; }
+// 1 when a or b might be a NaN: only an all-ones exponent can be one.
-s16 __nesf2(u32 a, u32 b) { return __cmpsf2(a, b) != 0; }
+// With the sign forced on, a high word of 0xFF80 or more has all exponent
-s16 __ltsf2(u32 a, u32 b) { return __cmpsf2(a, b);     }
+// bits set.  Spelled as a 16-bit compare because the equivalent mask test
-s16 __gtsf2(u32 a, u32 b) { return __cmpsf2(a, b);     }
+// is widened by the optimizer into a 32-bit one, several times dearer.
-s16 __lesf2(u32 a, u32 b) { return __cmpsf2(a, b);     }
+static u16 fmaybeNan(u32 a, u32 b) {
-s16 __gesf2(u32 a, u32 b) { return __cmpsf2(a, b);     }
+    return ((u16)(a >> 16) | 0x8000) >= 0xFF80 ||
+           ((u16)(b >> 16) | 0x8000) >= 0xFF80;
+}
+
+// The comparison when an operand has an all-ones exponent: `unord` for a
+// NaN, else the ordinary order (infinities).
+static COLD s16 fcmpSpecial(u32 a, u32 b, s16 unord) {
+    return __unordsf2(a, b) ? unord : forder(a, b);
+}
+
+// compiler-rt's contract, which LLVM's soft-float lowering assumes: the
+// helpers tested with `< 0` / `<= 0` / `== 0` / `!= 0` (lt, le, eq, ne,
+// cmp) answer 1 for unordered, the ones tested with `> 0` / `>= 0` (gt,
+// ge) answer -1, so every ordered predicate is false on a NaN and its
+// negation (!(a < b) is __ltsf2 >= 0) true.
+s16 __cmpsf2(u32 a, u32 b) {
+    return fmaybeNan(a, b) ? fcmpSpecial(a, b, 1) : forder(a, b);
+}
+s16 __eqsf2(u32 a, u32 b) {
+    return fmaybeNan(a, b) ? fcmpSpecial(a, b, 1) : forder(a, b);
+}
+s16 __nesf2(u32 a, u32 b) {
+    return fmaybeNan(a, b) ? fcmpSpecial(a, b, 1) : forder(a, b);
+}
+s16 __ltsf2(u32 a, u32 b) {
+    return fmaybeNan(a, b) ? fcmpSpecial(a, b, 1) : forder(a, b);
+}
+s16 __lesf2(u32 a, u32 b) {
+    return fmaybeNan(a, b) ? fcmpSpecial(a, b, 1) : forder(a, b);
+}
+s16 __gtsf2(u32 a, u32 b) {
+    return fmaybeNan(a, b) ? fcmpSpecial(a, b, -1) : forder(a, b);
+}
+s16 __gesf2(u32 a, u32 b) {
+    return fmaybeNan(a, b) ? fcmpSpecial(a, b, -1) : forder(a, b);
+}
 
 // __unordsf2: 1 if either arg is NaN, else 0. Used for IEEE 754
 // unordered comparisons (a < b is false if either is NaN, etc.).
@@ -262,7 +370,11 @@ s32 __fixsfsi(u32 a) {
     u32 sa, ma;
     s16 ea;
     u16 ca = fpClass(a, &sa, &ea, &ma);
-    if (ca == 0) return 0;
+    if (ca != FPC_NORMAL) {
+        // Zero and NaN convert to 0; infinities saturate like overflow.
+        if (ca != FPC_INF) return 0;
+        return sa ? -2147483647L - 1 : 2147483647L;
+    }
     if (ea < 0) return 0;          // |a| < 1
     if (ea >= 31) {                // overflow
         return sa ? -2147483647L - 1 : 2147483647L;
@@ -278,7 +390,11 @@ u32 __fixunssfsi(u32 a) {
     u32 sa, ma;
     s16 ea;
     u16 ca = fpClass(a, &sa, &ea, &ma);
-    if (ca == 0 || sa) return 0;   // negative -> 0
+    if (ca != FPC_NORMAL) {
+        // Zero, NaN and -inf convert to 0; +inf saturates.
+        return (ca == FPC_INF && sa == 0) ? 0xFFFFFFFFUL : 0;
+    }
+    if (sa) return 0;              // negative -> 0
     if (ea < 0) return 0;
     if (ea >= 32) return 0xFFFFFFFFUL;
     if (ea >= 23) return ma << (ea - 23);

scripts/common.sh

@@ -30,6 +30,16 @@ die() {
     exit 1
 }
 
+# MAME's cfg / nvram (the IIgs battery RAM) go to a throwaway directory.
+# Run from a directory with no mame.ini (tests/ubsan's runner cd's there),
+# MAME would otherwise leave cfg/ and nvram/ in it, and a saved BRAM would
+# carry settings from run to run.  Sets MAME_STATE and MAME_STATE_ARGS
+# (pass those to mame); the caller's EXIT trap removes "$MAME_STATE".
+makeMameState() {
+    MAME_STATE=$(mktemp -d)
+    MAME_STATE_ARGS=(-cfg_directory "$MAME_STATE" -nvram_directory "$MAME_STATE")
+}
+
 # Download to cache unless already present. $1=url, $2=dest filename in cache.
 fetchCached() {
     local url="$1"

scripts/runInMame.sh

@@ -71,7 +71,8 @@ fi
 
 [ -f "$BIN" ] || die "binary not found: $BIN"
 LUA_PATH=$(mktemp --suffix=.lua)
-trap 'rm -f "$LUA_PATH"' EXIT
+makeMameState
+trap 'rm -f "$LUA_PATH"; rm -rf "${MAME_STATE:?}"' EXIT
 
 cat > "$LUA_PATH" <<EOF
 local frame = 0
@@ -122,6 +123,7 @@ EOF
 # patched MAME and ROM set instead of the ones installMame.sh sets up.
 OUT=$(SDL_VIDEODRIVER=dummy SDL_AUDIODRIVER=dummy timeout 30 "${MAME:-mame}" apple2gs \
     -rompath "${MAME_ROMPATH:-$PROJECT_ROOT/tools/mame/roms}" \
+    "${MAME_STATE_ARGS[@]}" \
     -plugins -autoboot_script "$LUA_PATH" \
     -video none -sound none -nothrottle -seconds_to_run "$SECS" 2>&1 | grep "^MAME-")
 

scripts/runInMameCycles.sh

@@ -94,7 +94,8 @@ done
 [ -f "$BIN" ] || die "binary not found: $BIN"
 
 LUA_PATH=$(mktemp --suffix=.lua)
-trap 'rm -f "$LUA_PATH"' EXIT
+makeMameState
+trap 'rm -f "$LUA_PATH"; rm -rf "${MAME_STATE:?}"' EXIT
 
 cat > "$LUA_PATH" <<EOF
 local frame = 0
@@ -188,6 +189,7 @@ fi
 
 OUT=$(SDL_VIDEODRIVER=dummy SDL_AUDIODRIVER=dummy timeout 90 mame apple2gs \
     -rompath "$PROJECT_ROOT/tools/mame/roms" \
+    "${MAME_STATE_ARGS[@]}" \
     -plugins -autoboot_script "$LUA_PATH" \
     -video none -sound none -nothrottle -seconds_to_run "$SECS" 2>&1 | grep -E "$GREP_PAT")
 

scripts/runInMameWithGsosStub.sh

@@ -64,7 +64,8 @@ fi
 
 [ -f "$BIN" ] || die "binary not found: $BIN"
 LUA_PATH=$(mktemp --suffix=.lua)
-trap 'rm -f "$LUA_PATH"' EXIT
+makeMameState
+trap 'rm -f "$LUA_PATH"; rm -rf "${MAME_STATE:?}"' EXIT
 
 # Build the stub-write Lua statement (one mem:write_u8 per byte).
 STUB_BYTES=$(echo "$STUB_HEX" | tr -d ' ')
@@ -117,6 +118,7 @@ EOF
 
 OUT=$(SDL_VIDEODRIVER=dummy SDL_AUDIODRIVER=dummy timeout 30 mame apple2gs \
     -rompath "$PROJECT_ROOT/tools/mame/roms" \
+    "${MAME_STATE_ARGS[@]}" \
     -plugins -autoboot_script "$LUA_PATH" \
     -video none -sound none -nothrottle -seconds_to_run "$SECS" 2>&1 | grep "^MAME-")
 

scripts/runMultiSeg.sh

@@ -65,10 +65,7 @@ EOF
 [ -n "$PARSED" ] || die "manifest parse failed"
 
 LUA_PATH=$(mktemp --suffix=.lua)
-# MAME's cfg / nvram (the IIgs battery RAM) go to a throwaway directory:
+makeMameState
-# run from a test directory with no mame.ini, MAME would otherwise leave
-# them there, and a saved BRAM would carry settings from run to run.
-MAME_STATE=$(mktemp -d)
 trap 'rm -f "$LUA_PATH"; rm -rf "${MAME_STATE:?}"' EXIT
 
 # Build the per-segment load lua.
@@ -159,7 +156,7 @@ fi
 OUT=$(SDL_VIDEODRIVER=dummy SDL_AUDIODRIVER=dummy timeout "${MAME_TIMEOUT:-30}" mame apple2gs \
     "${RAMSIZE_ARG[@]}" \
     -rompath "$PROJECT_ROOT/tools/mame/roms" \
-    -cfg_directory "$MAME_STATE" -nvram_directory "$MAME_STATE" \
+    "${MAME_STATE_ARGS[@]}" \
     -plugins -autoboot_script "$LUA_PATH" \
     -video none -sound none -nothrottle -seconds_to_run "$SECS" 2>&1 | /usr/bin/grep -E "^(MAME-|SEG-)")
 

src/llvm/lib/Target/W65816/CMakeLists.txt

@@ -39,6 +39,7 @@ add_llvm_target(W65816CodeGen
   W65816ImgCalleeSave.cpp
   W65816NarrowI32Mul.cpp
   W65816CmpPrepare.cpp
+  W65816CmpSink.cpp
   W65816PromoteFiToImg.cpp
   W65816StackRelToImg.cpp
   W65816StackSlotMerge.cpp

src/llvm/lib/Target/W65816/W65816.h

@@ -131,6 +131,14 @@ FunctionPass *createW65816NarrowI32Mul();
 // one at a time.  See W65816CmpPrepare.cpp.
 FunctionPass *createW65816CmpPrepare();
 
+// Pre-RA pass, inserted right after the register coalescer: moves the
+// instruction that sets a conditional branch's flags down past the back-
+// edge COPYs PHI elimination left between them, so the copies (and the
+// allocator's spills) land above the compare instead of inside a PHP/PLP
+// wrap.  See W65816CmpSink.cpp.
+FunctionPass *createW65816CmpSink();
+extern char &W65816CmpSinkID;
+
 // Post-LSR IR pass: detect pointer-walking loops over GlobalAddress
 // bases and rewrite to forward-counter + indexed GEP form.  LSR's
 // transform is faster for pointer-arg loops (single ptr increment per
@@ -220,6 +228,7 @@ void initializeW65816LowerWide32Pass(PassRegistry &);
 void initializeW65816ImgCalleeSavePass(PassRegistry &);
 void initializeW65816NarrowI32MulPass(PassRegistry &);
 void initializeW65816CmpPreparePass(PassRegistry &);
+void initializeW65816CmpSinkPass(PassRegistry &);
 void initializeW65816PromoteFiToImgPass(PassRegistry &);
 void initializeW65816StackSlotMergePass(PassRegistry &);
 void initializeW65816StackRelToImgPass(PassRegistry &);

src/llvm/lib/Target/W65816/W65816AsmPrinter.cpp

@@ -404,6 +404,40 @@ void W65816AsmPrinter::emitInstruction(const MachineInstr *MI) {
     OutStreamer->emitLabel(SkipSym);
     return;
   }
+  case W65816::INC_HI_IF_CARRY_DP:
+  case W65816::DEC_HI_IF_BORROW_DP:
+  case W65816::DEC_HI_IF_BORROW_StackRel: {
+    // The carry / borrow into an i32's high half (see the .td).  Z comes
+    // from the low half: after its increment for INC (Z=1: it wrapped to
+    // 0), before its decrement for DEC (Z=1: it is about to wrap).
+    //   INC_HI_IF_CARRY_DP         bne L ; inc $dp ; L:
+    //   DEC_HI_IF_BORROW_DP        bne L ; dec $dp ; L:
+    //   DEC_HI_IF_BORROW_StackRel  bne L ; lda $off,s ; dec a ;
+    //                              sta $off,s ; lda #0 ; L:
+    // The stack form restores A = 0, the low half it found, for the
+    // `dec a` that follows.
+    int64_t Off = MI->getOperand(0).getImm();
+    MCSymbol *SkipSym = OutContext.createTempSymbol();
+    EmitToStreamer(*OutStreamer,
+        MCInstBuilder(W65816::BNE).addExpr(
+            MCSymbolRefExpr::create(SkipSym, OutContext)));
+    switch (MI->getOpcode()) {
+    case W65816::INC_HI_IF_CARRY_DP:
+      emitOpImm(W65816::INC_DP, Off);
+      break;
+    case W65816::DEC_HI_IF_BORROW_DP:
+      emitOpImm(W65816::DEC_DP, Off);
+      break;
+    default:
+      emitOpImm(W65816::LDA_StackRel, Off);
+      emitOp(W65816::DEA);
+      emitOpImm(W65816::STA_StackRel, Off);
+      emitOpImm(W65816::LDA_Imm16, 0);
+      break;
+    }
+    OutStreamer->emitLabel(SkipSym);
+    return;
+  }
   case W65816::ADJCALLSTACKDOWN: {
     // DOWN is a no-op in our scheme — the PUSH16 sequence in LowerCall
     // already shifted SP incrementally as args were pushed.  Nothing
@@ -1095,10 +1129,11 @@ void W65816AsmPrinter::emitInstruction(const MachineInstr *MI) {
     return;
   }
   case W65816::ASRA16: {
-    // PHA ; ASL A (sets carry from sign bit) ; PLA ; ROR A
+    // CMP #$8000 sets C exactly when bit 15 (the sign) is set; ROR A
-    emitOp(W65816::PHA);
+    // shifts it back in at the top.  5 cycles, where the PHA ; ASL A ;
-    emitOp(W65816::ASL_A);
+    // PLA ; ROR A it replaced took 13 and touched the stack.  The flags
-    emitOp(W65816::PLA);
+    // it leaves are ROR's either way.
+    emitOpImm(W65816::CMP_Imm16, 0x8000);
     emitOp(W65816::ROR_A);
     return;
   }

src/llvm/lib/Target/W65816/W65816CmpSink.cpp

@@ -0,0 +1,170 @@
+//===-- W65816CmpSink.cpp - Keep a compare next to its branch --------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// PHI elimination puts a loop's back-edge copies before the block's first
+// terminator.  Here that is the conditional branch; the compare that sets
+// its flags is an ordinary instruction above it, so the copies land between
+// the two:
+//
+//   CMPfi %next, %stack.5            ; p + 1 == end ?
+//   %s   = COPY %sum                 ; the loop-carried values
+//   %p   = COPY %next
+//   BNE %bb.loop
+//
+// Every copy becomes an LDA / STA / TXA after register allocation, which
+// rewrites N and Z, and W65816StackSlotCleanup then has to save the flags
+// around them with PHP / PLP: 7 cycles on every trip round a
+// `while (p != end)` loop.
+//
+// Right after the register coalescer, this pass moves the flag-setting
+// instruction down to a BEQ / BNE / BMI / BPL, past virtual-register COPYs
+// that do not define anything it reads or read anything it defines.  The allocator
+// then sees the compare at the end of the block and gives the copies (and
+// its own splits, which go before the compare's operands are needed) the
+// room above it.  LiveIntervals are updated in place.
+//
+//===----------------------------------------------------------------------===//
+
+#include "W65816.h"
+#include "W65816InstrInfo.h"
+#include "W65816Subtarget.h"
+#include "llvm/CodeGen/LiveIntervals.h"
+#include "llvm/CodeGen/MachineFunctionPass.h"
+#include "llvm/CodeGen/MachineRegisterInfo.h"
+#include "llvm/CodeGen/SlotIndexes.h"
+#include "llvm/InitializePasses.h"
+#include "llvm/Support/Debug.h"
+
+using namespace llvm;
+
+#define DEBUG_TYPE "w65816-cmp-sink"
+
+namespace {
+
+class W65816CmpSink : public MachineFunctionPass {
+public:
+  static char ID;
+  W65816CmpSink() : MachineFunctionPass(ID) {}
+  StringRef getPassName() const override {
+    return "W65816 sink compare to its branch";
+  }
+  void getAnalysisUsage(AnalysisUsage &AU) const override {
+    AU.setPreservesCFG();
+    AU.addRequired<LiveIntervalsWrapperPass>();
+    AU.addPreserved<LiveIntervalsWrapperPass>();
+    AU.addPreserved<SlotIndexesWrapperPass>();
+    MachineFunctionPass::getAnalysisUsage(AU);
+  }
+  bool runOnMachineFunction(MachineFunction &MF) override;
+};
+
+} // namespace
+
+char W65816CmpSink::ID = 0;
+char &llvm::W65816CmpSinkID = W65816CmpSink::ID;
+
+INITIALIZE_PASS_BEGIN(W65816CmpSink, DEBUG_TYPE,
+                      "W65816 sink compare to its branch", false, false)
+INITIALIZE_PASS_DEPENDENCY(LiveIntervalsWrapperPass)
+INITIALIZE_PASS_END(W65816CmpSink, DEBUG_TYPE,
+                    "W65816 sink compare to its branch", false, false)
+
+// A copy between two virtual registers: no flags, no memory, no physical
+// register the compare could care about.
+static bool isVirtualCopy(const MachineInstr &MI) {
+  return MI.isCopy() && MI.getOperand(0).getReg().isVirtual() &&
+         MI.getOperand(1).getReg().isVirtual();
+}
+
+// True when moving Test below Copy keeps every value the same: Copy does
+// not write a register Test reads or writes, and does not read one Test
+// writes.
+static bool canPass(const MachineInstr &Test, const MachineInstr &Copy) {
+  Register Dst = Copy.getOperand(0).getReg();
+  Register Src = Copy.getOperand(1).getReg();
+  for (const MachineOperand &MO : Test.operands()) {
+    if (!MO.isReg() || !MO.getReg())
+      continue;
+    if (MO.getReg() == Dst)
+      return false;
+    if (MO.isDef() && MO.getReg() == Src)
+      return false;
+  }
+  return true;
+}
+
+bool W65816CmpSink::runOnMachineFunction(MachineFunction &MF) {
+  if (skipFunction(MF.getFunction()))
+    return false;
+  LiveIntervals &LIS = getAnalysis<LiveIntervalsWrapperPass>().getLIS();
+  MachineRegisterInfo &MRI = MF.getRegInfo();
+  bool Changed = false;
+
+  for (MachineBasicBlock &MBB : MF) {
+    MachineBasicBlock::iterator Br = MBB.getFirstTerminator();
+    if (Br == MBB.end())
+      continue;
+    // Only a branch on N or Z: the copies' loads and transfers leave C and
+    // V alone, so a BCC / BCS / BVC / BVS never needed the wrap, and moving
+    // the end of a CMP / SBC carry chain away from its first half only
+    // costs the allocator a register (a TAX / TXA round trip).
+    unsigned BrOpc = Br->getOpcode();
+    if (BrOpc != W65816::BEQ && BrOpc != W65816::BNE &&
+        BrOpc != W65816::BMI && BrOpc != W65816::BPL)
+      continue;
+    // Walk up over the copies to the instruction that sets the flags.
+    MachineInstr *Test = nullptr;
+    SmallVector<MachineInstr *, 8> Copies;
+    for (MachineBasicBlock::iterator It = Br; It != MBB.begin();) {
+      MachineInstr &MI = *--It;
+      if (MI.isDebugInstr())
+        continue;
+      if (isVirtualCopy(MI)) {
+        Copies.push_back(&MI);
+        continue;
+      }
+      MachineOperand *PDef =
+          MI.findRegisterDefOperand(W65816::P, /*TRI=*/nullptr);
+      if (PDef && !PDef->isDead())
+        Test = &MI;
+      break;
+    }
+    if (!Test || Copies.empty())
+      continue;
+    // The copies touch neither memory nor physical registers, so only
+    // register dependencies can stop the move; a flag setter with an
+    // explicit physical register operand (or a call) is left alone.
+    if (Test->isCall())
+      continue;
+    bool PhysReg = false;
+    for (const MachineOperand &MO : Test->operands())
+      if (MO.isReg() && MO.getReg().isPhysical() &&
+          MO.getReg() != W65816::P && !MO.isImplicit())
+        PhysReg = true;
+    if (PhysReg)
+      continue;
+    bool Ok = true;
+    for (MachineInstr *C : Copies)
+      Ok = Ok && canPass(*Test, *C);
+    if (!Ok)
+      continue;
+    MBB.splice(Br, &MBB, Test->getIterator());
+    LIS.handleMove(*Test, /*UpdateFlags=*/true);
+    // A copy that used to be the last reader of one of the compare's
+    // operands no longer is; greedy works from the intervals, so dropping
+    // the stale kill flags is enough.
+    for (const MachineOperand &MO : Test->operands())
+      if (MO.isReg() && MO.isUse() && MO.getReg().isVirtual())
+        MRI.clearKillFlags(MO.getReg());
+    LLVM_DEBUG(dbgs() << "cmp-sink: " << *Test);
+    Changed = true;
+  }
+  return Changed;
+}
+
+FunctionPass *llvm::createW65816CmpSink() { return new W65816CmpSink(); }

src/llvm/lib/Target/W65816/W65816FarPtrStageReuse.cpp

@@ -79,6 +79,10 @@ enum Width : int8_t { WUnknown = -1, W8 = 0, W16 = 1 };
 constexpr unsigned StageLo = 0xE0;
 constexpr unsigned StageBank = 0xE2;
 
+// The IMG direct-page words ($C0..$CE, then $D0..$DE: one run of memory).
+constexpr unsigned ImgLow = 0xC0;
+constexpr unsigned ImgHigh = 0xDE;
+
 // Liveness bit set: 1 A, 2 X, 4 Y, 8 N/Z.
 constexpr unsigned LiveAll = 15;
 
@@ -153,6 +157,9 @@ private:
   // Results of the final marking walk.
   SmallPtrSet<MachineInstr *, 32> RedundantStores;
   SmallPtrSet<MachineInstr *, 32> RedundantLoads;
+  // [$E0],Y accesses whose pointer also sits, word for word, in an IMG
+  // direct-page pair: the pair's low address.
+  DenseMap<MachineInstr *, unsigned> Redirects;
   // M and X widths in force at each instruction (from the marking walk).
   DenseMap<const MachineInstr *, std::pair<Width, Width>> ModeAt;
   Width modeM(const MachineInstr &MI) const {
@@ -188,6 +195,10 @@ private:
   void computeLiveness();
   bool deleteDead();
   bool deleteDeadStackStores();
+  bool redirectToHomes();
+  unsigned stageReads(const MachineInstr &MI) const;
+  unsigned stageDefs(const MachineInstr &MI) const;
+  bool foldEmptyFlagSaves();
 };
 
 } // namespace
@@ -243,7 +254,7 @@ static bool isRMWName(StringRef N) {
   if (endsWithAny(N, {"_A"}))
     return false;
   return startsWithAny(N, {"INC_", "DEC_", "ASL_", "LSR_", "ROL_", "ROR_",
-                           "TSB_", "TRB_", "INC_HI_IF_CARRY"});
+                           "TSB_", "TRB_"});
 }
 
 static bool isNZBranch(StringRef N) {
@@ -294,7 +305,7 @@ static bool changesY(StringRef N) {
 static bool changesCarry(StringRef N) {
   return startsWithAny(N, {"ADC", "SBC", "ASL", "LSR", "ROL", "ROR", "CMP",
                            "CPX", "CPY", "ASLA", "LSRA", "ASRA", "NEG", "SHL",
-                           "SRL", "SRA", "INC_HI_IF_CARRY"});
+                           "SRL", "SRA"});
 }
 
 Effects W65816FarPtrStageReuse::effects(const MachineInstr &MI,
@@ -388,8 +399,10 @@ Effects W65816FarPtrStageReuse::effects(const MachineInstr &MI,
     E.UseA = E.UseX = E.UseY = true;
     E.DefA = E.DefX = E.DefY = false;
   }
-  // N/Z.
+  // N/Z.  The i32 carry / borrow pseudos branch on Z (bne inside).
-  if (isNZBranch(N) || N == "PHP")
+  bool CarryPseudo =
+      startsWithAny(N, {"INC_HI_IF_CARRY", "DEC_HI_IF_BORROW"});
+  if (isNZBranch(N) || N == "PHP" || CarryPseudo)
     E.UseNZ = true;
   if (N.starts_with("SELECT"))
     E.UseNZ = true;
@@ -401,7 +414,7 @@ Effects W65816FarPtrStageReuse::effects(const MachineInstr &MI,
                         "DEA", "INX", "DEX", "INY", "DEY", "XBA", "NEG", "SHL",
                         "SRL", "SRA", "ADD", "SUB"}) ||
       (N.starts_with("BIT") && N != "BIT_Imm16" && N != "BIT_Imm8");
-  if (N.starts_with("INC_HI_IF_CARRY"))
+  if (CarryPseudo)
     SetsNZ = false;  // conditional: may leave the flags alone
   if (SetsNZ)
     E.DefNZ = true;
@@ -822,6 +835,28 @@ void W65816FarPtrStageReuse::transfer(MachineInstr &MI, FlowState &S,
     return;
   }
 
+  // A far-pointer access through the staging words, when the same two
+  // words also sit in an IMG direct-page pair: it can use the pair in
+  // place (W65816StackRelToImg puts a pointer's halves side by side for
+  // this).  [dp],Y reads the low word at dp and the bank byte at dp+2.
+  if (Mark && HasImm && Imm == StageLo && N.contains("_DPIndLong")) {
+    auto Lo = S.DP.find(StageLo);
+    auto Bank = S.DP.find(StageBank);
+    if (Lo != S.DP.end() && Bank != S.DP.end() && Lo->second &&
+        Bank->second) {
+      for (const auto &KV : S.DP) {
+        unsigned D = KV.first;
+        if (D < ImgLow || D > ImgHigh - 2 || KV.second != Lo->second)
+          continue;
+        auto Hi = S.DP.find(D + 2);
+        if (Hi != S.DP.end() && Hi->second == Bank->second) {
+          Redirects[&MI] = D;
+          break;
+        }
+      }
+    }
+  }
+
   // Everything else.  Anything outside the families this file knows
   // forgets everything; the rest loses exactly what it can change.
   if (!isKnownFamily(N)) {
@@ -888,6 +923,7 @@ static FlowState meet(const FlowState &A, const FlowState &B) {
 void W65816FarPtrStageReuse::analyse() {
   RedundantStores.clear();
   RedundantLoads.clear();
+  Redirects.clear();
   ModeAt.clear();
 
   DenseMap<MachineBasicBlock *, FlowState> In, Out;
@@ -1212,9 +1248,135 @@ bool W65816FarPtrStageReuse::runRound() {
     analyse();  // ModeAt must describe the surviving instructions
   if (deleteDeadStackStores())
     Changed = true;
+  if (Changed)
+    analyse();  // and the redirects the instructions as they now stand
+  if (redirectToHomes())
+    Changed = true;
   return Changed;
 }
 
+// Staging words an instruction may read: bit 0 = $E0/$E1, bit 1 = $E2/$E3.
+// They are compiler scratch, never part of the ABI: a call does not read
+// them (it clobbers them, stageDefs; the runtime helpers that use $E0..
+// fill it themselves from their arguments, and an indirect call's vector
+// is $B8), nor does a plain return.  Other ways out and anything
+// unmodelled read both.
+unsigned W65816FarPtrStageReuse::stageReads(const MachineInstr &MI) const {
+  if (MI.isCall())
+    return 0;
+  if (MI.isInlineAsm())
+    return 3;
+  if (MI.isReturn())
+    return isPlainReturn(MI) ? 0 : 3;
+  StringRef N = name(MI);
+  // Register-only instructions (mode, stack, transfer and flag ops) are
+  // outside isKnownFamily because transfer() handles them first.  TCD and
+  // PLD are not here: they move the direct page itself.
+  static const char *const RegOnly[] = {
+      "REP", "SEP", "PHP", "PLP", "PHA", "PLA", "PHX", "PLX", "PHY", "PLY",
+      "PHB", "PLB", "PHK", "PHD", "CLC", "SEC", "CLI", "SEI", "CLD", "SED",
+      "CLV", "NOP", "TAX", "TAY", "TXA", "TYA", "TXY", "TYX", "TSC", "TCS",
+      "TSX", "TXS", "PEA", "PER", "PUSH16", "PUSH16X"};
+  for (const char *R : RegOnly)
+    if (N == R)
+      return 0;
+  if (!isKnownFamily(N))
+    return 3;
+  if (stageDefs(MI) || !N.contains("_DP"))
+    return 0;
+  if (isDPIndexed(N))
+    return 3;
+  int64_t Imm = 0;
+  if (!firstImm(MI, Imm))
+    return 3;
+  int64_t Len = N.contains("_DPIndLong") ? 3 : 2;
+  unsigned R = 0;
+  if (Imm < StageLo + 2 && Imm + Len > StageLo)
+    R |= 1;
+  if (Imm < StageBank + 2 && Imm + Len > StageBank)
+    R |= 2;
+  return R;
+}
+
+// Staging words a 16-bit store overwrites whole, or a call clobbers (same
+// bits).
+unsigned W65816FarPtrStageReuse::stageDefs(const MachineInstr &MI) const {
+  if (MI.isCall())
+    return 3;
+  StringRef N = name(MI);
+  Width W = N == "STA_DP" || N == "STZ_DP"   ? modeM(MI)
+            : N == "STX_DP" || N == "STY_DP" ? modeX(MI)
+                                             : WUnknown;
+  int64_t Imm = 0;
+  if (W != W16 || !firstImm(MI, Imm))
+    return 0;
+  return Imm == StageLo ? 1u : Imm == StageBank ? 2u : 0u;
+}
+
+// Point the accesses analyse() matched to their pointer's own DP pair,
+// then delete the staging stores nothing reads any more.  The loads that
+// fed them die with them, in the next round's deleteDead.
+bool W65816FarPtrStageReuse::redirectToHomes() {
+  if (Redirects.empty())
+    return false;
+  for (auto &KV : Redirects)
+    for (MachineOperand &MO : KV.first->operands())
+      if (MO.isImm()) {
+        MO.setImm(KV.second);
+        break;
+      }
+  Redirects.clear();
+
+  DenseMap<const MachineBasicBlock *, unsigned> LiveIn, Out;
+  auto outOf = [&](const MachineBasicBlock *MBB) {
+    unsigned O = 0;
+    if (MBB->succ_empty()) {
+      auto Last = MBB->getLastNonDebugInstr();
+      if (Last == MBB->end() || !isPlainReturn(*Last))
+        O = 3;
+    }
+    for (const MachineBasicBlock *Succ : MBB->successors())
+      O |= LiveIn.lookup(Succ);
+    return O;
+  };
+  bool Iterate = true;
+  while (Iterate) {
+    Iterate = false;
+    for (MachineBasicBlock *MBB : post_order(MF)) {
+      unsigned Live = outOf(MBB);
+      Out[MBB] = Live;
+      for (auto It = MBB->rbegin(); It != MBB->rend(); ++It) {
+        if (It->isMetaInstruction() || It->isDebugInstr())
+          continue;
+        Live = (Live & ~stageDefs(*It)) | stageReads(*It);
+      }
+      if (LiveIn.lookup(MBB) != Live || !LiveIn.count(MBB)) {
+        LiveIn[MBB] = Live;
+        Iterate = true;
+      }
+    }
+  }
+  SmallVector<MachineInstr *, 8> Doomed;
+  for (MachineBasicBlock &MBB : *MF) {
+    auto Found = Out.find(&MBB);
+    unsigned Live = Found != Out.end() ? Found->second : 3u;
+    for (auto It = MBB.rbegin(); It != MBB.rend(); ++It) {
+      if (It->isMetaInstruction() || It->isDebugInstr())
+        continue;
+      unsigned D = stageDefs(*It);
+      // Only the stores go; a call's clobber just ends the liveness.
+      if (D && !It->isCall() && !(Live & D)) {
+        Doomed.push_back(&*It);
+        continue;
+      }
+      Live = (Live & ~D) | stageReads(*It);
+    }
+  }
+  for (MachineInstr *MI : Doomed)
+    MI->eraseFromParent();
+  return true;
+}
+
 bool W65816FarPtrStageReuse::runOnMachineFunction(MachineFunction &Fn) {
   if (skipFunction(Fn.getFunction()) || getenv("W65816_DISABLE_FARPTR_REUSE"))
     return false;
@@ -1229,6 +1391,67 @@ bool W65816FarPtrStageReuse::runOnMachineFunction(MachineFunction &Fn) {
       break;
     Changed = true;
   }
+  Changed |= foldEmptyFlagSaves();
+  return Changed;
+}
+
+// W65816StackSlotCleanup keeps a branch's flags alive across the loads
+// between its compare and the branch with PHP / PLP or, when X is free,
+// with `TAX ... CPX #k` in place of the CMP #k.  Once this pass (or
+// StackRelToImg before it) has deleted every one of those loads, what is
+// left is a no-op PHP ; PLP (7 cycles) or a TAX ; CPX #k that CMP #k does
+// in 3, when nothing reads X after it.
+bool W65816FarPtrStageReuse::foldEmptyFlagSaves() {
+  bool Changed = false;
+  auto nextReal = [](MachineBasicBlock &MBB, MachineBasicBlock::iterator It) {
+    while (It != MBB.end() && It->isDebugInstr())
+      ++It;
+    return It;
+  };
+  auto xDeadFrom = [&](MachineBasicBlock &MBB, MachineBasicBlock::iterator It) {
+    for (; It != MBB.end(); ++It) {
+      if (It->isDebugInstr())
+        continue;
+      Effects E = effects(*It, W16);
+      if (E.UseX)
+        return false;
+      if (E.DefX)
+        return true;
+    }
+    for (MachineBasicBlock *Succ : MBB.successors())
+      if (Succ->isLiveIn(W65816::X))
+        return false;
+    return true;
+  };
+  for (MachineBasicBlock &MBB : *MF) {
+    for (auto It = MBB.begin(); It != MBB.end();) {
+      MachineInstr &MI = *It++;
+      unsigned Opc = MI.getOpcode();
+      if (Opc != W65816::PHP && Opc != W65816::TAX)
+        continue;
+      auto Next = nextReal(MBB, It);
+      if (Next == MBB.end())
+        continue;
+      if (Opc == W65816::PHP && Next->getOpcode() == W65816::PLP) {
+        It = std::next(Next);
+        Next->eraseFromParent();
+        MI.eraseFromParent();
+        Changed = true;
+        continue;
+      }
+      if (Opc != W65816::TAX || Next->getOpcode() != W65816::CPX_Imm16 ||
+          !Next->getOperand(0).isImm() || !xDeadFrom(MBB, std::next(Next)))
+        continue;
+      BuildMI(MBB, Next, Next->getDebugLoc(), TII->get(W65816::CMP_Imm16))
+          .addImm(Next->getOperand(0).getImm())
+          .addReg(W65816::A, RegState::Implicit)
+          .addReg(W65816::P, RegState::ImplicitDefine);
+      It = std::next(Next);
+      Next->eraseFromParent();
+      MI.eraseFromParent();
+      Changed = true;
+    }
+  }
   return Changed;
 }
 

src/llvm/lib/Target/W65816/W65816I32IncFold.cpp

@@ -1,40 +1,40 @@
-//===-- W65816I32IncFold.cpp - Fold i32 += 1 into INC + conditional skip --===//
+//===-- W65816I32IncFold.cpp - Fold i32 +/- 1 into INC/DEC + skip ---------===//
 //
 // Pre-emit peephole: detect the post-PEI 6-instruction sequence emitted
-// for `i32 += 1` on a Wide32 vreg whose halves spilled to two stack-rel
+// for `i32 += 1` or `i32 -= 1` on a value whose halves live in two stack-
-// slots, and rewrite to a tighter form using INA + a conditional skip
+// relative or direct-page slots, and rewrite it so the high half is only
-// over the hi half.
+// touched when the low half carries or borrows.
 //
-// Original (after PEI, pseudos still un-expanded):
+// Original (after PEI, pseudos still un-expanded), K = 1 / -1:
-//   $a = LDA_StackRel imm_lo            ; load lo half
+//   $a = LDA lo                         ; load lo half
-//   $a = ADCi16imm $a, 1                ; CLC + ADC #1  (5 cyc)
+//   $a = ADCi16imm $a, K                ; CLC + ADC #K
-//   STA_StackRel $a, imm_lo             ; store lo
+//   STA lo                              ; store lo
-//   $a = LDA_StackRel imm_hi            ; load hi half
+//   $a = LDA hi                         ; load hi half
-//   $a = ADCEi16imm $a, 0               ; ADC #0 (uses carry from lo)
+//   $a = ADCEi16imm $a, 0 / -1          ; ADC (carry from lo)
-//   STA_StackRel $a, imm_hi             ; store hi
+//   STA hi                              ; store hi
 //
-// Cycle cost: 5 + 2 + 3 + 5 + 5 + 3 + 5 = 28 cyc
+// Cost: 28 cycles with stack-relative halves, 24 in the direct page.
 //
-// Rewrite:
+// Increment:
-//   $a = LDA_StackRel imm_lo            ; load lo
+//   $a = LDA lo
-//   $a = INA_PSEUDO $a, $a              ; lo + 1 — sets Z based on result
+//   $a = INA_PSEUDO $a                  ; lo + 1 -- Z set iff it wrapped
-//   STA_StackRel $a, imm_lo             ; store lo (Z preserved)
+//   STA lo                              ; STA leaves Z alone
-//   INC_HI_IF_CARRY_StackRel imm_hi     ; AsmPrinter expands to:
+//   INC_HI_IF_CARRY hi                  ; bne L ; <hi + 1> ; L:
-//                                       ;   bne L_skip
-//                                       ;   lda imm_hi, s
-//                                       ;   inc a
-//                                       ;   sta imm_hi, s
-//                                       ; L_skip:
 //
-// Cycle cost (no carry, common case):
+// Decrement:
-//   5 + 2 + 5 + 3 (BNE taken) = 15 cyc — saves 13 cyc
+//   $a = LDA lo                         ; Z set iff lo is 0 ...
-// Cycle cost (with carry, rare case):
+//   DEC_HI_IF_BORROW hi                 ; ... so it borrows: bne L ; <hi - 1> ; L:
-//   5 + 2 + 5 + 2 (BNE not-taken) + 5 + 2 + 5 = 26 cyc — saves 2 cyc
+//   $a = DEA_PSEUDO $a
+//   STA lo
 //
-// The Z flag from `INA` survives the intervening STA_StackRel because
+// No carry / borrow (the common case): 15 cycles on the stack, 13 in the
-// STA does not modify the processor status register.  The BNE in the
+// direct page (where StackRelToImg also turns the increment's
-// expansion of INC_HI_IF_CARRY_StackRel reads that Z to decide whether
+// LDA / INA / STA into INC dp).
-// the hi half needs to be touched.
+//
+// The rewrite leaves A holding the low half (or, on the stack-relative
+// carry path, the high half) where the original left the high half, and
+// leaves different flags; it only fires when nothing reads either before
+// they are rewritten.
 //===----------------------------------------------------------------------===//
 
 #include "W65816.h"
@@ -55,92 +55,161 @@ public:
   W65816I32IncFold() : MachineFunctionPass(ID) {}
   bool runOnMachineFunction(MachineFunction &MF) override;
   StringRef getPassName() const override {
-    return "W65816 i32 += 1 → INC + conditional skip";
+    return "W65816 i32 +/- 1 -> INC/DEC + conditional skip";
   }
   void getAnalysisUsage(AnalysisUsage &AU) const override {
     MachineFunctionPass::getAnalysisUsage(AU);
   }
 };
+
+// Where one half lives: a stack-relative offset or a direct-page address.
+struct HalfLoc {
+  bool IsDP = false;
+  int64_t Addr = 0;
+  bool operator==(const HalfLoc &O) const {
+    return IsDP == O.IsDP && Addr == O.Addr;
+  }
+};
+
+// The matched sequence.
+struct I32Step {
+  bool IsDec = false;
+  HalfLoc Lo, Hi;
+  MachineInstr *LdaLo = nullptr, *AdcLo = nullptr, *StaLo = nullptr;
+  MachineInstr *LdaHi = nullptr, *AdcHi = nullptr, *StaHi = nullptr;
+  MachineBasicBlock::iterator End;
+};
 } // namespace
 
 char W65816I32IncFold::ID = 0;
 
 INITIALIZE_PASS(W65816I32IncFold, DEBUG_TYPE,
-                "W65816 i32 += 1 fold", false, false)
+                "W65816 i32 +/- 1 fold", false, false)
 
-// Match the 6-instruction sequence; returns the post-pattern iterator
+static bool loadLoc(const MachineInstr &MI, HalfLoc &L) {
-// and fills in the lo/hi stack-rel offsets if the pattern matches.
+  if (MI.getOpcode() != W65816::LDA_StackRel &&
-// Tolerates intervening TAX/TXA pairs (which regalloc inserts as
+      MI.getOpcode() != W65816::LDA_DP)
-// spurious A-save brackets around STAfi's conservative Defs=[A]).
+    return false;
-// They're collected into `KillMe` so the rewrite can erase them too.
+  if (MI.getNumOperands() < 1 || !MI.getOperand(0).isImm())
-static bool matchI32AddOnePattern(MachineBasicBlock::iterator It,
+    return false;
-                                  MachineBasicBlock::iterator End,
+  L.IsDP = MI.getOpcode() == W65816::LDA_DP;
-                                  int64_t &OffLo, int64_t &OffHi,
+  L.Addr = MI.getOperand(0).getImm();
-                                  MachineBasicBlock::iterator &PatEnd,
+  return true;
+}
+
+static bool storesTo(const MachineInstr &MI, const HalfLoc &L) {
+  unsigned Want = L.IsDP ? W65816::STA_DP : W65816::STA_StackRel;
+  return MI.getOpcode() == Want && MI.getNumOperands() >= 1 &&
+         MI.getOperand(0).isImm() && MI.getOperand(0).getImm() == L.Addr;
+}
+
+static bool adcImm(const MachineInstr &MI, unsigned Opc, int64_t &K) {
+  if (MI.getOpcode() != Opc || MI.getNumOperands() < 3 ||
+      !MI.getOperand(2).isImm())
+    return false;
+  K = MI.getOperand(2).getImm() & 0xFFFF;
+  return true;
+}
+
+// Match the 6-instruction sequence at It.  Tolerates TAX/TXA pairs where
+// the increment fold always has (regalloc inserts them as spurious A-save
+// brackets around STAfi's conservative Defs=[A]); they are collected into
+// KillMe so the rewrite can erase them too.
+static bool matchI32Step(MachineBasicBlock::iterator It,
+                         MachineBasicBlock::iterator End, I32Step &S,
                          SmallVectorImpl<MachineInstr *> &KillMe) {
   auto skipDebug = [&]() {
-    while (It != End && It->isDebugInstr()) ++It;
+    while (It != End && It->isDebugInstr())
+      ++It;
   };
   auto skipTaxTxa = [&]() {
     while (It != End && (It->isDebugInstr() ||
                          It->getOpcode() == W65816::TAX ||
                          It->getOpcode() == W65816::TXA)) {
-      if (It->getOpcode() == W65816::TAX || It->getOpcode() == W65816::TXA) {
+      if (!It->isDebugInstr())
         KillMe.push_back(&*It);
-      }
       ++It;
     }
   };
+  int64_t K = 0;
   skipDebug();
-  if (It == End) return false;
+  if (It == End || !loadLoc(*It, S.Lo))
-
+    return false;
-  // 1. LDA_StackRel imm_lo
+  S.LdaLo = &*It++;
-  if (It->getOpcode() != W65816::LDA_StackRel) return false;
-  if (It->getNumOperands() < 1 || !It->getOperand(0).isImm()) return false;
-  OffLo = It->getOperand(0).getImm();
-  ++It;
   skipTaxTxa();
-  if (It == End) return false;
+  if (It == End || !adcImm(*It, W65816::ADCi16imm, K) ||
-
+      (K != 1 && K != 0xFFFF))
-  // 2. ADCi16imm with imm == 1
+    return false;
-  if (It->getOpcode() != W65816::ADCi16imm) return false;
+  S.IsDec = K == 0xFFFF;
-  if (It->getNumOperands() < 3 || !It->getOperand(2).isImm()) return false;
+  S.AdcLo = &*It++;
-  if (It->getOperand(2).getImm() != 1) return false;
-  ++It;
   skipDebug();
-  if (It == End) return false;
+  if (It == End || !storesTo(*It, S.Lo))
-
+    return false;
-  // 3. STA_StackRel to same offset
+  S.StaLo = &*It++;
-  if (It->getOpcode() != W65816::STA_StackRel) return false;
-  if (It->getNumOperands() < 1 || !It->getOperand(0).isImm()) return false;
-  if (It->getOperand(0).getImm() != OffLo) return false;
-  ++It;
   skipTaxTxa();
-  if (It == End) return false;
+  if (It == End || !loadLoc(*It, S.Hi) || S.Hi == S.Lo)
-
+    return false;
-  // 4. LDA_StackRel imm_hi (different offset)
+  S.LdaHi = &*It++;
-  if (It->getOpcode() != W65816::LDA_StackRel) return false;
-  if (It->getNumOperands() < 1 || !It->getOperand(0).isImm()) return false;
-  OffHi = It->getOperand(0).getImm();
-  if (OffHi == OffLo) return false;
-  ++It;
   skipDebug();
-  if (It == End) return false;
+  if (It == End || !adcImm(*It, W65816::ADCEi16imm, K) ||
-
+      K != (S.IsDec ? 0xFFFF : 0))
-  // 5. ADCEi16imm with imm == 0
+    return false;
-  if (It->getOpcode() != W65816::ADCEi16imm) return false;
+  S.AdcHi = &*It++;
-  if (It->getNumOperands() < 3 || !It->getOperand(2).isImm()) return false;
-  if (It->getOperand(2).getImm() != 0) return false;
-  ++It;
   skipDebug();
-  if (It == End) return false;
+  if (It == End || !storesTo(*It, S.Hi))
+    return false;
+  S.StaHi = &*It++;
+  S.End = It;
+  // The TAX/TXA brackets only ever sat around the increment's stores; a
+  // decrement that needs them is left alone rather than trusting that.
+  return !S.IsDec || KillMe.empty();
+}
 
-  // 6. STA_StackRel to hi offset
+// A full 16-bit definition of A that reads nothing: after one of these
-  if (It->getOpcode() != W65816::STA_StackRel) return false;
+// neither the old A nor the old N/Z can be observed.
-  if (It->getNumOperands() < 1 || !It->getOperand(0).isImm()) return false;
+static bool isFreshALoad(unsigned Opc) {
-  if (It->getOperand(0).getImm() != OffHi) return false;
+  switch (Opc) {
-  ++It;
+  case W65816::LDA_StackRel: case W65816::LDA_DP: case W65816::LDA_Abs:
-  PatEnd = It;
+  case W65816::LDA_Long: case W65816::LDA_Imm16: case W65816::LDAi16imm:
+  case W65816::LDAabs: case W65816::LDA_DPIndLongY:
+  case W65816::TXA: case W65816::TYA: case W65816::PLA:
+    return true;
+  default:
+    return false;
+  }
+}
+
+// Stores and moves the scan may step over: they neither read A nor read
+// the flags.
+static bool isAAndFlagNeutral(unsigned Opc) {
+  switch (Opc) {
+  case W65816::STX_DP: case W65816::STY_DP: case W65816::STZ_DP:
+  case W65816::STX_Abs: case W65816::STY_Abs: case W65816::LDX_Imm16:
+  case W65816::LDY_Imm16: case W65816::LDXi16imm: case W65816::BRA:
+    return true;
+  default:
+    return false;
+  }
+}
+
+// True when A and the N/Z flags left by the sequence are dead: the next
+// thing that touches either is a fresh load of A, in this block or, past
+// its end, at the head of every successor.
+static bool accAndNZDeadFrom(MachineBasicBlock &MBB,
+                             MachineBasicBlock::iterator It, bool InSucc) {
+  for (; It != MBB.end(); ++It) {
+    if (It->isDebugInstr())
+      continue;
+    if (isFreshALoad(It->getOpcode()))
+      return true;
+    if (!isAAndFlagNeutral(It->getOpcode()))
+      return false;
+  }
+  if (InSucc || MBB.succ_empty())
+    return false;
+  for (MachineBasicBlock *Succ : MBB.successors())
+    if (!accAndNZDeadFrom(*Succ, Succ->begin(), true))
+      return false;
   return true;
 }
 
@@ -153,11 +222,19 @@ bool W65816I32IncFold::runOnMachineFunction(MachineFunction &MF) {
   for (MachineBasicBlock &MBB : MF) {
     auto It = MBB.begin();
     while (It != MBB.end()) {
-      int64_t OffLo = 0, OffHi = 0;
+      I32Step S;
-      MachineBasicBlock::iterator PatEnd;
       SmallVector<MachineInstr *, 4> KillMe;
-      auto Start = It;
+      if (!matchI32Step(It, MBB.end(), S, KillMe)) {
-      if (!matchI32AddOnePattern(It, MBB.end(), OffLo, OffHi, PatEnd, KillMe)) {
+        ++It;
+        continue;
+      }
+      // The high half's carry-out (C, V) must be dead too.  Its readers
+      // (a following ADC/SBC, a BCC/BCS) all model their use of P, so the
+      // def's dead flag is the proof.
+      MachineOperand *PDef =
+          S.AdcHi->findRegisterDefOperand(W65816::P, /*TRI=*/nullptr);
+      if (!PDef || !PDef->isDead() ||
+          !accAndNZDeadFrom(MBB, S.End, false)) {
         ++It;
         continue;
       }
@@ -165,52 +242,34 @@ bool W65816I32IncFold::runOnMachineFunction(MachineFunction &MF) {
       // the pattern.  These are dead because STAfi's Defs=[A] was
       // a conservative over-approximation; the A-source path preserves
       // A in the actual asm.
-      for (MachineInstr *MI : KillMe) MI->eraseFromParent();
+      for (MachineInstr *MI : KillMe)
-      // Found the 6-instruction pattern, [Start, PatEnd).  Rewrite
+        MI->eraseFromParent();
-      // in-place: keep the LDA_StackRel for lo, replace ADCi16imm
+      DebugLoc DL = S.AdcLo->getDebugLoc();
-      // with INA_PSEUDO, keep STA_StackRel for lo, then replace the
+      if (S.IsDec) {
-      // entire LDA-ADCE-STA hi-half triple with INC_HI_IF_CARRY_StackRel.
+        // lda lo ; DEC_HI_IF_BORROW hi ; dea ; sta lo
-      DebugLoc DL = Start->getDebugLoc();
+        BuildMI(MBB, S.AdcLo->getIterator(), DL,
-
+                TII->get(S.Hi.IsDP ? W65816::DEC_HI_IF_BORROW_DP
-      // Walk to the ADCi16imm (Start+1) and replace.  Build a fresh
+                                   : W65816::DEC_HI_IF_BORROW_StackRel))
-      // INA_PSEUDO with the same tied-def shape: dst=A, src=A.
+            .addImm(S.Hi.Addr);
-      auto AdcIt = std::next(Start);
+        BuildMI(MBB, S.AdcLo->getIterator(), DL, TII->get(W65816::DEA_PSEUDO),
-      while (AdcIt != PatEnd && AdcIt->isDebugInstr()) ++AdcIt;
+                W65816::A)
-      // INA_PSEUDO has constraint $src = $dst; emit with both as A.
-      // Operand layout: (outs Acc16:$dst), (ins Acc16:$src)
-      BuildMI(MBB, AdcIt, DL, TII->get(W65816::INA_PSEUDO), W65816::A)
             .addReg(W65816::A);
-      auto Erased = AdcIt;
+      } else {
-      ++AdcIt;
+        // lda lo ; ina ; sta lo ; INC_HI_IF_CARRY hi
-      Erased->eraseFromParent();
+        BuildMI(MBB, S.AdcLo->getIterator(), DL, TII->get(W65816::INA_PSEUDO),
-
+                W65816::A)
-      // Now find the start of the hi-half triple: it's at Start+3 (after
+            .addReg(W65816::A);
-      // skipping debug).  Walk past STA_StackRel (lo) which is now at
+        BuildMI(MBB, S.LdaHi->getIterator(), DL,
-      // AdcIt's position.
+                TII->get(S.Hi.IsDP ? W65816::INC_HI_IF_CARRY_DP
-      while (AdcIt != PatEnd && AdcIt->isDebugInstr()) ++AdcIt;
+                                   : W65816::INC_HI_IF_CARRY_StackRel))
-      // AdcIt should now point at STA_StackRel (lo).  Skip it.
+            .addImm(S.Hi.Addr);
-      ++AdcIt;
-      while (AdcIt != PatEnd && AdcIt->isDebugInstr()) ++AdcIt;
-      // AdcIt now points at LDA_StackRel (hi) — start of the hi triple.
-      MachineBasicBlock::iterator HiStart = AdcIt;
-
-      // Insert INC_HI_IF_CARRY_StackRel before the hi triple, then
-      // erase all three hi instructions.
-      BuildMI(MBB, HiStart, DL, TII->get(W65816::INC_HI_IF_CARRY_StackRel))
-          .addImm(OffHi);
-
-      // Erase the 3 hi instructions: LDA_StackRel, ADCEi16imm, STA_StackRel.
-      auto KillIt = HiStart;
-      for (int i = 0; i < 3 && KillIt != PatEnd; ) {
-        if (KillIt->isDebugInstr()) { ++KillIt; continue; }
-        auto Next = std::next(KillIt);
-        KillIt->eraseFromParent();
-        KillIt = Next;
-        ++i;
       }
-
+      S.AdcLo->eraseFromParent();
+      S.LdaHi->eraseFromParent();
+      S.AdcHi->eraseFromParent();
+      S.StaHi->eraseFromParent();
       Changed = true;
-      It = PatEnd;
+      It = S.End;
     }
   }
   return Changed;

src/llvm/lib/Target/W65816/W65816ISelLowering.cpp

@@ -1138,6 +1138,64 @@ SDValue W65816TargetLowering::LowerSignExtend(SDValue Op,
   return DAG.getNode(ISD::SUB, DL, MVT::i16, Xor, Sign);
 }
 
+// The abs-16 address of the global a pointer names, plus Extra bytes: a raw
+// GlobalAddress / ExternalSymbol, a Wide32 built from Wrapper +
+// WrapperBank of the same symbol, or one of those plus a constant.  Null for anything else (and for an
+// external symbol with Extra, which cannot carry an offset).  Our globals
+// live in the load segment crt0 pins to DBR, so the DBR-relative abs-16
+// forms (LDA / STA abs) reach them, not the 14-byte [dp],Y path.
+static SDValue foldGlobalAddr(SelectionDAG &DAG, const SDLoc &DL, SDValue Ptr,
+                              int64_t Extra = 0) {
+  auto wrapGlobal = [&](const GlobalAddressSDNode *GA) {
+    return DAG.getNode(W65816ISD::Wrapper, DL, MVT::i16,
+                       DAG.getTargetGlobalAddress(GA->getGlobal(), DL,
+                                                  MVT::i16,
+                                                  GA->getOffset() + Extra));
+  };
+  auto wrapExtern = [&](const ExternalSymbolSDNode *ES) {
+    if (Extra != 0)
+      return SDValue();
+    return DAG.getNode(W65816ISD::Wrapper, DL, MVT::i16,
+                       DAG.getTargetExternalSymbol(ES->getSymbol(),
+                                                   MVT::i16));
+  };
+  if (auto *GA = dyn_cast<GlobalAddressSDNode>(Ptr))
+    return wrapGlobal(GA);
+  if (auto *ES = dyn_cast<ExternalSymbolSDNode>(Ptr))
+    return wrapExtern(ES);
+  // A field or element: (add global, K), as an i32 load / store's second
+  // half is before the combiner folds K into the global's offset.
+  if (Ptr.getOpcode() == ISD::ADD)
+    if (auto *K = dyn_cast<ConstantSDNode>(Ptr.getOperand(1)))
+      return foldGlobalAddr(DAG, DL, Ptr.getOperand(0),
+                            Extra + K->getSExtValue());
+  if (!Ptr.getNode()->isMachineOpcode() ||
+      Ptr.getMachineOpcode() != TargetOpcode::REG_SEQUENCE)
+    return SDValue();
+  SDValue PLo, PHi;
+  for (unsigned i = 1; i + 1 < Ptr.getNumOperands(); i += 2) {
+    if (auto *CIdx = dyn_cast<ConstantSDNode>(Ptr.getOperand(i + 1))) {
+      if (CIdx->getZExtValue() == llvm::sub_lo) PLo = Ptr.getOperand(i);
+      else if (CIdx->getZExtValue() == llvm::sub_hi) PHi = Ptr.getOperand(i);
+    }
+  }
+  if (!PLo || !PHi || PLo.getOpcode() != W65816ISD::Wrapper ||
+      PHi.getOpcode() != W65816ISD::WrapperBank)
+    return SDValue();
+  SDValue WLo = PLo.getOperand(0);
+  SDValue WHi = PHi.getOperand(0);
+  auto *GLo = dyn_cast<GlobalAddressSDNode>(WLo);
+  auto *GHi = dyn_cast<GlobalAddressSDNode>(WHi);
+  auto *ELo = dyn_cast<ExternalSymbolSDNode>(WLo);
+  auto *EHi = dyn_cast<ExternalSymbolSDNode>(WHi);
+  if (GLo && GHi && GLo->getGlobal() == GHi->getGlobal() &&
+      GLo->getOffset() == GHi->getOffset())
+    return Extra == 0 ? PLo : wrapGlobal(GLo);
+  if (ELo && EHi && StringRef(ELo->getSymbol()) == EHi->getSymbol())
+    return Extra == 0 ? PLo : SDValue();
+  return SDValue();
+}
+
 // ptr32 foundation hook.  In ptr16 mode (PointerWidth=16, current
 // default) addresses are i16 and we return SDValue() so the legalizer
 // keeps the load and the existing LDAptr / STAptr selection patterns
@@ -1214,40 +1272,7 @@ SDValue W65816TargetLowering::LowerLoad(SDValue Op,
   // emit an abs-16 (DBR-relative) load (LDA / LDA8abs) instead of
   // the slower [dp],Y indirect-long.  Our globals are in the load
   // segment that crt0 pins to DBR.
-  SDValue FoldedLo;
+  SDValue FoldedLo = foldGlobalAddr(DAG, DL, Ptr);
-  if (auto *GA = dyn_cast<GlobalAddressSDNode>(Ptr)) {
-    FoldedLo = DAG.getNode(W65816ISD::Wrapper, DL, MVT::i16,
-        DAG.getTargetGlobalAddress(GA->getGlobal(), DL, MVT::i16,
-                                    GA->getOffset()));
-  } else if (auto *ES = dyn_cast<ExternalSymbolSDNode>(Ptr)) {
-    FoldedLo = DAG.getNode(W65816ISD::Wrapper, DL, MVT::i16,
-        DAG.getTargetExternalSymbol(ES->getSymbol(), MVT::i16));
-  } else if (Ptr.getNode()->isMachineOpcode() &&
-             Ptr.getMachineOpcode() == TargetOpcode::REG_SEQUENCE) {
-    SDValue PLo, PHi;
-    for (unsigned i = 1; i + 1 < Ptr.getNumOperands(); i += 2) {
-      if (auto *CIdx = dyn_cast<ConstantSDNode>(Ptr.getOperand(i + 1))) {
-        if (CIdx->getZExtValue() == llvm::sub_lo) PLo = Ptr.getOperand(i);
-        else if (CIdx->getZExtValue() == llvm::sub_hi) PHi = Ptr.getOperand(i);
-      }
-    }
-    if (PLo && PHi &&
-        PLo.getOpcode() == W65816ISD::Wrapper &&
-        PHi.getOpcode() == W65816ISD::WrapperBank) {
-      SDValue WLo = PLo.getOperand(0);
-      SDValue WHi = PHi.getOperand(0);
-      auto *GLo = dyn_cast<GlobalAddressSDNode>(WLo);
-      auto *GHi = dyn_cast<GlobalAddressSDNode>(WHi);
-      auto *ELo = dyn_cast<ExternalSymbolSDNode>(WLo);
-      auto *EHi = dyn_cast<ExternalSymbolSDNode>(WHi);
-      bool SameGlobal = (GLo && GHi && GLo->getGlobal() == GHi->getGlobal() &&
-                         GLo->getOffset() == GHi->getOffset());
-      bool SameExtern = (ELo && EHi &&
-                         StringRef(ELo->getSymbol()) == EHi->getSymbol());
-      if (SameGlobal || SameExtern)
-        FoldedLo = PLo;
-    }
-  }
   if (FoldedLo) {
     EVT MemVT = Ld->getMemoryVT();
     ISD::LoadExtType ExtType = Ld->getExtensionType();
@@ -1565,6 +1590,21 @@ SDValue W65816TargetLowering::LowerStore(SDValue Op,
         return StHi;
       }
     }
+    // A global's halves are two abs-16 stores, as its loads are two
+    // abs-16 loads.  Through ST_PTR they were two far-pointer stores,
+    // each staging `#g` and the program bank from $BE into $E0 / $E2
+    // first (~12x the cycles).  Plain i16 stores are safe here since
+    // canMergeStoresTo caps store merging at i16.
+    SDValue GLo = foldGlobalAddr(DAG, DL, Ptr);
+    SDValue GHi = foldGlobalAddr(DAG, DL, Ptr, 2);
+    if (GLo && GHi) {
+      MachineMemOperand::Flags Fl = St->getMemOperand()->getFlags();
+      SDValue StLo = DAG.getStore(Chain, DL, Lo, GLo, St->getPointerInfo(),
+                                  St->getAlign(), Fl);
+      return DAG.getStore(StLo, DL, Hi, GHi,
+                          St->getPointerInfo().getWithOffset(2),
+                          commonAlignment(St->getAlign(), 2), Fl);
+    }
     SDValue Two = DAG.getConstant(2, DL, PtrVT);
     SDValue Ptr2 = DAG.getNode(ISD::ADD, DL, PtrVT, Ptr, Two);
     if (PtrVT == MVT::i32) {
@@ -1607,40 +1647,7 @@ SDValue W65816TargetLowering::LowerStore(SDValue Op,
   // the load segment that crt0 pins to DBR, so abs-16 reaches them.
   // This avoids the 14-byte [dp],y indirect-long path AND re-enables
   // the STZ peephole that the indirect path defeats.
-  SDValue FoldedLo;
+  SDValue FoldedLo = foldGlobalAddr(DAG, DL, Ptr);
-  if (auto *GA = dyn_cast<GlobalAddressSDNode>(Ptr)) {
-    FoldedLo = DAG.getNode(W65816ISD::Wrapper, DL, MVT::i16,
-        DAG.getTargetGlobalAddress(GA->getGlobal(), DL, MVT::i16,
-                                    GA->getOffset()));
-  } else if (auto *ES = dyn_cast<ExternalSymbolSDNode>(Ptr)) {
-    FoldedLo = DAG.getNode(W65816ISD::Wrapper, DL, MVT::i16,
-        DAG.getTargetExternalSymbol(ES->getSymbol(), MVT::i16));
-  } else if (Ptr.getNode()->isMachineOpcode() &&
-             Ptr.getMachineOpcode() == TargetOpcode::REG_SEQUENCE) {
-    SDValue PLo, PHi;
-    for (unsigned i = 1; i + 1 < Ptr.getNumOperands(); i += 2) {
-      if (auto *CIdx = dyn_cast<ConstantSDNode>(Ptr.getOperand(i + 1))) {
-        if (CIdx->getZExtValue() == llvm::sub_lo) PLo = Ptr.getOperand(i);
-        else if (CIdx->getZExtValue() == llvm::sub_hi) PHi = Ptr.getOperand(i);
-      }
-    }
-    if (PLo && PHi &&
-        PLo.getOpcode() == W65816ISD::Wrapper &&
-        PHi.getOpcode() == W65816ISD::WrapperBank) {
-      SDValue WLo = PLo.getOperand(0);
-      SDValue WHi = PHi.getOperand(0);
-      auto *GLo = dyn_cast<GlobalAddressSDNode>(WLo);
-      auto *GHi = dyn_cast<GlobalAddressSDNode>(WHi);
-      auto *ELo = dyn_cast<ExternalSymbolSDNode>(WLo);
-      auto *EHi = dyn_cast<ExternalSymbolSDNode>(WHi);
-      bool SameGlobal = (GLo && GHi && GLo->getGlobal() == GHi->getGlobal() &&
-                         GLo->getOffset() == GHi->getOffset());
-      bool SameExtern = (ELo && EHi &&
-                         StringRef(ELo->getSymbol()) == EHi->getSymbol());
-      if (SameGlobal || SameExtern)
-        FoldedLo = PLo;
-    }
-  }
   if (FoldedLo) {
     // Preserve memVT — original may be a truncating store (e.g.,
     // i16 value into i8 memory).  getStore picks memVT from Val's
@@ -1959,69 +1966,60 @@ SDValue W65816TargetLowering::LowerShift(SDValue Op, SelectionDAG &DAG) const {
     SDValue R16 = DAG.getNode(Op.getOpcode(), DL, MVT::i16, X16, N16);
     return DAG.getNode(ISD::TRUNCATE, DL, MVT::i8, R16);
   }
-  // Fast path: shift-by-{1,2,3,4} have inline tablegen patterns.  Return
+  // Constant i16 shifts are inline: tablegen has SHL/SRL patterns for
-  // Op (the unchanged node) so the legalizer leaves it alone — the
+  // every count 1..15 (single-bit chains, XBA-and-mask for 8 and up) and
-  // pattern matcher catches it later.  Returning SDValue() instead
+  // SRA patterns for 1, 2 and 15.  Returning Op leaves the node for those
-  // would fall through to the generic Expand path, which generates a
+  // patterns; returning SDValue() would fall into the generic Expand path,
-  // BUILD_VECTOR-based magic-constant rewrite that we can't lower.
+  // which builds a BUILD_VECTOR magic-constant rewrite we cannot lower.
-  // Also allow `(srl x, 15)` through — pattern SRL15A handles it as
-  // `ASL A; LDA #0; ROL A` (3 bytes), much shorter than the libcall.
-  // The type-legalizer's i32-shift-by-1 expansion emits this exact
-  // node for the high-half "bit-from-low" slot.
-  // Everything else goes to a libcall (__ashlhi3 / __lshrhi3 / __ashrhi3).
-  // i16 only — i32 always routes to libcall (no inline i32 patterns).
   SDValue Amount = Op.getOperand(1);
   if (Op.getValueType() == MVT::i16) {
     if (auto *C = dyn_cast<ConstantSDNode>(Amount)) {
       uint64_t N = C->getZExtValue();
-      if ((Op.getOpcode() == ISD::SHL || Op.getOpcode() == ISD::SRL) &&
+      if (N >= 1 && N <= 15) {
-          N >= 1 && N <= 14)
+        if (Op.getOpcode() != ISD::SRA || N <= 2 || N == 15)
-        return Op;
-      if (N == 15 &&
-          (Op.getOpcode() == ISD::SRL || Op.getOpcode() == ISD::SHL))
-        return Op;
-      if (N == 1 && Op.getOpcode() == ISD::SRA)
-        return Op;
-      if (N == 15 && Op.getOpcode() == ISD::SRA)
           return Op;
+        // Arithmetic shift by 3..14: shift logically, then sign-extend the
+        // surviving 16-N bits, ((x >>u N) ^ M) - M with M the new sign
+        // bit.  Eight cycles over the logical shift, where the one-bit
+        // chain (`cmp #$8000 ; ror a`) costs five per bit and __ashrhi3
+        // is a call plus a loop.
+        SDLoc DL(Op);
+        SDValue Srl = DAG.getNode(ISD::SRL, DL, MVT::i16, Op.getOperand(0),
+                                  Amount);
+        SDValue M = DAG.getConstant(0x8000u >> N, DL, MVT::i16);
+        SDValue Flip = DAG.getNode(ISD::XOR, DL, MVT::i16, Srl, M);
+        return DAG.getNode(ISD::SUB, DL, MVT::i16, Flip, M);
+      }
     }
   }
 
   bool IsI32 = Op.getValueType() == MVT::i32;
 
-  // Inline i32 shift-by-small-constant.  The libcall path is ~140 cyc
+  // Constant i32 shifts are inline for every count: each is a few i16
-  // (post-tightening); unrolling N i16 ops plus carry propagation runs
+  // shifts of the halves, all of them inline (above).  The libcalls
-  // in ~30-90 cyc.  popcount, djb2-style hashes, BigInt-style code, and
+  // (__ashlsi3 & co.) shift a bit at a time, ~26 cycles a bit plus the
-  // CRC routines all hit this.  Larger N falls through to the libcall —
+  // call, so even the dearest inline form (a count of 7 or 15) wins by a
-  // the unrolled cost grows linearly while the libcall is constant.
+  // wide margin.  popcount, djb2-style hashes, CRCs and fixed-point
-  // Cutoff at N=5 chosen empirically: djb2's `(h << 5) + h` is the
+  // arithmetic all hit these.
-  // common one that benefits.  SRA needs an arithmetic-fill shift on
-  // the high half (i16 SRA by 1 is tablegen-supported); the low half is
-  // filled from the high's departing bit just like SRL.
   if (IsI32) {
     if (auto *C = dyn_cast<ConstantSDNode>(Amount)) {
       uint64_t N = C->getZExtValue();
       unsigned Op0 = Op.getOpcode();
-      if (N >= 1 && N <= 5 &&
-          (Op0 == ISD::SHL || Op0 == ISD::SRL || Op0 == ISD::SRA)) {
       SDLoc DL(Op);
       SDValue X = Op.getOperand(0);
+      if (N >= 1 && N <= 15) {
         SDValue Lo = extractWide32Lo(DAG, DL, X);
         SDValue Hi = extractWide32Hi(DAG, DL, X);
         SDValue ShN  = DAG.getConstant(N, DL, MVT::i16);
         SDValue ShCo = DAG.getConstant(16 - N, DL, MVT::i16);
         if (Op0 == ISD::SHL) {
           // (Hi:Lo) << N == ((Hi << N) | (Lo >> (16-N))) : (Lo << N)
-          // 4 SDAG ops instead of N iterations of 4 ops.  Lets the
-          // combiner / isel produce ASLA16-cascade + SRL8A+LSRA16-
-          // cascade + single OR, avoiding the bit-by-bit OR cascade
-          // that the unrolled form produced.
           SDValue NewLo  = DAG.getNode(ISD::SHL, DL, MVT::i16, Lo, ShN);
           SDValue HiTop  = DAG.getNode(ISD::SRL, DL, MVT::i16, Lo, ShCo);
           SDValue HiShl  = DAG.getNode(ISD::SHL, DL, MVT::i16, Hi, ShN);
           SDValue NewHi  = DAG.getNode(ISD::OR,  DL, MVT::i16, HiShl, HiTop);
           return buildWide32(DAG, DL, NewLo, NewHi);
-        } else {
+        }
         // SRL/SRA by N: NewHi = Hi >> N (logical or arithmetic);
         // NewLo = (Lo >> N) | (Hi << (16-N)).
         SDValue NewHi  = DAG.getNode(Op0, DL, MVT::i16, Hi, ShN);
@@ -2030,14 +2028,10 @@ SDValue W65816TargetLowering::LowerShift(SDValue Op, SelectionDAG &DAG) const {
         SDValue NewLo  = DAG.getNode(ISD::OR,  DL, MVT::i16, LoSrl, LoTop);
         return buildWide32(DAG, DL, NewLo, NewHi);
       }
-      }
+      // By 16 or more one half moves into the other and the rest is at
-      // By 16 or more one half moves into the other and the rest is at most
+      // most an i16 shift of it.  `(u16)(x >> 16)`, the high word, is the
-      // an i16 shift, which the i16 patterns do inline for 1..15 (SRA only
+      // high half itself.
-      // for 15, its sign fill).  `(u16)(x >> 16)`, the high word, was a
-      // __lshrsi3 call; now it is the high half itself.
       if (N >= 16 && N <= 31) {
-        SDLoc DL(Op);
-        SDValue X = Op.getOperand(0);
         SDValue Zero = DAG.getConstant(0, DL, MVT::i16);
         SDValue ShN = DAG.getConstant(N - 16, DL, MVT::i16);
         if (Op0 == ISD::SHL) {
@@ -2046,18 +2040,21 @@ SDValue W65816TargetLowering::LowerShift(SDValue Op, SelectionDAG &DAG) const {
               N == 16 ? Lo : DAG.getNode(ISD::SHL, DL, MVT::i16, Lo, ShN);
           return buildWide32(DAG, DL, Zero, NewHi);
         }
-        if (Op0 == ISD::SRL) {
         SDValue Hi = extractWide32Hi(DAG, DL, X);
+        if (Op0 == ISD::SRL) {
           SDValue NewLo =
               N == 16 ? Hi : DAG.getNode(ISD::SRL, DL, MVT::i16, Hi, ShN);
           return buildWide32(DAG, DL, NewLo, Zero);
         }
-        if (Op0 == ISD::SRA && (N == 16 || N == 31)) {
+        // SRA: the high half shifted arithmetically, and a high word of
-          SDValue Hi = extractWide32Hi(DAG, DL, X);
+        // copies of the sign.
         SDValue Fill = DAG.getNode(ISD::SRA, DL, MVT::i16, Hi,
                                    DAG.getConstant(15, DL, MVT::i16));
-          return buildWide32(DAG, DL, N == 16 ? Hi : Fill, Fill);
+        SDValue NewLo =
-        }
+            N == 16 ? Hi
+                    : N == 31 ? Fill
+                              : DAG.getNode(ISD::SRA, DL, MVT::i16, Hi, ShN);
+        return buildWide32(DAG, DL, NewLo, Fill);
       }
     }
   }
@@ -2070,28 +2067,8 @@ SDValue W65816TargetLowering::LowerShift(SDValue Op, SelectionDAG &DAG) const {
   default: llvm_unreachable("not a shift");
   }
 
-  SDValue Val = Op.getOperand(0);
+  // Variable counts only: every constant count is inline above.
-  if (IsI32 && Op.getOpcode() == ISD::SHL) {
+  SmallVector<SDValue, 2> Args = {Op.getOperand(0), Op.getOperand(1)};
-    // Force the high half of the input to be concretely zero when the
-    // shift count K is >= 16, so bits K..31 of the input are
-    // mathematically irrelevant.  SDAG legalisation can mark those bits
-    // as `undef` to give the regalloc freedom, but our libcall (a true
-    // 32-bit shift-and-rotate loop in libgcc.s) reads ALL 32 input
-    // bits and propagates garbage into the result's low half.  Caught
-    // by dadd via the dpack-inline `(u64 e) << 52` path which split
-    // into __ashlsi3(e_lo, 20) with X = undef → wrong mantissa.
-    // For SRL/SRA we'd zero/sign-extend the LOW half similarly when
-    // K >= 16, but those paths aren't exercising the bug yet.
-    if (auto *C = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
-      unsigned K = (unsigned)C->getZExtValue();
-      if (K >= 16) {
-        SDValue Lo = extractWide32Lo(DAG, SDLoc(Op), Val);
-        SDValue Zero = DAG.getConstant(0, SDLoc(Op), MVT::i16);
-        Val = buildWide32(DAG, SDLoc(Op), Lo, Zero);
-      }
-    }
-  }
-  SmallVector<SDValue, 2> Args = {Val, Op.getOperand(1)};
   TargetLowering::MakeLibCallOptions Opts;
   Opts.setIsSigned(Op.getOpcode() == ISD::SRA);
   return makeLibCall(DAG, LC, Op.getValueType(), Args, Opts, SDLoc(Op)).first;

src/llvm/lib/Target/W65816/W65816ISelLowering.h

@@ -148,6 +148,16 @@ public:
     return TargetLowering::getConstraintType(Constraint);
   }
 
+  // clang marks every `short` / `char` / `bool` return signext or zeroext;
+  // the default widens such a value to i32 and fills X with its sign or
+  // zeros, which no caller reads: the ABI returns 16 bits in A alone
+  // (LLVM_65816_DESIGN.md), and a caller that needs the value wider
+  // extends it itself.  Extend only to A's 16 bits.
+  EVT getTypeForExtReturn(LLVMContext &Context, EVT VT,
+                          ISD::NodeType ExtendKind) const override {
+    return VT.bitsLT(MVT::i16) ? EVT(MVT::i16) : VT;
+  }
+
   // Force i32 / i64 shifts through a libcall (__ashlsi3 / __lshrsi3 /
   // __ashrsi3) instead of LLVM's default ExpandToParts strategy, which
   // emits an SHL_PARTS node we have no pattern for.  ExpandToParts also

src/llvm/lib/Target/W65816/W65816InstrInfo.td

@@ -571,8 +571,7 @@ def LSRA8  : W65816Pseudo<(outs Acc8:$dst), (ins Acc8:$src),
                           [(set Acc8:$dst, (srl Acc8:$src, (i8 1)))]>;
 // Signed shift right by 1: copy A's high bit into carry, then ROR
 // to bring it back into A's high bit while halving the rest.  The
-// AsmPrinter expands this to the 4-instruction PHA;ASL;PLA;ROR
+// AsmPrinter expands this to `CMP #$8000 ; ROR A`.
-// sequence.
 def ASRA16 : W65816Pseudo<(outs Acc16:$dst), (ins Acc16:$src),
                           "# ASRA16 $dst, $src",
                           [(set Acc16:$dst, (sra Acc16:$src, (i16 1)))]> {
@@ -593,6 +592,9 @@ def : Pat<(shl Acc16:$src, (i16 4)),
           (ASLA16 (ASLA16 (ASLA16 (ASLA16 Acc16:$src))))>;
 
 def : Pat<(srl Acc16:$src, (i16 2)), (LSRA16 (LSRA16 Acc16:$src))>;
+// Arithmetic shift by 2: two ASRA16s (10 cycles) beat LowerShift's
+// logical-shift-and-sign-extend form (12) only at this count.
+def : Pat<(sra Acc16:$src, (i16 2)), (ASRA16 (ASRA16 Acc16:$src))>;
 def : Pat<(srl Acc16:$src, (i16 3)),
           (LSRA16 (LSRA16 (LSRA16 Acc16:$src)))>;
 def : Pat<(srl Acc16:$src, (i16 4)),
@@ -1870,11 +1872,37 @@ def EOR_StackRelIndY : InstStackRelIndY<0x53, "eor">;
 //   sta $imm, s
 // L_skip:
 let mayLoad = 1, mayStore = 1, hasSideEffects = 0,
-    Defs = [A] in {
+    Defs = [A], Size = 7 in {
 def INC_HI_IF_CARRY_StackRel : W65816Pseudo<(outs), (ins i16imm:$off),
                                             "# INC_HI_IF_CARRY_StackRel $off",
                                             []>;
 }
+// The same with the high half in the direct page: `bne L ; inc $dp ; L:`.
+// A is not touched.
+let mayLoad = 1, mayStore = 1, hasSideEffects = 0, Size = 4 in {
+def INC_HI_IF_CARRY_DP : W65816Pseudo<(outs), (ins i16imm:$dp),
+                                      "# INC_HI_IF_CARRY_DP $dp", []>;
+}
+
+// The borrow of an i32 decrement.  W65816I32IncFold emits
+//   lda lo ; DEC_HI_IF_BORROW hi ; dec a ; sta lo
+// so Z comes from the low half BEFORE the decrement: Z=1 means it is
+// about to wrap 0 -> $FFFF and the high half must go down by one.
+// The stack-relative form expands to
+//   bne L ; lda $off,s ; dec a ; sta $off,s ; lda #0 ; L:
+// and leaves A = 0 on the borrow path, which is the low half it found,
+// so the `dec a` that follows sees the same A on both paths.
+let mayLoad = 1, mayStore = 1, hasSideEffects = 0,
+    Defs = [A], Uses = [A], Size = 10 in {
+def DEC_HI_IF_BORROW_StackRel : W65816Pseudo<(outs), (ins i16imm:$off),
+                                             "# DEC_HI_IF_BORROW_StackRel $off",
+                                             []>;
+}
+// Direct-page form: `bne L ; dec $dp ; L:`, A untouched.
+let mayLoad = 1, mayStore = 1, hasSideEffects = 0, Size = 4 in {
+def DEC_HI_IF_BORROW_DP : W65816Pseudo<(outs), (ins i16imm:$dp),
+                                       "# DEC_HI_IF_BORROW_DP $dp", []>;
+}
 
 //===----------------------------------------------------------------------===//
 // Branch patterns (placed after the Bxx defs).

src/llvm/lib/Target/W65816/W65816StackSlotCleanup.cpp

@@ -31,6 +31,7 @@
 #include "W65816Subtarget.h"
 #include "llvm/ADT/BitVector.h"
 #include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/CodeGen/MachineFrameInfo.h"
 #include "llvm/CodeGen/MachineFunction.h"
@@ -1106,6 +1107,49 @@ bool W65816StackSlotCleanup::runOnMachineFunction(MachineFunction &MF) {
           continue;
         }
       }
+      // Compare through X instead of saving the flags, when X is free from
+      // the test to the end of the block and not live into a successor:
+      //   cmp #k ; php ; <loads> ; plp ; bxx     10 cycles, 5 bytes
+      //   tax ; <loads> ; cpx #k ; bxx           5 cycles, 4 bytes
+      // CPX sets N, Z and C from X - k exactly as CMP did from A - k, and
+      // the corrupters (loads and transfers into A) leave X alone.  Only a
+      // 16-bit compare, and only with no SEP/REP of the index width in the
+      // way (an 8-bit X would drop the high byte).
+      if (TestsNZ && Test->getOpcode() == W65816::CMPi16imm &&
+          Test->getOperand(0).isReg() &&
+          Test->getOperand(0).getReg() == W65816::A &&
+          Test->getOperand(1).isImm()) {
+        bool XFree = true;
+        for (auto It = Test->getIterator(); It != MBB.end() && XFree; ++It) {
+          if (It->isDebugInstr())
+            continue;
+          for (const MachineOperand &MO : It->operands())
+            if (MO.isReg() && MO.getReg() == W65816::X)
+              XFree = false;
+          if ((It->getOpcode() == W65816::SEP ||
+               It->getOpcode() == W65816::REP) &&
+              (!It->getOperand(0).isImm() ||
+               (It->getOperand(0).getImm() & 0x10)))
+            XFree = false;
+          if (It->isCall())
+            XFree = false;
+        }
+        for (MachineBasicBlock *Succ : MBB.successors())
+          if (Succ->isLiveIn(W65816::X))
+            XFree = false;
+        if (XFree) {
+          const TargetInstrInfo *TII = MF.getSubtarget().getInstrInfo();
+          DebugLoc DL = Test->getDebugLoc();
+          BuildMI(MBB, Test->getIterator(), DL, TII->get(W65816::TAX));
+          BuildMI(MBB, Br->getIterator(), DL, TII->get(W65816::CPX_Imm16))
+              .addImm(Test->getOperand(1).getImm())
+              .addReg(W65816::X, RegState::Implicit)
+              .addReg(W65816::P, RegState::ImplicitDefine);
+          Test->eraseFromParent();
+          Changed = true;
+          continue;
+        }
+      }
       // Stack-relative ops inside the wrap need one more byte of
       // displacement to compensate for PHP's S decrement.  Without it,
       // `lda 5,s` between PHP and PLP reads at (orig_S-1)+5 = orig_S+4,
@@ -1651,11 +1695,41 @@ bool W65816StackSlotCleanup::runOnMachineFunction(MachineFunction &MF) {
   // Pass 1c can only eliminate CMPi16imm $a, 0 if the preceding
   // A-modifier reliably sets N/Z to reflect A's final value.  LDAfi
   // under FP-rel expansion (`sty $fa ; ldy #imm ; lda [$f6],y ; ldy $fa`)
-  // ends with `ldy` that clobbers N/Z based on OLD Y, not loaded A — so
+  // ends with `ldy` that clobbers N/Z based on OLD Y, not loaded A — and
-  // in FP-rel functions (VLA / huge frame), the CMP is load-bearing.
+  // so do the other frame-index pseudos — so in FP-rel functions (VLA /
-  // Skip the whole pass for such functions (saves us from the sum_n
+  // huge frame) a CMP after one is load-bearing.  The PHP-wrap-aware
-  // VLA regression that the PHP-wrap-aware variant tripped).
+  // variant is off for such functions (the sum_n VLA regression it
+  // tripped), and a frame-index writer keeps its CMP there: with the
+  // loop's `cmp #0` moved past its back-edge copies to sit right after
+  // the counter's reload (W65816CmpSink), sum_n's second loop branched
+  // on the restored Y.
   bool ssCleanupSPRelOnly = !UsesFPRel;
+  auto hasFrameIndex = [](const MachineInstr &MI) {
+    for (const MachineOperand &MO : MI.operands())
+      if (MO.isFI())
+        return true;
+    return false;
+  };
+  // Whether any frame-index access can take the $F6 form: FrameLowering
+  // sets $F6 up only for a VLA or a final frame over 200 bytes.  The final
+  // frame holds at most the objects instructions still reference (plus
+  // alignment); estimateStackSize, behind UsesFPRel, also counts the dead
+  // staging slots every far-pointer access creates, and keeping `cmp #0`
+  // on its say-so cost JoeyLib's sprite paths 3 cycles a test.
+  bool FrameMayUseFP = MF.getFrameInfo().hasVarSizedObjects();
+  if (!FrameMayUseFP) {
+    const MachineFrameInfo &MFI = MF.getFrameInfo();
+    DenseSet<int> Seen;
+    uint64_t Bytes = 0;
+    for (MachineBasicBlock &MBB : MF)
+      for (MachineInstr &MI : MBB)
+        for (const MachineOperand &MO : MI.operands())
+          if (MO.isFI() && !MFI.isFixedObjectIndex(MO.getIndex()) &&
+              Seen.insert(MO.getIndex()).second)
+            Bytes += MFI.getObjectSize(MO.getIndex()) +
+                     MFI.getObjectAlign(MO.getIndex()).value() - 1;
+    FrameMayUseFP = Bytes > 200;
+  }
   for (MachineBasicBlock &MBB : MF) {
     SmallVector<MachineInstr *, 8> Cmps;
     for (MachineInstr &MI : MBB)
@@ -1672,6 +1746,7 @@ bool W65816StackSlotCleanup::runOnMachineFunction(MachineFunction &MF) {
       const MachineInstr *Writer =
           aWriterBefore(Cmp->getIterator(), MBB.begin());
       if (!Writer) continue;
+      if (FrameMayUseFP && hasFrameIndex(*Writer)) continue;
       // Only eliminate if there are NO LdaLike instructions between
       // this CMP and the next Bxx (or end of MBB).  Otherwise the
       // CMP is the only flag-setting marker between the test value
@@ -2000,7 +2075,6 @@ bool W65816StackSlotCleanup::runOnMachineFunction(MachineFunction &MF) {
       if (SlotB == NO_SLOT_MATCH || SlotB == SlotA) {
         continue;
       }
-      MachineInstr *Lda2 = &*It;
       ++It;
       if (!advancePastDebug(MBB, It)) {
         continue;
@@ -2008,7 +2082,6 @@ bool W65816StackSlotCleanup::runOnMachineFunction(MachineFunction &MF) {
       if (It->getOpcode() != W65816::STA_DP) {
         continue;
       }
-      MachineInstr *Sta1 = &*It;
       ++It;
       if (!advancePastDebug(MBB, It)) {
         continue;

src/llvm/lib/Target/W65816/W65816StackSlotMerge.cpp

@@ -204,6 +204,40 @@ static bool slotNotWrittenBetween(MachineBasicBlock::iterator Start,
 }
 
 
+// True when something between Start and End (both exclusive, same MBB)
+// may read the word at slot Slot.  A 16-bit access one byte either side
+// overlaps it, and inside a PHP/PLP wrap the same slot is spelled one
+// byte higher, so the neighbours count as well.
+static bool slotReadBetween(MachineBasicBlock::iterator Start,
+                            MachineBasicBlock::iterator End, int64_t Slot) {
+  for (auto It = std::next(Start); It != End; ++It) {
+    if (It->isDebugInstr() || It->getOpcode() == W65816::STA_StackRel)
+      continue;
+    if (It->isCall() || It->isInlineAsm())
+      return true;
+    if (It->getNumOperands() < 1 || !It->getOperand(0).isImm())
+      continue;
+    int64_t Off = It->getOperand(0).getImm();
+    bool StackUse = isStackRelOp(It->getOpcode()) ||
+                    It->getOpcode() == W65816::INC_HI_IF_CARRY_StackRel ||
+                    It->getOpcode() == W65816::DEC_HI_IF_BORROW_StackRel;
+    switch (It->getOpcode()) {
+    case W65816::LDA_StackRelIndY: case W65816::STA_StackRelIndY:
+    case W65816::ADC_StackRelIndY: case W65816::SBC_StackRelIndY:
+    case W65816::CMP_StackRelIndY: case W65816::AND_StackRelIndY:
+    case W65816::ORA_StackRelIndY: case W65816::EOR_StackRelIndY:
+      StackUse = true;
+      break;
+    default:
+      break;
+    }
+    if (StackUse && Off >= Slot - 1 && Off <= Slot + 1)
+      return true;
+  }
+  return false;
+}
+
+
 // Returns true if MI clobbers P (N/Z/C/V flags).  Mirrors LLVM's
 // operand-based check + an opcode whitelist for tablegen entries that
 // omit `Defs = [P]` (InstImplied, InstStackRel, etc.).
@@ -292,6 +326,16 @@ static MachineInstr *findTwin(MachineInstr *StaX,
     }
     if (!Earlier || !Later) continue;
     int64_t EOff = Earlier->getOperand(0).getImm();
+    // Merged, the two slots are one: the Earlier store then also
+    // overwrites the Later store's slot, early.  A read of that slot in
+    // between would see the new value instead of the old one.  The
+    // pointer walk `q = p + 1; ... *p ...; p = q` stores q first and reads
+    // p before the copy: merged, *p read q (strcpy copied every string
+    // one byte short).  Only a PHP/PLP wrap around the copy, which bumps
+    // its offsets past recognition, used to keep this shape out.
+    if (slotReadBetween(Earlier->getIterator(), Later->getIterator(),
+                        Later->getOperand(0).getImm()))
+      continue;
     // Case (2): if Later is preceded by `LDA_StackRel <Earlier's slot>`
     // (the PHI-copy reload), it's a PHI twin.  Also require slot
     // Earlier-slot wasn't re-written between Earlier and Later.
@@ -469,14 +513,16 @@ bool W65816StackSlotMerge::runOnMachineFunction(MachineFunction &MF) {
   DenseMap<int64_t, SmallVector<MachineInstr *, 4>> Stas;
   DenseMap<int64_t, SmallVector<MachineInstr *, 4>> Ldas;
   DenseMap<int64_t, unsigned> AllRefs;  // STA + LDA + ADC + ... count
-  // Slots an INC_HI_IF_CARRY_StackRel bumps.  That pseudo writes its slot
+  // Slots an INC_HI_IF_CARRY_StackRel or DEC_HI_IF_BORROW_StackRel bumps.
-  // without being an STA (so the twin check below can't see the write)
+  // Those pseudos write their slot without being an STA (so the twin check
-  // and is not a stack-rel MC op (so the rename would skip it): merging
+  // below can't see the write) and are not stack-rel MC ops (so the rename
-  // such a slot would leave the carry landing in the abandoned one.
+  // would skip them): merging such a slot would leave the carry landing in
+  // the abandoned one.
   DenseSet<int64_t> IncSlots;
   for (MachineBasicBlock &MBB : MF) {
     for (MachineInstr &MI : MBB) {
-      if (MI.getOpcode() == W65816::INC_HI_IF_CARRY_StackRel &&
+      if ((MI.getOpcode() == W65816::INC_HI_IF_CARRY_StackRel ||
+           MI.getOpcode() == W65816::DEC_HI_IF_BORROW_StackRel) &&
           MI.getNumOperands() >= 1 && MI.getOperand(0).isImm())
         IncSlots.insert(MI.getOperand(0).getImm());
       int64_t Off;

src/llvm/lib/Target/W65816/W65816TargetMachine.cpp

@@ -60,6 +60,7 @@ LLVMInitializeW65816Target() {
   initializeW65816NegYIndYPass(PR);
   initializeW65816SjLjFinalizePass(PR);
   initializeW65816LowerWide32Pass(PR);
+  initializeW65816CmpSinkPass(PR);
   initializeW65816I32IncFoldPass(PR);
   initializeW65816ImgCalleeSavePass(PR);
   initializeW65816NarrowI32MulPass(PR);
@@ -238,6 +239,10 @@ void W65816PassConfig::addPreRegAlloc() {
   // pass walks Acc16/Idx16/Img16 vregs; running this first means they
   // see the decomposed halves uniformly.
   addPass(createW65816LowerWide32());
+  // Keep each compare next to its conditional branch once PHI elimination
+  // has put the back-edge copies between them (see W65816CmpSink.cpp).
+  if (getOptLevel() != CodeGenOptLevel::None)
+    insertPass(&RegisterCoalescerID, &W65816CmpSinkID);
   addPass(createW65816ABridgeViaX());
   addPass(createW65816TiedDefSpill());
   addPass(createW65816WidenAcc16());

src/llvm/test/CodeGen/W65816/addframe-flag-wrap.mir

@@ -13,6 +13,10 @@
 # The BCC case pins the second half of the fix: ADDframe corrupts C, so a
 # carry branch needs the wrap too (LDA-like ops never did, and still don't).
 #
+# The branch targets take X live in, so the wrap cannot be replaced by the
+# TAX ... CPX #k form that StackSlotCleanup uses when X is free (pinned in
+# cmp-via-x.mir); these cases keep testing the PHP / PLP wrap itself.
+#
 # RUN: llc -mtriple=w65816 -run-pass=w65816-stack-slot-cleanup -o - %s | FileCheck %s
 # RUN: llc -mtriple=w65816 -run-pass=w65816-stack-slot-cleanup,prologepilog -o - %s | FileCheck %s --check-prefix=PEI
 --- |
@@ -49,8 +53,10 @@ body: |
     BEQ %bb.1, implicit $p
     BRA %bb.2
   bb.1:
+    liveins: $x
     RTL
   bb.2:
+    liveins: $x
     RTL
 ...
 ---
@@ -74,8 +80,10 @@ body: |
     BCC %bb.1, implicit $p
     BRA %bb.2
   bb.1:
+    liveins: $x
     RTL
   bb.2:
+    liveins: $x
     RTL
 ...
 ---
@@ -97,7 +105,9 @@ body: |
     BCC %bb.1, implicit $p
     BRA %bb.2
   bb.1:
+    liveins: $x
     RTL
   bb.2:
+    liveins: $x
     RTL
 ...

src/llvm/test/CodeGen/W65816/andi8-flag-corrupt-wrap.mir

@@ -30,6 +30,10 @@
 # The wrapped reload keeps its offset: eliminateFrameIndex adds the byte
 # PHP pushed (php-wrap-frame-index.mir).
 #
+# The branch targets take X live in, so the wrap cannot be replaced by the
+# TAX ... CPX #k form that StackSlotCleanup uses when X is free (pinned in
+# cmp-via-x.mir); these cases keep testing the PHP / PLP wrap itself.
+#
 # RUN: llc -mtriple=w65816 -run-pass=w65816-stack-slot-cleanup -o - %s | FileCheck %s
 --- |
   target triple = "w65816"
@@ -60,8 +64,10 @@ body: |
     BEQ %bb.1, implicit $p
     BRA %bb.2
   bb.1:
+    liveins: $x
     RTL
   bb.2:
+    liveins: $x
     RTL
 ...
 ---
@@ -87,8 +93,10 @@ body: |
     BNE %bb.1, implicit $p
     BRA %bb.2
   bb.1:
+    liveins: $x
     RTL
   bb.2:
+    liveins: $x
     RTL
 ...
 ---
@@ -109,7 +117,9 @@ body: |
     BEQ %bb.1, implicit $p
     BRA %bb.2
   bb.1:
+    liveins: $x
     RTL
   bb.2:
+    liveins: $x
     RTL
 ...

src/llvm/test/CodeGen/W65816/cmp-sink.ll

@@ -0,0 +1,47 @@
+; Pin: W65816CmpSink moves the compare of a BEQ / BNE / BMI / BPL down past
+; the back-edge COPYs that PHI elimination put between them, so the copies
+; (LDA / STA after allocation) land above the compare and the flags need no
+; PHP / PLP wrap.  `while (p != end) s += *p++;` on 16-bit elements: before
+; the pass both latch compares (the low and the high half of the pointer)
+; were wrapped.  A BCC / BCS keeps its place: loads leave C alone, and the
+; end of a CMP / SBC carry chain is better left next to its first half
+; (cmp-prepare.ll's postinc_ult pins that shape).
+;
+; RUN: llc -mtriple=w65816 -O2 -print-after=w65816-cmp-sink -o /dev/null < %s 2>&1 | FileCheck %s --check-prefix=MIR
+; RUN: llc -mtriple=w65816 -O2 < %s | FileCheck %s
+
+target datalayout = "e-m:e-p:32:16-i16:16-i32:16-i64:16-f32:16-f64:16-a:8-n8:16-S16"
+
+define i16 @sum16(ptr %p, ptr %end) {
+; MIR-LABEL: Machine code for function sum16
+; MIR:       ADCfi
+; MIR:       COPY
+; MIR-NEXT:  COPY
+; MIR-NEXT:  COPY
+; MIR-NEXT:  CMPfi
+; MIR-NEXT:  BNE
+; MIR:       COPY
+; MIR-NEXT:  COPY
+; MIR-NEXT:  COPY
+; MIR-NEXT:  CMPfi
+; MIR-NEXT:  BNE
+; CHECK-LABEL: sum16:
+; CHECK-NOT:   php
+; CHECK:       rtl
+entry:
+  %cmp.not3 = icmp eq ptr %p, %end
+  br i1 %cmp.not3, label %while.end, label %while.body
+
+while.body:
+  %s.05 = phi i16 [ %add, %while.body ], [ 0, %entry ]
+  %p.addr.04 = phi ptr [ %incdec.ptr, %while.body ], [ %p, %entry ]
+  %incdec.ptr = getelementptr inbounds nuw i8, ptr %p.addr.04, i32 2
+  %0 = load i16, ptr %p.addr.04, align 2
+  %add = add i16 %0, %s.05
+  %cmp.not = icmp eq ptr %incdec.ptr, %end
+  br i1 %cmp.not, label %while.end, label %while.body
+
+while.end:
+  %s.0.lcssa = phi i16 [ 0, %entry ], [ %add, %while.body ]
+  ret i16 %s.0.lcssa
+}

src/llvm/test/CodeGen/W65816/cmp-via-x.mir

@@ -0,0 +1,189 @@
+# Pin: when a load into A sits between a `cmp #k` and the BEQ / BNE / BMI /
+# BPL that reads it, W65816StackSlotCleanup compares through X instead of
+# saving the flags: `tax ; <loads> ; cpx #k` (5 cycles, 4 bytes) in place
+# of `cmp #k ; php ; <loads> ; plp` (10 cycles, 5 bytes).  CPX sets N, Z
+# and C from X - k as CMP did from A - k.  It needs X free from the test to
+# the end of the block and not live into a successor.
+#
+# via_x:         X free: TAX / CPX, no wrap.
+# x_live_out:    X live into a branch target: the wrap stays.
+# x_in_region:   the corrupter reads X (TXA): the wrap stays.
+# stack_operand: `cmp d,s` has no CPX form: the wrap stays.
+#
+# W65816FarPtrStageReuse, the last pass, folds what later deletions leave
+# empty (fold_*): `tax ; cpx #k` back into `cmp #k`, and `php ; plp` away.
+#
+# RUN: llc -mtriple=w65816 -run-pass=w65816-stack-slot-cleanup -o - %s | FileCheck %s
+# RUN: llc -mtriple=w65816 -run-pass=w65816-farptr-reuse -o - %s | FileCheck %s --check-prefix=FOLD
+--- |
+  target triple = "w65816"
+  define void @via_x() { ret void }
+  define void @x_live_out() { ret void }
+  define void @x_in_region() { ret void }
+  define void @stack_operand() { ret void }
+  define void @fold_tax_cpx() { ret void }
+  define void @fold_keeps_x_read() { ret void }
+  define void @fold_php_plp() { ret void }
+...
+---
+# CHECK-LABEL: name: via_x
+# CHECK:       $a = LDAfi %stack.0, 0
+# CHECK-NEXT:  TAX
+# CHECK-NEXT:  $a = LDAi16imm 7
+# CHECK-NEXT:  STAfi $a, %stack.1, 0
+# CHECK-NEXT:  CPX_Imm16 3, implicit $x, implicit-def $p
+# CHECK-NEXT:  BNE %bb.1
+# CHECK-NOT:   PHP
+name: via_x
+tracksRegLiveness: false
+stack:
+  - { id: 0, size: 2, alignment: 1 }
+  - { id: 1, size: 2, alignment: 1 }
+body: |
+  bb.0:
+    successors: %bb.1, %bb.2
+    $a = LDAfi %stack.0, 0
+    CMPi16imm killed $a, 3, implicit-def $p
+    $a = LDAi16imm 7
+    STAfi $a, %stack.1, 0, implicit-def $a
+    BNE %bb.1, implicit $p
+    BRA %bb.2
+  bb.1:
+    RTL
+  bb.2:
+    RTL
+...
+---
+# CHECK-LABEL: name: x_live_out
+# CHECK:       CMPi16imm killed $a, 3, implicit-def $p
+# CHECK-NEXT:  PHP
+# CHECK-NEXT:  $a = LDAi16imm 7
+# CHECK-NEXT:  PLP
+# CHECK-NEXT:  BEQ %bb.1
+name: x_live_out
+tracksRegLiveness: false
+stack:
+  - { id: 0, size: 2, alignment: 1 }
+body: |
+  bb.0:
+    successors: %bb.1, %bb.2
+    $a = LDAfi %stack.0, 0
+    CMPi16imm killed $a, 3, implicit-def $p
+    $a = LDAi16imm 7
+    BEQ %bb.1, implicit $p
+    BRA %bb.2
+  bb.1:
+    liveins: $x
+    RTL
+  bb.2:
+    RTL
+...
+---
+# CHECK-LABEL: name: x_in_region
+# CHECK:       CMPi16imm killed $a, 3, implicit-def $p
+# CHECK-NEXT:  PHP
+# CHECK-NEXT:  TXA
+# CHECK-NEXT:  PLP
+# CHECK-NEXT:  BEQ %bb.1
+name: x_in_region
+tracksRegLiveness: false
+stack:
+  - { id: 0, size: 2, alignment: 1 }
+body: |
+  bb.0:
+    successors: %bb.1, %bb.2
+    $a = LDAfi %stack.0, 0
+    CMPi16imm killed $a, 3, implicit-def $p
+    TXA implicit-def $a, implicit $x
+    BEQ %bb.1, implicit $p
+    BRA %bb.2
+  bb.1:
+    RTL
+  bb.2:
+    RTL
+...
+---
+# CHECK-LABEL: name: stack_operand
+# CHECK:       CMPfi $a, %stack.1, 0, implicit-def $p
+# CHECK-NEXT:  PHP
+# CHECK-NEXT:  $a = LDAi16imm 7
+# CHECK-NEXT:  PLP
+# CHECK-NEXT:  BEQ %bb.1
+name: stack_operand
+tracksRegLiveness: false
+stack:
+  - { id: 0, size: 2, alignment: 1 }
+  - { id: 1, size: 2, alignment: 1 }
+body: |
+  bb.0:
+    successors: %bb.1, %bb.2
+    $a = LDAfi %stack.0, 0
+    CMPfi $a, %stack.1, 0, implicit-def $p
+    $a = LDAi16imm 7
+    BEQ %bb.1, implicit $p
+    BRA %bb.2
+  bb.1:
+    RTL
+  bb.2:
+    RTL
+...
+---
+# FOLD-LABEL: name: fold_tax_cpx
+# FOLD:       LDA_StackRel 1
+# FOLD-NEXT:  CMP_Imm16 3, implicit $a, implicit-def $p
+# FOLD-NEXT:  BNE %bb.1
+name: fold_tax_cpx
+tracksRegLiveness: false
+body: |
+  bb.0:
+    successors: %bb.1, %bb.2
+    LDA_StackRel 1, implicit-def $a
+    TAX implicit-def $x, implicit $a
+    CPX_Imm16 3, implicit $x, implicit-def $p
+    BNE %bb.1, implicit $p
+    BRA %bb.2
+  bb.1:
+    RTL
+  bb.2:
+    RTL
+...
+---
+# FOLD-LABEL: name: fold_keeps_x_read
+# FOLD:       TAX
+# FOLD-NEXT:  CPX_Imm16 3
+name: fold_keeps_x_read
+tracksRegLiveness: false
+body: |
+  bb.0:
+    successors: %bb.1, %bb.2
+    LDA_StackRel 1, implicit-def $a
+    TAX implicit-def $x, implicit $a
+    CPX_Imm16 3, implicit $x, implicit-def $p
+    BNE %bb.1, implicit $p
+    BRA %bb.2
+  bb.1:
+    liveins: $x
+    RTL
+  bb.2:
+    RTL
+...
+---
+# FOLD-LABEL: name: fold_php_plp
+# FOLD:       CMP_Imm16 3
+# FOLD-NEXT:  BEQ %bb.1
+name: fold_php_plp
+tracksRegLiveness: false
+body: |
+  bb.0:
+    successors: %bb.1, %bb.2
+    LDA_StackRel 1, implicit-def $a
+    CMP_Imm16 3, implicit-def $p
+    PHP
+    PLP
+    BEQ %bb.1, implicit $p
+    BRA %bb.2
+  bb.1:
+    RTL
+  bb.2:
+    RTL
+...

src/llvm/test/CodeGen/W65816/cmp0-fprel-keep.mir

@@ -0,0 +1,60 @@
+# Pin: W65816StackSlotCleanup's Pass 1c keeps a `cmp #0` whose A comes
+# from a frame-index load in a frame that addresses its slots through the
+# $F6 frame pointer (a VLA, or a frame over 200 bytes).  There LDAfi
+# expands to `sty $fa ; ldy #off ; lda [$f6],y ; ldy $fa`, and the final
+# LDY leaves N / Z describing Y, not the loaded value.  Erased, the smoke
+# run's sum_n(3) VLA loop branched on the restored Y and returned 1; the
+# `cmp #0` only came to sit right after the load once W65816CmpSink moved
+# the loop test past its back-edge copies.
+#
+# vla_keeps: variable-sized frame: the CMP stays.
+# sp_drops:  the same load in an ordinary frame: the CMP goes.
+#
+# RUN: llc -mtriple=w65816 -run-pass=w65816-stack-slot-cleanup -o - %s | FileCheck %s
+--- |
+  target triple = "w65816"
+  define void @vla_keeps() { ret void }
+  define void @sp_drops() { ret void }
+...
+---
+# CHECK-LABEL: name: vla_keeps
+# CHECK:       $a = LDAfi %stack.1, 0
+# CHECK-NEXT:  CMPi16imm $a, 0
+# CHECK-NEXT:  BEQ
+name: vla_keeps
+tracksRegLiveness: false
+stack:
+  - { id: 0, type: variable-sized, offset: 0, alignment: 1 }
+  - { id: 1, size: 2, alignment: 1 }
+body: |
+  bb.0:
+    successors: %bb.1, %bb.2
+    $a = LDAfi %stack.1, 0
+    CMPi16imm $a, 0, implicit-def $p
+    BEQ %bb.2, implicit $p
+    BRA %bb.1
+  bb.1:
+    RTL
+  bb.2:
+    RTL
+...
+---
+# CHECK-LABEL: name: sp_drops
+# CHECK:       $a = LDAfi %stack.0, 0
+# CHECK-NEXT:  BEQ
+name: sp_drops
+tracksRegLiveness: false
+stack:
+  - { id: 0, size: 2, alignment: 1 }
+body: |
+  bb.0:
+    successors: %bb.1, %bb.2
+    $a = LDAfi %stack.0, 0
+    CMPi16imm $a, 0, implicit-def $p
+    BEQ %bb.2, implicit $p
+    BRA %bb.1
+  bb.1:
+    RTL
+  bb.2:
+    RTL
+...

src/llvm/test/CodeGen/W65816/extract-wide32-regseq.ll

@@ -16,8 +16,8 @@
 
 define void @clear_i32(ptr %p) {
 ; CHECK-LABEL: clear_i32:
-; CHECK:       sta [0xe0
+; CHECK:       sta [0x{{[0-9a-f]+}}], y
-; CHECK:       sta [0xe0
+; CHECK:       sta [0x{{[0-9a-f]+}}], y
 ; CHECK:       rtl
   store i32 0, ptr %p
   ret void
@@ -25,10 +25,10 @@ define void @clear_i32(ptr %p) {
 
 define void @clear_i32_pair(ptr %p, ptr %q) {
 ; CHECK-LABEL: clear_i32_pair:
-; CHECK:       sta [0xe0
+; CHECK:       sta [0x{{[0-9a-f]+}}], y
-; CHECK:       sta [0xe0
+; CHECK:       sta [0x{{[0-9a-f]+}}], y
-; CHECK:       sta [0xe0
+; CHECK:       sta [0x{{[0-9a-f]+}}], y
-; CHECK:       sta [0xe0
+; CHECK:       sta [0x{{[0-9a-f]+}}], y
 ; CHECK:       rtl
   store i32 0, ptr %p
   store i32 0, ptr %q

src/llvm/test/CodeGen/W65816/far-ptr-home.ll

@@ -0,0 +1,46 @@
+; Pin: in a leaf function W65816StackRelToImg puts each far pointer's low
+; and bank words in adjacent IMG direct-page words, and bumps an i32 high
+; half there too (INC_HI_IF_CARRY_DP), so W65816FarPtrStageReuse reads and
+; writes through the pointers in place: no $E0 / $E2 staging in the loop.
+; JoeyLib's old sfxCopyUnsigned loop (the refCopy of the IIgs sfx probe):
+; ~30 of its ~166 cycles a byte were the two pointers' restaging.
+;
+; RUN: llc -mtriple=w65816 -O2 < %s | FileCheck %s
+
+target datalayout = "e-m:e-p:32:16-i16:16-i32:16-i64:16-f32:16-f64:16-a:8-n8:16-S16"
+
+define void @refCopy(ptr %dst, ptr %src, i32 %length) {
+; CHECK-LABEL: refCopy:
+; CHECK:       .LBB0_{{[0-9]+}}: ; %for.body
+; CHECK-NOT:   0xe0
+; CHECK-NOT:   0xe2
+; CHECK:       inc 0x[[SRCLO:d[0-9a-f]]]
+; CHECK-NEXT:  bne
+; CHECK-NEXT:  inc 0x[[SRCHI:d[0-9a-f]]]
+; CHECK-NOT:   0xe0
+; CHECK:       sta [0x[[DSTLO:d[0-9a-f]]]], y
+; CHECK:       inc 0x[[DSTLO]]
+; CHECK-NEXT:  bne
+; CHECK-NOT:   0xe0
+; CHECK:       lda [0x[[SRCLO]]], y
+; CHECK:       rtl
+entry:
+  %cmp10.not = icmp eq i32 %length, 0
+  br i1 %cmp10.not, label %for.end, label %for.body
+
+for.body:
+  %i.011 = phi i32 [ %inc, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds nuw i8, ptr %src, i32 %i.011
+  %0 = load i8, ptr %arrayidx, align 1
+  %1 = xor i8 %0, -128
+  %cmp3 = icmp eq i8 %0, -128
+  %spec.store.select = select i1 %cmp3, i8 1, i8 %1
+  %arrayidx5 = getelementptr inbounds nuw i8, ptr %dst, i32 %i.011
+  store i8 %spec.store.select, ptr %arrayidx5, align 1
+  %inc = add nuw i32 %i.011, 1
+  %cmp = icmp ult i32 %inc, %length
+  br i1 %cmp, label %for.body, label %for.end
+
+for.end:
+  ret void
+}

src/llvm/test/CodeGen/W65816/far-ptr-home.mir

@@ -0,0 +1,106 @@
+# Pin: W65816FarPtrStageReuse addresses a far pointer where it lives when
+# its low and bank words sit side by side in the IMG direct page (which
+# W65816StackRelToImg arranges, far-ptr-home.ll): an access through the
+# $E0 / $E2 staging becomes [dp],Y on the pointer's own words, then the
+# staging stores nothing reads any more go, and the loads that fed them.
+#
+# in_place:        staged from $D2 / $D4 and used at once: [$D2],Y.
+# home_moved:      the low word is bumped after the staging: the staged
+#                  copy is the only right one, [$E0],Y stays.
+# not_adjacent:    staged from $D2 / $D6: no pair to point at.
+# stage_read_later: $E0 is also read directly after the access: the
+#                  access moves and the $E0 store stays; the $E2 store,
+#                  which nothing else reads, goes.
+#
+# RUN: llc -mtriple=w65816 -run-pass=w65816-farptr-reuse -o - %s | FileCheck %s
+--- |
+  target triple = "w65816"
+  define void @in_place() { ret void }
+  define void @home_moved() { ret void }
+  define void @not_adjacent() { ret void }
+  define void @stage_read_later() { ret void }
+...
+---
+# CHECK-LABEL: name: in_place
+# CHECK:       bb.0:
+# CHECK-NEXT:  REP 48
+# CHECK-NEXT:  LDY_Imm16 0
+# CHECK-NEXT:  LDA_DPIndLongY 210
+# CHECK-NEXT:  STA_DP 216
+# CHECK-NEXT:  RTL
+name: in_place
+tracksRegLiveness: false
+body: |
+  bb.0:
+    REP 48
+    LDA_DP 210, implicit-def $a
+    STA_DP 224, implicit $a
+    LDA_DP 212, implicit-def $a
+    STA_DP 226, implicit $a
+    LDY_Imm16 0, implicit-def $y
+    LDA_DPIndLongY 224, implicit-def $a, implicit $y
+    STA_DP 216, implicit $a
+    RTL
+...
+---
+# CHECK-LABEL: name: home_moved
+# CHECK:       STA_DP 224
+# CHECK:       STA_DP 226
+# CHECK:       INC_DP 210
+# CHECK:       LDA_DPIndLongY 224
+name: home_moved
+tracksRegLiveness: false
+body: |
+  bb.0:
+    REP 48
+    LDA_DP 210, implicit-def $a
+    STA_DP 224, implicit $a
+    LDA_DP 212, implicit-def $a
+    STA_DP 226, implicit $a
+    INC_DP 210
+    LDY_Imm16 0, implicit-def $y
+    LDA_DPIndLongY 224, implicit-def $a, implicit $y
+    STA_DP 216, implicit $a
+    RTL
+...
+---
+# CHECK-LABEL: name: not_adjacent
+# CHECK:       STA_DP 224
+# CHECK:       STA_DP 226
+# CHECK:       LDA_DPIndLongY 224
+name: not_adjacent
+tracksRegLiveness: false
+body: |
+  bb.0:
+    REP 48
+    LDA_DP 210, implicit-def $a
+    STA_DP 224, implicit $a
+    LDA_DP 214, implicit-def $a
+    STA_DP 226, implicit $a
+    LDY_Imm16 0, implicit-def $y
+    LDA_DPIndLongY 224, implicit-def $a, implicit $y
+    STA_DP 216, implicit $a
+    RTL
+...
+---
+# CHECK-LABEL: name: stage_read_later
+# CHECK:       STA_DP 224
+# CHECK-NOT:   STA_DP 226
+# CHECK:       LDA_DPIndLongY 210
+# CHECK:       LDA_DP 224
+name: stage_read_later
+tracksRegLiveness: false
+body: |
+  bb.0:
+    REP 48
+    LDA_DP 210, implicit-def $a
+    STA_DP 224, implicit $a
+    LDA_DP 212, implicit-def $a
+    STA_DP 226, implicit $a
+    LDY_Imm16 0, implicit-def $y
+    LDA_DPIndLongY 224, implicit-def $a, implicit $y
+    STA_DP 216, implicit $a
+    LDA_DP 224, implicit-def $a
+    STA_DP 218, implicit $a
+    RTL
+...

src/llvm/test/CodeGen/W65816/i32-compare-chain.ll

@@ -596,24 +596,24 @@ define i32 @sel32_uge(i32 %a, i32 %b, i32 %x, i32 %y) {
 
 ; The loop shape from JoeyLib's old sfxCopyUnsigned: the latch's i < n is
 ; the chain and one bcc back to the top, and every i32 += 1 keeps the
-; carry into its high half (the bne skips over it).
+; carry into its high half (the bne skips over it).  Both halves of the
+; pointer and of i live in the direct page, the pointer's side by side,
+; so the store goes through it in place (no $E0 / $E2 staging).
 define void @loop_ult(ptr %dst, i32 %n) {
 ; CHECK-LABEL: loop_ult:
 ; CHECK:       [[TOP:.LBB[0-9_]+]]: ; %body
 ; CHECK-NOT:   php
+; CHECK-NOT:   0xe0
+; CHECK:       sta [0xd2], y
 ; CHECK:       inc 0xd2
 ; CHECK-NEXT:  bne
-; CHECK-NEXT:  lda 0x3, s
+; CHECK-NEXT:  inc 0xd4
-; CHECK-NEXT:  inc a
+; CHECK:       inc 0xd6
-; CHECK-NEXT:  sta 0x3, s
-; CHECK:       inc 0xd4
 ; CHECK-NEXT:  bne
-; CHECK-NEXT:  lda 0x5, s
+; CHECK-NEXT:  inc 0xd8
-; CHECK-NEXT:  inc a
+; CHECK:       lda 0xd6
-; CHECK-NEXT:  sta 0x5, s
-; CHECK:       lda 0xd4
 ; CHECK-NEXT:  cmp 0xc, s
-; CHECK-NEXT:  lda 0x5, s
+; CHECK-NEXT:  lda 0xd8
 ; CHECK-NEXT:  sbc 0xe, s
 ; CHECK-NEXT:  bcc [[TOP]]
 ; CHECK:       rtl

src/llvm/test/CodeGen/W65816/i32-dec-fold.mir

@@ -0,0 +1,159 @@
+# Pin: W65816I32IncFold turns an in-place i32 -= 1 into
+#   lda lo ; DEC_HI_IF_BORROW hi ; dec a ; sta lo
+# (Z from the low half before it goes down: only a low half of 0 borrows),
+# and an i32 += 1 into lda lo ; inc a ; sta lo ; INC_HI_IF_CARRY hi, with
+# the halves in stack slots or in the direct page.  The folded form leaves
+# A holding the low half where the original left the high half, and other
+# flags, so it only fires when a fresh load of A comes first.
+#
+# dec_stack / dec_dp / inc_dp: the three fold shapes.
+# dec_a_read / inc_a_read:     A (the high half) is read after: no fold.
+#                              The increment fold never checked this.
+# dec_carry_read:              the high half's carry-out is live: no fold.
+#
+# RUN: llc -mtriple=w65816 -run-pass=w65816-i32-inc-fold -o - %s | FileCheck %s
+--- |
+  target triple = "w65816"
+  define void @dec_stack() { ret void }
+  define void @dec_dp() { ret void }
+  define void @inc_dp() { ret void }
+  define void @dec_a_read() { ret void }
+  define void @inc_a_read() { ret void }
+  define void @dec_carry_read() { ret void }
+...
+---
+# CHECK-LABEL: name: dec_stack
+# CHECK:       LDA_StackRel 5
+# CHECK-NEXT:  DEC_HI_IF_BORROW_StackRel 3
+# CHECK-NEXT:  $a = DEA_PSEUDO $a
+# CHECK-NEXT:  STA_StackRel 5
+# CHECK-NEXT:  LDA_StackRel 5
+# CHECK-NEXT:  ORA_StackRel 3
+name: dec_stack
+tracksRegLiveness: true
+body: |
+  bb.0:
+    successors: %bb.0, %bb.1
+    LDA_StackRel 5, implicit-def $a
+    $a = ADCi16imm killed $a, -1, implicit-def $p
+    STA_StackRel 5, implicit $a
+    LDA_StackRel 3, implicit-def $a
+    $a = ADCEi16imm killed $a, -1, implicit-def dead $p, implicit $p
+    STA_StackRel 3, implicit $a
+    LDA_StackRel 5, implicit-def $a
+    ORA_StackRel 3, implicit $a, implicit-def $a
+    BNE %bb.0, implicit $p
+
+  bb.1:
+    RTL
+...
+---
+# CHECK-LABEL: name: dec_dp
+# CHECK:       LDA_DP 210
+# CHECK-NEXT:  DEC_HI_IF_BORROW_DP 214
+# CHECK-NEXT:  $a = DEA_PSEUDO $a
+# CHECK-NEXT:  STA_DP 210
+name: dec_dp
+tracksRegLiveness: true
+body: |
+  bb.0:
+    successors: %bb.1
+    LDA_DP 210, implicit-def $a
+    $a = ADCi16imm killed $a, 65535, implicit-def $p
+    STA_DP 210, implicit $a
+    LDA_DP 214, implicit-def $a
+    $a = ADCEi16imm killed $a, 65535, implicit-def dead $p, implicit $p
+    STA_DP 214, implicit $a
+    BRA %bb.1
+
+  bb.1:
+    LDA_DP 210, implicit-def $a
+    ORA_DP 214, implicit $a, implicit-def $a
+    RTL
+...
+---
+# CHECK-LABEL: name: inc_dp
+# CHECK:       LDA_DP 210
+# CHECK-NEXT:  $a = INA_PSEUDO $a
+# CHECK-NEXT:  STA_DP 210
+# CHECK-NEXT:  INC_HI_IF_CARRY_DP 214
+# CHECK-NEXT:  STX_DP 216
+# CHECK-NEXT:  LDA_StackRel 1
+name: inc_dp
+tracksRegLiveness: true
+body: |
+  bb.0:
+    liveins: $x
+    LDA_DP 210, implicit-def $a
+    $a = ADCi16imm killed $a, 1, implicit-def $p
+    STA_DP 210, implicit $a
+    LDA_DP 214, implicit-def $a
+    $a = ADCEi16imm killed $a, 0, implicit-def dead $p, implicit $p
+    STA_DP 214, implicit $a
+    STX_DP 216
+    LDA_StackRel 1, implicit-def $a
+    RTL
+...
+---
+# CHECK-LABEL: name: dec_a_read
+# CHECK:       ADCi16imm
+# CHECK:       ADCEi16imm
+# CHECK-NOT:   DEC_HI_IF_BORROW
+name: dec_a_read
+tracksRegLiveness: true
+body: |
+  bb.0:
+    LDA_StackRel 5, implicit-def $a
+    $a = ADCi16imm killed $a, -1, implicit-def $p
+    STA_StackRel 5, implicit $a
+    LDA_StackRel 3, implicit-def $a
+    $a = ADCEi16imm killed $a, -1, implicit-def dead $p, implicit $p
+    STA_StackRel 3, implicit $a
+    TAX implicit-def $x, implicit $a
+    LDA_StackRel 5, implicit-def $a
+    RTL implicit $a, implicit $x
+...
+---
+# CHECK-LABEL: name: inc_a_read
+# CHECK:       ADCi16imm
+# CHECK:       ADCEi16imm
+# CHECK-NOT:   INC_HI_IF_CARRY
+name: inc_a_read
+tracksRegLiveness: true
+body: |
+  bb.0:
+    LDA_StackRel 5, implicit-def $a
+    $a = ADCi16imm killed $a, 1, implicit-def $p
+    STA_StackRel 5, implicit $a
+    LDA_StackRel 3, implicit-def $a
+    $a = ADCEi16imm killed $a, 0, implicit-def dead $p, implicit $p
+    STA_StackRel 3, implicit $a
+    TAX implicit-def $x, implicit $a
+    LDA_StackRel 5, implicit-def $a
+    RTL implicit $a, implicit $x
+...
+---
+# CHECK-LABEL: name: dec_carry_read
+# CHECK:       ADCi16imm
+# CHECK:       ADCEi16imm
+# CHECK-NOT:   DEC_HI_IF_BORROW
+name: dec_carry_read
+tracksRegLiveness: true
+body: |
+  bb.0:
+    successors: %bb.1, %bb.2
+    LDA_StackRel 5, implicit-def $a
+    $a = ADCi16imm killed $a, -1, implicit-def $p
+    STA_StackRel 5, implicit $a
+    LDA_StackRel 3, implicit-def $a
+    $a = ADCEi16imm killed $a, -1, implicit-def $p, implicit $p
+    STA_StackRel 3, implicit $a
+    LDA_StackRel 5, implicit-def $a
+    BCC %bb.2, implicit $p
+
+  bb.1:
+    RTL
+
+  bb.2:
+    RTL
+...

src/llvm/test/CodeGen/W65816/i32-inc-carry-slot.mir

@@ -1,29 +1,33 @@
 # Pin: a stack slot bumped by INC_HI_IF_CARRY_StackRel (W65816I32IncFold's
-# carry into the high half of an `i32 += 1`) is never promoted to a DP IMG
+# carry into the high half of an `i32 += 1`) keeps its carry through
-# slot by W65816StackRelToImg, nor renamed by W65816StackSlotMerge.
+# W65816StackRelToImg and W65816StackSlotMerge.
 #
-# The pseudo reads and writes its slot but is not one of the eight
+# The pseudo reads and writes its slot but is not one of the stack-rel MC
-# stack-rel MC ops those passes scan.  StackRelToImg promoted the hot
+# ops those passes scan.  StackRelToImg once promoted the hot high-half
-# high-half slot anyway, rewrote every LDA of it to DP, left the INC
+# slot anyway, rewrote every LDA of it to DP, left the INC bumping the
-# bumping the abandoned stack copy, and its dead-INC sweep then saw a slot
+# abandoned stack copy, and its dead-INC sweep then saw a slot nothing
-# nothing reads via stack-rel and erased the INC outright.  Every such
+# reads via stack-rel and erased the INC outright.  Every such `i32 += 1`
-# `i32 += 1` lost its carry: the loop below is the latch of JoeyLib's old
+# lost its carry: the loop below is the latch of JoeyLib's old
 # sfxCopyUnsigned, whose far pointers stopped crossing their bank and whose
 # 32-bit counter stuck below 64K.  StackSlotMerge had the same blind spot:
-# it renamed the slot everywhere except in the INC.
+# it renamed the slot everywhere except in the INC.  The fix then kept such
+# slots on the stack; since 2026-09-30 StackRelToImg promotes them and
+# rewrites the pseudo to INC_HI_IF_CARRY_DP with the slot's other sites.
 #
-# promote_keeps_carry: slots 7 (ptr lo) and 1 (i lo) may go to DP; slots 3
+# promote_keeps_carry: every slot goes to DP, each carry pseudo with its
-#                      and 5 (the high halves) must stay on the stack with
+#                      slot (3 -> $D4, 5 -> $D8), and the far pointer
-#                      both INC_HI_IF_CARRY_StackRel intact.
+#                      staged from slots 7 / 3 lands in adjacent words
+#                      ($D2 / $D4) for FarPtrStageReuse to use in place.
 # merge_keeps_carry:   `LDA 3 ; STA 5` and `LDA 5 ; STA 3` make slots 3 and
 #                      5 look like one PHI value, except that slot 3 is
 #                      also bumped by the INC: no rename.
-# sweep_keeps_wrapped: StackRelToImg's dead-INC sweep.  The only read of
+# sweep_keeps_wrapped: the only read of slot 5 sits inside a PHP/PLP wrap,
-#                      slot 5 sits inside a PHP/PLP wrap, so it is spelled
+#                      spelled `LDA_StackRel 6`.  StackRelToImg's dead-INC
-#                      `LDA_StackRel 6`; matching raw offsets called slot 5
+#                      sweep once matched raw offsets, called slot 5 unread
-#                      unread and erased the INC and the entry STA that
+#                      and erased the INC and the entry STA that initialised
-#                      initialised it (the runtime's strcpy, whose source
+#                      it (the runtime's strcpy, whose source bank word then
-#                      bank word then came from stack garbage).
+#                      came from stack garbage); promotion must likewise map
+#                      the wrapped read to slot 5's word.
 # sweep_drops_unread:  a slot nothing reads still loses its INC and STAs.
 #
 # RUN: llc -mtriple=w65816 -run-pass=w65816-stackrel-to-img -o - %s | FileCheck %s --check-prefix=IMG
@@ -38,14 +42,18 @@
 ---
 # IMG-LABEL: name: promote_keeps_carry
 # IMG:       bb.2:
-# IMG:       LDA_StackRel 3
+# IMG:       LDA_DP 210
-# IMG:       LDA_StackRel 3
+# IMG-NEXT:  STA_DP 224
-# IMG:       LDA_StackRel 5
+# IMG-NEXT:  LDA_DP 212
-# IMG:       INC_DP
+# IMG-NEXT:  STA_DP 226
-# IMG-NEXT:  INC_HI_IF_CARRY_StackRel 3
+# IMG:       LDA_DP 212
-# IMG:       INC_DP
+# IMG:       LDA_DP 216
-# IMG-NEXT:  INC_HI_IF_CARRY_StackRel 5
+# IMG:       INC_DP 210
-# IMG:       LDA_StackRel 5
+# IMG-NEXT:  INC_HI_IF_CARRY_DP 212
+# IMG-NEXT:  INC_DP 214
+# IMG-NEXT:  INC_HI_IF_CARRY_DP 216
+# IMG-NOT:   StackRel {{[1357]}},
+# IMG:       LDA_DP 216
 # IMG-NEXT:  SBC_StackRel 14
 # IMG-NEXT:  BCC %bb.2
 name: promote_keeps_carry
@@ -149,11 +157,11 @@ body: |
 ...
 ---
 # IMG-LABEL: name: sweep_keeps_wrapped
-# IMG:       STA_StackRel 5
+# IMG:       STA_DP 212
 # IMG:       bb.1:
-# IMG:       INC_HI_IF_CARRY_StackRel 5
+# IMG:       INC_HI_IF_CARRY_DP 212
 # IMG:       PHP
-# IMG-NEXT:  LDA_StackRel 6
+# IMG-NEXT:  LDA_DP 212
 # IMG-NEXT:  PLP
 name: sweep_keeps_wrapped
 tracksRegLiveness: true

src/llvm/test/CodeGen/W65816/i32-shift-16.ll

@@ -1,7 +1,7 @@
 ; Pin: i32 shifts by 16 or more are half moves plus at most an i16 shift,
 ; not __lshrsi3 / __ashlsi3 / __ashrsi3 calls.  `(u16)(x >> 16)`, the high
-; word, is the high half itself.  SRA by a count other than 16 or 31 still
+; word, is the high half itself.  (Counts below 16 are pinned in
-; calls the helper (the i16 SRA patterns only do 1 and 15).
+; shift-const-inline.ll.)
 ;
 ; RUN: llc -mtriple=w65816 -O2 < %s | FileCheck %s
 
@@ -57,9 +57,22 @@ define i32 @sar31(i32 %x) {
   ret i32 %s
 }
 
+; SRA by 17..30: the high half shifted arithmetically (a logical shift,
+; then ^ M - M to spread the new sign bit) and a high word of sign fill.
 define i32 @sar20(i32 %x) {
 ; CHECK-LABEL: sar20:
-; CHECK:       jsl __ashrsi3
+; CHECK-NOT:   jsl
+; CHECK:       asl a
+; CHECK-NEXT:  lda #0x0
+; CHECK-NEXT:  adc #0xffff
+; CHECK-NEXT:  eor #0xffff
+; CHECK:       lsr a
+; CHECK-NEXT:  lsr a
+; CHECK-NEXT:  lsr a
+; CHECK-NEXT:  lsr a
+; CHECK-NEXT:  eor #0x800
+; CHECK-NEXT:  clc
+; CHECK-NEXT:  adc #0xf800
 ; CHECK:       rtl
   %s = ashr i32 %x, 20
   ret i32 %s

src/llvm/test/CodeGen/W65816/ina-flag-corrupt-wrap.mir

@@ -22,6 +22,10 @@
 # no_test:  an INA_PSEUDO with no test before it gets no wrap (nothing to
 #           preserve, and the wrap costs 8 cycles).
 #
+# The branch targets take X live in, so the wrap cannot be replaced by the
+# TAX ... CPX #k form that StackSlotCleanup uses when X is free (pinned in
+# cmp-via-x.mir); these cases keep testing the PHP / PLP wrap itself.
+#
 # RUN: llc -mtriple=w65816 -run-pass=w65816-stack-slot-cleanup -o - %s | FileCheck %s
 --- |
   target triple = "w65816"
@@ -49,8 +53,10 @@ body: |
     BNE %bb.1, implicit $p
     BRA %bb.2
   bb.1:
+    liveins: $x
     RTL
   bb.2:
+    liveins: $x
     RTL
 ...
 ---
@@ -73,8 +79,10 @@ body: |
     BEQ %bb.1, implicit $p
     BRA %bb.2
   bb.1:
+    liveins: $x
     RTL
   bb.2:
+    liveins: $x
     RTL
 ...
 ---
@@ -94,7 +102,9 @@ body: |
     BNE %bb.1, implicit $p
     BRA %bb.2
   bb.1:
+    liveins: $x
     RTL
   bb.2:
+    liveins: $x
     RTL
 ...

src/llvm/test/CodeGen/W65816/loop-local-dp.ll

@@ -0,0 +1,72 @@
+; Pin: in a function that makes calls (so W65816StackRelToImg cannot move
+; slots into the direct page for the whole function), an innermost loop
+; that makes none still gets its far pointer and hot slots in free IMG
+; words for its duration: copied in at the end of the preheader, back at
+; the start of the exit.  The pointer then goes through [dp],Y in place
+; (W65816FarPtrStageReuse) and its bump is `inc lo ; bne ; inc hi`.
+; JoeyLib's sfx probe inlines its reference copy loop into main, which
+; calls: before this the loop restaged both pointers into $E0 / $E2 for
+; every byte.
+;
+; walk:       calls before and after the loop: promoted.
+; call_in_loop: the loop calls: left alone (still staged through $E0).
+;
+; RUN: llc -mtriple=w65816 -O2 < %s | FileCheck %s
+
+target datalayout = "e-m:e-p:32:16-i16:16-i32:16-i64:16-f32:16-f64:16-a:8-n8:16-S16"
+
+declare void @ext()
+declare void @use(i16)
+
+define i16 @walk(ptr %p, i16 %n) {
+; CHECK-LABEL: walk:
+; CHECK:       jsl ext
+; CHECK:       [[LOOP:.LBB[0-9_]+]]: ; %loop
+; CHECK-NOT:   0xe0
+; CHECK-NOT:   jsl
+; CHECK:       lda [0x{{d[0-9a-f]}}], y
+; CHECK-NOT:   0xe0
+; CHECK:       inc 0x{{d[0-9a-f]}}
+; CHECK-NEXT:  bne
+; CHECK-NEXT:  inc 0x{{d[0-9a-f]}}
+; CHECK:       bne [[LOOP]]
+; CHECK:       jsl ext
+entry:
+  call void @ext()
+  br label %loop
+loop:
+  %q = phi ptr [ %p, %entry ], [ %q1, %loop ]
+  %i = phi i16 [ %n, %entry ], [ %i1, %loop ]
+  %s = phi i16 [ 0, %entry ], [ %s1, %loop ]
+  %c = load i8, ptr %q, align 1
+  %cw = zext i8 %c to i16
+  %s1 = add i16 %s, %cw
+  %q1 = getelementptr inbounds i8, ptr %q, i32 1
+  %i1 = add i16 %i, -1
+  %done = icmp eq i16 %i1, 0
+  br i1 %done, label %exit, label %loop
+exit:
+  call void @ext()
+  ret i16 %s1
+}
+
+define void @call_in_loop(ptr %p, i16 %n) {
+; CHECK-LABEL: call_in_loop:
+; CHECK:       sta 0xe0
+; CHECK:       lda [0xe0], y
+; CHECK:       jsl use
+entry:
+  br label %loop
+loop:
+  %q = phi ptr [ %p, %entry ], [ %q1, %loop ]
+  %i = phi i16 [ %n, %entry ], [ %i1, %loop ]
+  %c = load i8, ptr %q, align 1
+  %cw = zext i8 %c to i16
+  call void @use(i16 %cw)
+  %q1 = getelementptr inbounds i8, ptr %q, i32 1
+  %i1 = add i16 %i, -1
+  %done = icmp eq i16 %i1, 0
+  br i1 %done, label %exit, label %loop
+exit:
+  ret void
+}

src/llvm/test/CodeGen/W65816/php-wrap-frame-index.mir

@@ -17,6 +17,10 @@
 #              $F6+266 inside the wrap too; the local stack.1 moves from
 #              `1,s` to `2,s`.
 #
+# The branch targets take X live in, so the wrap cannot be replaced by the
+# TAX ... CPX #k form that StackSlotCleanup uses when X is free (pinned in
+# cmp-via-x.mir); these cases keep testing the PHP / PLP wrap itself.
+#
 # RUN: llc -mtriple=w65816 -run-pass=w65816-stack-slot-cleanup,prologepilog -o - %s | FileCheck %s
 --- |
   target triple = "w65816"
@@ -53,8 +57,10 @@ body: |
     BEQ %bb.1, implicit $p
     BRA %bb.2
   bb.1:
+    liveins: $x
     RTL
   bb.2:
+    liveins: $x
     $a = LDAfi %stack.0, 0
     RTL
 ...

src/llvm/test/CodeGen/W65816/ret-ext-i16.ll

@@ -0,0 +1,56 @@
+; Pin: a signext / zeroext return (clang puts one on every short, char and
+; bool return) is extended to A's 16 bits only.  The default widened it to
+; i32 and filled X with the sign (`asl a ; lda #0 ; adc #$ffff ; eor
+; #$ffff ; tax`, plus a stack slot to keep A) or with zero (`ldx #0`),
+; which no caller reads: the ABI returns 16 bits in A, and a caller that
+; wants the value wider extends it itself (see `widen`).
+;
+; RUN: llc -mtriple=w65816 -O2 < %s | FileCheck %s
+
+target datalayout = "e-m:e-p:32:16-i16:16-i32:16-i64:16-f32:16-f64:16-a:8-n8:16-S16"
+
+define signext i16 @sret(i16 signext %x) {
+; CHECK-LABEL: sret:
+; CHECK:       rep #0x30
+; CHECK-NEXT:  lsr a
+; CHECK-NEXT:  lsr a
+; CHECK-NEXT:  lsr a
+; CHECK-NEXT:  eor #0x1000
+; CHECK-NEXT:  clc
+; CHECK-NEXT:  adc #0xf000
+; CHECK-NEXT:  rtl
+  %s = ashr i16 %x, 3
+  ret i16 %s
+}
+
+define zeroext i16 @zret(i16 zeroext %x) {
+; CHECK-LABEL: zret:
+; CHECK:       rep #0x30
+; CHECK-NEXT:  lsr a
+; CHECK-NEXT:  rtl
+  %s = lshr i16 %x, 1
+  ret i16 %s
+}
+
+define signext i8 @cret(i8 signext %x) {
+; CHECK-LABEL: cret:
+; CHECK-NOT:   tax
+; CHECK-NOT:   ldx
+; CHECK:       rtl
+  %s = add i8 %x, 1
+  ret i8 %s
+}
+
+declare signext i16 @sext16(i16 signext)
+
+; The caller widens the result itself; nothing reads X after the call.
+define i32 @widen(i16 %x) {
+; CHECK-LABEL: widen:
+; CHECK:       jsl sext16
+; CHECK-NOT:   txa
+; CHECK-NOT:   stx
+; CHECK:       rtl
+  %r = call signext i16 @sext16(i16 signext %x)
+  %w = sext i16 %r to i32
+  ret i32 %w
+}

src/llvm/test/CodeGen/W65816/seprep-ldy-elision-kill-flag.ll

@@ -21,8 +21,8 @@ define void @two_i32_stores_share_y(ptr %p) {
 ; or without the kill-flag fix.  The pin is `-verify-machineinstrs`
 ; not aborting.  Match a minimal shape so the test still has structure.
 ; CHECK:       ldy #0x0
-; CHECK:       sta [0xe0
+; CHECK:       sta [0x{{[0-9a-f]+}}], y
-; CHECK:       sta [0xe0
+; CHECK:       sta [0x{{[0-9a-f]+}}], y
 ; CHECK:       rtl
   store i32 0, ptr %p
   ret void

src/llvm/test/CodeGen/W65816/shift-const-inline.ll

@@ -0,0 +1,98 @@
+; Pin: every constant shift is inline.  i16 SRA by 1 and 2 is the
+; `cmp #$8000 ; ror a` pair (CMP puts the sign in C), by 3..14 a logical
+; shift and then ^ M - M with M the new sign bit (the combiner turns the
+; subtract into `clc ; adc #-M`), by 15 the sign fill.  i32 shifts by
+; 1..15 are i16 shifts of the halves and an OR; the __ashlsi3 / __lshrsi3
+; / __ashrsi3 loops (~26 cycles a bit plus the call) only serve variable
+; counts, and __ashrhi3 likewise for i16.
+;
+; RUN: llc -mtriple=w65816 -O2 < %s | FileCheck %s
+
+target datalayout = "e-m:e-p:32:16-i16:16-i32:16-i64:16-f32:16-f64:16-a:8-n8:16-S16"
+
+define i16 @sra1(i16 %x) {
+; CHECK-LABEL: sra1:
+; CHECK:       cmp #0x8000
+; CHECK-NEXT:  ror a
+; CHECK-NEXT:  rtl
+  %s = ashr i16 %x, 1
+  ret i16 %s
+}
+
+define i16 @sra2(i16 %x) {
+; CHECK-LABEL: sra2:
+; CHECK:       cmp #0x8000
+; CHECK-NEXT:  ror a
+; CHECK-NEXT:  cmp #0x8000
+; CHECK-NEXT:  ror a
+; CHECK-NEXT:  rtl
+  %s = ashr i16 %x, 2
+  ret i16 %s
+}
+
+define i16 @sra3(i16 %x) {
+; CHECK-LABEL: sra3:
+; CHECK:       lsr a
+; CHECK-NEXT:  lsr a
+; CHECK-NEXT:  lsr a
+; CHECK-NEXT:  eor #0x1000
+; CHECK-NEXT:  clc
+; CHECK-NEXT:  adc #0xf000
+; CHECK-NEXT:  rtl
+  %s = ashr i16 %x, 3
+  ret i16 %s
+}
+
+define i16 @sra9(i16 %x) {
+; CHECK-LABEL: sra9:
+; CHECK:       xba
+; CHECK-NEXT:  and #0xff
+; CHECK-NEXT:  lsr a
+; CHECK-NEXT:  eor #0x40
+; CHECK-NEXT:  clc
+; CHECK-NEXT:  adc #0xffc0
+; CHECK-NEXT:  rtl
+  %s = ashr i16 %x, 9
+  ret i16 %s
+}
+
+; A variable count still calls the helper.
+define i16 @srav(i16 %x, i16 %n) {
+; CHECK-LABEL: srav:
+; CHECK:       jsl __ashrhi3
+  %s = ashr i16 %x, %n
+  ret i16 %s
+}
+
+define i32 @sar9(i32 %x) {
+; CHECK-LABEL: sar9:
+; CHECK-NOT:   jsl
+; CHECK:       eor #0x40
+; CHECK-NEXT:  clc
+; CHECK-NEXT:  adc #0xffc0
+; CHECK-NEXT:  tax
+; CHECK:       rtl
+  %s = ashr i32 %x, 9
+  ret i32 %s
+}
+
+define i32 @shl11(i32 %x) {
+; CHECK-LABEL: shl11:
+; CHECK-NOT:   jsl
+; CHECK:       xba
+; CHECK-NEXT:  and #0xff00
+; CHECK-NEXT:  asl a
+; CHECK-NEXT:  asl a
+; CHECK-NEXT:  asl a
+; CHECK:       rtl
+  %s = shl i32 %x, 11
+  ret i32 %s
+}
+
+define i32 @shr7(i32 %x) {
+; CHECK-LABEL: shr7:
+; CHECK-NOT:   jsl
+; CHECK:       rtl
+  %s = lshr i32 %x, 7
+  ret i32 %s
+}

src/llvm/test/CodeGen/W65816/sign-test-flags.mir

@@ -11,6 +11,10 @@
 #     16-bit LDA already set N from bit 15.
 # keep_after_8bit:  an 8-bit op sets N from bit 7, so the `cmp #0` stays.
 #
+# The branch targets take X live in, so the wrap cannot be replaced by the
+# TAX ... CPX #k form that StackSlotCleanup uses when X is free (pinned in
+# cmp-via-x.mir); these cases keep testing the PHP / PLP wrap itself.
+#
 # RUN: llc -mtriple=w65816 -run-pass=w65816-stack-slot-cleanup -o - %s | FileCheck %s
 --- |
   target triple = "w65816"
@@ -39,8 +43,10 @@ body: |
     BMI %bb.1, implicit $p
     BRA %bb.2
   bb.1:
+    liveins: $x
     RTL
   bb.2:
+    liveins: $x
     RTL
 ...
 ---
@@ -62,8 +68,10 @@ body: |
     BPL %bb.1, implicit $p
     BRA %bb.2
   bb.1:
+    liveins: $x
     RTL
   bb.2:
+    liveins: $x
     RTL
 ...
 ---
@@ -86,8 +94,10 @@ body: |
     BMI %bb.1, implicit $p
     BRA %bb.2
   bb.1:
+    liveins: $x
     RTL
   bb.2:
+    liveins: $x
     RTL
 ...
 ---
@@ -106,8 +116,10 @@ body: |
     BMI %bb.1, implicit $p
     BRA %bb.2
   bb.1:
+    liveins: $x
     RTL
   bb.2:
+    liveins: $x
     RTL
 ...
 ---
@@ -125,7 +137,9 @@ body: |
     BMI %bb.1, implicit $p
     BRA %bb.2
   bb.1:
+    liveins: $x
     RTL
   bb.2:
+    liveins: $x
     RTL
 ...

src/llvm/test/CodeGen/W65816/srl8a-flag-corrupt-wrap.mir

@@ -23,6 +23,10 @@
 # The wrapped reload keeps its offset: eliminateFrameIndex adds the byte
 # PHP pushed (php-wrap-frame-index.mir).
 #
+# The branch targets take X live in, so the wrap cannot be replaced by the
+# TAX ... CPX #k form that StackSlotCleanup uses when X is free (pinned in
+# cmp-via-x.mir); these cases keep testing the PHP / PLP wrap itself.
+#
 # RUN: llc -mtriple=w65816 -run-pass=w65816-stack-slot-cleanup -o - %s | FileCheck %s
 --- |
   target triple = "w65816"
@@ -56,8 +60,10 @@ body: |
     BNE %bb.1, implicit $p
     BRA %bb.2
   bb.1:
+    liveins: $x
     RTL
   bb.2:
+    liveins: $x
     RTL
 ...
 ---
@@ -83,8 +89,10 @@ body: |
     BEQ %bb.1, implicit $p
     BRA %bb.2
   bb.1:
+    liveins: $x
     RTL
   bb.2:
+    liveins: $x
     RTL
 ...
 ---
@@ -109,7 +117,9 @@ body: |
     BEQ %bb.1, implicit $p
     BRA %bb.2
   bb.1:
+    liveins: $x
     RTL
   bb.2:
+    liveins: $x
     RTL
 ...

src/llvm/test/CodeGen/W65816/stack-slot-merge-read-between.mir

@@ -0,0 +1,94 @@
+# Pin: W65816StackSlotMerge does not merge a PHI-copy pair when the copy's
+# destination slot is read between the source slot's store and the copy.
+# Merged, the source's store would overwrite the destination early and
+# that read would see the new value.  The shape is strcpy's source walk,
+# `q = p + 1; c = *p; ...; p = q`: slot 9 is q's low word (bumped in
+# place), slot 3 p's, and p is staged into $E0 after q was bumped.  The
+# merge renamed 9 -> 3, so the load read q and every string was copied one
+# byte short (the smoke run's hash-table check).  The copy used to sit
+# inside a PHP/PLP wrap, whose offset bump kept the pass from matching it.
+#
+# keep_read_between: slot 3 is read after STA 9: no rename.
+# merge_no_read:     the same loop without that read still merges.
+#
+# RUN: llc -mtriple=w65816 -run-pass=w65816-stack-slot-merge -o - %s | FileCheck %s
+--- |
+  target triple = "w65816"
+  define void @keep_read_between() { ret void }
+  define void @merge_no_read() { ret void }
+...
+---
+# CHECK-LABEL: name: keep_read_between
+# CHECK:       bb.1:
+# CHECK:       LDA_StackRel 9
+# CHECK-NEXT:  $a = INA_PSEUDO $a
+# CHECK-NEXT:  STA_StackRel 9
+# CHECK:       LDA_StackRel 3
+# CHECK-NEXT:  STA_DP 224
+# CHECK:       LDA_StackRel 9
+# CHECK-NEXT:  STA_StackRel 3
+name: keep_read_between
+tracksRegLiveness: false
+frameInfo:
+  stackSize: 16
+body: |
+  bb.0:
+    successors: %bb.1
+    LDA_StackRel 20, implicit-def $a
+    STA_StackRel 3, implicit $a
+    STA_StackRel 9, implicit $a
+
+  bb.1:
+    successors: %bb.1, %bb.2
+    LDA_StackRel 9, implicit-def $a
+    $a = INA_PSEUDO $a
+    STA_StackRel 9, implicit $a
+    LDA_StackRel 3, implicit-def $a
+    STA_DP 224, implicit $a
+    LDA_StackRel 5, implicit-def $a
+    STA_DP 226, implicit $a
+    LDY_Imm16 0, implicit-def $y
+    LDA_DPIndLongY 224, implicit-def $a, implicit $y
+    STA_StackRel 15, implicit $a
+    LDA_StackRel 9, implicit-def $a
+    STA_StackRel 3, implicit $a
+    LDA_StackRel 15, implicit-def $a
+    BNE %bb.1, implicit $p
+
+  bb.2:
+    RTL
+...
+---
+# CHECK-LABEL: name: merge_no_read
+# CHECK:       bb.1:
+# CHECK:       LDA_StackRel 3
+# CHECK-NEXT:  $a = INA_PSEUDO $a
+# CHECK-NEXT:  STA_StackRel 3
+# CHECK-NOT:   StackRel 9
+# CHECK:       RTL
+name: merge_no_read
+tracksRegLiveness: false
+frameInfo:
+  stackSize: 16
+body: |
+  bb.0:
+    successors: %bb.1
+    LDA_StackRel 20, implicit-def $a
+    STA_StackRel 3, implicit $a
+    STA_StackRel 9, implicit $a
+
+  bb.1:
+    successors: %bb.1, %bb.2
+    LDA_StackRel 9, implicit-def $a
+    $a = INA_PSEUDO $a
+    STA_StackRel 9, implicit $a
+    LDA_StackRel 5, implicit-def $a
+    STA_DP 226, implicit $a
+    LDA_StackRel 9, implicit-def $a
+    STA_StackRel 3, implicit $a
+    LDA_StackRel 15, implicit-def $a
+    BNE %bb.1, implicit $p
+
+  bb.2:
+    RTL
+...

src/llvm/test/CodeGen/W65816/store-i32-global.ll

@@ -0,0 +1,51 @@
+; Pin: an i32 store to a global is two abs-16 stores (`sta g`, `sta g+2`),
+; as an i32 load from one is two abs-16 loads.  It went through the
+; far-pointer ST_PTR path instead: `#g` and the program bank from $BE
+; staged into $E0 / $E2 before each half, about 12x the cycles.  A field
+; or element (`add global, K`) folds the same way, loads included (the
+; high half of an i32 load used to take the far path when the combiner
+; had not yet folded its +2 into the global).
+;
+; RUN: llc -mtriple=w65816 -O2 < %s | FileCheck %s
+
+target datalayout = "e-m:e-p:32:16-i16:16-i32:16-i64:16-f32:16-f64:16-a:8-n8:16-S16"
+
+@gCount = global i32 0
+@gPair = global [2 x i32] zeroinitializer
+
+define void @setCount(i32 %v) {
+; CHECK-LABEL: setCount:
+; CHECK-NOT:   0xe0
+; CHECK:       sta gCount
+; CHECK-NEXT:  lda 0xd0
+; CHECK-NEXT:  sta gCount+2
+; CHECK-NEXT:  rtl
+  store i32 %v, ptr @gCount
+  ret void
+}
+
+define void @setSecond(i32 %v) {
+; CHECK-LABEL: setSecond:
+; CHECK-NOT:   0xe0
+; CHECK:       sta gPair+4
+; CHECK-NEXT:  lda 0xd0
+; CHECK-NEXT:  sta gPair+6
+; CHECK-NEXT:  rtl
+  %p = getelementptr inbounds [2 x i32], ptr @gPair, i32 0, i32 1
+  store i32 %v, ptr %p
+  ret void
+}
+
+define void @bump() {
+; CHECK-LABEL: bump:
+; CHECK-NOT:   0xe0
+; CHECK:       lda gCount+2
+; CHECK:       lda gCount
+; CHECK:       sta gCount
+; CHECK:       sta gCount+2
+; CHECK:       rtl
+  %v = load i32, ptr @gCount
+  %w = add i32 %v, 1
+  store i32 %w, ptr @gCount
+  ret void
+}

tests/coremark/coreMark.manifest.json

@@ -7,8 +7,8 @@
       "num": 1,
       "name": "SEG1",
       "base": "0x001000",
-      "size": 40728,
+      "size": 44791,
-      "bss_start": "0x00b700",
+      "bss_start": "0x00d800",
       "bss_size": 2034,
       "image": "coreMark.bin",
       "entry_offset": "0x0000"
@@ -17,7 +17,7 @@
       "num": 2,
       "name": "SEG2",
       "base": "0x040000",
-      "size": 16480,
+      "size": 10368,
       "bss_start": "0x000000",
       "bss_size": 0,
       "image": "coreMark.seg2.bin",

tests/fpNan/genFpTables.py

@@ -0,0 +1,589 @@
+#!/usr/bin/env python3
+# genFpTables.py - expected-value tables for the soft-float smoke checks in
+# scripts/smokeTest.sh ("float/double compares" and "float/double NaN and
+# infinity").  Paste the output over the matching table in smokeTest.sh.
+#
+#   python3 tests/fpNan/genFpTables.py cmp       # kFBits / kDBits / kExp
+#   python3 tests/fpNan/genFpTables.py special   # kCases
+#
+# Comparisons come from Python's own float compares, which are IEEE 754
+# (every ordered compare with a NaN is false).  Float operands are widened
+# to double exactly, so float and double share one truth table per value.
+#
+# The special-value table is a model of the documented runtime rules, not
+# of the host FPU: a NaN operand comes back quieted with its payload kept
+# (the first NaN operand when both are), an invalid operation gives the
+# default NaN 0x7FC00000 / 0x7FF8000000000000 (x86 would give the negative
+# one), and float/double to integer saturates with NaN converting to 0.
+# Finite cases use operands whose results are exact, because softFloat.c
+# truncates instead of rounding.
+#
+# The math.c cases follow C99 Annex F's special values (modelled here where
+# Python raises instead of answering).  The series-based functions (exp,
+# pow with a non-integer exponent, sinh, cbrt, ...) are only ~1e-6
+# accurate, so their ordinary-value cases carry an `approx` flag: the check
+# accepts a result within 1e-6 relative of the host's value.
+
+import math
+import struct
+import sys
+from fractions import Fraction
+
+F_QUIET = 0x00400000
+D_QUIET = 0x0008000000000000
+F_DNAN = 0x7FC00000
+D_DNAN = 0x7FF8000000000000
+F_INF = 0x7F800000
+D_INF = 0x7FF0000000000000
+F_SIGN = 0x80000000
+D_SIGN = 0x8000000000000000
+
+# The compare operand set: signed infinities and zeros, normals of both
+# signs, a quiet NaN, a negative quiet NaN and a signaling-pattern NaN.
+CMP_F = [0xFF800000, 0xC0200000, 0xBF800000, 0x80000000, 0x00000000,
+         0x3F800000, 0x41200000, 0x7149F2CA, 0x7F800000, 0x7FC00000,
+         0xFFC00000, 0x7F800001]
+# The same values as doubles (1e30f widened exactly, so both tables share
+# one truth table).
+CMP_D = [0xFFF0000000000000, 0xC004000000000000, 0xBFF0000000000000,
+         0x8000000000000000, 0x0000000000000000, 0x3FF0000000000000,
+         0x4024000000000000, 0x46293E5940000000, 0x7FF0000000000000,
+         0x7FF8000000000000, 0xFFF8000000000000, 0x7FF0000000000001]
+
+# Predicate order matches ALL() in the check: the six C operators, their
+# negations (the unordered LLVM predicates uge/ugt/ule/ult), then the C99
+# isunordered / !isunordered / islessgreater / !islessgreater.
+PREDS = [
+    lambda a, b: a < b,
+    lambda a, b: a <= b,
+    lambda a, b: a > b,
+    lambda a, b: a >= b,
+    lambda a, b: a == b,
+    lambda a, b: a != b,
+    lambda a, b: not (a < b),
+    lambda a, b: not (a <= b),
+    lambda a, b: not (a > b),
+    lambda a, b: not (a >= b),
+    lambda a, b: math.isnan(a) or math.isnan(b),
+    lambda a, b: not (math.isnan(a) or math.isnan(b)),
+    lambda a, b: a < b or a > b,
+    lambda a, b: not (a < b or a > b),
+]
+
+
+def fval(bits):
+    return struct.unpack('<f', struct.pack('<I', bits))[0]
+
+
+def dval(bits):
+    return struct.unpack('<d', struct.pack('<Q', bits))[0]
+
+
+def fbits(v):
+    return struct.unpack('<I', struct.pack('<f', v))[0]
+
+
+def dbits(v):
+    return struct.unpack('<Q', struct.pack('<d', v))[0]
+
+
+def emitCmp():
+    for f, d in zip(CMP_F, CMP_D):
+        assert (math.isnan(fval(f)) and math.isnan(dval(d))) or fval(f) == dval(d)
+    bits = []
+    for vals in ([fval(b) for b in CMP_F], [dval(b) for b in CMP_D]):
+        for p in PREDS:
+            for a in vals:
+                for b in vals:
+                    bits.append(1 if p(a, b) else 0)
+    out = []
+    for i in range(0, len(bits), 8):
+        byte = 0
+        for k, bit in enumerate(bits[i:i + 8]):
+            byte |= bit << k
+        out.append(byte)
+    print('#define NV %d' % len(CMP_F))
+    print('#define NP %d' % len(PREDS))
+    print('static const u32 kFBits[NV] = {')
+    print('    ' + ', '.join('0x%08XUL' % b for b in CMP_F))
+    print('};')
+    print('static const u64 kDBits[NV] = {')
+    for i in range(0, len(CMP_D), 3):
+        print('    ' + ', '.join('0x%016XULL' % b for b in CMP_D[i:i + 3]) + ',')
+    print('};')
+    print('static const u8 kExp[%d] = {' % len(out))
+    for i in range(0, len(out), 16):
+        print('    ' + ', '.join('0x%02X' % b for b in out[i:i + 16]) + ',')
+    print('};')
+
+
+# ---- Special-value model ------------------------------------------------
+
+def fclass(b):
+    e = (b >> 23) & 0xFF
+    m = b & 0x7FFFFF
+    if e == 0:
+        return 'zero'
+    if e == 0xFF:
+        return 'inf' if m == 0 else 'nan'
+    return 'normal'
+
+
+def dclass(b):
+    e = (b >> 52) & 0x7FF
+    m = b & 0xFFFFFFFFFFFFF
+    if e == 0:
+        return 'zero'
+    if e == 0x7FF:
+        return 'inf' if m == 0 else 'nan'
+    return 'normal'
+
+
+def binop(isD, op, a, b):
+    cls = dclass if isD else fclass
+    quiet, dnan, inf, sign = ((D_QUIET, D_DNAN, D_INF, D_SIGN) if isD else
+                              (F_QUIET, F_DNAN, F_INF, F_SIGN))
+    val, tobits = (dval, dbits) if isD else (fval, fbits)
+    if op == 'sub':
+        # __subXf3(a, b) is __addXf3(a, b with its sign flipped), as in
+        # compiler-rt, so a NaN b comes back with its sign flipped too.
+        b ^= sign
+        op = 'add'
+    ca, cb = cls(a), cls(b)
+    if ca == 'nan':
+        return a | quiet
+    if cb == 'nan':
+        return b | quiet
+    s = (a ^ b) & sign
+    if op == 'add':
+        if ca == 'inf':
+            return dnan if cb == 'inf' and s else a
+        if cb == 'inf':
+            return b
+        if ca == 'zero' and cb == 'zero':
+            return a & b & sign
+    elif op == 'mul':
+        if ca == 'inf' or cb == 'inf':
+            return dnan if 'zero' in (ca, cb) else s | inf
+        if ca == 'zero' or cb == 'zero':
+            return s
+    elif op == 'div':
+        if ca == 'inf':
+            return dnan if cb == 'inf' else s | inf
+        if cb == 'inf':
+            return s
+        if cb == 'zero':
+            return dnan if ca == 'zero' else s | inf
+        if ca == 'zero':
+            return s
+    x, y = val(a), val(b)
+    if op == 'add':
+        r = x + y
+    elif op == 'mul':
+        r = x * y
+    else:
+        r = x / y
+    if not isD and abs(r) > 3.4028234663852886e38:
+        r = math.copysign(math.inf, r)
+    got = tobits(r)
+    assert val(got) == r or math.isinf(r), (op, hex(a), hex(b), r)
+    return got
+
+
+def f2d(a):
+    if fclass(a) == 'nan':
+        return ((a & F_SIGN) << 32) | D_INF | ((a & 0x7FFFFF) << 29) | D_QUIET
+    return dbits(fval(a))
+
+
+def d2f(a):
+    if dclass(a) == 'nan':
+        return ((a >> 32) & F_SIGN) | F_DNAN | (((a & 0xFFFFFFFFFFFFF) >> 29) & 0x7FFFFF)
+    v = dval(a)
+    if abs(v) > 3.4028234663852886e38:
+        return fbits(math.copysign(math.inf, v))
+    got = fbits(v)
+    assert fval(got) == v
+    return got
+
+
+def toInt(v, unsigned):
+    if math.isnan(v):
+        return 0
+    lo, hi = (0, 0xFFFFFFFF) if unsigned else (-0x80000000, 0x7FFFFFFF)
+    if math.isinf(v):
+        r = hi if v > 0 else lo
+    else:
+        r = max(lo, min(hi, int(v)))
+    return r & 0xFFFFFFFF
+
+
+def classify(v, bitsSign):
+    nan = 1 if math.isnan(v) else 0
+    inf = 1 if math.isinf(v) else 0
+    fin = 1 if not (nan or inf) else 0
+    cls = 0 if nan else 1 if inf else 4 if v == 0 else 2
+    return nan | (inf << 1) | (fin << 2) | (bitsSign << 3) | (cls << 4)
+
+
+# Operation codes, matching the switch in the check's run().
+OPS = ['fadd', 'fsub', 'fmul', 'fdiv', 'dadd', 'dsub', 'dmul', 'ddiv',
+       'f2d', 'd2f', 'f2i', 'f2u', 'd2i', 'd2u', 'fcls', 'dcls',
+       'sqrt', 'log', 'asin', 'acos', 'atanh',
+       'floor', 'ceil', 'trunc', 'round', 'rint', 'nearbyint',
+       'floorf', 'ceilf', 'truncf', 'roundf', 'rintf',
+       'exp', 'exp2', 'expm1', 'sinh', 'cosh', 'tanh', 'log1p', 'cbrt',
+       'pow', 'fmod', 'fdim', 'hypot', 'remainder',
+       'nearbyintf', 'fmaxf', 'fminf']
+
+FQ = 0x7FC00123      # quiet NaN with a payload
+FS = 0x7F800005      # signaling-pattern NaN
+FNQ = 0xFFC00000     # negative quiet NaN
+DQ = 0x7FF8000000000123
+DS = 0x7FF0000000000005
+DNQ = 0xFFF8000000000000
+F1, FM1, F15, F2, F225, FM2, F3E38 = (fbits(1.0), fbits(-1.0), fbits(1.5),
+                                      fbits(2.0), fbits(2.25), fbits(-2.0),
+                                      fbits(3e38))
+D1, DM1, D15, D2, D225, DM2, D1E300 = (dbits(1.0), dbits(-1.0), dbits(1.5),
+                                       dbits(2.0), dbits(2.25), dbits(-2.0),
+                                       dbits(1e300))
+FZ, FNZ, FI, FNI = 0, F_SIGN, F_INF, F_INF | F_SIGN
+DZ, DNZ, DI, DNI = 0, D_SIGN, D_INF, D_INF | D_SIGN
+
+
+def fpairs():
+    return [(FQ, F1), (F1, FS), (FS, FQ), (FNQ, F2), (FZ, FQ), (FI, F1),
+            (FI, FI), (FI, FNI), (FNI, F2), (F2, FNI), (F1, FM1), (FM1, F1),
+            (FNZ, FNZ), (FNZ, FZ), (FZ, FNZ), (FZ, FZ), (F225, F15),
+            (F3E38, F3E38), (FZ, FI), (FI, FZ), (FNI, FM2), (FNI, FNI),
+            (FNZ, F2), (FZ, FZ), (F1, FZ), (FM1, FZ), (F1, FNZ), (F1, FI),
+            (FM1, FI), (FI, F2), (F15, F2)]
+
+
+def dpairs():
+    return [(DQ, D1), (D1, DS), (DS, DQ), (DNQ, D2), (DZ, DQ), (DI, D1),
+            (DI, DI), (DI, DNI), (DNI, D2), (D2, DNI), (D1, DM1), (DM1, D1),
+            (DNZ, DNZ), (DNZ, DZ), (DZ, DNZ), (DZ, DZ), (D225, D15),
+            (D1E300, D1E300), (DZ, DI), (DI, DZ), (DNI, DM2), (DNI, DNI),
+            (DNZ, D2), (DZ, DZ), (D1, DZ), (DM1, DZ), (D1, DNZ), (D1, DI),
+            (DM1, DI), (DI, D2), (D15, D2)]
+
+
+def emitSpecial():
+    cases = []
+    for isD, prefix, pairs in ((False, 'f', fpairs()), (True, 'd', dpairs())):
+        for op in ('add', 'sub', 'mul', 'div'):
+            for a, b in pairs:
+                cases.append((OPS.index(prefix + op), a, b, binop(isD, op, a, b)))
+    for a in (FQ, FS, 0xFFC00001, FI, FNI, FNZ, FZ, F15, FM2):
+        cases.append((OPS.index('f2d'), a, 0, f2d(a)))
+    for a in (DQ, 0xFFF8000020000000, DS, DI, DNI, DNZ, DZ, D15, DM2, D1E300,
+              dbits(-1e300)):
+        cases.append((OPS.index('d2f'), a, 0, d2f(a)))
+    fconv = (FQ, FNQ, FS, FI, FNI, fbits(3e9), fbits(-3e9), fbits(5e9),
+             fbits(-2.5), fbits(3.5), FM1, FNZ)
+    dconv = (DQ, DNQ, DS, DI, DNI, dbits(3e9), dbits(-3e9), dbits(5e9),
+             dbits(-2.5), dbits(3.5), DM1, DNZ)
+    for a in fconv:
+        cases.append((OPS.index('f2i'), a, 0, toInt(fval(a), False)))
+        cases.append((OPS.index('f2u'), a, 0, toInt(fval(a), True)))
+    for a in dconv:
+        cases.append((OPS.index('d2i'), a, 0, toInt(dval(a), False)))
+        cases.append((OPS.index('d2u'), a, 0, toInt(dval(a), True)))
+    for a in (FQ, FNQ, FS, FI, FNI, FZ, FNZ, F15, FM2):
+        cases.append((OPS.index('fcls'), a, 0, classify(fval(a), a >> 31)))
+    for a in (DQ, DNQ, DS, DI, DNI, DZ, DNZ, D15, DM2):
+        cases.append((OPS.index('dcls'), a, 0, classify(dval(a), a >> 63)))
+    # math.c's domain edges.  The check folds every NaN result to the
+    # default NaN, because how a NaN travels through a series (and so its
+    # sign) is not part of these functions' contract.
+    halfPi = dbits(math.pi / 2)
+    mathCases = [
+        ('sqrt', dbits(-1.0), D_DNAN), ('sqrt', DI, DI), ('sqrt', DNI, D_DNAN),
+        ('sqrt', DQ, D_DNAN), ('sqrt', dbits(4.0), dbits(2.0)),
+        ('sqrt', DNZ, DNZ),
+        ('log', DI, DI), ('log', DNI, D_DNAN), ('log', DQ, D_DNAN),
+        ('log', DM1, D_DNAN), ('log', DZ, DNI), ('log', D1, DZ),
+        ('asin', D2, D_DNAN), ('asin', DM2, D_DNAN), ('asin', D1, halfPi),
+        ('asin', DM1, halfPi | D_SIGN), ('asin', DQ, D_DNAN),
+        ('acos', D2, D_DNAN), ('acos', D1, DZ),
+        ('atanh', D2, D_DNAN), ('atanh', DM2, D_DNAN), ('atanh', D1, DI),
+        ('atanh', DM1, DNI), ('atanh', DQ, D_DNAN),
+    ]
+    cases = [c + (0,) for c in cases]
+    for name, a, want in mathCases:
+        cases.append((OPS.index(name), a, 0, want, 0))
+    # Two tables, one per build of the check: both together (with the math
+    # functions they pull in) overflow a bank-0 test binary.
+    for part, table in ((1, cases), (2, annexFCases())):
+        print('#if FP_TABLE == 1' if part == 1 else '#else')
+        print('#define NC %d' % len(table))
+        print('static const CaseT kCases[NC] = {')
+        for op, a, b, want, approx in table:
+            print('    { %-12s 0x%016XULL, 0x%016XULL, 0x%016XULL, %d },' %
+                  ('O_' + OPS[op].upper() + ',', a, b, want, approx))
+        print('};')
+    print('#endif')
+
+
+# ---- math.c Annex F cases ----------------------------------------------
+
+INF = math.inf
+NAN = math.nan
+
+
+def signedZero(r, x):
+    # An integral result of zero keeps x's sign (floor(-0.0) is -0.0,
+    # ceil(-0.5) is -0.0).
+    return math.copysign(0.0, x) if r == 0 else r
+
+
+def cfloor(x):
+    return x if not math.isfinite(x) else signedZero(float(math.floor(x)), x)
+
+
+def cceil(x):
+    return x if not math.isfinite(x) else signedZero(float(math.ceil(x)), x)
+
+
+def ctrunc(x):
+    return x if not math.isfinite(x) else signedZero(float(math.trunc(x)), x)
+
+
+def cround(x):
+    # C round: half away from zero, computed exactly.
+    if not math.isfinite(x):
+        return x
+    r = float(math.floor(Fraction(abs(x)) + Fraction(1, 2)))
+    return math.copysign(r, x)
+
+
+def crint(x):
+    # Round to nearest, ties to even (Python's round() on a float is exact).
+    if not math.isfinite(x):
+        return x
+    return math.copysign(float(round(x)), x)
+
+
+def guard(f, *args):
+    # Python raises where C returns: a domain error is NaN, an overflow is
+    # an infinity of the true result's sign.
+    try:
+        return f(*args)
+    except ValueError:
+        return NAN
+    except OverflowError:
+        return INF
+
+
+def isOddInt(y):
+    return math.isfinite(y) and y == math.floor(y) and abs(y) < 2.0 ** 53 and int(y) % 2 == 1
+
+
+def cpow(x, y):
+    # C99 F.9.4.4.
+    if y == 0 or x == 1.0:
+        return 1.0
+    if math.isnan(x) or math.isnan(y):
+        return NAN
+    if math.isinf(y):
+        if abs(x) == 1:
+            return 1.0
+        return INF if (abs(x) > 1) == (y > 0) else 0.0
+    if x == 0:
+        r = INF if y < 0 else 0.0
+        return math.copysign(r, x) if isOddInt(y) else r
+    if math.isinf(x):
+        r = INF if y > 0 else 0.0
+        return -r if (x < 0 and isOddInt(y)) else r
+    if x < 0 and y != math.floor(y):
+        return NAN
+    r = guard(math.pow, x, y)
+    # guard() cannot know an overflow's sign: a negative x to an odd power.
+    return -r if (math.isinf(r) and x < 0 and isOddInt(y)) else r
+
+
+def cfmod(x, y):
+    # C99 F.9.7.1.
+    if math.isnan(x) or math.isnan(y) or math.isinf(x) or y == 0:
+        return NAN
+    if math.isinf(y):
+        return x
+    return math.fmod(x, y)
+
+
+def cfdim(x, y):
+    if math.isnan(x) or math.isnan(y):
+        return NAN
+    return x - y if x > y else 0.0
+
+
+def cremainder(x, y):
+    # C99 F.9.7.2 (Python's math.remainder raises for the NaN cases).
+    if math.isnan(x) or math.isnan(y) or math.isinf(x) or y == 0:
+        return NAN
+    return math.remainder(x, y)
+
+
+def ccbrt(x):
+    if not math.isfinite(x) or x == 0:
+        return x
+    return math.copysign(abs(x) ** (1.0 / 3.0), x)
+
+
+def fround32(v):
+    return fval(fbits(v))
+
+
+MATH1 = {
+    'floor': cfloor, 'ceil': cceil, 'trunc': ctrunc, 'round': cround,
+    'rint': crint, 'nearbyint': crint,
+    'exp': lambda x: guard(math.exp, x),
+    'exp2': lambda x: guard(math.pow, 2.0, x),
+    'expm1': lambda x: guard(math.expm1, x),
+    'sinh': lambda x: guard(math.sinh, x) if not (math.isinf(x) or abs(x) > 700) else math.copysign(INF, x),
+    'cosh': lambda x: guard(math.cosh, x),
+    'tanh': math.tanh,
+    'log1p': lambda x: -INF if x == -1 else guard(math.log1p, x),
+    'cbrt': ccbrt,
+}
+MATH2 = {'pow': cpow, 'fmod': cfmod, 'fdim': cfdim, 'hypot': math.hypot,
+         'remainder': cremainder}
+FLOATOPS = {'floorf': cfloor, 'ceilf': cceil, 'truncf': ctrunc,
+            'roundf': cround, 'rintf': crint, 'nearbyintf': crint}
+
+
+def cfmax(x, y):
+    # C99 F.9.9.2: a NaN operand is ignored in favour of the other.
+    if math.isnan(x):
+        return y
+    if math.isnan(y):
+        return x
+    return max(x, y)
+
+
+def cfmin(x, y):
+    if math.isnan(x):
+        return y
+    if math.isnan(y):
+        return x
+    return min(x, y)
+
+
+FLOAT2 = {'fmaxf': cfmax, 'fminf': cfmin}
+
+# (op, x[, y], approx).  Every function gets its special values and a few
+# ordinary ones, so a normal-path regression shows too.
+MATH_CASES = [
+    ('floor', 2.7, 0), ('floor', -2.7, 0), ('floor', -0.0, 0), ('floor', 0.0, 0),
+    ('floor', -0.3, 0), ('floor', 0.3, 0), ('floor', INF, 0), ('floor', -INF, 0),
+    ('floor', NAN, 0), ('floor', 1e300, 0), ('floor', -0.5, 0),
+    ('ceil', 2.3, 0), ('ceil', -2.3, 0), ('ceil', 0.0, 0), ('ceil', -0.0, 0),
+    ('ceil', -0.5, 0), ('ceil', 0.5, 0), ('ceil', INF, 0), ('ceil', NAN, 0),
+    ('trunc', -2.7, 0), ('trunc', 2.7, 0), ('trunc', -0.5, 0), ('trunc', -0.0, 0),
+    ('trunc', INF, 0),
+    ('round', 2.5, 0), ('round', -2.5, 0), ('round', 0.49999999999999994, 0),
+    ('round', -0.49999999999999994, 0), ('round', 4503599627370497.0, 0),
+    ('round', -0.0, 0), ('round', 0.5, 0), ('round', 1.5, 0), ('round', INF, 0),
+    ('round', NAN, 0), ('round', -0.4, 0),
+    ('rint', 2.5, 0), ('rint', 3.5, 0), ('rint', -2.5, 0), ('rint', -0.3, 0),
+    ('rint', 0.3, 0), ('rint', 1.5, 0), ('rint', 4503599627370497.0, 0),
+    ('rint', INF, 0), ('rint', NAN, 0), ('rint', -0.0, 0),
+    ('nearbyint', 2.5, 0), ('nearbyint', -3.5, 0),
+    ('floorf', -0.0, 0), ('floorf', 2.75, 0), ('floorf', -2.75, 0),
+    ('ceilf', 0.0, 0), ('ceilf', -0.5, 0), ('truncf', -2.5, 0),
+    ('roundf', 2.5, 0), ('roundf', -0.0, 0),
+    ('rintf', 2.5, 0), ('rintf', 3.5, 0), ('rintf', -0.25, 0),
+    ('nearbyintf', 2.5, 0), ('nearbyintf', -1.5, 0),
+    ('fmaxf', 1.0, NAN, 0), ('fmaxf', NAN, -2.0, 0), ('fmaxf', -0.5, 2.0, 0),
+    ('fminf', 1.0, NAN, 0), ('fminf', -0.5, 2.0, 0), ('fminf', 3.0, -INF, 0),
+    ('exp', NAN, 0), ('exp', INF, 0), ('exp', -INF, 0), ('exp', 1e5, 0),
+    ('exp', -1e5, 0), ('exp', 1024.0, 0), ('exp', -1024.0, 0), ('exp', 800.0, 0),
+    ('exp', -800.0, 0), ('exp', 0.0, 0), ('exp', -0.0, 0), ('exp', 1.5, 1),
+    ('exp', -2.0, 1),
+    ('exp2', 3.0, 1), ('exp2', NAN, 0), ('exp2', INF, 0), ('exp2', -INF, 0),
+    ('expm1', -INF, 0), ('expm1', INF, 0), ('expm1', NAN, 0), ('expm1', -0.0, 0),
+    ('expm1', 0.5, 1),
+    ('sinh', INF, 0), ('sinh', -INF, 0), ('sinh', NAN, 0), ('sinh', -0.0, 0),
+    ('sinh', 800.0, 0), ('sinh', -800.0, 0), ('sinh', 1.0, 1), ('sinh', -1.0, 1),
+    ('cosh', INF, 0), ('cosh', -INF, 0), ('cosh', NAN, 0), ('cosh', 0.0, 0),
+    ('cosh', 1.0, 1),
+    ('tanh', INF, 0), ('tanh', -INF, 0), ('tanh', NAN, 0), ('tanh', -0.0, 0),
+    ('tanh', 40.0, 0), ('tanh', -40.0, 0), ('tanh', 400.0, 0), ('tanh', 0.5, 1),
+    ('tanh', -0.5, 1),
+    ('log1p', -0.0, 0), ('log1p', 0.0, 0), ('log1p', -1.0, 0), ('log1p', INF, 0),
+    ('log1p', NAN, 0), ('log1p', 1.0, 1),
+    ('cbrt', INF, 0), ('cbrt', -INF, 0), ('cbrt', NAN, 0), ('cbrt', -0.0, 0),
+    ('cbrt', 27.0, 1),
+    ('pow', NAN, 0.0, 0), ('pow', NAN, -0.0, 0), ('pow', 1.0, NAN, 0),
+    ('pow', 1.0, INF, 0), ('pow', -1.0, INF, 0), ('pow', -1.0, -INF, 0),
+    ('pow', 2.0, NAN, 0), ('pow', NAN, 2.0, 0), ('pow', 0.0, -3.0, 0),
+    ('pow', -0.0, -3.0, 0), ('pow', -0.0, -2.0, 0), ('pow', -0.0, -INF, 0),
+    ('pow', 0.0, -0.5, 0), ('pow', -0.0, 3.0, 0), ('pow', -0.0, 2.0, 0),
+    ('pow', 0.0, 0.5, 0), ('pow', 0.5, -INF, 0), ('pow', 2.0, -INF, 0),
+    ('pow', 0.5, INF, 0), ('pow', 2.0, INF, 0), ('pow', -INF, -3.0, 0),
+    ('pow', -INF, -2.0, 0), ('pow', -INF, 3.0, 0), ('pow', -INF, 2.0, 0),
+    ('pow', -INF, 0.5, 0), ('pow', INF, -1.0, 0), ('pow', INF, 2.0, 0),
+    ('pow', -8.0, 1.0 / 3.0, 0), ('pow', -2.0, 0.5, 0), ('pow', 2.0, 10.0, 0),
+    ('pow', 2.0, -3.0, 0), ('pow', -2.0, 3.0, 0), ('pow', -2.0, 2.0, 0),
+    ('pow', 10.0, 20.0, 0), ('pow', -2.0, 1e20, 0), ('pow', -2.0, 40001.0, 0),
+    ('pow', 2.0, 0.5, 1), ('pow', 2.0, 1.5, 1), ('pow', 10.0, 0.3, 1),
+    ('pow', 1.0000001, 100000.0, 1),
+    ('fmod', 7.5, 2.0, 0), ('fmod', -7.5, 2.0, 0), ('fmod', -4.0, 2.0, 0),
+    ('fmod', 4.0, 2.0, 0), ('fmod', 5.0, 3.0, 0), ('fmod', 1.0, INF, 0),
+    ('fmod', -1.0, -INF, 0), ('fmod', 1.0, 0.0, 0), ('fmod', 1.0, -0.0, 0),
+    ('fmod', INF, 2.0, 0), ('fmod', -INF, 2.0, 0), ('fmod', NAN, 2.0, 0),
+    ('fmod', 2.0, NAN, 0), ('fmod', 0.0, 5.0, 0), ('fmod', -0.0, 5.0, 0),
+    ('fmod', 0.0, 0.0, 0), ('fmod', INF, INF, 0),
+    ('fdim', NAN, 1.0, 0), ('fdim', 1.0, NAN, 0), ('fdim', 5.0, 3.0, 0),
+    ('fdim', 3.0, 5.0, 0), ('fdim', INF, 1.0, 0), ('fdim', 1.0, INF, 0),
+    ('fdim', INF, INF, 0),
+    ('hypot', INF, NAN, 0), ('hypot', NAN, -INF, 0), ('hypot', -INF, 1.0, 0),
+    ('hypot', NAN, 1.0, 0), ('hypot', 3.0, 4.0, 1), ('hypot', 0.0, 0.0, 0),
+    ('remainder', 5.0, 2.0, 0), ('remainder', 7.0, 2.0, 0),
+    ('remainder', 7.0, 4.0, 0), ('remainder', 1.0, INF, 0),
+    ('remainder', -3.0, -INF, 0), ('remainder', 1.0, 0.0, 0),
+    ('remainder', INF, 2.0, 0), ('remainder', NAN, 1.0, 0),
+]
+
+
+def nanFold(bits, isF):
+    # The check folds every NaN a math function returns to the default NaN.
+    v = fval(bits) if isF else dval(bits)
+    if math.isnan(v):
+        return F_DNAN if isF else D_DNAN
+    return bits
+
+
+def annexFCases():
+    out = []
+    for c in MATH_CASES:
+        name = c[0]
+        if name in FLOAT2:
+            x, y, approx = fround32(c[1]), fround32(c[2]), c[3]
+            want = fround32(FLOAT2[name](x, y))
+            out.append((OPS.index(name), fbits(x), fbits(y), nanFold(fbits(want), True), approx))
+            continue
+        if name in MATH2:
+            x, y, approx = c[1], c[2], c[3]
+            want = MATH2[name](x, y)
+            a, b = dbits(x), dbits(y)
+        else:
+            x, approx = c[1], c[2]
+            y = 0.0
+            if name in FLOATOPS:
+                x = fround32(x)
+                want = fround32(FLOATOPS[name](x))
+                out.append((OPS.index(name), fbits(x), 0, nanFold(fbits(want), True), approx))
+                continue
+            want = MATH1[name](x)
+            a, b = dbits(x), 0
+        out.append((OPS.index(name), a, b, nanFold(dbits(want), False), approx))
+    return out
+
+
+if __name__ == '__main__':
+    if len(sys.argv) != 2 or sys.argv[1] not in ('cmp', 'special'):
+        sys.exit('usage: genFpTables.py cmp|special')
+    emitCmp() if sys.argv[1] == 'cmp' else emitSpecial()

tests/lua/luaTest.manifest.json

@@ -7,8 +7,8 @@
       "num": 1,
       "name": "SEG1",
       "base": "0x001000",
-      "size": 43790,
+      "size": 44531,
-      "bss_start": "0x00e200",
+      "bss_start": "0x00e300",
       "bss_size": 4248,
       "image": "luaTest.bin",
       "entry_offset": "0x0000"
@@ -17,7 +17,7 @@
       "num": 2,
       "name": "SEG2",
       "base": "0x040000",
-      "size": 43204,
+      "size": 44940,
       "bss_start": "0x000000",
       "bss_size": 0,
       "image": "luaTest.seg2.bin",
@@ -27,7 +27,7 @@
       "num": 3,
       "name": "SEG3",
       "base": "0x050000",
-      "size": 41056,
+      "size": 41839,
       "bss_start": "0x000000",
       "bss_size": 0,
       "image": "luaTest.seg3.bin",
@@ -37,7 +37,7 @@
       "num": 4,
       "name": "SEG4",
       "base": "0x060000",
-      "size": 28810,
+      "size": 28814,
       "bss_start": "0x000000",
       "bss_size": 0,
       "image": "luaTest.seg4.bin",

tests/regress/genShiftDec.py

@@ -0,0 +1,160 @@
+#!/usr/bin/env python3
+# Emits the C source of the smoke check "constant shifts and i32 +/- 1",
+# with kExp holding the expected results computed here.
+V32 = [0x00000000, 0xFFFFFFFF, 0x80000000, 0x7FFFFFFF, 0x12345678, 0xFEDCBA98, 0x00018001, 0x8000FFFF]
+V16 = [0x0000, 0xFFFF, 0x8000, 0x7FFF, 0x1234, 0xFEDC, 0x8001, 0x4000]
+# (start, count) for the down-counting loops: each decrements a u32 `count`
+# times from `start` and returns the final value's halves and a checksum.
+DOWN = [(0x00010002, 5), (0x00000002, 4), (0x00020000, 1), (0x80000001, 3), (0x00000000, 1), (0x00030001, 2)]
+UP = [(0x0000FFFE, 5), (0xFFFFFFFE, 4), (0x0001FFFF, 1), (0x7FFFFFFF, 2)]
+
+def s32(x):
+    x &= 0xFFFFFFFF
+    return x - (1 << 32) if x & 0x80000000 else x
+
+def s16(x):
+    x &= 0xFFFF
+    return x - (1 << 16) if x & 0x8000 else x
+
+exp = []
+for op in ("shl", "shr", "sar"):
+    for n in range(1, 32):
+        for v in V32:
+            if op == "shl":
+                r = (v << n) & 0xFFFFFFFF
+            elif op == "shr":
+                r = v >> n
+            else:
+                r = (s32(v) >> n) & 0xFFFFFFFF
+            exp += [r & 0xFFFF, r >> 16]
+for n in range(1, 16):
+    for v in V16:
+        exp.append((s16(v) >> n) & 0xFFFF)
+# Down loops: final value and the sum (mod 2^16) of every low half seen.
+for start, cnt in DOWN:
+    v = start
+    s = 0
+    for _ in range(cnt):
+        v = (v - 1) & 0xFFFFFFFF
+        s = (s + (v & 0xFFFF) + (v >> 16)) & 0xFFFF
+    exp += [v & 0xFFFF, v >> 16, s]
+for start, cnt in UP:
+    v = start
+    s = 0
+    for _ in range(cnt):
+        v = (v + 1) & 0xFFFFFFFF
+        s = (s + (v & 0xFFFF) + (v >> 16)) & 0xFFFF
+    exp += [v & 0xFFFF, v >> 16, s]
+
+def rows(vals, per, fmt):
+    out = []
+    for i in range(0, len(vals), per):
+        out.append("    " + ", ".join(fmt % x for x in vals[i:i + per]) + ",")
+    return "\n".join(out)
+
+print("""// Every constant i32 shift (<< >>u >>s by 1..31) on 8 values, every
+// constant i16 >>s (1..15) on 8 values, and in-place i32 -= 1 / += 1 loop
+// counters across 0x0000 / 0x10000 / 0x80000000 / 0xFFFFFFFF.  kExp is
+// computed by genShiftDec.py.  Mailbox: 0x025000 = mismatches, 0x025002 =
+// first failing index, 0x025004 = 0xC0DE done.  int is 16 bits, long 32.
+typedef unsigned short u16;
+typedef short          s16;
+typedef unsigned long  u32;
+typedef long           s32;
+#define NOI __attribute__((noinline)) static
+#define NV 8
+static const u32 kV32[NV] = { %s };
+static const u16 kV16[NV] = { %s };""" % (", ".join("0x%08XUL" % v for v in V32), ", ".join("0x%04Xu" % v for v in V16)))
+print("static const u16 kExp[%d] = {\n%s\n};" % (len(exp), rows(exp, 12, "0x%04X")))
+print(r"""#define SH(n) \
+    NOI u32 shl##n(u32 x) { return x << n; } \
+    NOI u32 shr##n(u32 x) { return x >> n; } \
+    NOI u32 sar##n(u32 x) { return (u32)((s32)x >> n); }
+#define L7(M, a, b, c, d, e, f, g) M(a) M(b) M(c) M(d) M(e) M(f) M(g)
+#define L8(M, a, b, c, d, e, f, g, h) L7(M, a, b, c, d, e, f, g) M(h)
+L8(SH, 1, 2, 3, 4, 5, 6, 7, 8) L8(SH, 9, 10, 11, 12, 13, 14, 15, 16)
+L8(SH, 17, 18, 19, 20, 21, 22, 23, 24) L7(SH, 25, 26, 27, 28, 29, 30, 31)
+#define S16(n) NOI u16 sa16_##n(u16 x) { return (u16)((s16)x >> n); }
+L8(S16, 1, 2, 3, 4, 5, 6, 7, 8) L7(S16, 9, 10, 11, 12, 13, 14, 15)
+typedef u32 (*Fn32)(u32);
+typedef u16 (*Fn16)(u16);
+#define P31(p) p##1, p##2, p##3, p##4, p##5, p##6, p##7, p##8, p##9, p##10, \
+    p##11, p##12, p##13, p##14, p##15, p##16, p##17, p##18, p##19, p##20, \
+    p##21, p##22, p##23, p##24, p##25, p##26, p##27, p##28, p##29, p##30, p##31
+static Fn32 const kFn32[93] = { P31(shl), P31(shr), P31(sar) };
+static Fn16 const kFn16[15] = { sa16_1, sa16_2, sa16_3, sa16_4, sa16_5, sa16_6, sa16_7,
+    sa16_8, sa16_9, sa16_10, sa16_11, sa16_12, sa16_13, sa16_14, sa16_15 };
+// The counter lives in a frame / direct-page slot across the loop, where
+// W65816I32IncFold turns its -= 1 / += 1 into the conditional-carry form.
+static volatile u16 gSink;
+NOI u32 down(u32 v, u16 n, u16 *sum) {
+    u16 s = 0;
+    while (n != 0u) {
+        v--;
+        s = (u16)(s + (u16)v + (u16)(v >> 16));
+        gSink = s;
+        n--;
+    }
+    *sum = s;
+    return v;
+}
+NOI u32 up(u32 v, u16 n, u16 *sum) {
+    u16 s = 0;
+    while (n != 0u) {
+        v++;
+        s = (u16)(s + (u16)v + (u16)(v >> 16));
+        gSink = s;
+        n--;
+    }
+    *sum = s;
+    return v;
+}""")
+print("static const u32 kDownStart[%d] = { %s };" % (len(DOWN), ", ".join("0x%08XUL" % d[0] for d in DOWN)))
+print("static const u16 kDownCount[%d] = { %s };" % (len(DOWN), ", ".join("%du" % d[1] for d in DOWN)))
+print("static const u32 kUpStart[%d] = { %s };" % (len(UP), ", ".join("0x%08XUL" % d[0] for d in UP)))
+print("static const u16 kUpCount[%d] = { %s };" % (len(UP), ", ".join("%du" % d[1] for d in UP)))
+print(r"""static u16 gBad;
+static u16 gFirst = 0xFFFFu;
+static u16 gIdx;
+static void check(u16 got) {
+    if (got != kExp[gIdx] && gBad++ == 0u) {
+        gFirst = gIdx;
+    }
+    gIdx++;
+}
+int main(void) {
+    u16 f;
+    u16 i;
+    u16 s;
+    u32 r;
+    for (f = 0u; f < 93u; f++) {
+        for (i = 0u; i < NV; i++) {
+            r = kFn32[f](kV32[i]);
+            check((u16)r);
+            check((u16)(r >> 16));
+        }
+    }
+    for (f = 0u; f < 15u; f++) {
+        for (i = 0u; i < NV; i++) {
+            check(kFn16[f](kV16[i]));
+        }
+    }""")
+print("    for (i = 0u; i < %du; i++) {" % len(DOWN))
+print(r"""        r = down(kDownStart[i], kDownCount[i], &s);
+        check((u16)r);
+        check((u16)(r >> 16));
+        check(s);
+    }""")
+print("    for (i = 0u; i < %du; i++) {" % len(UP))
+print(r"""        r = up(kUpStart[i], kUpCount[i], &s);
+        check((u16)r);
+        check((u16)(r >> 16));
+        check(s);
+    }
+    *(volatile u16 *)0x025000UL = gBad;
+    *(volatile u16 *)0x025002UL = gFirst;
+    *(volatile u16 *)0x025004UL = 0xC0DEu;
+    for (;;) {
+    }
+    return 0;
+}""")