; Minimal libgcc-equivalent runtime for the W65816 / Apple IIgs.
; Provides the helpers that the LLVM backend lowers integer multiply,
; shift, divide, and modulo operations to.  Implementations are
; correct-but-unoptimised; they exist to unblock end-to-end testing,
; not to compete with hand-tuned 65816 math libraries.  The exception is
; memcpy / memmove / memset at the end of this file (MVN / MVP block
; moves, in their own section): the backend lowers struct copies and
; clears to them too, so they live here rather than in libc.c.
;
; Calling convention (matches W65816ISelLowering::LowerCall):
;   - Arg 0 in A (16-bit M).
;   - Arg 1 pushed via PHA before the JSL.  Reads as (4,S) inside the
;     callee (3-byte JSL return address sits at 1..3,S).
;   - Return value in A.  Caller releases pushed args.
;   - Routines run in 16-bit M, 16-bit X (REP #$30 by convention).
;
; Direct-page scratch lives at DP+$E0..DP+$EF (16 bytes).  Programs
; that use this runtime must keep DP=0 or remap accordingly.
;
; Assembled with: tools/llvm-mos-build/bin/llvm-mc -arch=w65816 \
;                                                 -filetype=obj
;                                                 runtime/src/libgcc.s
;                                                 -o runtime/libgcc.o

	.text

; --------------------------------------------------------------------
; Indirect-call trampoline (REENTRANT).  An indirect call (function
; pointer) stores its 24-bit target -- 16-bit offset to __indirTarget
; ($B8/$B9) and bank byte to $BA -- then JSLs here.  __jsl_indir reads
; that target DP-RELATIVE and RTL-jumps to it with the original caller's
; JSL frame still on the stack, so the target's own RTL returns straight
; back to the caller (one frame, no double-RTL).  The instruction-level
; idiom is documented at the __jsl_indir label below.
;
; Caller emit sequence in W65816ISelLowering::LowerCall:
;   sta __indirTarget    ; store ptr offset (must precede any A clobber for args)
;   ...store target bank to $BA, arg pushes...
;   jsl __jsl_indir
;
; No self-modified operand:  the target is read from the caller's OWN
; direct page ($B8/$B9 offset + $BA bank), and DP is per-thread -- GNO
; context-switches it every task swap, and both the GS/OS Loader and GNO
; keep DP at a non-zero, process-private page (never bank-0 zero page).
; So each thread dispatches through ITS OWN target, and a fork()ed child's
; function pointer no longer wild-jumps through the parent thread's page.
; (An earlier form JMP'd through a single global operand that crt0 self-
; modified to the main thread's DP+$B8; it aliased under preemptive multi-
; tasking and is gone -- there is no __jsl_indir_op now, and the crt0
; variants no longer patch anything.)
;
; Keeping DP non-zero also parks every codegen scratch slot (indirect-call
; anchor $B8, PBR stash $BE, libcall scratch $E0..$FF) in reserved /
; process-private bank-0 memory instead of on system zero page that ROM
; IRQ handlers and GS/OS dispatcher state freely scribble.  Forcing DP=0
; used to crash clang-built S16 apps with C++ ctors in the init_array
; indirect-dispatch loop for exactly that reason.  DP is always in bank 0
; on the 65816, so the $B8/$BA reads resolve to bank 0 regardless of DP --
; bare-metal (crt0.s) keeps DP=0 and the same reads land at $00B8/$00BA,
; needing no patch.
;
; The fetched pointer is honored BANK-EXPLICIT via the $BA bank byte, so a
; cross-bank function pointer -- e.g. &func the linker placed in a
; different segment/bank and redirected through a seg1 JML trampoline --
; lands in the correct bank.
; --------------------------------------------------------------------
	.globl __indirTarget
__indirTarget = 0xb8

	.text
	.globl __jsl_indir
__jsl_indir:
	; REENTRANT dispatch: read the 24-bit target DP-RELATIVE (offset $B8-$B9,
	; bank $BA -- both stored by clang at the call site) and RTL to it, with NO
	; self-modified global operand.  DP is per-thread (context-switched by GNO
	; every task swap; process-private under GS/OS), so each caller reads ITS
	; OWN target -- a fork()ed child dispatching a function pointer no longer
	; wild-jumps through the parent thread's page.  Preserves A/X/Y (C-ABI arg
	; regs) and the caller's JSL frame (the target's RTL returns to the JSL
	; site).  $BC is a per-thread DP temp to free A; assumes the 16-bit native
	; call state (M=0/X=0) every clang indirect call + the ctor loop use.  RTL
	; adds 1 to the pulled 16-bit PC, hence offset-1.
	sta 0xbc           ; save arg0 (A) to a per-thread DP temp
	sep #0x20          ; 8-bit A for the 1-byte bank push
	lda 0xba           ; target bank
	pha                ; push PBR
	rep #0x20          ; back to 16-bit A
	lda 0xb8           ; target offset (16-bit)
	dec a              ; offset-1 (RTL adds 1 to the pulled PC)
	pha                ; push PCH:PCL
	lda 0xbc           ; restore arg0 (A)
	rtl                ; -> bank:offset; caller's RTL frame remains below

; --------------------------------------------------------------------
; __run_cxa_atexit — weak no-op fallback.
;
; Each crt0 variant calls `jsl __run_cxa_atexit` after main() returns,
; so pure-C programs that don't link libcxxabi must still resolve the
; symbol.  libcxxabi.c overrides this with a strong implementation that
; walks the registered dtor table.
; --------------------------------------------------------------------
	.weak __run_cxa_atexit
__run_cxa_atexit:
	rtl

; --------------------------------------------------------------------
; exit / _exit -- weak fallbacks.
;
; crt0Gsos.s and crt0Gno.s end a program by calling exit() with main()'s
; value in A.  libc.c has the real exit (atexit handlers, then C++ dtors,
; then _exit); this weak one serves programs that link no libc: dtors,
; then _exit.  _exit is the platform's way out: crt0Gsos.s (GS/OS QUIT)
; and libcGno.c (GNO QUIT) override the weak halt below, which is bare
; metal's -- BRK so MAME's debugger catches it, then spin.
; --------------------------------------------------------------------
	.weak exit
exit:
	pha                ; status
	jsl __run_cxa_atexit
	pla
	jsl _exit          ; does not return

	.weak _exit
_exit:
	.byte 0x00, 0x00   ; BRK
.L_exitSpin:
	bra .L_exitSpin

; --------------------------------------------------------------------
; __heapLock / __heapUnlock -- weak no-op heap-serialization hooks.
;
; malloc()/free() (libc.c) bracket their free-list critical sections with these.
; Single-threaded runtimes (bare-metal crt0.s, GS/OS crt0Gsos.s) keep the no-op:
; there are no concurrent threads to serialize.  libcGno.c overrides both with a
; GNO binary-semaphore mutex so malloc/free are safe across preemptive threads.
; --------------------------------------------------------------------
	.weak __heapLock
__heapLock:
	rtl
	.weak __heapUnlock
__heapUnlock:
	rtl

; --------------------------------------------------------------------
; __srandInitFromTime — weak no-op fallback.
;
; crt0Gsos / crt0Gno call `jsl __srandInitFromTime` after .init_array to
; seed rand() from ReadTimeHex.  Programs that don't link extras.o (smoke
; harness link-tests) must still resolve the symbol; the no-op fallback
; leaves rand() at its deterministic seed-1 starting state.
; --------------------------------------------------------------------
	.weak __srandInitFromTime
__srandInitFromTime:
	rtl

; --------------------------------------------------------------------
; __heapInitMM — weak no-op fallback.
;
; crt0Gsos calls `jsl __heapInitMM` (Memory-Manager user ID in A) before
; .init_array to back the malloc heap with a locked MM handle.  GS/OS
; programs that don't link libc.o (bare probes / link tests) must still
; resolve the symbol; the no-op leaves malloc on its link-time
; [__heap_start,__heap_end) window.  libc.o overrides with the strong
; version.  A is ignored.
; --------------------------------------------------------------------
	.weak __heapInitMM
__heapInitMM:
	rtl

; --------------------------------------------------------------------
; 16x16 multiplies.  One method, inlined in __mulhi3 and __umulhisi3 and
; a JSR subroutine (__umul16_core) for __mulhisi3 and __mulsi3: the
; multiplier in $e0, the multiplicand in $e2, the product's low word
; returned in A and its high word in X.  Clobbers Y and $e0..$e5 (never
; the IMG words $c0..$df: W65816StackRelToImg keeps values there across
; these calls).
;   Both operands below 256: quarter squares, a*b = T[a+b] - T[|a-b|]
;     with T[k] = k*k/4 (__umulhisi3_qsq), ~60 cycles.
;   One below 256: eight steps with it as the multiplier, ~150 cycles.
;   Otherwise sixteen steps, ~255 cycles.
; A step of the shift-right method: the multiplier's next bit is in C; if
; set, the product's high word gains the multiplicand (the carry out in
; C); then the high word and the multiplier word shift right together,
; the product's low bits filling the multiplier word from the top as its
; own bits leave at the bottom.  The loop this replaced shifted the
; multiplicand left in 32 bits and added in 32 bits, ~44 cycles a
; multiplier bit: ~730 cycles for a 16-bit multiplier.
; --------------------------------------------------------------------

; __mulhi3 -- 16-bit multiply, A * (4,S) -> A.  Signed and unsigned are
; the same low word.
	.globl __mulhi3
__mulhi3:
	sta	0xe0
	ora	0x4, s
	and	#0xff00
	bne	.Lmh_wide
	; Both below 256: a*b = T[a+b] - T[|a-b|], T[k] = k*k/4, the two
	; entries indexed by Y and X.
	lda	0xe0
	clc
	adc	0x4, s
	asl	a
	tay
	lda	0xe0
	sec
	sbc	0x4, s
	bcs	.Lmh_qpos
	eor	#0xffff
	inc	a
.Lmh_qpos:
	asl	a
	tax
	lda	__umulhisi3_qsq, y
	sec
	sbc	__umulhisi3_qsq, x
	rtl
.Lmh_wide:
	lda	0x4, s
	sta	0xe2
	lda	0xe0
	and	#0xff00
	beq	.Lmh_m8
	lda	0xe2
	and	#0xff00
	bne	.Lmh_m16
	; The multiplicand is the short one: swap, so eight steps do.
	lda	0xe0
	ldx	0xe2
	stx	0xe0
	sta	0xe2
.Lmh_m8:
	lda	#0x0
	lsr	0xe0
	bcc	.Lmh_e0
	clc
	adc	0xe2
.Lmh_e0:
	ror	a
	ror	0xe0
	bcc	.Lmh_e1
	clc
	adc	0xe2
.Lmh_e1:
	ror	a
	ror	0xe0
	bcc	.Lmh_e2
	clc
	adc	0xe2
.Lmh_e2:
	ror	a
	ror	0xe0
	bcc	.Lmh_e3
	clc
	adc	0xe2
.Lmh_e3:
	ror	a
	ror	0xe0
	bcc	.Lmh_e4
	clc
	adc	0xe2
.Lmh_e4:
	ror	a
	ror	0xe0
	bcc	.Lmh_e5
	clc
	adc	0xe2
.Lmh_e5:
	ror	a
	ror	0xe0
	bcc	.Lmh_e6
	clc
	adc	0xe2
.Lmh_e6:
	ror	a
	ror	0xe0
	bcc	.Lmh_e7
	clc
	adc	0xe2
.Lmh_e7:
	ror	a
	ror	0xe0
	; A = product bits 23..8, $e1 = bits 7..0.
	sta	0xe4
	xba
	and	#0xff00
	sta	0xe4
	lda	0xe0
	xba
	and	#0xff
	ora	0xe4
	rtl
.Lmh_m16:
	lda	#0x0
	lsr	0xe0
	bcc	.Lmh_s0
	clc
	adc	0xe2
.Lmh_s0:
	ror	a
	ror	0xe0
	bcc	.Lmh_s1
	clc
	adc	0xe2
.Lmh_s1:
	ror	a
	ror	0xe0
	bcc	.Lmh_s2
	clc
	adc	0xe2
.Lmh_s2:
	ror	a
	ror	0xe0
	bcc	.Lmh_s3
	clc
	adc	0xe2
.Lmh_s3:
	ror	a
	ror	0xe0
	bcc	.Lmh_s4
	clc
	adc	0xe2
.Lmh_s4:
	ror	a
	ror	0xe0
	bcc	.Lmh_s5
	clc
	adc	0xe2
.Lmh_s5:
	ror	a
	ror	0xe0
	bcc	.Lmh_s6
	clc
	adc	0xe2
.Lmh_s6:
	ror	a
	ror	0xe0
	bcc	.Lmh_s7
	clc
	adc	0xe2
.Lmh_s7:
	ror	a
	ror	0xe0
	bcc	.Lmh_s8
	clc
	adc	0xe2
.Lmh_s8:
	ror	a
	ror	0xe0
	bcc	.Lmh_s9
	clc
	adc	0xe2
.Lmh_s9:
	ror	a
	ror	0xe0
	bcc	.Lmh_s10
	clc
	adc	0xe2
.Lmh_s10:
	ror	a
	ror	0xe0
	bcc	.Lmh_s11
	clc
	adc	0xe2
.Lmh_s11:
	ror	a
	ror	0xe0
	bcc	.Lmh_s12
	clc
	adc	0xe2
.Lmh_s12:
	ror	a
	ror	0xe0
	bcc	.Lmh_s13
	clc
	adc	0xe2
.Lmh_s13:
	ror	a
	ror	0xe0
	bcc	.Lmh_s14
	clc
	adc	0xe2
.Lmh_s14:
	ror	a
	ror	0xe0
	bcc	.Lmh_s15
	clc
	adc	0xe2
.Lmh_s15:
	ror	a
	ror	0xe0
	lda	0xe0
	rtl

; __umulhisi3 -- unsigned 16x16 -> 32: A * (4,S) -> A (low) : X (high).
; W65816NarrowI32Mul calls it for an i32 multiply of two values whose high
; words are known zero.
	.globl __umulhisi3
__umulhisi3:
	sta	0xe0
	ora	0x4, s
	and	#0xff00
	bne	.Lmu_wide
	; Both below 256: a*b = T[a+b] - T[|a-b|], T[k] = k*k/4, the two
	; entries indexed by Y and X.
	lda	0xe0
	clc
	adc	0x4, s
	asl	a
	tay
	lda	0xe0
	sec
	sbc	0x4, s
	bcs	.Lmu_qpos
	eor	#0xffff
	inc	a
.Lmu_qpos:
	asl	a
	tax
	lda	__umulhisi3_qsq, y
	sec
	sbc	__umulhisi3_qsq, x
	ldx	#0x0
	rtl
.Lmu_wide:
	lda	0x4, s
	sta	0xe2
	lda	0xe0
	and	#0xff00
	beq	.Lmu_m8
	lda	0xe2
	and	#0xff00
	bne	.Lmu_m16
	; The multiplicand is the short one: swap, so eight steps do.
	lda	0xe0
	ldx	0xe2
	stx	0xe0
	sta	0xe2
.Lmu_m8:
	lda	#0x0
	lsr	0xe0
	bcc	.Lmu_e0
	clc
	adc	0xe2
.Lmu_e0:
	ror	a
	ror	0xe0
	bcc	.Lmu_e1
	clc
	adc	0xe2
.Lmu_e1:
	ror	a
	ror	0xe0
	bcc	.Lmu_e2
	clc
	adc	0xe2
.Lmu_e2:
	ror	a
	ror	0xe0
	bcc	.Lmu_e3
	clc
	adc	0xe2
.Lmu_e3:
	ror	a
	ror	0xe0
	bcc	.Lmu_e4
	clc
	adc	0xe2
.Lmu_e4:
	ror	a
	ror	0xe0
	bcc	.Lmu_e5
	clc
	adc	0xe2
.Lmu_e5:
	ror	a
	ror	0xe0
	bcc	.Lmu_e6
	clc
	adc	0xe2
.Lmu_e6:
	ror	a
	ror	0xe0
	bcc	.Lmu_e7
	clc
	adc	0xe2
.Lmu_e7:
	ror	a
	ror	0xe0
	; A = product bits 23..8, $e1 = bits 7..0.
	sta	0xe4
	xba
	and	#0xff
	tax
	lda	0xe4
	xba
	and	#0xff00
	sta	0xe4
	lda	0xe0
	xba
	and	#0xff
	ora	0xe4
	rtl
.Lmu_m16:
	lda	#0x0
	lsr	0xe0
	bcc	.Lmu_s0
	clc
	adc	0xe2
.Lmu_s0:
	ror	a
	ror	0xe0
	bcc	.Lmu_s1
	clc
	adc	0xe2
.Lmu_s1:
	ror	a
	ror	0xe0
	bcc	.Lmu_s2
	clc
	adc	0xe2
.Lmu_s2:
	ror	a
	ror	0xe0
	bcc	.Lmu_s3
	clc
	adc	0xe2
.Lmu_s3:
	ror	a
	ror	0xe0
	bcc	.Lmu_s4
	clc
	adc	0xe2
.Lmu_s4:
	ror	a
	ror	0xe0
	bcc	.Lmu_s5
	clc
	adc	0xe2
.Lmu_s5:
	ror	a
	ror	0xe0
	bcc	.Lmu_s6
	clc
	adc	0xe2
.Lmu_s6:
	ror	a
	ror	0xe0
	bcc	.Lmu_s7
	clc
	adc	0xe2
.Lmu_s7:
	ror	a
	ror	0xe0
	bcc	.Lmu_s8
	clc
	adc	0xe2
.Lmu_s8:
	ror	a
	ror	0xe0
	bcc	.Lmu_s9
	clc
	adc	0xe2
.Lmu_s9:
	ror	a
	ror	0xe0
	bcc	.Lmu_s10
	clc
	adc	0xe2
.Lmu_s10:
	ror	a
	ror	0xe0
	bcc	.Lmu_s11
	clc
	adc	0xe2
.Lmu_s11:
	ror	a
	ror	0xe0
	bcc	.Lmu_s12
	clc
	adc	0xe2
.Lmu_s12:
	ror	a
	ror	0xe0
	bcc	.Lmu_s13
	clc
	adc	0xe2
.Lmu_s13:
	ror	a
	ror	0xe0
	bcc	.Lmu_s14
	clc
	adc	0xe2
.Lmu_s14:
	ror	a
	ror	0xe0
	bcc	.Lmu_s15
	clc
	adc	0xe2
.Lmu_s15:
	ror	a
	ror	0xe0
	tax
	lda	0xe0
	rtl

; __umul16_core -- the same as a JSR subroutine: $e0 * $e2 -> A : X.
__umul16_core:
	lda	0xe0
	ora	0xe2
	and	#0xff00
	bne	.Lmc_wide
	; Both below 256: a*b = T[a+b] - T[|a-b|], T[k] = k*k/4, the two
	; entries indexed by Y and X.
	lda	0xe0
	clc
	adc	0xe2
	asl	a
	tay
	lda	0xe0
	sec
	sbc	0xe2
	bcs	.Lmc_qpos
	eor	#0xffff
	inc	a
.Lmc_qpos:
	asl	a
	tax
	lda	__umulhisi3_qsq, y
	sec
	sbc	__umulhisi3_qsq, x
	ldx	#0x0
	rts
.Lmc_wide:
	lda	0xe0
	and	#0xff00
	beq	.Lmc_m8
	lda	0xe2
	and	#0xff00
	bne	.Lmc_m16
	; The multiplicand is the short one: swap, so eight steps do.
	lda	0xe0
	ldx	0xe2
	stx	0xe0
	sta	0xe2
.Lmc_m8:
	lda	#0x0
	lsr	0xe0
	bcc	.Lmc_e0
	clc
	adc	0xe2
.Lmc_e0:
	ror	a
	ror	0xe0
	bcc	.Lmc_e1
	clc
	adc	0xe2
.Lmc_e1:
	ror	a
	ror	0xe0
	bcc	.Lmc_e2
	clc
	adc	0xe2
.Lmc_e2:
	ror	a
	ror	0xe0
	bcc	.Lmc_e3
	clc
	adc	0xe2
.Lmc_e3:
	ror	a
	ror	0xe0
	bcc	.Lmc_e4
	clc
	adc	0xe2
.Lmc_e4:
	ror	a
	ror	0xe0
	bcc	.Lmc_e5
	clc
	adc	0xe2
.Lmc_e5:
	ror	a
	ror	0xe0
	bcc	.Lmc_e6
	clc
	adc	0xe2
.Lmc_e6:
	ror	a
	ror	0xe0
	bcc	.Lmc_e7
	clc
	adc	0xe2
.Lmc_e7:
	ror	a
	ror	0xe0
	; A = product bits 23..8, $e1 = bits 7..0.
	sta	0xe4
	xba
	and	#0xff
	tax
	lda	0xe4
	xba
	and	#0xff00
	sta	0xe4
	lda	0xe0
	xba
	and	#0xff
	ora	0xe4
	rts
.Lmc_m16:
	lda	#0x0
	lsr	0xe0
	bcc	.Lmc_s0
	clc
	adc	0xe2
.Lmc_s0:
	ror	a
	ror	0xe0
	bcc	.Lmc_s1
	clc
	adc	0xe2
.Lmc_s1:
	ror	a
	ror	0xe0
	bcc	.Lmc_s2
	clc
	adc	0xe2
.Lmc_s2:
	ror	a
	ror	0xe0
	bcc	.Lmc_s3
	clc
	adc	0xe2
.Lmc_s3:
	ror	a
	ror	0xe0
	bcc	.Lmc_s4
	clc
	adc	0xe2
.Lmc_s4:
	ror	a
	ror	0xe0
	bcc	.Lmc_s5
	clc
	adc	0xe2
.Lmc_s5:
	ror	a
	ror	0xe0
	bcc	.Lmc_s6
	clc
	adc	0xe2
.Lmc_s6:
	ror	a
	ror	0xe0
	bcc	.Lmc_s7
	clc
	adc	0xe2
.Lmc_s7:
	ror	a
	ror	0xe0
	bcc	.Lmc_s8
	clc
	adc	0xe2
.Lmc_s8:
	ror	a
	ror	0xe0
	bcc	.Lmc_s9
	clc
	adc	0xe2
.Lmc_s9:
	ror	a
	ror	0xe0
	bcc	.Lmc_s10
	clc
	adc	0xe2
.Lmc_s10:
	ror	a
	ror	0xe0
	bcc	.Lmc_s11
	clc
	adc	0xe2
.Lmc_s11:
	ror	a
	ror	0xe0
	bcc	.Lmc_s12
	clc
	adc	0xe2
.Lmc_s12:
	ror	a
	ror	0xe0
	bcc	.Lmc_s13
	clc
	adc	0xe2
.Lmc_s13:
	ror	a
	ror	0xe0
	bcc	.Lmc_s14
	clc
	adc	0xe2
.Lmc_s14:
	ror	a
	ror	0xe0
	bcc	.Lmc_s15
	clc
	adc	0xe2
.Lmc_s15:
	ror	a
	ror	0xe0
	tax
	lda	0xe0
	rts

; __mulhisi3 -- signed 16x16 -> 32: A * (4,S) -> A (low) : X (high).
; W65816NarrowI32Mul calls it for an i32 multiply of two sign-extended
; 16-bit values (and __mulsi3 jumps here for them).  The magnitudes go
; through the unsigned core, so small negative operands take its short
; paths too; the product is negated when the signs differ.
	.globl __mulhisi3
__mulhisi3:
	sta	0xe0
	eor	0x4, s
	sta	0xe6		; bit 15: the product is negative
	lda	0xe0
	bpl	.Lmhs_apos
	eor	#0xffff
	inc	a
	sta	0xe0
.Lmhs_apos:
	lda	0x4, s
	bpl	.Lmhs_bpos
	eor	#0xffff
	inc	a
.Lmhs_bpos:
	sta	0xe2
	jsr	__umul16_core
	ldy	0xe6
	bpl	.Lmhs_done
	eor	#0xffff
	clc
	adc	#0x1
	sta	0xe4
	txa
	eor	#0xffff
	adc	#0x0
	tax
	lda	0xe4
.Lmhs_done:
	rtl

; --------------------------------------------------------------------
; __ashlhi3 — A << (4,S) -> A.  Shift count is i16 but only the low 4
; bits are meaningful (counts >=16 are undefined behaviour in C).
; --------------------------------------------------------------------
	.globl __ashlhi3
__ashlhi3:
	pha			; save value on stack so we can free A
	lda	0x6, s		; arg 1 sits at 6,s now (PHA shifted by 2)
	tax
	pla			; restore value
.Lashl_loop:
	cpx	#0x0
	beq	.Lashl_done
	asl	a
	dex
	bra	.Lashl_loop
.Lashl_done:
	rtl

; --------------------------------------------------------------------
; __lshrhi3 — A logical >> (4,S) -> A.  Same shape as __ashlhi3 with
; LSR instead of ASL.
; --------------------------------------------------------------------
	.globl __lshrhi3
__lshrhi3:
	pha
	lda	0x6, s
	tax
	pla
.Llshr_loop:
	cpx	#0x0
	beq	.Llshr_done
	lsr	a
	dex
	bra	.Llshr_loop
.Llshr_done:
	rtl

; --------------------------------------------------------------------
; __ashrhi3 — A arithmetic >> (4,S) -> A.  Sign bit is preserved by
; copying it into carry before each ROR via CMP #$8000 (which sets
; carry exactly when the sign bit is set on a 16-bit unsigned compare).
; --------------------------------------------------------------------
	.globl __ashrhi3
__ashrhi3:
	pha
	lda	0x6, s
	tax
	pla
.Lashr_loop:
	cpx	#0x0
	beq	.Lashr_done
	cmp	#0x8000
	ror	a
	dex
	bra	.Lashr_loop
.Lashr_done:
	rtl

; --------------------------------------------------------------------
; 16-bit divide and modulo.  __udivmod16 (JSR): $e0 / $e2 -> quotient in
; $e0, remainder in A.  Each step shifts the dividend's top bit into the
; remainder, held in A; a 17th bit carried out of A means the remainder is
; past any divisor, so it subtracts without comparing.  The quotient's
; bits enter $e0 from the bottom as the dividend's leave at the top.  A
; dividend below 256 starts eight bits in.  ~28 cycles a step (the loop
; this replaced shifted three words a step, ~45).
; Signed: magnitudes through the same core; the quotient negated when the
; signs differ, the remainder given the dividend's sign (C99).
; --------------------------------------------------------------------
__udivmod16:
	ldy	#0x10
	lda	0xe0
	and	#0xff00
	bne	.Lud16_go
	lda	0xe0
	xba
	sta	0xe0
	ldy	#0x8
.Lud16_go:
	lda	#0x0
.Lud16_loop:
	asl	0xe0
	rol	a
	bcs	.Lud16_sub
	cmp	0xe2
	bcc	.Lud16_skip
.Lud16_sub:
	sbc	0xe2
	inc	0xe0
.Lud16_skip:
	dey
	bne	.Lud16_loop
	rts

	.globl __udivhi3
__udivhi3:
	sta	0xe0
	lda	0x4, s
	sta	0xe2
	jsr	__udivmod16
	lda	0xe0
	rtl

	.globl __umodhi3
__umodhi3:
	sta	0xe0
	lda	0x4, s
	sta	0xe2
	jsr	__udivmod16
	rtl

	.globl __divhi3
__divhi3:
	sta	0xe0
	eor	0x4, s
	sta	0xe4		; bit 15: the quotient is negative
	lda	0xe0
	bpl	.Ldh_apos
	eor	#0xffff
	inc	a
	sta	0xe0
.Ldh_apos:
	lda	0x4, s
	bpl	.Ldh_bpos
	eor	#0xffff
	inc	a
.Ldh_bpos:
	sta	0xe2
	jsr	__udivmod16
	lda	0xe0
	ldy	0xe4
	bpl	.Ldh_done
	eor	#0xffff
	inc	a
.Ldh_done:
	rtl

	.globl __modhi3
__modhi3:
	sta	0xe0
	sta	0xe4		; bit 15: the dividend (and remainder) is negative
	lda	0xe0
	bpl	.Lmh_apos
	eor	#0xffff
	inc	a
	sta	0xe0
.Lmh_apos:
	lda	0x4, s
	bpl	.Lmh_bpos
	eor	#0xffff
	inc	a
.Lmh_bpos:
	sta	0xe2
	jsr	__udivmod16
	ldy	0xe4
	bpl	.Lmh_done
	eor	#0xffff
	inc	a
.Lmh_done:
	rtl

; ====================================================================
; 32-bit (long / si) helpers.
;
; ABI for these is the natural extension of the i16 libcalls:
;   - arg0_lo in A
;   - arg0_hi at (4,s)
;   - arg1_lo at (6,s)         (or shift count, for the shift helpers)
;   - arg1_hi at (8,s)
;   - return: result_lo in A, result_hi in X
;
; All are correct-but-unoptimised; goal is unblocking end-to-end builds,
; not winning a 65816 codegolf.
;
; Direct-page scratch for these:
;   $e0..$e3  = a (lo, hi)        [renamed from $e0/$e2 for the i16 ones]
;   $e4..$e7  = b (lo, hi)
;   $e8..$eb  = result / quotient (lo, hi)
;   $ec..$ef  = remainder (lo, hi)
; ====================================================================

; --------------------------------------------------------------------
; __mulsi3 -- 32-bit multiply, (a * b) mod 2^32.
;
; ABI: A = a_lo, X = a_hi (the i32-first-arg in A:X convention),
;      (4,s) = b_lo, (6,s) = b_hi.  Result returned in A:X (lo:hi).
;
; Both high words zero: one 16x16 multiply.  Both operands sign-extended
; 16-bit values (`(long)s16 * s16`, and the small negatives the old
; 32-step loop paid ~1100 cycles for): __mulhisi3, whose (4,s) is our
; b_lo.  Otherwise a_lo*b_lo + ((a_lo*b_hi + a_hi*b_lo) << 16), three
; trips through __umul16_core.
; --------------------------------------------------------------------
	.globl __mulsi3
__mulsi3:
	sta	0xe8		; a_lo
	stx	0xea		; a_hi
	lda	0x4, s
	sta	0xec		; b_lo
	lda	0x6, s
	sta	0xee		; b_hi
	ora	0xea
	bne	.Lmsi_wide
	lda	0xe8
	sta	0xe0
	lda	0xec
	sta	0xe2
	jsr	__umul16_core
	rtl
.Lmsi_wide:
	; a_hi == sign(a_lo) and b_hi == sign(b_lo)?  `asl ; lda #0 ; adc
	; #$ffff ; eor #$ffff` makes $ffff from a set bit 15, 0 from a clear one.
	lda	0xe8
	asl	a
	lda	#0x0
	adc	#0xffff
	eor	#0xffff
	cmp	0xea
	bne	.Lmsi_full
	lda	0xec
	asl	a
	lda	#0x0
	adc	#0xffff
	eor	#0xffff
	cmp	0xee
	bne	.Lmsi_full
	lda	0xe8
	brl	__mulhisi3
.Lmsi_full:
	lda	0xe8
	sta	0xe0
	lda	0xee
	sta	0xe2
	jsr	__umul16_core	; a_lo * b_hi, low word
	sta	0xe6
	lda	0xea
	sta	0xe0
	lda	0xec
	sta	0xe2
	jsr	__umul16_core	; a_hi * b_lo, low word
	clc
	adc	0xe6
	sta	0xe6
	lda	0xe8
	sta	0xe0
	lda	0xec
	sta	0xe2
	jsr	__umul16_core	; a_lo * b_lo
	sta	0xe4
	txa
	clc
	adc	0xe6
	tax
	lda	0xe4
	rtl

; --------------------------------------------------------------------
; __ashlsi3 — (A:X) << (4,s) -> A:X.  Shift count is i16 in low byte;
; counts >= 32 are UB in C.  Uses a per-bit loop (cheap on 65816 — one
; ASL + ROL per bit).
;
; ABI: A = a_lo, X = a_hi (i32-first-arg in A:X), (4,s) = count.
; --------------------------------------------------------------------
	.globl __ashlsi3
__ashlsi3:
	sta	0xe0			; lo
	stx	0xe2			; hi
	lda	0x4, s
	tay				; count -> Y; sets Z if 0
	beq	.Lashlsi_done		; count==0: skip loop
.Lashlsi_loop:
	; Per-bit: 7+7+2+3 = 19 cyc (was 25 with cpy/beq/bra).
	asl	0xe0
	rol	0xe2
	dey
	bne	.Lashlsi_loop
.Lashlsi_done:
	ldx	0xe2
	lda	0xe0
	rtl

; --------------------------------------------------------------------
; __lshrsi3 — logical >> shift.  LSR hi, ROR lo: hi gets a 0, lo gets
; hi's old bit 0.  Per-bit loop.
; --------------------------------------------------------------------
	.globl __lshrsi3
__lshrsi3:
	sta	0xe0
	stx	0xe2
	lda	0x4, s
	tay
	beq	.Llshrsi_done
.Llshrsi_loop:
	lsr	0xe2
	ror	0xe0
	dey
	bne	.Llshrsi_loop
.Llshrsi_done:
	ldx	0xe2
	lda	0xe0
	rtl

; --------------------------------------------------------------------
; __ashrsi3 — arithmetic >> shift.  Sign bit must be preserved on each
; iteration: copy bit 15 of hi into carry (via CMP #$8000), then ROR
; hi, ROR lo.  Per-bit loop.
; --------------------------------------------------------------------
	.globl __ashrsi3
__ashrsi3:
	sta	0xe0
	stx	0xe2
	lda	0x4, s
	tay
	beq	.Lashrsi_done
.Lashrsi_loop:
	; CMP #$8000 sets C iff the unsigned value >= 0x8000, i.e. bit 15
	; is set — exactly the sign bit.
	lda	0xe2
	cmp	#0x8000
	ror	0xe2
	ror	0xe0
	dey
	bne	.Lashrsi_loop
.Lashrsi_done:
	ldx	0xe2
	lda	0xe0
	rtl

; --------------------------------------------------------------------
; __udivmodsi_core -- internal 32-bit unsigned divide (JSR / RTS).
; Dividend in $e0/$e2, divisor in $e4/$e6; quotient out in $e0/$e2,
; remainder in $e8/$ea.
;
; A divisor below 2^16 keeps the remainder in A, as __udivmod16 does:
; the dividend's high word first (skipped, quotient high word 0, when it
; is already below the divisor), then its low word, 16 steps each.  A
; divisor of 2^16 or more makes a quotient below 2^16: the remainder
; starts as the dividend's high word and 16 steps shift the low word
; through a 32-bit remainder (a bit carried out of it means the remainder
; is past the divisor).  ~450..900 cycles for the first form and ~1000
; for the second, against ~2400 for the 32 three-word steps before.
; --------------------------------------------------------------------
__udivmodsi_core:
	lda	0xe6
	bne	.Lud32_wide
	lda	0xe2
	cmp	0xe4
	bcs	.Lud32_hi
	stz	0xe2
	bra	.Lud32_lo
.Lud32_hi:
	lda	#0x0
	ldy	#0x10
.Lud32_hloop:
	asl	0xe2
	rol	a
	bcs	.Lud32_hsub
	cmp	0xe4
	bcc	.Lud32_hskip
.Lud32_hsub:
	sbc	0xe4
	inc	0xe2
.Lud32_hskip:
	dey
	bne	.Lud32_hloop
.Lud32_lo:
	ldy	#0x10
.Lud32_lloop:
	asl	0xe0
	rol	a
	bcs	.Lud32_lsub
	cmp	0xe4
	bcc	.Lud32_lskip
.Lud32_lsub:
	sbc	0xe4
	inc	0xe0
.Lud32_lskip:
	dey
	bne	.Lud32_lloop
	sta	0xe8
	stz	0xea
	rts
.Lud32_wide:
	lda	0xe2
	sta	0xe8
	stz	0xea
	stz	0xe2
	ldy	#0x10
.Lud32_wloop:
	asl	0xe0
	rol	0xe8
	rol	0xea
	bcs	.Lud32_wsub
	lda	0xea
	cmp	0xe6
	bcc	.Lud32_wskip
	bne	.Lud32_wsub
	lda	0xe8
	cmp	0xe4
	bcc	.Lud32_wskip
.Lud32_wsub:
	lda	0xe8
	sec
	sbc	0xe4
	sta	0xe8
	lda	0xea
	sbc	0xe6
	sta	0xea
	inc	0xe0
.Lud32_wskip:
	dey
	bne	.Lud32_wloop
	rts

; --------------------------------------------------------------------
; __udivsi3 -- unsigned 32/32 -> 32 divide.
; --------------------------------------------------------------------
	.globl __udivsi3
__udivsi3:
	; ABI: A = a_lo, X = a_hi, (4,s) = b_lo, (6,s) = b_hi.
	sta	0xe0
	stx	0xe2
	lda	0x4, s
	sta	0xe4
	lda	0x6, s
	sta	0xe6
	jsr	__udivmodsi_core
	ldx	0xe2
	lda	0xe0
	rtl

; --------------------------------------------------------------------
; __umodsi3 -- unsigned 32/32 -> 32 modulo.
; --------------------------------------------------------------------
	.globl __umodsi3
__umodsi3:
	sta	0xe0
	stx	0xe2
	lda	0x4, s
	sta	0xe4
	lda	0x6, s
	sta	0xe6
	jsr	__udivmodsi_core
	ldx	0xea
	lda	0xe8
	rtl

; --------------------------------------------------------------------
; __divsi3 / __modsi3 -- signed 32-bit divide / modulo.  Strategy mirrors
; the i16 helpers: stash signs, take abs, run unsigned core, negate
; result(s) as needed.  Sign tracker bits in $f0:
;   bit 0 = dividend was negative (modulo result sign)
;   bit 1 = quotient sign (sign(a) XOR sign(b))
; --------------------------------------------------------------------
	.globl __divsi3
__divsi3:
	jsr	__divmodsi_setup
	jsr	__udivmodsi_core
	; Quotient at $e0/$e2.  Negate if bit 1 of $f0 is set.
	lda	0xf0
	and	#0x2
	beq	.Ldivsi_pos
	lda	0xe0
	eor	#0xffff
	clc
	adc	#0x1
	sta	0xe0
	lda	0xe2
	eor	#0xffff
	adc	#0x0
	sta	0xe2
.Ldivsi_pos:
	ldx	0xe2
	lda	0xe0
	rtl

	.globl __modsi3
__modsi3:
	jsr	__divmodsi_setup
	jsr	__udivmodsi_core
	; Remainder at $e8/$ea.  Negate if bit 0 of $f0 set (dividend
	; was negative -- C99 remainder takes dividend's sign).
	lda	0xf0
	and	#0x1
	beq	.Lmodsi_pos
	lda	0xe8
	eor	#0xffff
	clc
	adc	#0x1
	sta	0xe8
	lda	0xea
	eor	#0xffff
	adc	#0x0
	sta	0xea
.Lmodsi_pos:
	ldx	0xea
	lda	0xe8
	rtl

; --------------------------------------------------------------------
; __divmodsi_setup — common prologue for __divsi3 / __modsi3.
; Reads A=a_lo, X=a_hi (i32-first-arg ABI), (4,s)=b_lo, (6,s)=b_hi.
; Writes |a| to $e0/$e2, |b| to $e4/$e6, sign bits to $f0.  JSR/RTS.
; After JSR's 2-byte ret push, callee-relative offsets are (6,s)=b_lo,
; (8,s)=b_hi.
; --------------------------------------------------------------------
__divmodsi_setup:
	; Clear sign tracker.  STZ preserves A.
	stz	0xf0
	; |a|: A=a_lo, X=a_hi.  Save them first (we need a_hi for sign test).
	sta	0xe0			; tentative a_lo (may negate below)
	stx	0xe2			; tentative a_hi
	cpx	#0x8000
	bcc	.Lsetsi_a_pos
	; a is negative.  Set sign tracker bits 0+1 and negate.
	lda	0xf0
	ora	#0x3
	sta	0xf0
	; 32-bit negate: invert + 1.
	lda	0xe0
	eor	#0xffff
	clc
	adc	#0x1
	sta	0xe0
	lda	0xe2
	eor	#0xffff
	adc	#0x0
	sta	0xe2
.Lsetsi_a_pos:
	; |b|.  Args shifted by 2 (the JSR ret push).
	lda	0x6, s
	sta	0xe4
	lda	0x8, s
	sta	0xe6
	cmp	#0x8000
	bcc	.Lsetsi_b_pos
	; b is negative.  Flip bit 1 of $f0.
	lda	0xf0
	eor	#0x2
	sta	0xf0
	lda	0xe4
	eor	#0xffff
	clc
	adc	#0x1
	sta	0xe4
	lda	0xe6
	eor	#0xffff
	adc	#0x0
	sta	0xe6
.Lsetsi_b_pos:
	rts

; ====================================================================
; i64 (long long) helpers.
;
; Calling convention (i64 first arg is split via i32-first-arg path):
;   A   = arg0_lo[0..15]    (lowest word)
;   X   = arg0_lo[16..31]
;   4,S = arg0_hi[0..15]
;   6,S = arg0_hi[16..31]   (highest word)
;   For binary ops (mul/div/mod), arg1 follows on the stack:
;   8,S = arg1_lo[0..15]
;   10,S = arg1_lo[16..31]
;   12,S = arg1_hi[0..15]
;   14,S = arg1_hi[16..31]
;   For shift ops, the count occupies a single i16 at 8,S.
;
; Return ABI (matches LowerReturn for i64):
;   A   = result_lo[0..15]
;   X   = result_lo[16..31]
;   Y   = result_hi[0..15]
;   DP $F0..$F1 = result_hi[16..31]
;
; Scratch DP layout (per-libcall, no overlap between concurrent calls):
;   $E0..$E7 = a (8 bytes; 4 16-bit words)
;   $E8..$EF = b OR product (8 bytes)
;
; All routines run with REP #$30 (M=0, X=0).
; ====================================================================

; --------------------------------------------------------------------
; __divmoddi4_stash — common entry point.  Stashes a -> $E0..$E7,
; b -> $E8..$EF.  Used by __udivdi3 / __umoddi3 / __divdi3 / __moddi3
; setup; signed variants flip signs around it.
; --------------------------------------------------------------------
__divmoddi4_stash:
	; Called via JSR from another libgcc helper that was itself
	; called via JSL.  Stack layout inside this routine:
	;   slot 1..2  = JSR return address (2 bytes, same-bank)
	;   slot 3..5  = JSL return address (3 bytes, long)
	;   slot 6..7  = first 16-bit stack arg (caller's first push)
	;   slot 8..9  = second
	;   ... etc.
	; Earlier code read slots 4, 6, 8, 10, 12, 14 — which lands on
	; the JSL ret address bytes, treating them as args.  Caught by
	; `u64mul(0x12, 0x12)` returning the result at $E2 (mid-low)
	; instead of $E0 (lo) plus 0x678-shaped garbage at $E6.
	sta	0xe0			; a_lo_lo
	stx	0xe2			; a_lo_hi
	lda	0x6, s
	sta	0xe4			; a_hi_lo
	lda	0x8, s
	sta	0xe6			; a_hi_hi
	lda	0xa, s
	sta	0xe8			; b_lo_lo
	lda	0xc, s
	sta	0xea			; b_lo_hi
	lda	0xe, s
	sta	0xec			; b_hi_lo
	lda	0x10, s
	sta	0xee			; b_hi_hi
	rts

; --------------------------------------------------------------------
; Helper: pack the result at $E0..$E7 into the i64 return ABI.
; Trashes A, Y.  Caller falls through to RTL.
; --------------------------------------------------------------------
__retdi:
	lda	0xe6
	sta	0xf0
	lda	0xe4
	tay
	lda	0xe2
	tax
	lda	0xe0
	rtl

; --------------------------------------------------------------------
; __ashldi3 — i64 left shift by n.  Per-bit loop.  Y holds count.
; --------------------------------------------------------------------
	.globl __ashldi3
__ashldi3:
	sta	0xe0
	stx	0xe2
	lda	0x4, s
	sta	0xe4
	lda	0x6, s
	sta	0xe6
	lda	0x8, s
	tay				; Y = count
.Lashldi_loop:
	cpy	#0x0
	beq	.Lashldi_done
	asl	0xe0
	rol	0xe2
	rol	0xe4
	rol	0xe6
	dey
	bra	.Lashldi_loop
.Lashldi_done:
	brl	__retdi

; --------------------------------------------------------------------
; __lshrdi3 — i64 logical right shift.  LSR top word, ROR rest.
; --------------------------------------------------------------------
	.globl __lshrdi3
__lshrdi3:
	sta	0xe0
	stx	0xe2
	lda	0x4, s
	sta	0xe4
	lda	0x6, s
	sta	0xe6
	lda	0x8, s
	tay
.Llshrdi_loop:
	cpy	#0x0
	beq	.Llshrdi_done
	lsr	0xe6
	ror	0xe4
	ror	0xe2
	ror	0xe0
	dey
	bra	.Llshrdi_loop
.Llshrdi_done:
	brl	__retdi

; --------------------------------------------------------------------
; __ashrdi3 — i64 arithmetic right shift.  Same as lshrdi3 but the top
; bit replicates: sign-extend by ASL/ROR which would clear; instead
; take a copy of the sign and OR it back, OR use cmp/sbc trick — use
; the standard idiom: capture sign before LSR via "asl; ror" so C is
; preserved.  Simpler: copy bit 15 of $E7 into C before each shift.
; --------------------------------------------------------------------
	.globl __ashrdi3
__ashrdi3:
	sta	0xe0
	stx	0xe2
	lda	0x4, s
	sta	0xe4
	lda	0x6, s
	sta	0xe6
	lda	0x8, s
	tay
.Lashrdi_loop:
	cpy	#0x0
	beq	.Lashrdi_done
	; "ASL $E6" sets C from bit 15 (the sign), then we ROR $E6 back.
	; Net effect on $E6: arithmetic right shift by 1 (sign preserved).
	; The carry chain into $E4..$E0 is the new bit 15.
	lda	0xe6
	asl	a			; C = sign bit; A = (sign<<1) | rest
	ror	0xe6			; $E6: (sign << 15) | ($E6 >> 1)
	ror	0xe4
	ror	0xe2
	ror	0xe0
	dey
	bra	.Lashrdi_loop
.Lashrdi_done:
	brl	__retdi

; --------------------------------------------------------------------
; __muldi3 — i64 multiply (low 64 bits of 64x64 product).
; Shift-and-add over a (64 bits).  Product accumulates at $F2..$F9
; (above the return DP slot, scratch).  Need a fresh 8-byte product
; slot since $E0..$EF holds operands.
; --------------------------------------------------------------------
	.globl __muldi3
__muldi3:
	jsr	__divmoddi4_stash
	; Clear product P0..P3 at $F2..$F8.
	stz	0xf2
	stz	0xf4
	stz	0xf6
	stz	0xf8
	; Short-circuit when a's high half ($E4/$E6) is zero: bits 32..63
	; of a are 0, so the 32 high iterations would add nothing.  Saves
	; ~50% of __muldi3 cost in mulhi64Aligned (softDouble.c), which
	; passes only u32-wide operands.  b's high half is irrelevant for
	; this short-circuit — even if b is full-width, iters 32..63 only
	; shift b and add zero.
	lda	0xe4
	ora	0xe6
	bne	.Lmuldi_long
	ldy	#0x20
	bra	.Lmuldi_loop
.Lmuldi_long:
	ldy	#0x40
.Lmuldi_loop:
	; Right-shift the 64-bit `a` by 1.  $E0=lo..$E6=hi (matches the
	; stash + __retdi convention).  Must shift HI first (LSR loses
	; bit 63 of $E6) so each ROR carries the previous half's bit 0
	; INTO the top of the next-LOWER half — that's the actual
	; right-shift direction in a $E0=lo layout.  After the chain,
	; C = orig $E0_b0 = bit 0 of the 64-bit value, which drops out
	; and is what we want to BCC on.  The earlier code shifted lo
	; first which ran the shift in the WRONG direction (lo → hi)
	; and tested $E6_b0 (bit 48) instead of bit 0 — every multiply
	; involving bits 16+ came back garbage.
	lda	0xe6
	lsr	a
	sta	0xe6
	lda	0xe4
	ror	a
	sta	0xe4
	lda	0xe2
	ror	a
	sta	0xe2
	lda	0xe0
	ror	a
	sta	0xe0
	bcc	.Lmuldi_noadd
	; Add b ($E8..$EE) to product ($F2..$F8).
	clc
	lda	0xf2
	adc	0xe8
	sta	0xf2
	lda	0xf4
	adc	0xea
	sta	0xf4
	lda	0xf6
	adc	0xec
	sta	0xf6
	lda	0xf8
	adc	0xee
	sta	0xf8
.Lmuldi_noadd:
	; Shift b left by 1 (so each iteration uses next bit position).
	asl	0xe8
	rol	0xea
	rol	0xec
	rol	0xee
	dey
	bne	.Lmuldi_loop
	; Move product into return slots ($E0..$E7) and tail-call __retdi.
	lda	0xf2
	sta	0xe0
	lda	0xf4
	sta	0xe2
	lda	0xf6
	sta	0xe4
	lda	0xf8
	sta	0xe6
	brl	__retdi

; --------------------------------------------------------------------
; __ucmpdi2 — unsigned i64 compare.  Returns 0 if a<b, 1 if a==b,
; 2 if a>b (libgcc convention).  We emit i16 result in A (with the
; high bytes don't-care).
; --------------------------------------------------------------------
	.globl __ucmpdi2
__ucmpdi2:
	; Compare from MSB downwards.  Stash a/b first so we have a stable
	; layout.
	jsr	__divmoddi4_stash
	; Compare $E6 vs $EE (a_hi_hi vs b_hi_hi).
	lda	0xe6
	cmp	0xee
	bne	.Lucmpdi_decided
	lda	0xe4
	cmp	0xec
	bne	.Lucmpdi_decided
	lda	0xe2
	cmp	0xea
	bne	.Lucmpdi_decided
	lda	0xe0
	cmp	0xe8
	bne	.Lucmpdi_decided
	; Equal.
	lda	#0x1
	rtl
.Lucmpdi_decided:
	; Carry clear  -> a < b -> return 0.
	; Carry set, Z clear -> a > b -> return 2.
	bcc	.Lucmpdi_lt
	lda	#0x2
	rtl
.Lucmpdi_lt:
	lda	#0x0
	rtl

; --------------------------------------------------------------------
; __cmpdi2 — signed i64 compare.  Same {0,1,2} return convention.
; Implemented by flipping the high-word sign bits before doing an
; unsigned compare ($N XOR $8000 swaps the signed-int order to
; unsigned-int order).
; --------------------------------------------------------------------
	.globl __cmpdi2
__cmpdi2:
	jsr	__divmoddi4_stash
	lda	0xe6
	eor	#0x8000
	sta	0xe6
	lda	0xee
	eor	#0x8000
	sta	0xee
	; Unsigned compare on the rewritten values.
	lda	0xe6
	cmp	0xee
	bne	.Lcmpdi_decided
	lda	0xe4
	cmp	0xec
	bne	.Lcmpdi_decided
	lda	0xe2
	cmp	0xea
	bne	.Lcmpdi_decided
	lda	0xe0
	cmp	0xe8
	bne	.Lcmpdi_decided
	lda	#0x1
	rtl
.Lcmpdi_decided:
	bcc	.Lcmpdi_lt
	lda	#0x2
	rtl
.Lcmpdi_lt:
	lda	#0x0
	rtl

; --------------------------------------------------------------------
; __udivdi3 / __umoddi3 — unsigned 64-bit divide / modulo.  Restoring
; division: shift dividend left into a remainder register, conditionally
; subtract the divisor.  The two libcalls share the core; quotient
; lands at $E0..$E7, remainder at $F2..$F8.  Each entry sets a flag in
; X to select which to return.
; --------------------------------------------------------------------
	.globl __udivdi3
__udivdi3:
	jsr	__divmoddi4_stash
	jsr	__udivmoddi_core
	brl	__retdi

	.globl __umoddi3
__umoddi3:
	jsr	__divmoddi4_stash
	jsr	__udivmoddi_core
	; Move remainder ($F2..$F8) -> $E0..$E7 for return.
	lda	0xf2
	sta	0xe0
	lda	0xf4
	sta	0xe2
	lda	0xf6
	sta	0xe4
	lda	0xf8
	sta	0xe6
	brl	__retdi

; Core: dividend at $E0..$E6, divisor at $E8..$EE.
; Output: quotient at $E0..$E6, remainder at $F2..$F8.
__udivmoddi_core:
	; Clear remainder $F2..$F8.
	stz	0xf2
	stz	0xf4
	stz	0xf6
	stz	0xf8
	ldy	#0x40
.Ludivmoddi_loop:
	; Shift left: dividend (becomes quotient) and remainder together
	; as a 128-bit register.  bit shifted out of dividend top -> remainder LSB.
	asl	0xe0
	rol	0xe2
	rol	0xe4
	rol	0xe6
	rol	0xf2
	rol	0xf4
	rol	0xf6
	rol	0xf8
	; Try remainder - divisor.  If no borrow, accept and set quotient bit.
	sec
	lda	0xf2
	sbc	0xe8
	sta	0xfa			; tentative subtract result at $FA..$
	lda	0xf4
	sbc	0xea
	sta	0xfc
	lda	0xf6
	sbc	0xec
	sta	0xfe
	lda	0xf8
	sbc	0xee
	; A holds new high word.  C = !borrow.
	bcc	.Ludivmoddi_skip
	; Accept: remainder = remainder - divisor, quotient bit 0 = 1.
	sta	0xf8
	lda	0xfe
	sta	0xf6
	lda	0xfc
	sta	0xf4
	lda	0xfa
	sta	0xf2
	; Set bit 0 of dividend (which we shifted left, so position is open).
	lda	0xe0
	ora	#0x1
	sta	0xe0
.Ludivmoddi_skip:
	dey
	bne	.Ludivmoddi_loop
	rts

; --------------------------------------------------------------------
; __divdi3 / __moddi3 — signed 64-bit divide / modulo.  Take absolute
; values, run the unsigned core, fix up the sign.
;   div: sign(quotient) = sign(a) XOR sign(b)
;   mod: sign(remainder) = sign(a)
; --------------------------------------------------------------------
	.globl __divdi3
__divdi3:
	jsr	__divmoddi4_stash
	; Track signs: bit 15 of $E6 (a) and $EE (b).  Save XOR in a temp.
	lda	0xe6
	eor	0xee
	and	#0x8000
	sta	0xfa			; sign of quotient at $FA
	; Abs(a)
	jsr	__absdi_a
	; Abs(b)
	jsr	__absdi_b
	jsr	__udivmoddi_core
	; Fix quotient sign: if $FA != 0, negate $E0..$E6.
	lda	0xfa
	beq	.Ldivdi_pos
	jsr	__negdi_a
.Ldivdi_pos:
	brl	__retdi

	.globl __moddi3
__moddi3:
	jsr	__divmoddi4_stash
	; Mod sign = sign of a.
	lda	0xe6
	and	#0x8000
	sta	0xfa
	jsr	__absdi_a
	jsr	__absdi_b
	jsr	__udivmoddi_core
	; Move remainder to $E0..$E6.
	lda	0xf2
	sta	0xe0
	lda	0xf4
	sta	0xe2
	lda	0xf6
	sta	0xe4
	lda	0xf8
	sta	0xe6
	; Apply sign.
	lda	0xfa
	beq	.Lmoddi_pos
	jsr	__negdi_a
.Lmoddi_pos:
	brl	__retdi

; --- subroutines used by signed div/mod ---

; __absdi_a: if $E6 has sign bit set, negate $E0..$E6.
__absdi_a:
	lda	0xe6
	bpl	.Labsdi_a_done
	jsr	__negdi_a
.Labsdi_a_done:
	rts

; __absdi_b: if $EE has sign bit set, negate $E8..$EE.
__absdi_b:
	lda	0xee
	bpl	.Labsdi_b_done
	jsr	__negdi_b
.Labsdi_b_done:
	rts

; __negdi_a: 2's complement negate $E0..$E6.
__negdi_a:
	sec
	lda	#0x0
	sbc	0xe0
	sta	0xe0
	lda	#0x0
	sbc	0xe2
	sta	0xe2
	lda	#0x0
	sbc	0xe4
	sta	0xe4
	lda	#0x0
	sbc	0xe6
	sta	0xe6
	rts

; __negdi_b: 2's complement negate $E8..$EE.
__negdi_b:
	sec
	lda	#0x0
	sbc	0xe8
	sta	0xe8
	lda	#0x0
	sbc	0xea
	sta	0xea
	lda	#0x0
	sbc	0xec
	sta	0xec
	lda	#0x0
	sbc	0xee
	sta	0xee
	rts

; --------------------------------------------------------------------
; setjmp(jmp_buf env) - save calling environment, return 0
; longjmp(jmp_buf env, int val) - restore environment, return val (or 1 if val == 0)
;
; jmp_buf layout (8 bytes):
;   [0..1]  = caller's stack pointer (SP+3 at entry to setjmp)
;   [2..3]  = return address PC lo:hi (16 bits)
;   [4]     = return address bank (1 byte)
;   [5..6]  = direct page register (DP)
;   [7]     = reserved / padding
;
; Caller-save convention: longjmp doesn't restore X / Y / A — caller's
; setjmp returned 0 with all-callee-savable regs already preserved by
; setjmp's caller.
; --------------------------------------------------------------------
; setjmp / longjmp use the (dp),y indirect mode (opcodes 0x91/0xb1)
; to write through the jmp_buf pointer in $E0.  Y is set explicitly
; before each indirect access; M=0 except where noted.
	.globl setjmp
setjmp:
	sta	0xe0		; jmp_buf addr -> DP scratch
	tsc			; A = current SP
	clc
	adc	#0x3		; A = caller's SP (undo JSL push)
	ldy	#0
	sta (0xe0), y		; env[0..1] = caller SP (16-bit M)
	lda	0x1, s		; A = retaddr lo:hi
	ldy	#2
	sta (0xe0), y		; env[2..3] = retaddr lo:hi
	sep	#0x20
	lda	0x3, s		; A_lo = bank
	ldy	#4
	sta (0xe0), y		; env[4] = bank (8-bit M)
	rep	#0x20
	tdc			; A = DP
	ldy	#5
	sta (0xe0), y		; env[5..6] = DP
	lda	#0		; setjmp returns 0
	rtl

	.globl longjmp
longjmp:
	sta	0xe0		; jmp_buf addr -> DP scratch
	lda	0x4, s		; A = val (2nd arg, on stack)
	sta	0xe2		; save val
	; Restore SP: env[0..1].  TCS directly — the PHAs below consume
	; 3 bytes (1 bank + 2 retaddr); RTL pulls them back; net post-RTL
	; S = saved_SP.  (Earlier this subtracted 3 before TCS, leaving
	; S = saved_SP - 3 after RTL, mangling caller's stack-relative
	; reads — caught by an SJLJ EH test where main's locals shifted
	; by 3 bytes after longjmp.)
	ldy	#0
	lda (0xe0), y		; A = saved SP (16-bit)
	tcs			; SP = saved_SP
	; Push retaddr: bank, then 16-bit lo:hi.  RTL pulls lo, hi, bank.
	sep	#0x20
	ldy	#4
	lda (0xe0), y		; bank (8-bit)
	pha
	rep	#0x20
	ldy	#2
	lda (0xe0), y		; lo:hi (16-bit)
	pha
	; Restore DP.
	ldy	#5
	lda (0xe0), y		; DP (16-bit)
	tcd
	; Compute return value: val if nonzero, else 1.
	lda	0xe2
	bne	.Llj_done
	lda	#1
.Llj_done:
	rtl

; --------------------------------------------------------------------
; __umulhisi3_qsq: quarter-square lookup table.
; T[k] = floor(k² / 4) for k ∈ [0, 510].  Used by __umulhisi3's fast
; 8x8 path: a*b = T[a+b] - T[|a-b|].  511 entries × 2 bytes = 1022 B.
; --------------------------------------------------------------------
__umulhisi3_qsq:
	.short	0x0000, 0x0000, 0x0001, 0x0002, 0x0004, 0x0006, 0x0009, 0x000c
	.short	0x0010, 0x0014, 0x0019, 0x001e, 0x0024, 0x002a, 0x0031, 0x0038
	.short	0x0040, 0x0048, 0x0051, 0x005a, 0x0064, 0x006e, 0x0079, 0x0084
	.short	0x0090, 0x009c, 0x00a9, 0x00b6, 0x00c4, 0x00d2, 0x00e1, 0x00f0
	.short	0x0100, 0x0110, 0x0121, 0x0132, 0x0144, 0x0156, 0x0169, 0x017c
	.short	0x0190, 0x01a4, 0x01b9, 0x01ce, 0x01e4, 0x01fa, 0x0211, 0x0228
	.short	0x0240, 0x0258, 0x0271, 0x028a, 0x02a4, 0x02be, 0x02d9, 0x02f4
	.short	0x0310, 0x032c, 0x0349, 0x0366, 0x0384, 0x03a2, 0x03c1, 0x03e0
	.short	0x0400, 0x0420, 0x0441, 0x0462, 0x0484, 0x04a6, 0x04c9, 0x04ec
	.short	0x0510, 0x0534, 0x0559, 0x057e, 0x05a4, 0x05ca, 0x05f1, 0x0618
	.short	0x0640, 0x0668, 0x0691, 0x06ba, 0x06e4, 0x070e, 0x0739, 0x0764
	.short	0x0790, 0x07bc, 0x07e9, 0x0816, 0x0844, 0x0872, 0x08a1, 0x08d0
	.short	0x0900, 0x0930, 0x0961, 0x0992, 0x09c4, 0x09f6, 0x0a29, 0x0a5c
	.short	0x0a90, 0x0ac4, 0x0af9, 0x0b2e, 0x0b64, 0x0b9a, 0x0bd1, 0x0c08
	.short	0x0c40, 0x0c78, 0x0cb1, 0x0cea, 0x0d24, 0x0d5e, 0x0d99, 0x0dd4
	.short	0x0e10, 0x0e4c, 0x0e89, 0x0ec6, 0x0f04, 0x0f42, 0x0f81, 0x0fc0
	.short	0x1000, 0x1040, 0x1081, 0x10c2, 0x1104, 0x1146, 0x1189, 0x11cc
	.short	0x1210, 0x1254, 0x1299, 0x12de, 0x1324, 0x136a, 0x13b1, 0x13f8
	.short	0x1440, 0x1488, 0x14d1, 0x151a, 0x1564, 0x15ae, 0x15f9, 0x1644
	.short	0x1690, 0x16dc, 0x1729, 0x1776, 0x17c4, 0x1812, 0x1861, 0x18b0
	.short	0x1900, 0x1950, 0x19a1, 0x19f2, 0x1a44, 0x1a96, 0x1ae9, 0x1b3c
	.short	0x1b90, 0x1be4, 0x1c39, 0x1c8e, 0x1ce4, 0x1d3a, 0x1d91, 0x1de8
	.short	0x1e40, 0x1e98, 0x1ef1, 0x1f4a, 0x1fa4, 0x1ffe, 0x2059, 0x20b4
	.short	0x2110, 0x216c, 0x21c9, 0x2226, 0x2284, 0x22e2, 0x2341, 0x23a0
	.short	0x2400, 0x2460, 0x24c1, 0x2522, 0x2584, 0x25e6, 0x2649, 0x26ac
	.short	0x2710, 0x2774, 0x27d9, 0x283e, 0x28a4, 0x290a, 0x2971, 0x29d8
	.short	0x2a40, 0x2aa8, 0x2b11, 0x2b7a, 0x2be4, 0x2c4e, 0x2cb9, 0x2d24
	.short	0x2d90, 0x2dfc, 0x2e69, 0x2ed6, 0x2f44, 0x2fb2, 0x3021, 0x3090
	.short	0x3100, 0x3170, 0x31e1, 0x3252, 0x32c4, 0x3336, 0x33a9, 0x341c
	.short	0x3490, 0x3504, 0x3579, 0x35ee, 0x3664, 0x36da, 0x3751, 0x37c8
	.short	0x3840, 0x38b8, 0x3931, 0x39aa, 0x3a24, 0x3a9e, 0x3b19, 0x3b94
	.short	0x3c10, 0x3c8c, 0x3d09, 0x3d86, 0x3e04, 0x3e82, 0x3f01, 0x3f80
	.short	0x4000, 0x4080, 0x4101, 0x4182, 0x4204, 0x4286, 0x4309, 0x438c
	.short	0x4410, 0x4494, 0x4519, 0x459e, 0x4624, 0x46aa, 0x4731, 0x47b8
	.short	0x4840, 0x48c8, 0x4951, 0x49da, 0x4a64, 0x4aee, 0x4b79, 0x4c04
	.short	0x4c90, 0x4d1c, 0x4da9, 0x4e36, 0x4ec4, 0x4f52, 0x4fe1, 0x5070
	.short	0x5100, 0x5190, 0x5221, 0x52b2, 0x5344, 0x53d6, 0x5469, 0x54fc
	.short	0x5590, 0x5624, 0x56b9, 0x574e, 0x57e4, 0x587a, 0x5911, 0x59a8
	.short	0x5a40, 0x5ad8, 0x5b71, 0x5c0a, 0x5ca4, 0x5d3e, 0x5dd9, 0x5e74
	.short	0x5f10, 0x5fac, 0x6049, 0x60e6, 0x6184, 0x6222, 0x62c1, 0x6360
	.short	0x6400, 0x64a0, 0x6541, 0x65e2, 0x6684, 0x6726, 0x67c9, 0x686c
	.short	0x6910, 0x69b4, 0x6a59, 0x6afe, 0x6ba4, 0x6c4a, 0x6cf1, 0x6d98
	.short	0x6e40, 0x6ee8, 0x6f91, 0x703a, 0x70e4, 0x718e, 0x7239, 0x72e4
	.short	0x7390, 0x743c, 0x74e9, 0x7596, 0x7644, 0x76f2, 0x77a1, 0x7850
	.short	0x7900, 0x79b0, 0x7a61, 0x7b12, 0x7bc4, 0x7c76, 0x7d29, 0x7ddc
	.short	0x7e90, 0x7f44, 0x7ff9, 0x80ae, 0x8164, 0x821a, 0x82d1, 0x8388
	.short	0x8440, 0x84f8, 0x85b1, 0x866a, 0x8724, 0x87de, 0x8899, 0x8954
	.short	0x8a10, 0x8acc, 0x8b89, 0x8c46, 0x8d04, 0x8dc2, 0x8e81, 0x8f40
	.short	0x9000, 0x90c0, 0x9181, 0x9242, 0x9304, 0x93c6, 0x9489, 0x954c
	.short	0x9610, 0x96d4, 0x9799, 0x985e, 0x9924, 0x99ea, 0x9ab1, 0x9b78
	.short	0x9c40, 0x9d08, 0x9dd1, 0x9e9a, 0x9f64, 0xa02e, 0xa0f9, 0xa1c4
	.short	0xa290, 0xa35c, 0xa429, 0xa4f6, 0xa5c4, 0xa692, 0xa761, 0xa830
	.short	0xa900, 0xa9d0, 0xaaa1, 0xab72, 0xac44, 0xad16, 0xade9, 0xaebc
	.short	0xaf90, 0xb064, 0xb139, 0xb20e, 0xb2e4, 0xb3ba, 0xb491, 0xb568
	.short	0xb640, 0xb718, 0xb7f1, 0xb8ca, 0xb9a4, 0xba7e, 0xbb59, 0xbc34
	.short	0xbd10, 0xbdec, 0xbec9, 0xbfa6, 0xc084, 0xc162, 0xc241, 0xc320
	.short	0xc400, 0xc4e0, 0xc5c1, 0xc6a2, 0xc784, 0xc866, 0xc949, 0xca2c
	.short	0xcb10, 0xcbf4, 0xccd9, 0xcdbe, 0xcea4, 0xcf8a, 0xd071, 0xd158
	.short	0xd240, 0xd328, 0xd411, 0xd4fa, 0xd5e4, 0xd6ce, 0xd7b9, 0xd8a4
	.short	0xd990, 0xda7c, 0xdb69, 0xdc56, 0xdd44, 0xde32, 0xdf21, 0xe010
	.short	0xe100, 0xe1f0, 0xe2e1, 0xe3d2, 0xe4c4, 0xe5b6, 0xe6a9, 0xe79c
	.short	0xe890, 0xe984, 0xea79, 0xeb6e, 0xec64, 0xed5a, 0xee51, 0xef48
	.short	0xf040, 0xf138, 0xf231, 0xf32a, 0xf424, 0xf51e, 0xf619, 0xf714
	.short	0xf810, 0xf90c, 0xfa09, 0xfb06, 0xfc04, 0xfd02, 0xfe01


; --------------------------------------------------------------------
; memcpy / memmove / memset -- block moves with MVN / MVP.
;
; C signatures (size_t is 32 bits):
;   void *memcpy (void *dst, const void *src, size_t n);
;   void *memmove(void *dst, const void *src, size_t n);
;   void *memset (void *dst, int c, size_t n);
; dst arrives in A (offset) : X (bank, pad byte 0); the other arguments
; are on the stack after the 3-byte return address: src at 4,s and n at
; 8,s, or c at 4,s and n at 6,s.  All three return dst in A:X.
;
; Counts below MEM_SHORT copy through [dp],y with 16-bit loads and stores
; (10.5 cycles a byte): MVN's setup costs as much as that many bytes of
; the loop.  A long-indirect-indexed address is a full 24-bit sum, so
; these loops cross bank boundaries on their own.
;
; Longer copies run MVN (forward) or MVP (backward), 7 cycles a byte.
; Both wrap X and Y inside their banks, so a copy is cut into runs that
; stop at every 64 KB boundary of the source and of the destination (most
; copies are one run).  The bank operands are immediates, so each call
; builds its own 4-byte stub on the stack -- MVN/MVP, dst bank, src bank,
; RTL -- and RTLs into it; the stack is always bank 0, so this works from
; any code bank.  Nothing shared is modified: an interrupt that lands
; mid-move (MVN/MVP resume after RTI) may itself call these routines, as
; may another GNO thread, without disturbing the interrupted move.  MVN /
; MVP set DBR to the destination bank; the caller's DBR is saved on entry
; and restored on exit.  D is never changed; the direct-page bytes used
; are the $E0-$EB libcall scratch, so an interrupt handler that runs C
; must switch D (or save that scratch) like it must for any C code.
;
; memset needs no block move: a 16-bit store loop fills at 5.9 cycles a
; byte, faster than MVN, without its setup.  It fills in passes of at
; most MEM_PASS bytes so the 16-bit index never wraps.
; --------------------------------------------------------------------

MEM_SRC   = 0xe0            ; DP: source (memset: fill) far pointer
MEM_DST   = 0xe4            ; DP: destination far pointer, the result
MEM_REM   = 0xe8            ; DP: memset bytes still to fill, 32 bits
MEM_SHORT = 34              ; from this count on, copies use MVN / MVP
MEM_PASS  = 0x8000          ; memset pass length for fills of 64 KB up
OP_MVN    = 0x54
OP_MVP    = 0x44
OP_RTL    = 0x6b
OP_FWD    = 0x10            ; the opcode bit that tells MVN from MVP

; The multi-run frame, as stack offsets from its base S0 (the stack
; pointer at the top of every run).
F_CNT     = 1               ; bytes - 1 in this run (MVN/MVP's A)
F_REM     = 3               ; bytes still to move, 32 bits
F_STUB    = 7               ; stub: opcode, dst bank, src bank, RTL
F_DBR     = 11              ; the caller's DBR

	.section .text.memcpy,"ax",@progbits
	.globl memcpy
	.globl memmove
	.globl memset

memcpy:
	rep #0x30
	sta MEM_DST
	stx MEM_DST+2
	lda 4, s
	sta MEM_SRC
	lda 6, s
	sta MEM_SRC+2
	lda 10, s                   ; n.hi
	bne .LmemFwdFar
	lda 8, s                    ; n.lo
	cmp #MEM_SHORT
	bcs .LmemFwdLong
	tay
	beq .LmemRet
; Copy [0, Y) top down in words, an odd last byte first: right for any
; copy but an overlapping one with dst below src.  Y < $8000.
.LmemDown:
	tya
	lsr a                       ; C = odd count
	bcc .LmemDownPair
	dey
	sep #0x20
	.a8
	lda [MEM_SRC], y
	sta [MEM_DST], y
	rep #0x20
	.a16
.LmemDownPair:
	dey
	dey                         ; the last whole word, or $FFFE: none left
	bmi .LmemRet
.LmemDownWords:
	lda [MEM_SRC], y
	sta [MEM_DST], y
	dey
	dey
	bpl .LmemDownWords
.LmemRet:
	lda MEM_DST
	ldx MEM_DST+2
	rtl

; One MVN when neither block crosses a bank boundary.  A = n < 64 KB.
.LmemFwdLong:
	dec a
	tax                         ; n - 1
	clc
	adc MEM_SRC
	bcs .LmemFwdFar
	txa
	clc
	adc MEM_DST
	bcs .LmemFwdFar
	ldx MEM_SRC
	ldy MEM_DST
	lda #OP_MVN
; One MVN / MVP (A = its opcode) of n bytes from X to Y, in the source's
; and the destination's banks.  The stub's RTL returns to .LmemOneBack.
.LmemOne:
	phb
	sep #0x20
	.a8
	xba                         ; keep the opcode in B
	lda #OP_RTL
	pha
	lda MEM_SRC+2
	pha
	lda MEM_DST+2
	pha
	xba
	pha                         ; the stub is 4,s..7,s from here
	phk
	per .LmemOneBack-1
	lda #0
	pha                         ; the stack's bank
	rep #0x20
	.a16
	tsc
	clc
	adc #4                      ; the stub's address - 1
	pha
	lda 19, s                   ; n.lo (n was 8,s on entry, 11 bytes ago)
	dec a
	rtl
.LmemOneBack:
	pla                         ; drop the stub
	pla
	plb
	lda MEM_DST
	ldx MEM_DST+2
	rtl
.LmemFwdFar:
	brl .LmemFwdRuns

memmove:
	rep #0x30
	sta MEM_DST
	stx MEM_DST+2
	lda 4, s
	sta MEM_SRC
	lda 6, s
	sta MEM_SRC+2
	; dst below src moves forward, dst above src backward; the pad
	; bytes above the banks take no part in the compare.
	sep #0x20
	.a8
	lda MEM_DST+2
	cmp MEM_SRC+2
	rep #0x20
	.a16
	bne .LmemOrdered
	lda MEM_DST
	cmp MEM_SRC
	beq .LmemMoveRet
.LmemOrdered:
	bcc .LmemMoveUp
	lda 10, s                   ; n.hi
	bne .LmemBwdRuns
	lda 8, s                    ; n.lo
	cmp #MEM_SHORT
	bcs .LmemBwdLong
	tay
	beq .LmemMoveRet
	brl .LmemDown
.LmemMoveUp:
	lda 10, s                   ; n.hi
	bne .LmemMoveFwdRuns
	lda 8, s                    ; n.lo
	cmp #MEM_SHORT
	bcs .LmemMoveFwdLong
	tay
	beq .LmemMoveRet
; Copy [0, n) bottom up in words, an odd last byte last: right for an
; overlapping copy with dst below src.  A = n; C survives the loop.
	ldy #0
	lsr a                       ; C = odd count, A = whole words
	tax
	beq .LmemUpOdd
.LmemUpWords:
	lda [MEM_SRC], y
	sta [MEM_DST], y
	iny
	iny
	dex
	bne .LmemUpWords
.LmemUpOdd:
	bcc .LmemMoveRet
	sep #0x20
	.a8
	lda [MEM_SRC], y
	sta [MEM_DST], y
	rep #0x20
	.a16
.LmemMoveRet:
	lda MEM_DST
	ldx MEM_DST+2
	rtl
.LmemMoveFwdLong:
	brl .LmemFwdLong
.LmemMoveFwdRuns:
	brl .LmemFwdRuns

; One MVP from the last bytes when neither block crosses a bank
; boundary.  A = n < 64 KB.
.LmemBwdLong:
	dec a
	tay                         ; n - 1
	clc
	adc MEM_SRC
	bcs .LmemBwdRuns
	tax                         ; the source's last byte
	tya
	clc
	adc MEM_DST
	bcs .LmemBwdRuns
	tay                         ; the destination's last byte
	lda #OP_MVP
	brl .LmemOne

; Several runs, backward: X / Y = the last byte's offset in its bank, the
; stub's bank bytes = that byte's bank.
.LmemBwdRuns:
	phb
	pea OP_RTL << 8
	pea OP_MVP
	; S0 + 6 here: the stub is 1,s..4,s and n 13,s..16,s.
	lda MEM_SRC
	clc
	adc 13, s
	tax
	sep #0x20
	.a8
	lda MEM_SRC+2
	adc 15, s
	sta 3, s
	rep #0x20
	.a16
	txa
	bne .LmemBwdSrc
	sep #0x20
	.a8
	lda 3, s
	dec a
	sta 3, s
	rep #0x20
	.a16
.LmemBwdSrc:
	dex
	lda MEM_DST
	clc
	adc 13, s
	tay
	sep #0x20
	.a8
	lda MEM_DST+2
	adc 15, s
	sta 2, s
	rep #0x20
	.a16
	tya
	bne .LmemBwdDst
	sep #0x20
	.a8
	lda 2, s
	dec a
	sta 2, s
	rep #0x20
	.a16
.LmemBwdDst:
	dey
	bra .LmemRem

; Several runs, forward: X / Y = the first byte's offset.
.LmemFwdRuns:
	phb
	pea OP_RTL << 8
	pea OP_MVN
	sep #0x20
	.a8
	lda MEM_SRC+2
	sta 3, s
	lda MEM_DST+2
	sta 2, s
	rep #0x20
	.a16
	ldx MEM_SRC
	ldy MEM_DST
.LmemRem:
	lda 15, s                   ; n.hi
	pha
	lda 15, s                   ; n.lo
	pha
	pha                         ; F_CNT
; One run: as many bytes as fit before the source or the destination
; leaves its bank, or the rest of the copy if that is less.
.LmemRun:
	lda F_STUB, s
	and #OP_FWD
	beq .LmemRunDown
	txa
	eor #0xffff                 ; bytes - 1 up to the source bank's end
	sta F_CNT, s
	tya
	eor #0xffff                 ; and to the destination bank's end
	bra .LmemRunMin
.LmemRunDown:
	txa                         ; bytes - 1 down to the source bank's start
	sta F_CNT, s
	tya                         ; and to the destination bank's start
.LmemRunMin:
	cmp F_CNT, s
	bcs .LmemRunRem
	sta F_CNT, s
.LmemRunRem:
	lda F_REM+2, s
	bne .LmemRunGo              ; 64 KB or more to go: a bank edge is first
	lda F_REM, s
	dec a
	cmp F_CNT, s
	bcs .LmemRunGo
	sta F_CNT, s
.LmemRunGo:
	lda F_CNT, s
	eor #0xffff                 ; -(bytes in this run)
	clc
	adc F_REM, s
	sta F_REM, s
	lda F_REM+2, s
	adc #0xffff
	sta F_REM+2, s
	; RTL into the stub (bank 0), whose RTL comes back here.
	phk
	per .LmemRunBack-1
	sep #0x20
	.a8
	lda #0
	pha
	rep #0x20
	.a16
	tsc
	clc
	adc #F_STUB+3               ; S0 - 4 now: the stub's address - 1
	pha
	lda F_CNT+6, s
	rtl
.LmemRunBack:
	lda F_REM, s
	ora F_REM+2, s
	beq .LmemRunsDone
	; X / Y wrapped to 0 (forward) or $FFFF (backward) where the source /
	; destination left its bank: step that bank byte.
	lda F_STUB, s
	and #OP_FWD
	sep #0x20
	.a8
	beq .LmemStepDown
	cpx #0
	bne .LmemStepUpDst
	lda F_STUB+2, s
	inc a
	sta F_STUB+2, s
.LmemStepUpDst:
	cpy #0
	bne .LmemStepDone
	lda F_STUB+1, s
	inc a
	sta F_STUB+1, s
	bra .LmemStepDone
.LmemStepDown:
	cpx #0xffff
	bne .LmemStepDownDst
	lda F_STUB+2, s
	dec a
	sta F_STUB+2, s
.LmemStepDownDst:
	cpy #0xffff
	bne .LmemStepDone
	lda F_STUB+1, s
	dec a
	sta F_STUB+1, s
.LmemStepDone:
	rep #0x20
	.a16
	bra .LmemRun
.LmemRunsDone:
	tsc
	clc
	adc #F_DBR-1                ; drop the run count, the rest and the stub
	tcs
	plb
	lda MEM_DST
	ldx MEM_DST+2
	rtl

memset:
	rep #0x30
	sta MEM_DST
	stx MEM_DST+2
	sta MEM_SRC                 ; the fill pointer moves; MEM_DST is the result
	stx MEM_SRC+2
	lda 4, s                    ; c: only its low byte counts
	sep #0x20
	.a8
	xba
	lda 4, s
	rep #0x20
	.a16
	tax                         ; X = the fill byte, twice
	lda 8, s                    ; n.hi
	sta MEM_REM+2
	bne .LmemSetBig
	lda 6, s                    ; n.lo
	tay
	beq .LmemSetRet
; Fill [0, Y) top down: an odd last byte, words down to a multiple of 8
; bytes, then 8 bytes a loop.
.LmemSetFill:
	tya
	lsr a                       ; C = odd count
	bcc .LmemSetPeel
	dey
	txa
	sep #0x20
	.a8
	sta [MEM_SRC], y
	rep #0x20
	.a16
.LmemSetPeel:
	tya
	bit #0x0006
	beq .LmemSetOcts
	txa
	dey
	dey
	sta [MEM_SRC], y
	bra .LmemSetPeel
.LmemSetOcts:
	txa
	cpy #0
	beq .LmemSetDone
.LmemSetOct:
	dey
	dey
	sta [MEM_SRC], y
	dey
	dey
	sta [MEM_SRC], y
	dey
	dey
	sta [MEM_SRC], y
	dey
	dey
	sta [MEM_SRC], y
	bne .LmemSetOct
.LmemSetDone:
	lda MEM_REM+2
	bne .LmemSetStep
.LmemSetRet:
	lda MEM_DST
	ldx MEM_DST+2
	rtl
; 64 KB or more: MEM_PASS bytes a pass while the rest's high word is
; nonzero (so it is nonzero after every such pass's fill), then the rest.
.LmemSetBig:
	lda 6, s                    ; n.lo
	sta MEM_REM
.LmemSetPass:
	ldy #MEM_PASS
	bra .LmemSetFill
.LmemSetStep:
	lda MEM_SRC
	clc
	adc #MEM_PASS
	sta MEM_SRC
	lda MEM_SRC+2
	adc #0
	sta MEM_SRC+2
	lda MEM_REM
	sec
	sbc #MEM_PASS
	sta MEM_REM
	lda MEM_REM+2
	sbc #0
	sta MEM_REM+2
	bne .LmemSetPass
	ldy MEM_REM
	bne .LmemSetFill
	bra .LmemSetRet