From d4b2f4b53596e30e2a5a839d3395227af4eb1b52 Mon Sep 17 00:00:00 2001 From: Scott Duensing Date: Tue, 29 Sep 2026 19:30:56 -0500 Subject: [PATCH] Yet more fixes! --- demos/rsrcProbe.apl | Bin 47302 -> 47302 bytes runtime/include/c++/cmath | 20 + runtime/include/math.h | 12 + runtime/src/math.c | 421 +++++- runtime/src/softDouble.c | 176 ++- runtime/src/softFloat.c | 172 ++- scripts/common.sh | 10 + scripts/runInMame.sh | 4 +- scripts/runInMameCycles.sh | 4 +- scripts/runInMameWithGsosStub.sh | 4 +- scripts/runMultiSeg.sh | 7 +- scripts/smokeTest.sh | 1037 +++++++++++++- src/llvm/lib/Target/W65816/CMakeLists.txt | 1 + src/llvm/lib/Target/W65816/W65816.h | 9 + .../lib/Target/W65816/W65816AsmPrinter.cpp | 43 +- src/llvm/lib/Target/W65816/W65816CmpSink.cpp | 170 +++ .../Target/W65816/W65816FarPtrStageReuse.cpp | 233 +++- .../lib/Target/W65816/W65816I32IncFold.cpp | 331 +++-- .../lib/Target/W65816/W65816ISelLowering.cpp | 275 ++-- .../lib/Target/W65816/W65816ISelLowering.h | 10 + src/llvm/lib/Target/W65816/W65816InstrInfo.td | 34 +- .../lib/Target/W65816/W65816StackRelToImg.cpp | 1218 ++++++++++++----- .../Target/W65816/W65816StackSlotCleanup.cpp | 85 +- .../Target/W65816/W65816StackSlotMerge.cpp | 56 +- .../lib/Target/W65816/W65816TargetMachine.cpp | 5 + .../CodeGen/W65816/addframe-flag-wrap.mir | 10 + .../W65816/andi8-flag-corrupt-wrap.mir | 10 + src/llvm/test/CodeGen/W65816/cmp-sink.ll | 47 + src/llvm/test/CodeGen/W65816/cmp-via-x.mir | 189 +++ .../test/CodeGen/W65816/cmp0-fprel-keep.mir | 60 + .../CodeGen/W65816/extract-wide32-regseq.ll | 12 +- src/llvm/test/CodeGen/W65816/far-ptr-home.ll | 46 + src/llvm/test/CodeGen/W65816/far-ptr-home.mir | 106 ++ .../test/CodeGen/W65816/i32-compare-chain.ll | 20 +- src/llvm/test/CodeGen/W65816/i32-dec-fold.mir | 159 +++ .../CodeGen/W65816/i32-inc-carry-slot.mir | 66 +- src/llvm/test/CodeGen/W65816/i32-shift-16.ll | 19 +- .../CodeGen/W65816/ina-flag-corrupt-wrap.mir | 10 + src/llvm/test/CodeGen/W65816/loop-local-dp.ll | 72 + .../CodeGen/W65816/php-wrap-frame-index.mir | 6 + src/llvm/test/CodeGen/W65816/ret-ext-i16.ll | 56 + .../W65816/seprep-ldy-elision-kill-flag.ll | 4 +- .../test/CodeGen/W65816/shift-const-inline.ll | 98 ++ .../test/CodeGen/W65816/sign-test-flags.mir | 14 + .../W65816/srl8a-flag-corrupt-wrap.mir | 10 + .../W65816/stack-slot-merge-read-between.mir | 94 ++ .../test/CodeGen/W65816/store-i32-global.ll | 51 + tests/coremark/coreMark.manifest.json | 6 +- tests/fpNan/genFpTables.py | 589 ++++++++ tests/lua/luaTest.manifest.json | 10 +- tests/regress/genShiftDec.py | 160 +++ 51 files changed, 5372 insertions(+), 889 deletions(-) create mode 100644 src/llvm/lib/Target/W65816/W65816CmpSink.cpp create mode 100644 src/llvm/test/CodeGen/W65816/cmp-sink.ll create mode 100644 src/llvm/test/CodeGen/W65816/cmp-via-x.mir create mode 100644 src/llvm/test/CodeGen/W65816/cmp0-fprel-keep.mir create mode 100644 src/llvm/test/CodeGen/W65816/far-ptr-home.ll create mode 100644 src/llvm/test/CodeGen/W65816/far-ptr-home.mir create mode 100644 src/llvm/test/CodeGen/W65816/i32-dec-fold.mir create mode 100644 src/llvm/test/CodeGen/W65816/loop-local-dp.ll create mode 100644 src/llvm/test/CodeGen/W65816/ret-ext-i16.ll create mode 100644 src/llvm/test/CodeGen/W65816/shift-const-inline.ll create mode 100644 src/llvm/test/CodeGen/W65816/stack-slot-merge-read-between.mir create mode 100644 src/llvm/test/CodeGen/W65816/store-i32-global.ll create mode 100644 tests/fpNan/genFpTables.py create mode 100644 tests/regress/genShiftDec.py diff --git a/demos/rsrcProbe.apl b/demos/rsrcProbe.apl index 74552e45c68b46f5f43b80ba89c2f02833b746aa..119b3c1b9d4ba5326fbf002b2b169d4a7c4a95b0 100644 GIT binary patch delta 9806 zcmb_idwkPXmcPIJnx<(9ZHQ@`hO~K5($WVl4=K;`D9|EL5mG<~u)Mkz9hr}x5nLyj z-vqV#b8zlD#hGCzcsEGu#7)X5s3T&F&eYG08?n^d!EPGsQYBUDQb~Ko(`VN8pIH|%@9hr|67BaN>vd}C z$h8abXV3Ze3g^fY5OOvW;(T3C8>J1lqLsnpts(15=hk$xTm0!HWKgr`x^v$`dQK|R z0&m%LI!fcw3{0X;03TJ-lbYXQZ?|mKJX20vpqOOipzM2u zga}azLf41ng4U2k`a!phBvXjIsu-CmrUM>6v;q~lG0x3I5nE_-n{4~`C}vdC0wFS+ z57mM|e5h-9tO;ZY$PhIppr!^`rkWmLCRG!lucnyO4`_lh#mdaGmG-5K(Kza8iuSnl zC)%N%B~7B5)Y)myI<(nWL>Hx6T{+C$kfWHG-rvs*{sEQ&Zf)rYokZw#I_v1+RHxR$ zv<(*eAE~b{f#J;D%1no+=Qg5Kn%XN4Pp`U|508ogy_1RFG0K_XorMxEo-fvjYq7D+ zst`AMfq;e_?N@YQ zA%Ira4NhR0xeZ!HV#WrYqGwWWIGd+TS9+KpFTBa`^B8nN7ESm>Mu#8q_aBeRr zzNLg|a~m?1Or~ovDn{5l#o%Sqpcc2r01pI>H3(zD=Vwi@6IsHaIPWlxsO0SVt(;eyg(t7$>`gnPJOAn2u zkCU?LU(@GF8)$XLT|kS`_KY!Ew_k^yB(+TTpV$tDR?*KhRIdii84*x)uv6Y4=P1}% z$+?(70^x?1;tKR)1*{6(I;lfu(-bYc`Rp`s);^5mPKf6;e;>P9(FXgllfot`GTI5u zJ!Rl{e^+Q|$ln+2_V)+R_y>;m<7U``T^+WV?DZ{_Sw>!G79@pFmXwSuqhPO{HFadS zzpvTf8+s<7lmz>78%l8PN^!8aq1fLY`b9u-Bh?LRk1K^hE%f&WV=zHC#0Ull%mEtN zP_Pe(z45^D$2ia5ySWRfF|ftk@9#kq`yl$egD3oFf_*S-`Ox{^nC5&eNxac?e^*cp z33e$3!5--LbrhrpyNbYtqlSXLxs4L&3&t7=to(d8=s5!hi0Q;^iUF+Z4R-lsY?PsO zz#by)_I@l2zIU$s=#kWsO_JB&H((Dxf3V#S)`8;VBpAzW903}Gr%-V>lUurwlt%nL zg4D|R7KK z;BcSGgH&V=wG*e3!H zOGYVuBx|g6jYhI=&$LRwRtlL5*4ddtsLi-iGpmg5G(M-%)E#k-dRPLazcuB0e~!Uu zOs11japOn_b7NIiT*84xBiJwG{Wnswfdc8ZKOPr~8zYN ze33$cFt_3Y zC56oLt>W;~b>E{yXkQL9#7_+3B%xylG>eG+^NK(=!}2Ym4Qmt=6p{^Vcp(9&UXVEb z0YQHtNskHouLS*9N%}ECe^Jn1Owt2_{;r_Eo1{N1dHDoa1>CFqZ?_3{ZGO)|bD+@O#c5K7Yf1^o|#{)Z&} zkAi+q(9b355kZ&NasBeT!~*sUx%0lZ9>J&59>coc{j4-VMjqk;pKZ-FR=5Oo8oyez=3Yh-4EkhZ`X!vIwTN`60- ze3P<)S(No5nLzNzX*u}*fY3A3O?hT>d=N|m2QN4i@;=>T$ z;~#+IMIf8djbj{cm{~*MkY+}C1J0Ct!*J-1!t}P#6zh1S5tPAs7qebiNNcQBH?3nC z^14n)rx`cc`Gm_v16F%kCRPJ}mR-X##Ttlz;@V_oQUjume;mbp@&Q5RZ8Kv zK8axCTbT~m1Yue?1aI4w;KP;05?VG^3RipJJU!E+6ow~y;Pb2!ybH$+g(xe)kXE+B z-`@;D<8JBaRDTSCzG<`TrhDLfWvYi>wiW88coc1TvWMF8=SXI{AwN&bp}XN<1AQ{z zWq7!h@WHKo?Cbe+b*+#$U=qzN$o9fi%lK5w_*6VuxVW+eU2%&@vH1K-9Y%2=4I|j) z94zHWXxaB5QP&=Z_po~VVHl%sJgja$tp4DzddFdP>tV&>yX))t`&k!%hq^`CmwhWX ze-S6~^MoUrZ&{E;xRHqgXbT>cz-#$mFjp&c3?Ityc-8dXgh}L^F#rqQIL6`)p1`!r zZG@zRG-~8~Zsj=#uQBZMoP+Gn5nMC8i!mehA|tiK*@41t>_Xiz3SkF?HVWIA1GzH8 z40S+$^JK@$02_1O?yN|ZrIbX~N)UJ<01LrI#S(;xE?%*!9OF|V)j<)ig8r*3)tead zt_Ap^9TwAMM!yR!^qWpL531rRR*Iqrp! z7GsCGA&i9O(?L-yn?7tggcFeBp~d5h4Og_OP|>EsN4`N{nUMZjm-YGYwAc?-~a;GJHLsQ2% zD`Ab}`6RGzt_lccK!D9iq)ka;+5%9qi>?_n&ihyy@qPJ)_*mti+nGWBxy@?i1#lw+ ze$+9zlL5?tWyNG)Zwi|s&$^NF#}qbOo_izZg%mcc7XI>KPZI+77tX*fa4De_G(&98 zR^*NXHdoP}wbB0>GkQ@CtbRe_>!JW60X}13rgtQU9ifyur$68Pr64uPPuqlv^R61#_)s-*f<%$iG*U(VVigJ*vIYQ1Xw9uVn zC+Nj&pn!(QF7+tDXj^i6v>2u@z$*}bjwuj(bSq(&PO>LS zjm%}4C{u{nByfu7*Cg@d%6HrZ=|JyuJ7&BnL5LJXv4cHq#q%}P|4<4ZbHJ5WynnK? zoU;XBSHgXx(HK3BqNm+;DKOSSokb66(#zEi6YB}7`L4B8bjv65oSq_wTW`W`^!j0GCc%bRv?#pLP%o#1tld9;S zipQ?J1UrJRn24^JmrH^asxdyn$y;=QDX}kSd=V9`JWGC>a2{}hO=fwmkZJ6^m=e)x?UKnH_X8kv3 zeViCnI?U0Q;Y7N!_(WO5oC3~qBsimnIa4`jdV*sb=B(tLRb}m7VfRhLtR0+ncVdub znDY?lJd)t#4s#B24oh&dhdD2E&MOH{&M@aR=Xl>uur$N0PdV#yf|D}L(U+qaGRos# zNFV07IHxedNgd|Q`}Yo9fq1U2ygxGx@dY=USH=*iS2cCCv$Cw=9x2ov zQS=yWy5Mo&Xu|ZljoEmJF*E|rz;tW+9Ix2-@5v3sfRz$JN1yEHEN#QpliYa_D0=NGrAUsLmo;p+V(9CIfdq?x@Elc=P z1Q?l82u^yIFo{A(BZXw_WMk(lfl3XXQtVtHaH@gMT>(ioc0R+- zk4Oj(xR!8Q8+5jkP$qWrv9pGJpL+(UB;EMeA)n*>W$1m=KbZFESp0FL6N+pF)54D) z&YimNm4>NOVXNYM^YxH#kUlZPQq<+(zm334)zpk1pB;`t*>`Y=Wp>!$VgJyTY{IM% z#z#)gSSuOCHUnexHQ#bNW!Ct*XsYjcDps_K;m0EJ*TSHNO$39oWZ$+_xNARN8+vbD z?V%W(>Z~{7>w99eQpqODl`+}ZkcxRyhIta6RZ=><7}zK+tE^>{Yay?$XbAiZv(&=j z2`c6e;_PzJAI#2`BGgtrM+2RW)w$AbbXWC5*g;Mkwa(@o?5w6g79AA3M&Frze`3sL z+BhdUrk!TYO^(@0w+(mFX~*2FCdj!Gc``qM#czje=$rE#nx;kc<9YRClGek*`Ut*# z3vo9>Uk@q8tg@9&bV6b_*8CIZ=joaiw0Qo#XhvXuuIAU*siQWZ_RPP^y{?4%4ulcYgwTu4F-0VnO z?d#I$$hG#0+VG?{c&8Oh;wUOL%Muv3QlbsZr+D9rsE9@Ex2XE_sQP$R{pYB9UsT;1Rqu$Z%cJVz zs5&jG-W*lOL{)oK9T`>Aqw1Ft^-4q?h^T!L^;ATCETZPOscCKMXHoSGZCzgIRbvr# zUqtXFK~^)TQ51b-*V_u@In*5nn3G^EY*MB4zy2DclKg+_QEq^-l?n=F+ z*|wMt+_nW_&nh><&Q+5~Zd+_Wzr=U+x}87J=T~i7yKQms{QOUf$sYeTyLYMmp`{=Z zyf!G?f40;&0R@BSzr*|teyq9zYyaH;Vem`;`Gm0cdhmP}4%-;~1Tehx$PZWlSDJUt zNa<9)Xti8ZvRdauePm*`RfDu|{IU?v$^0x%D59sm}S z*YyBVvNau`gX~)fFi4h72dI+IpzJP2j|=slU-c z@6M^8jsmVGf^u!yK;jM|1e}56bDd49yLicz+3eIz7E!#6Np3BE0RnhH%=Gy9(evPbtFtp5+K1cs3xM^W24S z(X$ufV-H2R>b6fcXOI1S}D-Lcl5kYY-s+){E{Y0b2w#3fLjw9s&0X z*d^cr0S^lB3wT6;D&PqLPYQTSzySdVIe1A>bVCBd0-^$*7x1EhR|LE!;Pv_~=(Wyz x?HYhT*SiqjuAhT&x<1RXyI(6!C@9XCcub(%BW+`P_JULYQ>{HD{Pc?@!b_{hA5+3rubi(yo zJ-Kui{_a0>sMK|+5QJQ(jx)Ugkh zd?WC(Dp{)GM#JxBur?rv%h_v&-(qR2YBy{zWqZ_O?OI}gV#ff@=rMFsYY*w9X{vQZ z^?g8sgwXQfl~J`kA*hi25fuSzki0_ExOKn;9Q*-oCIBOUK(H;rgicxphVe50P`m#W zPlMj<8N!++!Rftf$v0Kar)iqdHNE$0UC>k~Eka#I5nZ4tu9HqhXsUofFJSSVv;eV! z7%ZuidJyx-sumD~{^cQ*Cn2je%bvzF`p;{DfsYCHc(>~NgCQ_L-8@5evmJ4h48!&8 zH*p7*Pg$FBu5ttGHqQUi>3S6P)hawOtJMHb;mNJg!whcCYB5s_@Th}lv^bzoiT)v; z41O?2{Lo2r0tcy`=KyX8JYtv)z)WrfFcY1&XUKKBo|VSCj81NBak3lZUs(o?@{Det z)>VY|ZaXhh`C%fjB5?_#Tb9YvYg__%GGIY-h zL8U2#TDUS|MBm{LkeM|MBGQ-EVFyN@a9%h14nr#$1`wjnj$>+E1F;N&-5^1TnluE{WL*dma5i{2;_Er+Yg3?)Dm(fDeX0qZ)MjuE z*OMO5me~W%*tn@1_!aHoxC-`0!lb#-xPKsU(0`PhvRYwKpt^7t8Z{f+nl0NhNt@#JouDPR@)BsM2)bzl**}A99Q!h{ zN?FD#lkUiXF!ltF(!5(SVFUW4$S+DAR7DwL5Psdz%j7L>OXa02qVNE7&vSnP&MBcHE=S!KZSr>FB1RB z3#lY<(iOjgZ1oSc`v-%2aY_$lwRmtaVJZ){IBs3AH*-&vX(om!wFlP|PG3!)r zkQ~a*ABb0d?_cqC?!PjwP0?uPsob6ypE;BZ+Cl3j0$pq!2kHVxQR}h5aoNGZdwYhE zl|cL_Bx^8mLUxmvN>&`?C&7X~SfvTP5LqC06k;L~0uSsx$!87V>=QpU0#K)woXXuZ zbSO7)vNJ{h32hmmbF!_Yx{^R2coTFU7u@54W8W|m=V0K3V4VmIB5TaZX?q|NB1se- zCXLL-5#hyPnLsjSq+;gFXVsR;%4POr%Q|b0g6ur*1}F8$k%WBqj%BqWy@VB}Ja2fd ze*ejopDMRORP(&C`isUg=d&2nX)>f?Hm6Y!s&jKse{@OnO&`afQ@dh-NdSNoyCShg zJ=pdJ3@{@bNKMzU`dxBGgmL@G41S~*7-Hp6E;UE1Dws$`)u|*Q!o4P+V%DuuxL`En zxKB}{RWJoZTVNz=gjS}K5fd65`AvKVC8%g)ttHFA`uC4|jqzBw85V91ILxVAJVjh_PHg z8Ma8i>QU~@rcOu+?raFPG)QB8uvf@q^%>)}!}bI|5tE1sq%etqLU$|Zwe!4eng^8} z*&zHKZRArIpFk(c5yy6pT=6|IiYD2(1+9QFitNx#+%f=@8FP*dcub}lLFam^Ss~{* z*NdD3Dn3#KW8Qj-_`E3orom~SN8-a0ACBUh#5JW^^gwCWdvJ?nI3(_f;`I`* zlz64YN3Ijrua)512nah~7~OjS#AtBCNZunkA}V0tKBD^KOk(=a0N;OU5NkvRWM50C zLS+8uR=SC2(A$G5fnbkdwyaiIS@L)$E3;)};KBlxpbeNEn9{M#R%A`g(kD$EraUL+ z`gp9C!FFVoCW4X#U5O*xp>KdY6Hqt@g_ET47gBhfF6^AG>m3)>dyNon74r}`JRazT zQ>;sRpJ4ORIi79G&Mx6@$p8Zk1h(m7>9JsgSut9mn2NJ6gALkBtgtEM7BRPB?!)g) zmYzM!+Q!Xl8zitXTi0io7bIhPN=BEr!fv`1oy}9}Bt9PfJwFbmK{L9cFS|&y;&H>L z#0A$C%Q<-q_o%mg*Lw7>+Ow`0wSj7!=l`MNhRT5rIjM){QS28_gY6KUGx}PbV7w=r zdLVUR29fzAa`YI>QCb3@ls}T%PZaL_7^-`~oIL8>{Sxsd-@3W(-eHu(k$PYQ!3f;|O1t?6xykm3JwAgyjbO4@Cpb)BH^O1+ zt48+Gggn!ijnvq4v5}?aESQi9k?jx>} z*mbME*FjZqeHXnE&kr{WJy2H!dGKbfi#8zMpyRNyYVM+|5nCOD{iutsL~P|4#yuT3 zb%CKB>$*T+hqsGn_!eA_tRJSxV()=9MfH_jj1bs(28D^%nCQw2G5n$z+c^T%~%xi__Dg4q%2s zS#V8(0wHONq{6d~k~l~nL3YmR)O6u*WJ{Snb%k>L>!IOpUN442ZactWumbb5r*i@} zWX4P*j>*d)6D~7f|Z~LT8Fv@>Cr($XOxv zP4^E){3<2OpWJwoTJlCa0^pg!mr5D39Os@vLHj zB&zJ_<4e`$GUc0KbMF{XeaT`=Zsv>B8Yqp%;UJoMt$IC_&d2eZTKFfhXW@?P%BZV( z6HYcK{bB>o(IOPh0e5qAvN>{fmL zQLeW>9#-~#LP}L%CxDTqczG-I%_i3i{U~pbR!POpfDMHq0c3C?oYt9yD0^X&-F=ga z7gsrW#ns~IMmgAwWJ0M>~c=D zd9)ascO<|-r3)|M=&T~#)=%ySEDbQ~2rc9myd)~ZOCoW10}~~(tNNCcaD~WW7xNr* zF5;vvgTMh5(c8~B8Oq!~Yd8n8Vdk&{?9$yg0gXw-T5Q$ix;Us?z@DD`kYT8p<$LC8 zdx~(8~oc3;<;%@&? z9p25sdK!dj??envH3-2y7Ei+^M-@}wit}6g-wiU6$ThW7v(Fl_jg5G+mHEt^KY8W5 zFuds5a`bGu9>AC(`&I1DdAL=S(LAF1LMo1inb(MT1#ZZ|Yx)f+cUI{an}`nfVE!Y9 zdrR2YQzjR}#Sn)3n+D&z;@-gwQB*UsBnbOB_<UaE zN)Nkz$|Htip(qX&v4W{D8JbJj@YIz#g~X173L}+ATq{a(?~n%@Eiuq;wz4pNJ`}+{ zB4J>*SGX~W8l{-n`sIMf#S)t&{-_F9u*NDym$9=zp(EACjj9b;3RPv*Om?hrY69qn zE6`3O3l`c_(?BLIR>sO^6wQP^nNhXq+LA%qB%1^88LQ0H2wn?`-Hb7~rx|VuVL0L5 z8xx?u1$nX=ECnq1=70F7knrOW>~!IUT^! z+B+WASqWpTy`oZ3=j4uYUKO0fI;U`q^H;%nSLdXRaXuHEi#jKDjAPMC&^l{L#JaRG zR)OG5)j8>7oGQVo);YE@PNU#7>71-F&KAMBN9T;!IoiIC#{`S%tW2G?uVbI!^ynPJ z80U!KyrFaA#yIZ_&IdXtX^iuw;C!WX;sqyJj6_-~dLg5f-B7aI@;t=gb<}ua5`m-` zzk+>MQj~I+5aj!sPAiDR5#u@v_bVm809r;E#AX`+{cg+Ad}RJK*lA@_YNRxZREfeS?WNrZ%pvYsIQwP z%zWGZqS7-EqK=+Fhp1h}b>P=Z&+8$Y*7I5j-qK*6Wj~u$X=qT`oil&Hem*Np%Yq9$ zSloI(2?5ir1Ysj;>mgw%Tp>Xf3*)hHo&@3KZR;t-!f6sTVqqZ`-XTF)&ul$E#KIp* z(1e9MuGgoC!N=RqulNH7r#zrX?~LEJ3&gu)t>9w$LFmR?8JP7;Izx2@+S7Vag% zWGws*3!6v~_TRRigkC5#lb{6)6S2@hu6EGK^r#!Z8udB8OCk*<*bB4YT^-8D_fY-t zBD0tqySdd>lkgq#&`<&lRAg)}kNW<0g{ReWOD%WSa!>6ALM|wTTj5oatJTduxGpV? zk8;c5oDtRci%|?4I|S{1Q^ie+IdbK8$0gql?2b7li=U78y&R9@P~P)!2<#B6!-#>G z!|xQ>(!}HY$Jz_OZ056FPo?5+puC-y@p82cjB1X@8Bv5Y;>?_>$zz>?>ohj8GRFYc zR90pwFR&j}F0i+{v9It)3)Pi)wzAEt((IypeT|6%yIy`Jbl-VdA)9vQAZ}L+_z~i&@-4r=e&i^DO+~ zq-gJe@d4qmKIh?R1uu`n`^K_vUhaaq%opBUr$NMb}3Vc=wfE-9BN=Yu~Q^$II+|)g*Shx?0y!eOc&u zuqLXbbN|ORM-;8+(xFm#tMxds_gvcxYifZ)y0N!EZ&!Z~hRlx-!n9?$VzF;7WRHaC zV+cQ{!?1xjX6Lv$BnyZl;YafHf+y>usnJ7Wa%d+9@nJrskD z@1=gk{4rQ+FQtgl7>xAN2M~K82D==l8e*D`Y0&T|VY(f`?K%k9nK0dk*tQsKFih`3 z?4B5GAWXL)wj~BT7^a&M+Z==adzjvh*xfN$SD4-@thqA={B@XaLcvWj*w4cBcEoOv z!5#_IHpJRuuzSO_6|vSB>|et47Q}9e!G08`8xh;+AJXLB1bW;YrY#7z#0ahk(+!Ai zh{2YIX%k}diUvXj>RuP7jfgeI$ju1TwTP{a!6t?28pPn{Cep4p6wD2O>#hbAUK76h zo+c@L^*zlOp{wp`f)_%x9vhLD1=7G%A-W8)WzmL*U}>aF0vdqw^e+Wrp{hf{x){OT zA-Wi`#WC1#LbMvO>KH5(`qr@@I`Z4l)o1?VUK%~~YmpT-IHmXMLw{B;t-wa)DPJ0x z-W&U}Tw;b^Iuqq0=XxU-;^8P<|i2`Wuw)aO}BA)_pFFAE9XaS11zv*D$=phKAu>q+1x=9gaO0 zNxA#N)PZu27`d%s{GvqvC_>8J8peH){#As;ZVuyXt;lB)n!M2pt_tJ3^lyF`u{<36 zVT7zaCmj1-gv4fr;k`1{h5M+e(UZe)NdcH826KgJGGfUwn3fWzW(3VK;JEO&jt&q= zmqM}U2HEi6L-2h8)HTM`JsqMdVrmRF6ru`ZN(?qS+L7Dak;IPHmaw1HO;5-B$c}4! z55fB|g0GihL3n~WuTM7&*0Y({&&_DfAuWv20 ztPFh8=0C&Z{AY9F_or>SCs)9a!A}C`3Jhc<@m8pTZ}Hl0jd0lHCC}x6#=af~?eG)heS#HNE(0`i8`_JS8b`D!Pz6yS_b4y)D@oRs^ zvP8vDu$tvA--zVkau1Tw^6A$Utj_Ja!FS+FuK2+|TfX6@g4KaD)#nPx{r*e23mS6k z8bBg&X++Ik+2EUof`K#NV*brP6u1Ch_`Cmf;0ym5UD$Xfa3%$-tq+_7x~XCR^YtGk zX!e^3al=yqPXjy~;n@z)Zg{%kc@>_w;Q0uiOYqp&5>fz91w3mS30VxZ37$51w!`x< zJimfxA3VQ@=L9_G;7MIah!dVtcxvHkfTwL8(a2W#u>+pH@SwsY5qccxd3cO~A^!jK zX#e+$Jz(q`O@w?1kGh_a<4`{FkDkORW54~Te_5q{u97UG#RDlNuhj#oB!>!tRFPH! zWHI?m5|A+2Y6fzctf>PsLb57>)G5E40pwZba5<14!dHVpHp5qgKu)Q{sLW+Njbw); zcf^ZQfhh|`cAMswey*vcIYEf_CkbYYSy*wWc@QgZF)u|czA(qH0J0(_y#xrc3X%3y zLHg4L?};35G4SS3d>zRuXDv$Za1|qIaW^CRgS&YZkTZEjD}n6v41uIZ?#mZwxZqxt zd2FhvvAIyxc)M@~TDrGXG%>xbVH%YFsjMae$fdH|EkOQxn&1sh6X2iAi>?LmlX3yR zd)?CpD9x@2}fiVyk9n*K=QOCT#~(#?32Xyf~5N; OIUvc)K-kSMHUAH{fOvxd diff --git a/runtime/include/c++/cmath b/runtime/include/c++/cmath index c9316a3..01c0c0b 100644 --- a/runtime/include/c++/cmath +++ b/runtime/include/c++/cmath @@ -26,11 +26,23 @@ #undef isinf #undef isfinite #undef signbit +#undef isgreater +#undef isgreaterequal +#undef isless +#undef islessequal +#undef islessgreater +#undef isunordered inline bool isnan(double x) { return ::__isnan_d(x) != 0; } inline bool isinf(double x) { return ::__isinf_d(x) != 0; } inline bool isfinite(double x) { return ::__isfinite_d(x) != 0; } inline bool signbit(double x) { return ::__signbit_d(x) != 0; } +inline bool isgreater(double x, double y) { return __builtin_isgreater(x, y); } +inline bool isgreaterequal(double x, double y) { return __builtin_isgreaterequal(x, y); } +inline bool isless(double x, double y) { return __builtin_isless(x, y); } +inline bool islessequal(double x, double y) { return __builtin_islessequal(x, y); } +inline bool islessgreater(double x, double y) { return __builtin_islessgreater(x, y); } +inline bool isunordered(double x, double y) { return __builtin_isunordered(x, y); } namespace std { @@ -166,6 +178,14 @@ using ::scalblnf; // ---- Classification (function form) --------------------------------- using ::fpclassify; +// ---- Comparison (function form per C++ ) --------------------- +using ::isgreater; +using ::isgreaterequal; +using ::isless; +using ::islessequal; +using ::islessgreater; +using ::isunordered; + } // namespace std #endif // _W65816_CXX_CMATH diff --git a/runtime/include/math.h b/runtime/include/math.h index c23a10d..b996b44 100644 --- a/runtime/include/math.h +++ b/runtime/include/math.h @@ -25,6 +25,18 @@ int __signbit_d (double x); #define isfinite(x) __isfinite_d((double)(x)) #define signbit(x) __signbit_d((double)(x)) +// ---- C99 comparison macros ------------------------------------------ +// False, never an error, when either operand is a NaN. The soft-float +// compares raise nothing, so clang's builtins (which LLVM lowers to the +// ordinary compare helpers, plus __unordsf2 / __unorddf2 for the +// unordered tests) are all these need to be. +#define isgreater(x, y) __builtin_isgreater((x), (y)) +#define isgreaterequal(x, y) __builtin_isgreaterequal((x), (y)) +#define isless(x, y) __builtin_isless((x), (y)) +#define islessequal(x, y) __builtin_islessequal((x), (y)) +#define islessgreater(x, y) __builtin_islessgreater((x), (y)) +#define isunordered(x, y) __builtin_isunordered((x), (y)) + // ---- Absolute / sign ------------------------------------------------ double fabs (double x); float fabsf (float x); diff --git a/runtime/src/math.c b/runtime/src/math.c index 0398c8c..1e3e24c 100644 --- a/runtime/src/math.c +++ b/runtime/src/math.c @@ -46,6 +46,73 @@ static float fFromBits(uint32_t b) { } +#define D_SIGN 0x8000000000000000ULL +#define D_INF 0x7FF0000000000000ULL +#define D_ONE 0x3FF0000000000000ULL + + +// Used before their definitions (this file does not include math.h). +double exp(double x); +double log(double x); +double rint(double x); + + +// clang's library-call simplifier rewrites (float)floor((double)x) into +// floorf(x), which is exact for the rounding and min/max functions; inside +// floorf itself that is a call to itself, so every floorf, ceilf, truncf, +// roundf, fmaxf and fminf recursed until the stack overran the program. +// no_builtin("f") stops the rewrite inside the one wrapper that needs it. +#define NO_SHRINK(f) __attribute__((no_builtin(#f))) + + +// Special-value paths kept out of line: inlined, they grew the ordinary +// path's frame and register traffic by several percent. +#define COLD __attribute__((noinline)) + + +// The top word of a double's bits: sign, exponent and the mantissa's top +// four bits. Free to extract, so exponent tests go through it instead of +// 64-bit compares. +static unsigned topWord(uint64_t b) { + return (unsigned)(b >> 48); +} + + +// 1 when the double with bits b is an infinity or a NaN. The top word +// holds the whole exponent, so this is one masked compare. +static int isNonFinite(uint64_t b) { + return (topWord(b) & 0x7FF0u) == 0x7FF0u; +} + + +// 1 when the double with bits b is a NaN (its magnitude is above +// infinity's). A 64-bit compare, so callers screen with isNonFinite. +static int isNanBits(uint64_t b) { + return (b & ~D_SIGN) > D_INF; +} + + +// 1 when the double with bits b is +0 or -0. Spelled word by word: the +// optimizer turns a 64-bit `(b & ~sign) == 0` back into `x == 0.0`, which +// on this target is a soft-float libcall compare (~500 cycles). +static int isZeroBits(uint64_t b) { + return (((unsigned)(b >> 48) & 0x7FFFu) | (unsigned)(b >> 32) | + (unsigned)(b >> 16) | (unsigned)b) == 0u; +} + + +// sqrt and log of an infinity or a NaN: +inf maps to itself, a NaN to +// itself quieted (x + x does both), -inf is a domain error like any other +// negative argument. Without this the series below turned +inf into a +// finite value and a NaN into a plausible-looking number. +static double nonFiniteArg(double x, uint64_t b) { + if (b == 0xFFF0000000000000ULL) { + return __builtin_nan(""); + } + return x + x; +} + + double fabs(double x) { return dFromBits(dToBits(x) & ~((uint64_t)1 << 63)); } @@ -75,7 +142,12 @@ double floor(double x) { uint64_t b = dToBits(x); int e = (int)((b >> 52) & 0x7FF) - 1023; if (e < 0) { - // |x| < 1: floor(positive) = 0, floor(negative) = -1. + // |x| < 1: floor(positive) = 0, floor(negative) = -1, and a zero + // keeps its sign. -1 for -0 made ceil(+0), which is -floor(-0), + // come out as 1. + if (isZeroBits(b)) { + return x; + } return (b >> 63) ? -1.0 : 0.0; } if (e >= 52) { @@ -100,30 +172,54 @@ double ceil(double x) { } -float floorf(float x) { +NO_SHRINK(floor) float floorf(float x) { return (float)floor((double)x); } -float ceilf(float x) { +NO_SHRINK(ceil) float ceilf(float x) { return (float)ceil((double)x); } -// fmod(x, y) = x - trunc(x / y) * y. trunc(q) = round-toward-zero. +// fmod(x, y) = x - trunc(x / y) * y, which takes the sign of x. The +// special cases follow C99 Annex F: a NaN operand, an infinite x or a zero +// y give NaN; an infinite y or a zero x give x back. They are screened on +// the bits, which also replaced a `y == 0.0` libcall compare. +static COLD double fmodSpecial(double x, double y) { + uint64_t xb = dToBits(x); + uint64_t yb = dToBits(y); + if (isNanBits(xb) || isNanBits(yb)) { + return x + y; // the NaN, quieted + } + if (isNonFinite(xb) || isZeroBits(yb)) { + return __builtin_nan(""); // domain error + } + return x; // infinite y, or a zero x +} + + double fmod(double x, double y) { - if (y == 0.0) { - return 0.0; // implementation-defined; we return 0 + uint64_t xb = dToBits(x); + uint64_t yb = dToBits(y); + if (isNonFinite(xb) || isNonFinite(yb) || isZeroBits(yb) || isZeroBits(xb)) { + return fmodSpecial(x, y); } double q = x / y; - // trunc: floor for positives, ceil for negatives. + // trunc: floor for positives, ceil for negatives. The sign test reads + // the bits, which is cheaper than a `<` libcall compare. double t; - if (q < 0.0) { + if (dToBits(q) >> 63) { t = -floor(-q); } else { t = floor(q); } - return x - t * y; + double r = x - t * y; + // An exact multiple leaves x + -x, which is +0; the result takes x's sign. + if (isZeroBits(dToBits(r))) { + return (xb >> 63) ? -0.0 : 0.0; + } + return r; } @@ -146,6 +242,9 @@ double sqrt(double x) { if ((b & ~((uint64_t)1 << 63)) == 0) { return x; } + if (isNonFinite(b)) { + return nonFiniteArg(x, b); + } if (b & ((uint64_t)1 << 63)) { return 0.0 / 0.0; // NaN for negatives } @@ -168,38 +267,60 @@ float sqrtf(float x) { } -// pow(x, y) supports two restricted regimes: -// 1. integer y (positive, negative, or zero) — repeated multiply. -// 2. y == 0.5 — returns sqrt(x). -// Anything else (general real exponent) needs exp/log; not yet -// landed. Returning 0 is the documented behaviour for now. -double pow(double x, double y) { - if (y == 0.0) { - return 1.0; +// 0 when the finite, nonzero y is not an integer, 1 when it is an odd one +// and 2 when it is an even one. e is y's unbiased exponent; only pow's +// rare paths need this, the |y| < 2^15 integer path tests (int)y instead. +static int intClass(double y, int e) { + if (e < 0) { + return 0; // 0 < |y| < 1 } - if (y == 0.5) { - return sqrt(x); + if (e >= 53) { + return 2; // every double from 2^53 up is even } - // Test for integer y by floor-equal trick (no soft-fp comparison). - double yi = floor(y); - uint64_t yi_b, y_b; - __builtin_memcpy(&yi_b, &yi, sizeof(yi_b)); - __builtin_memcpy(&y_b, &y, sizeof(y_b)); - if (yi_b != y_b) { - return 0.0; // non-integer, non-0.5 y not supported yet + if (dToBits(floor(y)) != dToBits(y)) { + return 0; } - // y is a whole number; convert via __fixdfsi. Range -32768..32767 - // covers any practical exponent. Use unsigned for the magnitude - // to avoid signed-overflow UB on INT_MIN. - int sn = (int)yi; - int neg = 0; - unsigned int n; - if (sn < 0) { - neg = 1; - n = 0u - (unsigned int)sn; - } else { - n = (unsigned int)sn; + double h = y * 0.5; + return (dToBits(floor(h)) == dToBits(h)) ? 2 : 1; +} + + +// The high word's mantissa bits below the unit of a double whose unbiased +// exponent is 0..14: nonzero ones make it a non-integer. +static const uint32_t kPowFracHi[15] = { + 0xFFFFFUL, 0x7FFFFUL, 0x3FFFFUL, 0x1FFFFUL, 0xFFFFUL, 0x7FFFUL, 0x3FFFUL, + 0x1FFFUL, 0xFFFUL, 0x7FFUL, 0x3FFUL, 0x1FFUL, 0xFFUL, 0x7FUL, 0x3FUL +}; + + +// pow when x is a zero or an infinity or either operand is a NaN or y is +// an infinity (y is not a zero, x is not +1). C99 F.9.4.4: a NaN operand +// gives NaN; pow(-1, +-inf) is 1, otherwise an infinite y gives +inf when +// |x| > 1 agrees with y > 0 and +0 when not; a zero or infinite x gives +// +inf or +0 by y's sign (inverted for a zero x), negated for a negative x +// raised to an odd integer. +static COLD double powSpecial(double x, double y, uint64_t xb, uint64_t yb) { + if (isNanBits(xb) || isNanBits(yb)) { + return x + y; // the NaN, quieted } + uint64_t xa = xb & ~D_SIGN; + int yNeg = (int)(yb >> 63); + if ((yb & ~D_SIGN) == D_INF) { + if (xa == D_ONE) { + return 1.0; // pow(-1, +-inf) + } + return ((xa > D_ONE) != yNeg) ? __builtin_inf() : 0.0; + } + int e = (int)((topWord(yb) >> 4) & 0x7FFu) - 1023; + int huge = isZeroBits(xb) ? yNeg : !yNeg; + double r = huge ? __builtin_inf() : 0.0; + return ((xb >> 63) && intClass(y, e) == 1) ? -r : r; +} + + +// x raised to the integer n (1 / x^n when neg) by repeated squaring, +// exact whenever the result is representable. +static double powInt(double x, unsigned int n, int neg) { double r = 1.0; double base = x; while (n > 0) { @@ -216,6 +337,53 @@ double pow(double x, double y) { } +// pow(x, y). C99 F.9.4.4's special cases first: pow(x, +-0) and pow(+1, y) +// are 1 even when the other operand is a NaN; zeros, infinities and NaNs go +// to powSpecial. An integer y below 2^15 in magnitude is repeated +// multiplication (exact whenever the result is representable); any other y +// is exp(y * log|x|), as accurate as those (~1e-6), and a negative x +// raised to a non-integer is a domain error. Every non-integer y except +// 0.5 used to return 0. +double pow(double x, double y) { + uint64_t xb = dToBits(x); + uint64_t yb = dToBits(y); + if (isZeroBits(yb) || xb == D_ONE) { + return 1.0; + } + if (isNonFinite(xb) || isNonFinite(yb) || isZeroBits(xb)) { + return powSpecial(x, y, xb, yb); + } + int e = (int)((topWord(yb) >> 4) & 0x7FFu) - 1023; + int xNeg = (int)(xb >> 63); + if (e < 15) { + // 1 <= |y| < 2^15: y is an integer when every mantissa bit below + // its unit is clear. From the bits: (int)y and back through + // __fixdfsi / __floatsidf normalize bit by bit and cost more than + // the multiplications of pow(2, 10). The high word's bit 20 is the + // implicit leading 1; bits 5..20 of it, as 16 bits, hold every bit + // of an integer below 2^15. + uint32_t hi = (uint32_t)(yb >> 32); + if (e >= 0 && (uint32_t)yb == 0 && (hi & kPowFracHi[e]) == 0) { + unsigned int n = (((unsigned int)(hi >> 5) & 0x7FFFu) | 0x8000u) >> (15 - e); + return powInt(x, n, (int)(yb >> 63)); + } + if (xNeg) { + return __builtin_nan(""); // negative x, non-integer y + } + if (yb == 0x3FE0000000000000ULL) { + return sqrt(x); // y == 0.5 + } + return exp(y * log(x)); + } + int yc = intClass(y, e); + if (xNeg && yc == 0) { + return __builtin_nan(""); + } + double r = exp(y * log(fabs(x))); + return (xNeg && yc == 1) ? -r : r; +} + + float powf(float x, float y) { return (float)pow((double)x, (double)y); } @@ -300,15 +468,37 @@ float tanf(float x) { // exp via 2^k * e^r where x = k*ln2 + r, |r| < ln2/2. Then Taylor // series for e^r converges in ~10 terms. k * 2 multiplication uses // the IEEE-754 layout (add k to exponent field). +// 1/n for the Taylor terms: a table of the correctly rounded constants, +// which the run-time 1.0 / n computed identically at ~60000 cycles each. +static const double kInvN[13] = { + 0.0, 1.0, 1.0 / 2.0, 1.0 / 3.0, 1.0 / 4.0, 1.0 / 5.0, 1.0 / 6.0, + 1.0 / 7.0, 1.0 / 8.0, 1.0 / 9.0, 1.0 / 10.0, 1.0 / 11.0, 1.0 / 12.0 +}; + + double exp(double x) { - double k_d = floor(x / M_LN2 + 0.5); + uint64_t xb = dToBits(x); + // |x| >= 1024, the infinities and the NaNs: the result is surely past + // the double range (and k below would not fit an int), so exp(+inf) + // and every large x overflow to +inf and exp(-inf) and every large + // negative x underflow to +0. A NaN comes back quieted. + if ((topWord(xb) & 0x7FF0u) >= 0x4090u) { + if (isNanBits(xb)) { + return x + x; + } + return (xb >> 63) ? 0.0 : __builtin_inf(); + } + // k = round(x / ln2); multiplying by 1/ln2 instead of dividing saves a + // soft division, and any k within one of the exact one still leaves r + // well inside the series' fast-converging range. + double k_d = floor(x * 1.44269504088896340736 + 0.5); double r = x - k_d * M_LN2; int k = (int)k_d; - // Taylor: 1 + r + r^2/2 + r^3/6 + ... + r^10/3628800 + // Taylor: 1 + r + r^2/2 + r^3/6 + ... + r^12/479001600 double term = 1.0; double s = 1.0; for (int n = 1; n <= 12; n++) { - term = term * r * (1.0 / (double)n); + term = term * r * kInvN[n]; s = s + term; } // Multiply by 2^k by adding k to the IEEE-754 exponent field. @@ -345,6 +535,9 @@ double log(double x) { if ((b & ~((uint64_t)1 << 63)) == 0) { return -1.0 / 0.0; } + if (isNonFinite(b)) { + return nonFiniteArg(x, b); + } if (b & ((uint64_t)1 << 63)) { return 0.0 / 0.0; // log(negative) = NaN (domain error) } @@ -353,13 +546,19 @@ double log(double x) { b = (b & ~((uint64_t)0x7FF << 52)) | ((uint64_t)1023 << 52); double m; __builtin_memcpy(&m, &b, sizeof(m)); + // 1/(2n+1) for the series: a table of the correctly rounded constants + // (the run-time division gave the same values at ~60000 cycles each). + static const double kInvOdd[9] = { + 1.0, 1.0 / 3.0, 1.0 / 5.0, 1.0 / 7.0, 1.0 / 9.0, 1.0 / 11.0, + 1.0 / 13.0, 1.0 / 15.0, 1.0 / 17.0 + }; double u = (m - 1.0) / (m + 1.0); double u2 = u * u; double term = u; double s = term; for (int n = 1; n <= 8; n++) { term = term * u2; - s = s + term * (1.0 / (double)(2 * n + 1)); + s = s + term * kInvOdd[n]; } return 2.0 * s + (double)e * M_LN2; } @@ -447,7 +646,10 @@ float atan2f(float y, float x) { double asin(double x) { double r = 1.0 - x * x; if (r <= 0.0) { - // |x| >= 1; clamp to ±pi/2. + // |x| > 1 is a domain error; |x| == 1 is exactly +-pi/2. + if (r < 0.0) { + return __builtin_nan(""); + } uint64_t b; __builtin_memcpy(&b, &x, sizeof(b)); return (b >> 63) ? -M_HALF_PI : M_HALF_PI; @@ -471,7 +673,12 @@ float acosf(float x) { } +// sinh(+-0) is +-0 (the formula gives +0 for both); infinities and NaNs +// come through exp: sinh(+-inf) = +-inf. double sinh(double x) { + if (isZeroBits(dToBits(x))) { + return x; + } double e = exp(x); return (e - 1.0 / e) * 0.5; } @@ -493,7 +700,20 @@ float coshf(float x) { } +// tanh is +-1 to double precision from |x| = 22 on; from 32 (a top-word +// test) the formula would reach inf / inf = NaN once exp(2x) overflows, so +// return +-1 there, which also covers tanh(+-inf). tanh(+-0) is +-0. double tanh(double x) { + uint64_t b = dToBits(x); + if ((topWord(b) & 0x7FF0u) >= 0x4040u) { + if (isNanBits(b)) { + return x + x; + } + return (b >> 63) ? -1.0 : 1.0; + } + if (isZeroBits(b)) { + return x; + } double e2 = exp(x + x); return (e2 - 1.0) / (e2 + 1.0); } @@ -540,18 +760,29 @@ double trunc(double x) { } -float truncf(float x) { +NO_SHRINK(trunc) float truncf(float x) { return (float)trunc((double)x); } -// round: round half away from zero. +// round: round half away from zero. floor(|x| + 0.5) is exact only in +// between: below 0.5 the addition can round up to 1 (0.49999999999999994 +// rounded to 1), and from 2^52 on, where every double is an integer, it +// can round an odd x up to the next even one. double round(double x) { - return (dToBits(x) >> 63) ? -floor(-x + 0.5) : floor(x + 0.5); + uint64_t b = dToBits(x); + unsigned e = (topWord(b) >> 4) & 0x7FFu; + if (e < 1022u) { + return (b >> 63) ? -0.0 : 0.0; // |x| < 0.5, zeros keep their sign + } + if (e >= 1075u) { + return x + 0.0; // integer, infinity or NaN (quieted) + } + return (b >> 63) ? -floor(-x + 0.5) : floor(x + 0.5); } -float roundf(float x) { +NO_SHRINK(round) float roundf(float x) { return (float)round((double)x); } @@ -573,12 +804,19 @@ double fmin(double x, double y) { double fdim(double x, double y) { - return x > y ? x - y : 0.0; + if (x > y) { + return x - y; + } + // Unordered compares false too: a NaN operand gives NaN, not +0. + if (isNanBits(dToBits(x)) || isNanBits(dToBits(y))) { + return x + y; + } + return 0.0; } -float fmaxf(float x, float y) { return (float)fmax((double)x, (double)y); } -float fminf(float x, float y) { return (float)fmin((double)x, (double)y); } +NO_SHRINK(fmax) float fmaxf(float x, float y) { return (float)fmax((double)x, (double)y); } +NO_SHRINK(fmin) float fminf(float x, float y) { return (float)fmin((double)x, (double)y); } float fdimf(float x, float y) { return (float)fdim((double)x, (double)y); } @@ -692,12 +930,19 @@ float exp2f(float x) { return (float)exp2((double)x); } // log1p(x) = log(1 + x). For |x| small the naive form loses precision; // we accept that for now since this is a standalone runtime, not a // numerics library. -double log1p(double x) { return log(1.0 + x); } +double log1p(double x) { + // log1p(+-0) is +-0; log(1 + -0) would be +0. + return isZeroBits(dToBits(x)) ? x : log(1.0 + x); +} float log1pf(float x) { return (float)log1p((double)x); } // expm1(x) = exp(x) - 1. Same loss-of-precision caveat near 0. -double expm1(double x) { return exp(x) - 1.0; } +double expm1(double x) { + // expm1(+-0) is +-0; exp(-0) - 1 would be +0. expm1(-inf) = -1 and + // expm1(+inf) = +inf come through exp. + return isZeroBits(dToBits(x)) ? x : exp(x) - 1.0; +} float expm1f(float x) { return (float)expm1((double)x); } @@ -708,6 +953,13 @@ float expm1f(float x) { return (float)expm1((double)x); } // past that you'd want the standard scale-by-max trick. double hypot(double x, double y) { + uint64_t xb = dToBits(x); + uint64_t yb = dToBits(y); + // C99 F.9.4.3: an infinite operand gives +inf even when the other is a + // NaN; otherwise a NaN operand gives NaN (through the arithmetic). + if ((xb & ~D_SIGN) == D_INF || (yb & ~D_SIGN) == D_INF) { + return __builtin_inf(); + } double xx = x * x; double yy = y * y; double s = xx + yy; @@ -728,7 +980,15 @@ float hypotf(float x, float y) { // known builtin and either inlines it (with bad fold of pow(x,1/3)) // or DCEs the call entirely (cbrt body collapses to "return 0"). double cbrt(double x) { - if (x == 0.0) return x; + uint64_t xb = dToBits(x); + // cbrt(+-inf) = +-inf and a NaN comes back quieted (x + x does both; + // the iteration below turned inf into inf / inf = NaN); cbrt(+-0) = +-0. + if (isNonFinite(xb)) { + return x + x; + } + if (isZeroBits(xb)) { + return x; + } int neg = (int)(dToBits(x) >> 63) & 1; double a = neg ? -x : x; double r = a; // crude initial guess @@ -768,8 +1028,8 @@ double acosh(double x) { double atanh(double x) { // atanh(x) = 0.5 * log((1+x)/(1-x)) for |x| < 1. Returns +/-inf // at x = +/-1 and NaN outside [-1, 1]. - if (x >= 1.0) return __builtin_inff(); - if (x <= -1.0) return -__builtin_inff(); + if (x >= 1.0) return (x == 1.0) ? __builtin_inff() : __builtin_nan(""); + if (x <= -1.0) return (x == -1.0) ? -__builtin_inff() : __builtin_nan(""); return 0.5 * log((1.0 + x) / (1.0 - x)); } @@ -800,10 +1060,16 @@ float nanf(const char *tagp) { (void)tagp; return __builtin_nanf(""); } double remainder(double x, double y) { - // IEEE 754 remainder: x - n*y where n = round-to-nearest-even(x/y). - // Falls back to fmod for non-finite cases via the existing primitives. + // IEEE 754 remainder: x - n*y where n = round-to-nearest-even(x/y), + // which is rint (round() rounded ties away: remainder(5, 2) was -1). + // remainder(x, +-inf) is x for a finite x (C99 F.9.7.2); x / y would + // make it 0 * inf = NaN. if (y == 0.0) return __builtin_nanf(""); - double quotient = round(x / y); + uint64_t yb = dToBits(y); + if ((yb & ~D_SIGN) == D_INF && !isNonFinite(dToBits(x))) { + return x; + } + double quotient = rint(x / y); return x - quotient * y; } @@ -811,10 +1077,39 @@ double remainder(double x, double y) { float remainderf(float x, float y) { return (float)remainder((double)x, (double)y); } -double rint(double x) { return round(x); } -float rintf(float x) { return roundf(x); } -double nearbyint(double x) { return round(x); } -float nearbyintf(float x) { return roundf(x); } +// rint / nearbyint round in the current rounding mode, which is always +// to nearest, ties to even, here (the soft-float ignores fesetround). They +// used to be round(), which rounds ties away from zero (rint(2.5) was 3). +// Done on the bits, like floor: the fraction bits below the unit are +// dropped, and one unit is added when they exceed half a unit or equal it +// with an odd integer part. A carry out of the mantissa steps the +// exponent, which is exactly the next power of two. +double rint(double x) { + uint64_t b = dToBits(x); + int e = (int)((topWord(b) >> 4) & 0x7FFu) - 1023; + if (e >= 52) { + return x + 0.0; // integer, infinity or NaN (quieted) + } + uint64_t sign = b & D_SIGN; + if (e < 0) { + // |x| < 1: +-0 up to a half (ties go to the even 0), +-1 above it. + uint64_t mag = b & ~D_SIGN; + return dFromBits(sign | ((mag > 0x3FE0000000000000ULL) ? D_ONE : 0)); + } + uint64_t unit = (uint64_t)1 << (52 - e); + uint64_t half = unit >> 1; + uint64_t frac = b & (unit - 1); + b &= ~(unit - 1); + // For e == 0 the unit bit is the exponent's lowest bit, which is 1 for + // 1023: the integer part 1 is odd, as it should be. + if (frac > half || (frac == half && (b & unit))) { + b += unit; + } + return dFromBits(b); +} +NO_SHRINK(rint) float rintf(float x) { return (float)rint((double)x); } +double nearbyint(double x) { return rint(x); } +NO_SHRINK(rint) float nearbyintf(float x) { return (float)rint((double)x); } long lround(double x) { @@ -828,12 +1123,12 @@ long lroundf(float x) { long lrint(double x) { - return (long)round(x); + return (long)rint(x); } long lrintf(float x) { - return (long)roundf(x); + return (long)rintf(x); } diff --git a/runtime/src/softDouble.c b/runtime/src/softDouble.c index 398656b..e294f2e 100644 --- a/runtime/src/softDouble.c +++ b/runtime/src/softDouble.c @@ -5,6 +5,13 @@ // Earlier attempts crashed the Register Coalescer; the greedy // regalloc landing fixed the underlying register pressure problem. // Each routine is broken into small helpers to keep frames shallow. +// +// NaN and infinity follow IEEE 754 and compiler-rt's libcall contract, +// as softFloat.c does: a NaN operand comes back quieted with its payload +// kept, invalid operations return the default NaN, and comparisons with a +// NaN are unordered. Double to integer saturates: NaN converts to 0, +// +inf and too-large values to the maximum, -inf and too-negative values +// to the minimum (0 for unsigned). // Local typedefs (no stdint.h — clang's host stdint pulls glibc). typedef unsigned long long u64; @@ -21,6 +28,19 @@ typedef unsigned char u8; #define DMANT_LEAD 0x0010000000000000ULL #define DEXP_SHIFT 52 #define DEXP_BIAS 1023 +#define DQUIET_BIT 0x0008000000000000ULL // mantissa MSB: set = quiet NaN +#define DDEFAULT_NAN 0x7FF8000000000000ULL // what an invalid op returns +#define DPC_ZERO 0 // dclass results +#define DPC_NORMAL 1 +#define DPC_INF 2 +#define DPC_NAN 3 +#define DPC_SPECIAL 2 // set for inf and NaN only +// The special-value paths are kept out of line: inlined, they grew the +// hot functions' frames and the compiler speculated their tests into the +// normal path. +#define COLD __attribute__((noinline)) + +COLD s16 __unorddf2(u64 a, u64 b); // Pack sign / unbiased-exp / mantissa-with-leading-bit into IEEE-754 // double. Returns sign for zero or underflow; sign|inf for overflow. @@ -45,16 +65,57 @@ static u16 dclass(u64 x, u64 *out_sign, s16 *out_exp, u64 *out_mant) { if (e == 0) { *out_exp = 0; *out_mant = 0; - return 0; + return DPC_ZERO; } if (e == 0x7FF) { *out_exp = 0x7FF; *out_mant = m; - return (m == 0) ? 2 : 3; + return (m == 0) ? DPC_INF : DPC_NAN; } *out_exp = e - DEXP_BIAS; *out_mant = m | DMANT_LEAD; - return 1; + return DPC_NORMAL; +} + +// a op b when an operand is a zero, an infinity or a NaN, apart from the +// zero cases the hot routines answer themselves (op: DSP_ADD / DSP_MUL / +// DSP_DIV). One out-of-line routine for all three keeps the rarely used +// code small, and it reclassifies the operands itself so the call sites +// stay small: bigger ones perturbed the hot routines' register allocation +// enough to cost the ordinary path 2-4%. +#define DSP_ADD 0 +#define DSP_MUL 1 +#define DSP_DIV 2 +static COLD u64 dpSpecial(u16 op, u64 a, u64 b) { + u64 sa, sb, ma, mb; + s16 ea, eb; + u16 ca = dclass(a, &sa, &ea, &ma); + u16 cb = dclass(b, &sb, &eb, &mb); + // The first NaN operand wins, quieted; keeping its payload lets a + // NaN's origin survive arithmetic. + if (ca == DPC_NAN) return a | DQUIET_BIT; + if (cb == DPC_NAN) return b | DQUIET_BIT; + u64 sign = (a ^ b) & DSIGN_BIT; + if (op == DSP_ADD) { + // Opposite infinities cancel to nothing representable: invalid. + if (ca == DPC_INF) return (cb == DPC_INF && sign) ? DDEFAULT_NAN : a; + if (cb == DPC_INF) return b; + // 0 + 0 is -0 only when both are -0 (round to nearest). + if (ca == DPC_ZERO) return (cb == DPC_ZERO) ? (a & b & DSIGN_BIT) : b; + return a; // x + 0 + } + if (op == DSP_MUL) { + // 0 * inf has no meaningful magnitude: invalid. + if (ca == DPC_INF || cb == DPC_INF) { + return (ca == DPC_ZERO || cb == DPC_ZERO) ? DDEFAULT_NAN : (sign | DEXP_MASK); + } + return sign; // a zero operand + } + if (ca == DPC_INF) return (cb == DPC_INF) ? DDEFAULT_NAN : (sign | DEXP_MASK); + if (cb == DPC_INF) return sign; // finite / inf = signed zero + // x / 0 is a signed infinity, but 0 / 0 is invalid. + if (cb == DPC_ZERO) return (ca == DPC_ZERO) ? DDEFAULT_NAN : (sign | DEXP_MASK); + return sign; // 0 / x } u64 __adddf3(u64 a, u64 b) { @@ -62,8 +123,11 @@ u64 __adddf3(u64 a, u64 b) { s16 ea, eb; u16 ca = dclass(a, &sa, &ea, &ma); u16 cb = dclass(b, &sb, &eb, &mb); - if (ca == 0) return b; - if (cb == 0) return a; + // x + 0 and 0 + x stay fast; every other zero, infinity or NaN case + // goes out of line. + if (ca == DPC_ZERO) return (cb == DPC_NORMAL) ? b : dpSpecial(DSP_ADD, a, b); + if (cb == DPC_ZERO) return (ca == DPC_NORMAL) ? a : dpSpecial(DSP_ADD, a, b); + if (__builtin_expect((ca | cb) & DPC_SPECIAL, 0)) return dpSpecial(DSP_ADD, a, b); // Shift mantissas left by 3 to reserve guard / round / sticky bits // below position 0. Lead bit is now at position 55 instead of 52. // The sticky bit is preserved by ORing it into the LSB whenever a @@ -215,7 +279,12 @@ u64 __muldf3(u64 a, u64 b) { u16 ca = dclass(a, &sa, &ea, &ma); u16 cb = dclass(b, &sb, &eb, &mb); u64 sr = sa ^ sb; - if (ca == 0 || cb == 0) return sr; + // x * 0 and 0 * x stay fast; every other zero, infinity or NaN case + // goes out of line. + if ((ca == DPC_ZERO && cb == DPC_NORMAL) || (cb == DPC_ZERO && ca == DPC_NORMAL)) { + return sr; + } + if (__builtin_expect(ca != DPC_NORMAL || cb != DPC_NORMAL, 0)) return dpSpecial(DSP_MUL, a, b); u16 carry; u16 round_bits; u64 mr = mulhi64Aligned(ma, mb, &carry, &round_bits); @@ -239,8 +308,10 @@ u64 __divdf3(u64 a, u64 b) { u16 ca = dclass(a, &sa, &ea, &ma); u16 cb = dclass(b, &sb, &eb, &mb); u64 sr = sa ^ sb; - if (ca == 0) return sr; - if (cb == 0) return sr | DEXP_MASK; // div-by-zero → inf + // 0 / x stays fast; every other zero, infinity or NaN case goes out of + // line. + if (ca == DPC_ZERO && cb == DPC_NORMAL) return sr; + if (__builtin_expect(ca != DPC_NORMAL || cb != DPC_NORMAL, 0)) return dpSpecial(DSP_DIV, a, b); // Long division: handle the leading quotient bit explicitly (since // we need to "consume" the dividend's leading 1 by subtracting), // then generate 52 more fractional bits by shifting r left and @@ -283,7 +354,10 @@ u64 __divdf3(u64 a, u64 b) { return dpack(sr, er, q); } -s16 __cmpdf2(u64 a, u64 b) { +// Order of a against b: -1, 0 or 1. Neither may be a NaN. Infinities +// need no case of their own: their bit patterns are the largest +// magnitudes. +static s16 dorder(u64 a, u64 b) { u64 sa = a & DSIGN_BIT; u64 sb = b & DSIGN_BIT; if (sa != sb) { @@ -296,6 +370,45 @@ s16 __cmpdf2(u64 a, u64 b) { return a < b ? -1 : 1; } +// 1 when a or b might be a NaN: only an all-ones exponent can be one. +// With the sign forced on, a top word of 0xFFF0 or more has all exponent +// bits set. Spelled as a 16-bit compare because the equivalent mask test +// is widened by the optimizer into a 64-bit one, several times dearer. +static u16 dmaybeNan(u64 a, u64 b) { + return ((u16)(a >> 48) | 0x8000) >= 0xFFF0 || + ((u16)(b >> 48) | 0x8000) >= 0xFFF0; +} + +// 1 when x is a NaN: its magnitude is above infinity's. +static u16 disNan(u64 x) { + return (x & ~DSIGN_BIT) > DEXP_MASK; +} + +// Order of a against b, or `unord` when either is a NaN. Forced inline +// into each helper, and the NaN test kept inline behind the cheap screen: +// a call anywhere in a helper stops the stack-slot promotion that keeps +// the ordinary compare fast. +static inline __attribute__((always_inline)) s16 dcmp(u64 a, u64 b, s16 unord) { + if (__builtin_expect(dmaybeNan(a, b), 0) && (disNan(a) || disNan(b))) { + return unord; + } + return dorder(a, b); +} + +// 0 when a == b, else 1 (unordered included). eq / ne only need that, +// and it is less than half the code of the three-way order. +static inline __attribute__((always_inline)) s16 dne(u64 a, u64 b) { + if (__builtin_expect(dmaybeNan(a, b), 0) && (disNan(a) || disNan(b))) { + return 1; + } + if (a == b) return 0; + return ((a | b) << 1) != 0; // +0 == -0 +} + +s16 __cmpdf2(u64 a, u64 b) { + return dcmp(a, b, 1); +} + s16 __unorddf2(u64 a, u64 b) { // Returns nonzero if either is NaN. u64 ea = (a >> DEXP_SHIFT) & 0x7FF; @@ -311,14 +424,19 @@ s16 __unorddf2(u64 a, u64 b) { // return the three-way order, as softFloat.c's do. They used to return C // booleans, which made every double `<` false, `>=` always true and `<=` // inverted (CoreMark's "must run 10 s" check, fmod of a negative quotient -// and asin all went wrong). eq/ne are tested with == 0 / != 0, for which -// "nonzero when different" is enough. -s16 __eqdf2(u64 a, u64 b) { return __cmpdf2(a, b) != 0; } -s16 __nedf2(u64 a, u64 b) { return __cmpdf2(a, b) != 0; } -s16 __ltdf2(u64 a, u64 b) { return __cmpdf2(a, b); } -s16 __ledf2(u64 a, u64 b) { return __cmpdf2(a, b); } -s16 __gtdf2(u64 a, u64 b) { return __cmpdf2(a, b); } -s16 __gedf2(u64 a, u64 b) { return __cmpdf2(a, b); } +// and asin all went wrong). For a NaN operand they follow compiler-rt: +// the helpers tested with `< 0` / `<= 0` / `== 0` / `!= 0` (lt, le, eq, ne) +// answer 1, the ones tested with `> 0` / `>= 0` (gt, ge) answer -1, so +// every ordered predicate is false and its negation true. The unordered +// predicates (ult is !(a >= b), ugt is !(a <= b), ...) are lowered to +// these same helpers with the test inverted, and ueq / one add a +// __unorddf2 call. +s16 __eqdf2(u64 a, u64 b) { return dne(a, b); } +s16 __nedf2(u64 a, u64 b) { return dne(a, b); } +s16 __ltdf2(u64 a, u64 b) { return dcmp(a, b, 1); } +s16 __ledf2(u64 a, u64 b) { return dcmp(a, b, 1); } +s16 __gtdf2(u64 a, u64 b) { return dcmp(a, b, -1); } +s16 __gedf2(u64 a, u64 b) { return dcmp(a, b, -1); } // double <-> float conversions. u64 __extendsfdf2(u32 x) { @@ -327,7 +445,12 @@ u64 __extendsfdf2(u32 x) { u32 m = x & 0x7FFFFFUL; if (e == 0) return sign; if (e == 0xFF) { - return sign | DEXP_MASK | ((u64)m << 29); + // Infinity, or a NaN quieted with its payload moved up to the + // top of the wider mantissa (so a float NaN's payload survives a + // round trip through double). + u64 r = sign | DEXP_MASK | ((u64)m << 29); + if (m != 0) r |= DQUIET_BIT; + return r; } s16 unbiased = e - 127; return dpack(sign, unbiased, ((u64)m << 29) | DMANT_LEAD); @@ -339,7 +462,10 @@ u32 __truncdfsf2(u64 x) { u64 m = x & DMANT_MASK; if (e == 0) return (u32)sign; if (e == 0x7FF) { - return (u32)sign | 0x7F800000UL | (u32)(m >> 29); + // A NaN whose payload lives only in the low 29 bits would lose it + // all and become infinity; setting the quiet bit keeps it a NaN. + if (m == 0) return (u32)sign | 0x7F800000UL; + return (u32)sign | 0x7FC00000UL | (u32)(m >> 29); } s16 unbiased = e - DEXP_BIAS; s16 fexp = unbiased + 127; @@ -373,7 +499,11 @@ s32 __fixdfsi(u64 x) { u64 sign = x & DSIGN_BIT; s16 e = (s16)((x >> DEXP_SHIFT) & 0x7FF); if (e == 0) return 0; - if (e == 0x7FF) return sign ? (s32)0x80000000 : 0x7FFFFFFF; + if (e == 0x7FF) { + // NaN converts to 0; infinities saturate like overflow. + if (x & DMANT_MASK) return 0; + return sign ? (s32)0x80000000 : 0x7FFFFFFF; + } s16 unbiased = e - DEXP_BIAS; if (unbiased < 0) return 0; if (unbiased > 30) return sign ? (s32)0x80000000 : 0x7FFFFFFF; @@ -385,13 +515,13 @@ s32 __fixdfsi(u64 x) { // __fixunsdfsi: unsigned double → uint32. Saturates to 0 for negative -// inputs, to 0xFFFFFFFF for inputs >= 2^32. Used by clang when casting -// double values to unsigned integer types. +// inputs and NaN, to 0xFFFFFFFF for inputs >= 2^32 and +inf. Used by clang +// when casting double values to unsigned integer types. u32 __fixunsdfsi(u64 x) { if (x & DSIGN_BIT) return 0; // negative → 0 u16 e = (u16)((x >> DEXP_SHIFT) & 0x7FF); if (e == 0) return 0; - if (e == 0x7FF) return 0xFFFFFFFF; + if (e == 0x7FF) return (x & DMANT_MASK) ? 0 : 0xFFFFFFFF; s16 unbiased = (s16)e - DEXP_BIAS; if (unbiased < 0) return 0; if (unbiased > 31) return 0xFFFFFFFF; diff --git a/runtime/src/softFloat.c b/runtime/src/softFloat.c index 0cd2d3e..4760980 100644 --- a/runtime/src/softFloat.c +++ b/runtime/src/softFloat.c @@ -16,14 +16,18 @@ // // Limitations (V1): // - No subnormal / denormal handling — values flush to zero. -// - No NaN / Inf handling — operations on these give garbage but -// don't crash. // - Round-to-zero (truncation) only; no banker's rounding. // - Add/sub use a 24-bit mantissa; underflow rounding is crude. // -// These are correct enough for end-to-end test programs that do -// "normal" arithmetic in the representable range. Production-grade -// IEEE compliance is a significantly bigger project. +// NaN and infinity follow IEEE 754 (and compiler-rt, whose libcall +// contract LLVM lowers to): a NaN operand comes back quieted with its +// payload kept, an invalid operation (inf - inf, 0 * inf, 0 / 0, +// inf / inf) returns the default NaN, infinities add, multiply and divide +// as IEEE says, and every comparison with a NaN is unordered. The checks +// hang off the class fpClass already computes, so ordinary operands pay +// one branch. Float to integer saturates: NaN converts to 0, +inf and +// too-large values to the type's maximum, -inf and too-negative values to +// its minimum (0 for unsigned). typedef unsigned long u32; typedef long s32; @@ -37,6 +41,19 @@ typedef int s16; #define EXP_BIAS 127 #define MANT_MASK 0x007FFFFFUL #define MANT_LEAD 0x00800000UL // implicit leading 1 +#define QUIET_BIT 0x00400000UL // mantissa MSB: set = quiet NaN +#define DEFAULT_NAN 0x7FC00000UL // what an invalid operation returns +#define FPC_ZERO 0 // fpClass results +#define FPC_NORMAL 1 +#define FPC_INF 2 +#define FPC_NAN 3 +#define FPC_SPECIAL 2 // set in the class of an inf or NaN only +// The special-value paths are kept out of line: inlined, they grew the +// hot functions' frames and the compiler speculated their tests into the +// normal path. +#define COLD __attribute__((noinline)) + +COLD s16 __unordsf2(u32 a, u32 b); static u16 fpClass(u32 x, u32 *out_sign, s16 *out_exp, u32 *out_mant) { *out_sign = x & SIGN_BIT; @@ -46,18 +63,58 @@ static u16 fpClass(u32 x, u32 *out_sign, s16 *out_exp, u32 *out_mant) { // Zero or subnormal — treat as zero (flush). *out_exp = 0; *out_mant = 0; - return 0; // zero + return FPC_ZERO; } if (e == 0xFF) { // Inf or NaN — return as-is, caller decides. *out_exp = 0xFF; *out_mant = m; - return (m == 0) ? 2 : 3; // 2=inf, 3=nan + return (m == 0) ? FPC_INF : FPC_NAN; } // Normal — restore implicit leading 1. *out_exp = e - EXP_BIAS; *out_mant = m | MANT_LEAD; - return 1; // normal + return FPC_NORMAL; +} + +// a op b when an operand is a zero, an infinity or a NaN, apart from the +// zero cases the hot routines answer themselves (op: SP_ADD / SP_MUL / +// SP_DIV). One out-of-line routine for all three keeps the rarely used +// code small, and it reclassifies the operands itself so the call sites +// stay small (see softDouble.c's dpSpecial). +#define SP_ADD 0 +#define SP_MUL 1 +#define SP_DIV 2 +static COLD u32 fpSpecial(u16 op, u32 a, u32 b) { + u32 sa, sb, ma, mb; + s16 ea, eb; + u16 ca = fpClass(a, &sa, &ea, &ma); + u16 cb = fpClass(b, &sb, &eb, &mb); + // The first NaN operand wins, quieted; keeping its payload lets a + // NaN's origin survive arithmetic. + if (ca == FPC_NAN) return a | QUIET_BIT; + if (cb == FPC_NAN) return b | QUIET_BIT; + u32 sign = (a ^ b) & SIGN_BIT; + if (op == SP_ADD) { + // Opposite infinities cancel to nothing representable: invalid. + if (ca == FPC_INF) return (cb == FPC_INF && sign) ? DEFAULT_NAN : a; + if (cb == FPC_INF) return b; + // 0 + 0 is -0 only when both are -0 (round to nearest). + if (ca == FPC_ZERO) return (cb == FPC_ZERO) ? (a & b & SIGN_BIT) : b; + return a; // x + 0 + } + if (op == SP_MUL) { + // 0 * inf has no meaningful magnitude: invalid. + if (ca == FPC_INF || cb == FPC_INF) { + return (ca == FPC_ZERO || cb == FPC_ZERO) ? DEFAULT_NAN : (sign | EXP_MASK); + } + return sign; // a zero operand + } + if (ca == FPC_INF) return (cb == FPC_INF) ? DEFAULT_NAN : (sign | EXP_MASK); + if (cb == FPC_INF) return sign; // finite / inf = signed zero + // x / 0 is a signed infinity, but 0 / 0 is invalid. + if (cb == FPC_ZERO) return (ca == FPC_ZERO) ? DEFAULT_NAN : (sign | EXP_MASK); + return sign; // 0 / x } static u32 fpPack(u32 sign, s16 exp, u32 mant) { @@ -82,8 +139,11 @@ u32 __addsf3(u32 a, u32 b) { s16 ea, eb; u16 ca = fpClass(a, &sa, &ea, &ma); u16 cb = fpClass(b, &sb, &eb, &mb); - if (ca == 0) return b; - if (cb == 0) return a; + // x + 0 and 0 + x stay fast; every other zero, infinity or NaN case + // goes out of line. + if (ca == FPC_ZERO) return (cb == FPC_NORMAL) ? b : fpSpecial(SP_ADD, a, b); + if (cb == FPC_ZERO) return (ca == FPC_NORMAL) ? a : fpSpecial(SP_ADD, a, b); + if (__builtin_expect((ca | cb) & FPC_SPECIAL, 0)) return fpSpecial(SP_ADD, a, b); // Align: shift smaller-exp mantissa right. if (ea > eb) { @@ -103,12 +163,15 @@ u32 __addsf3(u32 a, u32 b) { u32 m = ma + mb; return fpPack(sa, ea, m); } else { - // Different signs — subtract the smaller magnitude. - if (ma >= mb) { + // Different signs -- subtract the smaller magnitude. Exact + // cancellation is +0 whichever operand was negative. + if (ma > mb) { return fpPack(sa, ea, ma - mb); - } else { + } + if (mb > ma) { return fpPack(sb, eb, mb - ma); } + return 0; } } @@ -126,7 +189,13 @@ u32 __mulsf3(u32 a, u32 b) { u16 ca = fpClass(a, &sa, &ea, &ma); u16 cb = fpClass(b, &sb, &eb, &mb); u32 sign = sa ^ sb; - if (ca == 0 || cb == 0) return sign; // zero + // x * 0 and 0 * x stay fast; every other zero, infinity or NaN case + // goes out of line. (This spelling measured faster than softDouble.c's + // for this routine: the difference is register allocation, not work.) + if (ca == FPC_ZERO || cb == FPC_ZERO) { + return ((ca | cb) & FPC_SPECIAL) ? fpSpecial(SP_MUL, a, b) : sign; + } + if (__builtin_expect((ca | cb) & FPC_SPECIAL, 0)) return fpSpecial(SP_MUL, a, b); // 24-bit x 24-bit -> 48-bit product. Take top 24 bits. // We approximate by multiplying the 16-bit halves and combining. @@ -171,8 +240,10 @@ u32 __divsf3(u32 a, u32 b) { u16 ca = fpClass(a, &sa, &ea, &ma); u16 cb = fpClass(b, &sb, &eb, &mb); u32 sign = sa ^ sb; - if (cb == 0) return sign | EXP_MASK; // div-by-zero -> inf - if (ca == 0) return sign; + // 0 / x stays fast; every other zero, infinity or NaN case goes out of + // line. + if (ca == FPC_ZERO && cb == FPC_NORMAL) return sign; + if (__builtin_expect(ca != FPC_NORMAL || cb != FPC_NORMAL, 0)) return fpSpecial(SP_DIV, a, b); // Long division: quotient = ma/mb, in 24+1 bits. We shift ma left // until larger than mb, accumulating quotient bits. Use a 32-bit @@ -192,10 +263,12 @@ u32 __divsf3(u32 a, u32 b) { return fpPack(sign, new_exp, q); } -s16 __cmpsf2(u32 a, u32 b) { - // Returns -1 if ab. - // For NaN, libgcc returns 1 from cmpsf2 (no-NaN convention). We - // skip NaN handling. +// Order of a against b: -1, 0 or 1. Neither may be a NaN. Kept free of +// calls: one would push its values into callee-saved slots and, on this +// target, stop the stack-slot promotion for the whole function. +// Infinities need no case of their own: their bit patterns are the +// largest magnitudes. +static s16 forder(u32 a, u32 b) { if (a == b) return 0; u32 sa = a & SIGN_BIT; u32 sb = b & SIGN_BIT; @@ -211,12 +284,47 @@ s16 __cmpsf2(u32 a, u32 b) { return sa ? -r : r; } -s16 __eqsf2(u32 a, u32 b) { return __cmpsf2(a, b) != 0; } -s16 __nesf2(u32 a, u32 b) { return __cmpsf2(a, b) != 0; } -s16 __ltsf2(u32 a, u32 b) { return __cmpsf2(a, b); } -s16 __gtsf2(u32 a, u32 b) { return __cmpsf2(a, b); } -s16 __lesf2(u32 a, u32 b) { return __cmpsf2(a, b); } -s16 __gesf2(u32 a, u32 b) { return __cmpsf2(a, b); } +// 1 when a or b might be a NaN: only an all-ones exponent can be one. +// With the sign forced on, a high word of 0xFF80 or more has all exponent +// bits set. Spelled as a 16-bit compare because the equivalent mask test +// is widened by the optimizer into a 32-bit one, several times dearer. +static u16 fmaybeNan(u32 a, u32 b) { + return ((u16)(a >> 16) | 0x8000) >= 0xFF80 || + ((u16)(b >> 16) | 0x8000) >= 0xFF80; +} + +// The comparison when an operand has an all-ones exponent: `unord` for a +// NaN, else the ordinary order (infinities). +static COLD s16 fcmpSpecial(u32 a, u32 b, s16 unord) { + return __unordsf2(a, b) ? unord : forder(a, b); +} + +// compiler-rt's contract, which LLVM's soft-float lowering assumes: the +// helpers tested with `< 0` / `<= 0` / `== 0` / `!= 0` (lt, le, eq, ne, +// cmp) answer 1 for unordered, the ones tested with `> 0` / `>= 0` (gt, +// ge) answer -1, so every ordered predicate is false on a NaN and its +// negation (!(a < b) is __ltsf2 >= 0) true. +s16 __cmpsf2(u32 a, u32 b) { + return fmaybeNan(a, b) ? fcmpSpecial(a, b, 1) : forder(a, b); +} +s16 __eqsf2(u32 a, u32 b) { + return fmaybeNan(a, b) ? fcmpSpecial(a, b, 1) : forder(a, b); +} +s16 __nesf2(u32 a, u32 b) { + return fmaybeNan(a, b) ? fcmpSpecial(a, b, 1) : forder(a, b); +} +s16 __ltsf2(u32 a, u32 b) { + return fmaybeNan(a, b) ? fcmpSpecial(a, b, 1) : forder(a, b); +} +s16 __lesf2(u32 a, u32 b) { + return fmaybeNan(a, b) ? fcmpSpecial(a, b, 1) : forder(a, b); +} +s16 __gtsf2(u32 a, u32 b) { + return fmaybeNan(a, b) ? fcmpSpecial(a, b, -1) : forder(a, b); +} +s16 __gesf2(u32 a, u32 b) { + return fmaybeNan(a, b) ? fcmpSpecial(a, b, -1) : forder(a, b); +} // __unordsf2: 1 if either arg is NaN, else 0. Used for IEEE 754 // unordered comparisons (a < b is false if either is NaN, etc.). @@ -262,7 +370,11 @@ s32 __fixsfsi(u32 a) { u32 sa, ma; s16 ea; u16 ca = fpClass(a, &sa, &ea, &ma); - if (ca == 0) return 0; + if (ca != FPC_NORMAL) { + // Zero and NaN convert to 0; infinities saturate like overflow. + if (ca != FPC_INF) return 0; + return sa ? -2147483647L - 1 : 2147483647L; + } if (ea < 0) return 0; // |a| < 1 if (ea >= 31) { // overflow return sa ? -2147483647L - 1 : 2147483647L; @@ -278,7 +390,11 @@ u32 __fixunssfsi(u32 a) { u32 sa, ma; s16 ea; u16 ca = fpClass(a, &sa, &ea, &ma); - if (ca == 0 || sa) return 0; // negative -> 0 + if (ca != FPC_NORMAL) { + // Zero, NaN and -inf convert to 0; +inf saturates. + return (ca == FPC_INF && sa == 0) ? 0xFFFFFFFFUL : 0; + } + if (sa) return 0; // negative -> 0 if (ea < 0) return 0; if (ea >= 32) return 0xFFFFFFFFUL; if (ea >= 23) return ma << (ea - 23); diff --git a/scripts/common.sh b/scripts/common.sh index 3b1cb3c..0dc3cb5 100755 --- a/scripts/common.sh +++ b/scripts/common.sh @@ -30,6 +30,16 @@ die() { exit 1 } +# MAME's cfg / nvram (the IIgs battery RAM) go to a throwaway directory. +# Run from a directory with no mame.ini (tests/ubsan's runner cd's there), +# MAME would otherwise leave cfg/ and nvram/ in it, and a saved BRAM would +# carry settings from run to run. Sets MAME_STATE and MAME_STATE_ARGS +# (pass those to mame); the caller's EXIT trap removes "$MAME_STATE". +makeMameState() { + MAME_STATE=$(mktemp -d) + MAME_STATE_ARGS=(-cfg_directory "$MAME_STATE" -nvram_directory "$MAME_STATE") +} + # Download to cache unless already present. $1=url, $2=dest filename in cache. fetchCached() { local url="$1" diff --git a/scripts/runInMame.sh b/scripts/runInMame.sh index 15c0fc4..2974c7d 100755 --- a/scripts/runInMame.sh +++ b/scripts/runInMame.sh @@ -71,7 +71,8 @@ fi [ -f "$BIN" ] || die "binary not found: $BIN" LUA_PATH=$(mktemp --suffix=.lua) -trap 'rm -f "$LUA_PATH"' EXIT +makeMameState +trap 'rm -f "$LUA_PATH"; rm -rf "${MAME_STATE:?}"' EXIT cat > "$LUA_PATH" <&1 | grep "^MAME-") diff --git a/scripts/runInMameCycles.sh b/scripts/runInMameCycles.sh index c182b18..897aaf6 100755 --- a/scripts/runInMameCycles.sh +++ b/scripts/runInMameCycles.sh @@ -94,7 +94,8 @@ done [ -f "$BIN" ] || die "binary not found: $BIN" LUA_PATH=$(mktemp --suffix=.lua) -trap 'rm -f "$LUA_PATH"' EXIT +makeMameState +trap 'rm -f "$LUA_PATH"; rm -rf "${MAME_STATE:?}"' EXIT cat > "$LUA_PATH" <&1 | grep -E "$GREP_PAT") diff --git a/scripts/runInMameWithGsosStub.sh b/scripts/runInMameWithGsosStub.sh index f38bbf4..0eef55e 100755 --- a/scripts/runInMameWithGsosStub.sh +++ b/scripts/runInMameWithGsosStub.sh @@ -64,7 +64,8 @@ fi [ -f "$BIN" ] || die "binary not found: $BIN" LUA_PATH=$(mktemp --suffix=.lua) -trap 'rm -f "$LUA_PATH"' EXIT +makeMameState +trap 'rm -f "$LUA_PATH"; rm -rf "${MAME_STATE:?}"' EXIT # Build the stub-write Lua statement (one mem:write_u8 per byte). STUB_BYTES=$(echo "$STUB_HEX" | tr -d ' ') @@ -117,6 +118,7 @@ EOF OUT=$(SDL_VIDEODRIVER=dummy SDL_AUDIODRIVER=dummy timeout 30 mame apple2gs \ -rompath "$PROJECT_ROOT/tools/mame/roms" \ + "${MAME_STATE_ARGS[@]}" \ -plugins -autoboot_script "$LUA_PATH" \ -video none -sound none -nothrottle -seconds_to_run "$SECS" 2>&1 | grep "^MAME-") diff --git a/scripts/runMultiSeg.sh b/scripts/runMultiSeg.sh index 9a03129..8217a84 100755 --- a/scripts/runMultiSeg.sh +++ b/scripts/runMultiSeg.sh @@ -65,10 +65,7 @@ EOF [ -n "$PARSED" ] || die "manifest parse failed" LUA_PATH=$(mktemp --suffix=.lua) -# MAME's cfg / nvram (the IIgs battery RAM) go to a throwaway directory: -# run from a test directory with no mame.ini, MAME would otherwise leave -# them there, and a saved BRAM would carry settings from run to run. -MAME_STATE=$(mktemp -d) +makeMameState trap 'rm -f "$LUA_PATH"; rm -rf "${MAME_STATE:?}"' EXIT # Build the per-segment load lua. @@ -159,7 +156,7 @@ fi OUT=$(SDL_VIDEODRIVER=dummy SDL_AUDIODRIVER=dummy timeout "${MAME_TIMEOUT:-30}" mame apple2gs \ "${RAMSIZE_ARG[@]}" \ -rompath "$PROJECT_ROOT/tools/mame/roms" \ - -cfg_directory "$MAME_STATE" -nvram_directory "$MAME_STATE" \ + "${MAME_STATE_ARGS[@]}" \ -plugins -autoboot_script "$LUA_PATH" \ -video none -sound none -nothrottle -seconds_to_run "$SECS" 2>&1 | /usr/bin/grep -E "^(MAME-|SEG-)") diff --git a/scripts/smokeTest.sh b/scripts/smokeTest.sh index f5914e3..3c55fc4 100755 --- a/scripts/smokeTest.sh +++ b/scripts/smokeTest.sh @@ -886,12 +886,16 @@ EOF # ~65536 iterations reading out of bounds (the caeb78e follow-up # course_floor_piece hang, reported by the external SNES user). The pass # inserts a PHP only when it wraps -- greping for it distinguishes fixed from - # buggy on a scheduling-independent MIR input. - log "check: W65816StackSlotCleanup wraps a flag-corrupting EOR before a guard branch" + # buggy on a scheduling-independent MIR input. With X free it protects + # the flags by comparing through X instead (`tax ; ... ; cpx #k` in + # place of the CMP, cmp-via-x.mir): either form proves the EOR was seen. + log "check: W65816StackSlotCleanup protects a guard branch from a flag-corrupting EOR" ssMirOut="$(mktemp --suffix=.mir)" "$LLC" -march=w65816 -run-pass=w65816-stack-slot-cleanup \ "$PROJECT_ROOT/tests/regress/flagWrapLoopGuard.mir" -o "$ssMirOut" 2>/dev/null - if ! grep -qE '^[[:space:]]*PHP$' "$ssMirOut"; then + if ! grep -qE '^[[:space:]]*PHP$' "$ssMirOut" && + ! { grep -qE '^[[:space:]]*TAX ' "$ssMirOut" && + grep -qE '^[[:space:]]*CPX_Imm16 ' "$ssMirOut"; }; then cat "$ssMirOut" >&2 die "StackSlotCleanup did not wrap the flag corruptor (flag-wrap regression)" fi @@ -1633,54 +1637,113 @@ EOF fi rm -f "$cDblMame" "$oDblMame" "$oSdMame" "$binDblMame" - # Every ordered float and double predicate (< <= > >= == !=) on eight - # values, signed zeros included, as a branch and as a 0/1 value; kExp is - # the host-computed expected bit table. Pins two 2026-09-29 fixes: the - # backend read the compare helpers' 16-bit int result as an i32 and - # tested the sign of X (getCmpLibcallReturnType), and softDouble.c's - # __ltdf2 / __ledf2 / __gedf2 returned booleans where LLVM tests a - # three-way order. Before them 396 of the 768 answers were wrong. - log "check: MAME float/double compares: every ordered predicate, signed zeros" + # Every C float and double comparison on twelve values -- signed zeros, + # signed infinities, a quiet NaN, a negative quiet NaN and a + # signaling-pattern NaN -- as a branch and as a 0/1 value. The fourteen + # predicates are the six operators, their negations (LLVM's unordered + # uge/ugt/ule/ult, lowered to the same helpers with the test inverted) + # and C99's isunordered / islessgreater with their negations (uno, ord, + # one, ueq). kExp is host-computed: tests/fpNan/genFpTables.py cmp. + # Pins the 2026-09-29 fixes (the backend read the compare helpers' + # 16-bit result as an i32; __ltdf2 and co returned booleans where LLVM + # tests a three-way order: 396 of 768 answers were wrong) and the NaN + # contract: lt/le/eq/ne answer 1 for unordered, gt/ge answer -1. + log "check: MAME float/double compares: every predicate, signed zeros, infinities, NaNs" cFpCmpFile="$(mktemp --suffix=.c)" oFpCmpFile="$(mktemp --suffix=.o)" binFpCmpFile="$(mktemp --suffix=.bin)" cat > "$cFpCmpFile" <<'EOF' -// Every ordered float and double predicate on 8 values (signed zeros -// included) as a branch and as a 0/1 value. kExp is the host-computed -// expected bit table. Mailbox: 0x025000 = mismatches, 0x025002 = first -// failing case, 0x025004 = 0xC0DE done. -typedef unsigned char u8; -typedef unsigned short u16; -#define NV 8 -#define NP 6 +// Every float and double comparison on 12 values (signed zeros, signed +// infinities, NaNs) as a branch and as a 0/1 value. kExp is the +// host-computed expected bit table (tests/fpNan/genFpTables.py cmp). +// Mailbox: 0x025000 = mismatches, 0x025002 = first failing case (bit +// index, | 0x8000 for the 0/1 form), 0x025004 = 0xC0DE done. +#include +typedef unsigned char u8; +typedef unsigned short u16; +typedef unsigned long u32; +typedef unsigned long long u64; #define BAR() __asm__ volatile("" ::: "memory") #define NOI __attribute__((noinline)) static -static const float kF[NV] = { -2.5f, -1.0f, -0.0f, 0.0f, 1.0f, 1.5f, 10.0f, 1e30f }; -static const double kD[NV] = { -2.5, -1.0, -0.0, 0.0, 1.0, 1.5, 10.0, 1e30 }; -static const u8 kExp[96] = { - 0xFE, 0xFC, 0xF0, 0xF0, 0xE0, 0xC0, 0x80, 0x00, 0xFF, 0xFE, 0xFC, 0xFC, 0xF0, 0xE0, 0xC0, 0x80, - 0x00, 0x01, 0x03, 0x03, 0x0F, 0x1F, 0x3F, 0x7F, 0x01, 0x03, 0x0F, 0x0F, 0x1F, 0x3F, 0x7F, 0xFF, - 0x01, 0x02, 0x0C, 0x0C, 0x10, 0x20, 0x40, 0x80, 0xFE, 0xFD, 0xF3, 0xF3, 0xEF, 0xDF, 0xBF, 0x7F, - 0xFE, 0xFC, 0xF0, 0xF0, 0xE0, 0xC0, 0x80, 0x00, 0xFF, 0xFE, 0xFC, 0xFC, 0xF0, 0xE0, 0xC0, 0x80, - 0x00, 0x01, 0x03, 0x03, 0x0F, 0x1F, 0x3F, 0x7F, 0x01, 0x03, 0x0F, 0x0F, 0x1F, 0x3F, 0x7F, 0xFF, - 0x01, 0x02, 0x0C, 0x0C, 0x10, 0x20, 0x40, 0x80, 0xFE, 0xFD, 0xF3, 0xF3, 0xEF, 0xDF, 0xBF, 0x7F, +#define NV 12 +#define NP 14 +static const u32 kFBits[NV] = { + 0xFF800000UL, 0xC0200000UL, 0xBF800000UL, 0x80000000UL, 0x00000000UL, 0x3F800000UL, 0x41200000UL, 0x7149F2CAUL, 0x7F800000UL, 0x7FC00000UL, 0xFFC00000UL, 0x7F800001UL }; -#define ALL(M, T, p) M(T, p, lt, <) M(T, p, le, <=) M(T, p, gt, >) M(T, p, ge, >=) M(T, p, eq, ==) M(T, p, ne, !=) -#define BR(T, p, n, op) NOI u16 br##p##_##n(T a, T b) { if (a op b) { BAR(); return 1u; } return 0u; } -#define VA(T, p, n, op) NOI u16 va##p##_##n(T a, T b) { return (u16)(a op b); } +static const u64 kDBits[NV] = { + 0xFFF0000000000000ULL, 0xC004000000000000ULL, 0xBFF0000000000000ULL, + 0x8000000000000000ULL, 0x0000000000000000ULL, 0x3FF0000000000000ULL, + 0x4024000000000000ULL, 0x46293E5940000000ULL, 0x7FF0000000000000ULL, + 0x7FF8000000000000ULL, 0xFFF8000000000000ULL, 0x7FF0000000000001ULL, +}; +static const u8 kExp[504] = { + 0xFE, 0xC1, 0x1F, 0xF8, 0x01, 0x1E, 0xE0, 0x01, 0x1C, 0x80, 0x01, 0x10, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0xFF, 0xE1, 0x1F, 0xFC, 0x81, 0x1F, 0xF8, 0x01, 0x1E, 0xC0, 0x01, 0x18, 0x00, 0x01, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x00, 0x03, 0x70, 0x00, 0x07, 0xF0, 0x01, 0x3F, 0xF0, 0x07, + 0xFF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x30, 0x00, 0x07, 0xF0, 0x01, 0x1F, 0xF0, 0x03, 0x7F, + 0xF0, 0x0F, 0xFF, 0x01, 0x00, 0x00, 0x00, 0x00, 0x01, 0x20, 0x00, 0x04, 0x80, 0x01, 0x18, 0x00, + 0x02, 0x40, 0x00, 0x08, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0xFE, 0xDF, 0xFF, 0xFB, 0x7F, 0xFE, + 0xE7, 0xFF, 0xFD, 0xBF, 0xFF, 0xF7, 0xFF, 0xFE, 0xFF, 0xFF, 0xFF, 0xFF, 0x01, 0x3E, 0xE0, 0x07, + 0xFE, 0xE1, 0x1F, 0xFE, 0xE3, 0x7F, 0xFE, 0xEF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x1E, + 0xE0, 0x03, 0x7E, 0xE0, 0x07, 0xFE, 0xE1, 0x3F, 0xFE, 0xE7, 0xFF, 0xFE, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xEF, 0xFF, 0xFC, 0x8F, 0xFF, 0xF8, 0x0F, 0xFE, 0xC0, 0x0F, 0xF8, 0x00, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFE, 0xCF, 0xFF, 0xF8, 0x0F, 0xFE, 0xE0, 0x0F, 0xFC, 0x80, 0x0F, 0xF0, 0x00, 0xFE, + 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x0E, 0xE0, 0x00, 0x0E, 0xE0, 0x00, 0x0E, 0xE0, 0x00, 0x0E, 0xE0, + 0x00, 0xFE, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xF1, 0x1F, 0xFF, 0xF1, 0x1F, 0xFF, 0xF1, 0x1F, 0xFF, + 0xF1, 0x1F, 0xFF, 0x01, 0x00, 0x00, 0x00, 0x00, 0xFE, 0xD1, 0x1F, 0xFB, 0x71, 0x1E, 0xE7, 0xF1, + 0x1D, 0xBF, 0xF1, 0x17, 0xFF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x2E, 0xE0, 0x04, 0x8E, 0xE1, + 0x18, 0x0E, 0xE2, 0x40, 0x0E, 0xE8, 0x00, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFE, 0xC1, 0x1F, 0xF8, + 0x01, 0x1E, 0xE0, 0x01, 0x1C, 0x80, 0x01, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xFF, 0xE1, + 0x1F, 0xFC, 0x81, 0x1F, 0xF8, 0x01, 0x1E, 0xC0, 0x01, 0x18, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x10, 0x00, 0x03, 0x70, 0x00, 0x07, 0xF0, 0x01, 0x3F, 0xF0, 0x07, 0xFF, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x01, 0x30, 0x00, 0x07, 0xF0, 0x01, 0x1F, 0xF0, 0x03, 0x7F, 0xF0, 0x0F, 0xFF, 0x01, + 0x00, 0x00, 0x00, 0x00, 0x01, 0x20, 0x00, 0x04, 0x80, 0x01, 0x18, 0x00, 0x02, 0x40, 0x00, 0x08, + 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0xFE, 0xDF, 0xFF, 0xFB, 0x7F, 0xFE, 0xE7, 0xFF, 0xFD, 0xBF, + 0xFF, 0xF7, 0xFF, 0xFE, 0xFF, 0xFF, 0xFF, 0xFF, 0x01, 0x3E, 0xE0, 0x07, 0xFE, 0xE1, 0x1F, 0xFE, + 0xE3, 0x7F, 0xFE, 0xEF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x1E, 0xE0, 0x03, 0x7E, 0xE0, + 0x07, 0xFE, 0xE1, 0x3F, 0xFE, 0xE7, 0xFF, 0xFE, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xEF, 0xFF, 0xFC, + 0x8F, 0xFF, 0xF8, 0x0F, 0xFE, 0xC0, 0x0F, 0xF8, 0x00, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFE, 0xCF, + 0xFF, 0xF8, 0x0F, 0xFE, 0xE0, 0x0F, 0xFC, 0x80, 0x0F, 0xF0, 0x00, 0xFE, 0xFF, 0xFF, 0xFF, 0xFF, + 0x00, 0x0E, 0xE0, 0x00, 0x0E, 0xE0, 0x00, 0x0E, 0xE0, 0x00, 0x0E, 0xE0, 0x00, 0xFE, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xF1, 0x1F, 0xFF, 0xF1, 0x1F, 0xFF, 0xF1, 0x1F, 0xFF, 0xF1, 0x1F, 0xFF, 0x01, + 0x00, 0x00, 0x00, 0x00, 0xFE, 0xD1, 0x1F, 0xFB, 0x71, 0x1E, 0xE7, 0xF1, 0x1D, 0xBF, 0xF1, 0x17, + 0xFF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x2E, 0xE0, 0x04, 0x8E, 0xE1, 0x18, 0x0E, 0xE2, 0x40, + 0x0E, 0xE8, 0x00, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, +}; +#define ALL(M, T, p) \ + M(T, p, lt, a < b) M(T, p, le, a <= b) M(T, p, gt, a > b) \ + M(T, p, ge, a >= b) M(T, p, eq, a == b) M(T, p, ne, a != b) \ + M(T, p, nlt, !(a < b)) M(T, p, nle, !(a <= b)) M(T, p, ngt, !(a > b)) \ + M(T, p, nge, !(a >= b)) M(T, p, uno, isunordered(a, b)) \ + M(T, p, ord, !isunordered(a, b)) M(T, p, lg, islessgreater(a, b)) \ + M(T, p, nlg, !islessgreater(a, b)) +#define BR(T, p, n, e) NOI u16 br##p##_##n(T a, T b) { if (e) { BAR(); return 1u; } return 0u; } +#define VA(T, p, n, e) NOI u16 va##p##_##n(T a, T b) { return (u16)(e); } ALL(BR, float, f) ALL(VA, float, f) ALL(BR, double, d) ALL(VA, double, d) typedef u16 (*FnF)(float, float); typedef u16 (*FnD)(double, double); -#define R(p, k) p##k##_lt, p##k##_le, p##k##_gt, p##k##_ge, p##k##_eq, p##k##_ne +#define R(p, k) p##k##_lt, p##k##_le, p##k##_gt, p##k##_ge, p##k##_eq, \ + p##k##_ne, p##k##_nlt, p##k##_nle, p##k##_ngt, p##k##_nge, \ + p##k##_uno, p##k##_ord, p##k##_lg, p##k##_nlg static FnF const kBrF[NP] = { R(br, f) }; static FnF const kVaF[NP] = { R(va, f) }; static FnD const kBrD[NP] = { R(br, d) }; static FnD const kVaD[NP] = { R(va, d) }; static u16 gBad; static u16 gFirst = 0xFFFFu; +static float fv(u16 i) { + union { u32 u; float f; } c; + c.u = kFBits[i]; + return c.f; +} +static double dv(u16 i) { + union { u64 u; double d; } c; + c.u = kDBits[i]; + return c.d; +} static void check(u16 got, u16 want, u16 id) { if (got != want) { if (gBad == 0u) { @@ -1698,8 +1761,8 @@ int main(void) { for (i = 0u; i < NV; i++) { for (j = 0u; j < NV; j++) { u16 want = (u16)((kExp[bit >> 3] >> (bit & 7u)) & 1u); - check(kBrF[p](kF[i], kF[j]), want, bit); - check(kVaF[p](kF[i], kF[j]) != 0u, want, (u16)(bit | 0x4000u)); + check(kBrF[p](fv(i), fv(j)), want, bit); + check(kVaF[p](fv(i), fv(j)) != 0u, want, (u16)(bit | 0x8000u)); bit++; } } @@ -1708,8 +1771,8 @@ int main(void) { for (i = 0u; i < NV; i++) { for (j = 0u; j < NV; j++) { u16 want = (u16)((kExp[bit >> 3] >> (bit & 7u)) & 1u); - check(kBrD[p](kD[i], kD[j]), want, bit); - check(kVaD[p](kD[i], kD[j]) != 0u, want, (u16)(bit | 0x4000u)); + check(kBrD[p](dv(i), dv(j)), want, bit); + check(kVaD[p](dv(i), dv(j)) != 0u, want, (u16)(bit | 0x8000u)); bit++; } } @@ -1722,8 +1785,8 @@ int main(void) { return 0; } EOF - "$CLANG" --target=w65816 -O2 -ffunction-sections -c \ - "$cFpCmpFile" -o "$oFpCmpFile" + "$CLANG" --target=w65816 -O2 -ffunction-sections \ + -I"$PROJECT_ROOT/runtime/include" -c "$cFpCmpFile" -o "$oFpCmpFile" "$PROJECT_ROOT/tools/link816" -o "$binFpCmpFile" --text-base 0x1000 \ "$oCrt0File" "$oFpCmpFile" "$PROJECT_ROOT/runtime/softFloat.o" \ "$PROJECT_ROOT/runtime/softDouble.o" "$oLibgccFile" >/dev/null 2>&1 @@ -1734,6 +1797,776 @@ EOF fi rm -f "$cFpCmpFile" "$oFpCmpFile" "$binFpCmpFile" + # NaN and infinity through the soft-float arithmetic, conversions, + # classification and math.c (C99 Annex F special values, plus ordinary + # values for every math function touched): a NaN operand comes back + # quieted with its payload (the first NaN when both are), inf - inf, + # 0 * inf, 0 / 0 and inf / inf give the default NaN, infinities add, + # multiply and divide as IEEE says, x + -x is +0, float <-> double keeps + # NaN payloads, infinities and signed zeros, float/double to integer + # saturates with NaN converting to 0, isnan / isinf / isfinite / + # signbit / fpclassify see through the float-to-double widening, and + # sqrt / log / asin / acos / atanh return NaN (or +-inf) at their + # domain edges; floor / ceil / trunc / round keep signed zeros, rint is + # ties-to-even, exp / sinh / cosh / tanh / expm1 / cbrt / pow / fmod / + # fdim / hypot / remainder give Annex F's answers for zeros, + # infinities and NaNs, and the float wrappers (floorf ... fminf) return + # at all (clang had turned each into a call to itself). Series-based + # results are checked to 1e-6 relative. kCases is host-computed: + # tests/fpNan/genFpTables.py special (two tables, run as two checks). + log "check: MAME float/double NaN and infinity: arithmetic, conversions, classification" + cFpNanFile="$(mktemp --suffix=.c)" + oFpNanFile="$(mktemp --suffix=.o)" + binFpNanFile="$(mktemp --suffix=.bin)" + cat > "$cFpNanFile" <<'EOF' +// Each case runs one operation on bit-pattern operands and compares the +// result's bits with the host-computed expectation. Mailbox: 0x025000 = +// mismatches, 0x025002 = first failing case, 0x025004 = 0xC0DE done. +#include +typedef unsigned char u8; +typedef unsigned short u16; +typedef unsigned long u32; +typedef unsigned long long u64; +#define NOI __attribute__((noinline)) static +enum { + O_FADD, O_FSUB, O_FMUL, O_FDIV, O_DADD, O_DSUB, O_DMUL, O_DDIV, + O_F2D, O_D2F, O_F2I, O_F2U, O_D2I, O_D2U, O_FCLS, O_DCLS, + O_SQRT, O_LOG, O_ASIN, O_ACOS, O_ATANH, + O_FLOOR, O_CEIL, O_TRUNC, O_ROUND, O_RINT, O_NEARBYINT, + O_FLOORF, O_CEILF, O_TRUNCF, O_ROUNDF, O_RINTF, + O_EXP, O_EXP2, O_EXPM1, O_SINH, O_COSH, O_TANH, O_LOG1P, O_CBRT, + O_POW, O_FMOD, O_FDIM, O_HYPOT, O_REMAINDER, + O_NEARBYINTF, O_FMAXF, O_FMINF +}; +// approx: the result of a series-based function, accepted within 1e-6 +// relative of the host's value (the documented accuracy). +typedef struct { + u8 op; + u64 a; + u64 b; + u64 want; + u8 approx; +} CaseT; +#if FP_TABLE == 1 +#define NC 358 +static const CaseT kCases[NC] = { + { O_FADD, 0x000000007FC00123ULL, 0x000000003F800000ULL, 0x000000007FC00123ULL, 0 }, + { O_FADD, 0x000000003F800000ULL, 0x000000007F800005ULL, 0x000000007FC00005ULL, 0 }, + { O_FADD, 0x000000007F800005ULL, 0x000000007FC00123ULL, 0x000000007FC00005ULL, 0 }, + { O_FADD, 0x00000000FFC00000ULL, 0x0000000040000000ULL, 0x00000000FFC00000ULL, 0 }, + { O_FADD, 0x0000000000000000ULL, 0x000000007FC00123ULL, 0x000000007FC00123ULL, 0 }, + { O_FADD, 0x000000007F800000ULL, 0x000000003F800000ULL, 0x000000007F800000ULL, 0 }, + { O_FADD, 0x000000007F800000ULL, 0x000000007F800000ULL, 0x000000007F800000ULL, 0 }, + { O_FADD, 0x000000007F800000ULL, 0x00000000FF800000ULL, 0x000000007FC00000ULL, 0 }, + { O_FADD, 0x00000000FF800000ULL, 0x0000000040000000ULL, 0x00000000FF800000ULL, 0 }, + { O_FADD, 0x0000000040000000ULL, 0x00000000FF800000ULL, 0x00000000FF800000ULL, 0 }, + { O_FADD, 0x000000003F800000ULL, 0x00000000BF800000ULL, 0x0000000000000000ULL, 0 }, + { O_FADD, 0x00000000BF800000ULL, 0x000000003F800000ULL, 0x0000000000000000ULL, 0 }, + { O_FADD, 0x0000000080000000ULL, 0x0000000080000000ULL, 0x0000000080000000ULL, 0 }, + { O_FADD, 0x0000000080000000ULL, 0x0000000000000000ULL, 0x0000000000000000ULL, 0 }, + { O_FADD, 0x0000000000000000ULL, 0x0000000080000000ULL, 0x0000000000000000ULL, 0 }, + { O_FADD, 0x0000000000000000ULL, 0x0000000000000000ULL, 0x0000000000000000ULL, 0 }, + { O_FADD, 0x0000000040100000ULL, 0x000000003FC00000ULL, 0x0000000040700000ULL, 0 }, + { O_FADD, 0x000000007F61B1E6ULL, 0x000000007F61B1E6ULL, 0x000000007F800000ULL, 0 }, + { O_FADD, 0x0000000000000000ULL, 0x000000007F800000ULL, 0x000000007F800000ULL, 0 }, + { O_FADD, 0x000000007F800000ULL, 0x0000000000000000ULL, 0x000000007F800000ULL, 0 }, + { O_FADD, 0x00000000FF800000ULL, 0x00000000C0000000ULL, 0x00000000FF800000ULL, 0 }, + { O_FADD, 0x00000000FF800000ULL, 0x00000000FF800000ULL, 0x00000000FF800000ULL, 0 }, + { O_FADD, 0x0000000080000000ULL, 0x0000000040000000ULL, 0x0000000040000000ULL, 0 }, + { O_FADD, 0x0000000000000000ULL, 0x0000000000000000ULL, 0x0000000000000000ULL, 0 }, + { O_FADD, 0x000000003F800000ULL, 0x0000000000000000ULL, 0x000000003F800000ULL, 0 }, + { O_FADD, 0x00000000BF800000ULL, 0x0000000000000000ULL, 0x00000000BF800000ULL, 0 }, + { O_FADD, 0x000000003F800000ULL, 0x0000000080000000ULL, 0x000000003F800000ULL, 0 }, + { O_FADD, 0x000000003F800000ULL, 0x000000007F800000ULL, 0x000000007F800000ULL, 0 }, + { O_FADD, 0x00000000BF800000ULL, 0x000000007F800000ULL, 0x000000007F800000ULL, 0 }, + { O_FADD, 0x000000007F800000ULL, 0x0000000040000000ULL, 0x000000007F800000ULL, 0 }, + { O_FADD, 0x000000003FC00000ULL, 0x0000000040000000ULL, 0x0000000040600000ULL, 0 }, + { O_FSUB, 0x000000007FC00123ULL, 0x000000003F800000ULL, 0x000000007FC00123ULL, 0 }, + { O_FSUB, 0x000000003F800000ULL, 0x000000007F800005ULL, 0x00000000FFC00005ULL, 0 }, + { O_FSUB, 0x000000007F800005ULL, 0x000000007FC00123ULL, 0x000000007FC00005ULL, 0 }, + { O_FSUB, 0x00000000FFC00000ULL, 0x0000000040000000ULL, 0x00000000FFC00000ULL, 0 }, + { O_FSUB, 0x0000000000000000ULL, 0x000000007FC00123ULL, 0x00000000FFC00123ULL, 0 }, + { O_FSUB, 0x000000007F800000ULL, 0x000000003F800000ULL, 0x000000007F800000ULL, 0 }, + { O_FSUB, 0x000000007F800000ULL, 0x000000007F800000ULL, 0x000000007FC00000ULL, 0 }, + { O_FSUB, 0x000000007F800000ULL, 0x00000000FF800000ULL, 0x000000007F800000ULL, 0 }, + { O_FSUB, 0x00000000FF800000ULL, 0x0000000040000000ULL, 0x00000000FF800000ULL, 0 }, + { O_FSUB, 0x0000000040000000ULL, 0x00000000FF800000ULL, 0x000000007F800000ULL, 0 }, + { O_FSUB, 0x000000003F800000ULL, 0x00000000BF800000ULL, 0x0000000040000000ULL, 0 }, + { O_FSUB, 0x00000000BF800000ULL, 0x000000003F800000ULL, 0x00000000C0000000ULL, 0 }, + { O_FSUB, 0x0000000080000000ULL, 0x0000000080000000ULL, 0x0000000000000000ULL, 0 }, + { O_FSUB, 0x0000000080000000ULL, 0x0000000000000000ULL, 0x0000000080000000ULL, 0 }, + { O_FSUB, 0x0000000000000000ULL, 0x0000000080000000ULL, 0x0000000000000000ULL, 0 }, + { O_FSUB, 0x0000000000000000ULL, 0x0000000000000000ULL, 0x0000000000000000ULL, 0 }, + { O_FSUB, 0x0000000040100000ULL, 0x000000003FC00000ULL, 0x000000003F400000ULL, 0 }, + { O_FSUB, 0x000000007F61B1E6ULL, 0x000000007F61B1E6ULL, 0x0000000000000000ULL, 0 }, + { O_FSUB, 0x0000000000000000ULL, 0x000000007F800000ULL, 0x00000000FF800000ULL, 0 }, + { O_FSUB, 0x000000007F800000ULL, 0x0000000000000000ULL, 0x000000007F800000ULL, 0 }, + { O_FSUB, 0x00000000FF800000ULL, 0x00000000C0000000ULL, 0x00000000FF800000ULL, 0 }, + { O_FSUB, 0x00000000FF800000ULL, 0x00000000FF800000ULL, 0x000000007FC00000ULL, 0 }, + { O_FSUB, 0x0000000080000000ULL, 0x0000000040000000ULL, 0x00000000C0000000ULL, 0 }, + { O_FSUB, 0x0000000000000000ULL, 0x0000000000000000ULL, 0x0000000000000000ULL, 0 }, + { O_FSUB, 0x000000003F800000ULL, 0x0000000000000000ULL, 0x000000003F800000ULL, 0 }, + { O_FSUB, 0x00000000BF800000ULL, 0x0000000000000000ULL, 0x00000000BF800000ULL, 0 }, + { O_FSUB, 0x000000003F800000ULL, 0x0000000080000000ULL, 0x000000003F800000ULL, 0 }, + { O_FSUB, 0x000000003F800000ULL, 0x000000007F800000ULL, 0x00000000FF800000ULL, 0 }, + { O_FSUB, 0x00000000BF800000ULL, 0x000000007F800000ULL, 0x00000000FF800000ULL, 0 }, + { O_FSUB, 0x000000007F800000ULL, 0x0000000040000000ULL, 0x000000007F800000ULL, 0 }, + { O_FSUB, 0x000000003FC00000ULL, 0x0000000040000000ULL, 0x00000000BF000000ULL, 0 }, + { O_FMUL, 0x000000007FC00123ULL, 0x000000003F800000ULL, 0x000000007FC00123ULL, 0 }, + { O_FMUL, 0x000000003F800000ULL, 0x000000007F800005ULL, 0x000000007FC00005ULL, 0 }, + { O_FMUL, 0x000000007F800005ULL, 0x000000007FC00123ULL, 0x000000007FC00005ULL, 0 }, + { O_FMUL, 0x00000000FFC00000ULL, 0x0000000040000000ULL, 0x00000000FFC00000ULL, 0 }, + { O_FMUL, 0x0000000000000000ULL, 0x000000007FC00123ULL, 0x000000007FC00123ULL, 0 }, + { O_FMUL, 0x000000007F800000ULL, 0x000000003F800000ULL, 0x000000007F800000ULL, 0 }, + { O_FMUL, 0x000000007F800000ULL, 0x000000007F800000ULL, 0x000000007F800000ULL, 0 }, + { O_FMUL, 0x000000007F800000ULL, 0x00000000FF800000ULL, 0x00000000FF800000ULL, 0 }, + { O_FMUL, 0x00000000FF800000ULL, 0x0000000040000000ULL, 0x00000000FF800000ULL, 0 }, + { O_FMUL, 0x0000000040000000ULL, 0x00000000FF800000ULL, 0x00000000FF800000ULL, 0 }, + { O_FMUL, 0x000000003F800000ULL, 0x00000000BF800000ULL, 0x00000000BF800000ULL, 0 }, + { O_FMUL, 0x00000000BF800000ULL, 0x000000003F800000ULL, 0x00000000BF800000ULL, 0 }, + { O_FMUL, 0x0000000080000000ULL, 0x0000000080000000ULL, 0x0000000000000000ULL, 0 }, + { O_FMUL, 0x0000000080000000ULL, 0x0000000000000000ULL, 0x0000000080000000ULL, 0 }, + { O_FMUL, 0x0000000000000000ULL, 0x0000000080000000ULL, 0x0000000080000000ULL, 0 }, + { O_FMUL, 0x0000000000000000ULL, 0x0000000000000000ULL, 0x0000000000000000ULL, 0 }, + { O_FMUL, 0x0000000040100000ULL, 0x000000003FC00000ULL, 0x0000000040580000ULL, 0 }, + { O_FMUL, 0x000000007F61B1E6ULL, 0x000000007F61B1E6ULL, 0x000000007F800000ULL, 0 }, + { O_FMUL, 0x0000000000000000ULL, 0x000000007F800000ULL, 0x000000007FC00000ULL, 0 }, + { O_FMUL, 0x000000007F800000ULL, 0x0000000000000000ULL, 0x000000007FC00000ULL, 0 }, + { O_FMUL, 0x00000000FF800000ULL, 0x00000000C0000000ULL, 0x000000007F800000ULL, 0 }, + { O_FMUL, 0x00000000FF800000ULL, 0x00000000FF800000ULL, 0x000000007F800000ULL, 0 }, + { O_FMUL, 0x0000000080000000ULL, 0x0000000040000000ULL, 0x0000000080000000ULL, 0 }, + { O_FMUL, 0x0000000000000000ULL, 0x0000000000000000ULL, 0x0000000000000000ULL, 0 }, + { O_FMUL, 0x000000003F800000ULL, 0x0000000000000000ULL, 0x0000000000000000ULL, 0 }, + { O_FMUL, 0x00000000BF800000ULL, 0x0000000000000000ULL, 0x0000000080000000ULL, 0 }, + { O_FMUL, 0x000000003F800000ULL, 0x0000000080000000ULL, 0x0000000080000000ULL, 0 }, + { O_FMUL, 0x000000003F800000ULL, 0x000000007F800000ULL, 0x000000007F800000ULL, 0 }, + { O_FMUL, 0x00000000BF800000ULL, 0x000000007F800000ULL, 0x00000000FF800000ULL, 0 }, + { O_FMUL, 0x000000007F800000ULL, 0x0000000040000000ULL, 0x000000007F800000ULL, 0 }, + { O_FMUL, 0x000000003FC00000ULL, 0x0000000040000000ULL, 0x0000000040400000ULL, 0 }, + { O_FDIV, 0x000000007FC00123ULL, 0x000000003F800000ULL, 0x000000007FC00123ULL, 0 }, + { O_FDIV, 0x000000003F800000ULL, 0x000000007F800005ULL, 0x000000007FC00005ULL, 0 }, + { O_FDIV, 0x000000007F800005ULL, 0x000000007FC00123ULL, 0x000000007FC00005ULL, 0 }, + { O_FDIV, 0x00000000FFC00000ULL, 0x0000000040000000ULL, 0x00000000FFC00000ULL, 0 }, + { O_FDIV, 0x0000000000000000ULL, 0x000000007FC00123ULL, 0x000000007FC00123ULL, 0 }, + { O_FDIV, 0x000000007F800000ULL, 0x000000003F800000ULL, 0x000000007F800000ULL, 0 }, + { O_FDIV, 0x000000007F800000ULL, 0x000000007F800000ULL, 0x000000007FC00000ULL, 0 }, + { O_FDIV, 0x000000007F800000ULL, 0x00000000FF800000ULL, 0x000000007FC00000ULL, 0 }, + { O_FDIV, 0x00000000FF800000ULL, 0x0000000040000000ULL, 0x00000000FF800000ULL, 0 }, + { O_FDIV, 0x0000000040000000ULL, 0x00000000FF800000ULL, 0x0000000080000000ULL, 0 }, + { O_FDIV, 0x000000003F800000ULL, 0x00000000BF800000ULL, 0x00000000BF800000ULL, 0 }, + { O_FDIV, 0x00000000BF800000ULL, 0x000000003F800000ULL, 0x00000000BF800000ULL, 0 }, + { O_FDIV, 0x0000000080000000ULL, 0x0000000080000000ULL, 0x000000007FC00000ULL, 0 }, + { O_FDIV, 0x0000000080000000ULL, 0x0000000000000000ULL, 0x000000007FC00000ULL, 0 }, + { O_FDIV, 0x0000000000000000ULL, 0x0000000080000000ULL, 0x000000007FC00000ULL, 0 }, + { O_FDIV, 0x0000000000000000ULL, 0x0000000000000000ULL, 0x000000007FC00000ULL, 0 }, + { O_FDIV, 0x0000000040100000ULL, 0x000000003FC00000ULL, 0x000000003FC00000ULL, 0 }, + { O_FDIV, 0x000000007F61B1E6ULL, 0x000000007F61B1E6ULL, 0x000000003F800000ULL, 0 }, + { O_FDIV, 0x0000000000000000ULL, 0x000000007F800000ULL, 0x0000000000000000ULL, 0 }, + { O_FDIV, 0x000000007F800000ULL, 0x0000000000000000ULL, 0x000000007F800000ULL, 0 }, + { O_FDIV, 0x00000000FF800000ULL, 0x00000000C0000000ULL, 0x000000007F800000ULL, 0 }, + { O_FDIV, 0x00000000FF800000ULL, 0x00000000FF800000ULL, 0x000000007FC00000ULL, 0 }, + { O_FDIV, 0x0000000080000000ULL, 0x0000000040000000ULL, 0x0000000080000000ULL, 0 }, + { O_FDIV, 0x0000000000000000ULL, 0x0000000000000000ULL, 0x000000007FC00000ULL, 0 }, + { O_FDIV, 0x000000003F800000ULL, 0x0000000000000000ULL, 0x000000007F800000ULL, 0 }, + { O_FDIV, 0x00000000BF800000ULL, 0x0000000000000000ULL, 0x00000000FF800000ULL, 0 }, + { O_FDIV, 0x000000003F800000ULL, 0x0000000080000000ULL, 0x00000000FF800000ULL, 0 }, + { O_FDIV, 0x000000003F800000ULL, 0x000000007F800000ULL, 0x0000000000000000ULL, 0 }, + { O_FDIV, 0x00000000BF800000ULL, 0x000000007F800000ULL, 0x0000000080000000ULL, 0 }, + { O_FDIV, 0x000000007F800000ULL, 0x0000000040000000ULL, 0x000000007F800000ULL, 0 }, + { O_FDIV, 0x000000003FC00000ULL, 0x0000000040000000ULL, 0x000000003F400000ULL, 0 }, + { O_DADD, 0x7FF8000000000123ULL, 0x3FF0000000000000ULL, 0x7FF8000000000123ULL, 0 }, + { O_DADD, 0x3FF0000000000000ULL, 0x7FF0000000000005ULL, 0x7FF8000000000005ULL, 0 }, + { O_DADD, 0x7FF0000000000005ULL, 0x7FF8000000000123ULL, 0x7FF8000000000005ULL, 0 }, + { O_DADD, 0xFFF8000000000000ULL, 0x4000000000000000ULL, 0xFFF8000000000000ULL, 0 }, + { O_DADD, 0x0000000000000000ULL, 0x7FF8000000000123ULL, 0x7FF8000000000123ULL, 0 }, + { O_DADD, 0x7FF0000000000000ULL, 0x3FF0000000000000ULL, 0x7FF0000000000000ULL, 0 }, + { O_DADD, 0x7FF0000000000000ULL, 0x7FF0000000000000ULL, 0x7FF0000000000000ULL, 0 }, + { O_DADD, 0x7FF0000000000000ULL, 0xFFF0000000000000ULL, 0x7FF8000000000000ULL, 0 }, + { O_DADD, 0xFFF0000000000000ULL, 0x4000000000000000ULL, 0xFFF0000000000000ULL, 0 }, + { O_DADD, 0x4000000000000000ULL, 0xFFF0000000000000ULL, 0xFFF0000000000000ULL, 0 }, + { O_DADD, 0x3FF0000000000000ULL, 0xBFF0000000000000ULL, 0x0000000000000000ULL, 0 }, + { O_DADD, 0xBFF0000000000000ULL, 0x3FF0000000000000ULL, 0x0000000000000000ULL, 0 }, + { O_DADD, 0x8000000000000000ULL, 0x8000000000000000ULL, 0x8000000000000000ULL, 0 }, + { O_DADD, 0x8000000000000000ULL, 0x0000000000000000ULL, 0x0000000000000000ULL, 0 }, + { O_DADD, 0x0000000000000000ULL, 0x8000000000000000ULL, 0x0000000000000000ULL, 0 }, + { O_DADD, 0x0000000000000000ULL, 0x0000000000000000ULL, 0x0000000000000000ULL, 0 }, + { O_DADD, 0x4002000000000000ULL, 0x3FF8000000000000ULL, 0x400E000000000000ULL, 0 }, + { O_DADD, 0x7E37E43C8800759CULL, 0x7E37E43C8800759CULL, 0x7E47E43C8800759CULL, 0 }, + { O_DADD, 0x0000000000000000ULL, 0x7FF0000000000000ULL, 0x7FF0000000000000ULL, 0 }, + { O_DADD, 0x7FF0000000000000ULL, 0x0000000000000000ULL, 0x7FF0000000000000ULL, 0 }, + { O_DADD, 0xFFF0000000000000ULL, 0xC000000000000000ULL, 0xFFF0000000000000ULL, 0 }, + { O_DADD, 0xFFF0000000000000ULL, 0xFFF0000000000000ULL, 0xFFF0000000000000ULL, 0 }, + { O_DADD, 0x8000000000000000ULL, 0x4000000000000000ULL, 0x4000000000000000ULL, 0 }, + { O_DADD, 0x0000000000000000ULL, 0x0000000000000000ULL, 0x0000000000000000ULL, 0 }, + { O_DADD, 0x3FF0000000000000ULL, 0x0000000000000000ULL, 0x3FF0000000000000ULL, 0 }, + { O_DADD, 0xBFF0000000000000ULL, 0x0000000000000000ULL, 0xBFF0000000000000ULL, 0 }, + { O_DADD, 0x3FF0000000000000ULL, 0x8000000000000000ULL, 0x3FF0000000000000ULL, 0 }, + { O_DADD, 0x3FF0000000000000ULL, 0x7FF0000000000000ULL, 0x7FF0000000000000ULL, 0 }, + { O_DADD, 0xBFF0000000000000ULL, 0x7FF0000000000000ULL, 0x7FF0000000000000ULL, 0 }, + { O_DADD, 0x7FF0000000000000ULL, 0x4000000000000000ULL, 0x7FF0000000000000ULL, 0 }, + { O_DADD, 0x3FF8000000000000ULL, 0x4000000000000000ULL, 0x400C000000000000ULL, 0 }, + { O_DSUB, 0x7FF8000000000123ULL, 0x3FF0000000000000ULL, 0x7FF8000000000123ULL, 0 }, + { O_DSUB, 0x3FF0000000000000ULL, 0x7FF0000000000005ULL, 0xFFF8000000000005ULL, 0 }, + { O_DSUB, 0x7FF0000000000005ULL, 0x7FF8000000000123ULL, 0x7FF8000000000005ULL, 0 }, + { O_DSUB, 0xFFF8000000000000ULL, 0x4000000000000000ULL, 0xFFF8000000000000ULL, 0 }, + { O_DSUB, 0x0000000000000000ULL, 0x7FF8000000000123ULL, 0xFFF8000000000123ULL, 0 }, + { O_DSUB, 0x7FF0000000000000ULL, 0x3FF0000000000000ULL, 0x7FF0000000000000ULL, 0 }, + { O_DSUB, 0x7FF0000000000000ULL, 0x7FF0000000000000ULL, 0x7FF8000000000000ULL, 0 }, + { O_DSUB, 0x7FF0000000000000ULL, 0xFFF0000000000000ULL, 0x7FF0000000000000ULL, 0 }, + { O_DSUB, 0xFFF0000000000000ULL, 0x4000000000000000ULL, 0xFFF0000000000000ULL, 0 }, + { O_DSUB, 0x4000000000000000ULL, 0xFFF0000000000000ULL, 0x7FF0000000000000ULL, 0 }, + { O_DSUB, 0x3FF0000000000000ULL, 0xBFF0000000000000ULL, 0x4000000000000000ULL, 0 }, + { O_DSUB, 0xBFF0000000000000ULL, 0x3FF0000000000000ULL, 0xC000000000000000ULL, 0 }, + { O_DSUB, 0x8000000000000000ULL, 0x8000000000000000ULL, 0x0000000000000000ULL, 0 }, + { O_DSUB, 0x8000000000000000ULL, 0x0000000000000000ULL, 0x8000000000000000ULL, 0 }, + { O_DSUB, 0x0000000000000000ULL, 0x8000000000000000ULL, 0x0000000000000000ULL, 0 }, + { O_DSUB, 0x0000000000000000ULL, 0x0000000000000000ULL, 0x0000000000000000ULL, 0 }, + { O_DSUB, 0x4002000000000000ULL, 0x3FF8000000000000ULL, 0x3FE8000000000000ULL, 0 }, + { O_DSUB, 0x7E37E43C8800759CULL, 0x7E37E43C8800759CULL, 0x0000000000000000ULL, 0 }, + { O_DSUB, 0x0000000000000000ULL, 0x7FF0000000000000ULL, 0xFFF0000000000000ULL, 0 }, + { O_DSUB, 0x7FF0000000000000ULL, 0x0000000000000000ULL, 0x7FF0000000000000ULL, 0 }, + { O_DSUB, 0xFFF0000000000000ULL, 0xC000000000000000ULL, 0xFFF0000000000000ULL, 0 }, + { O_DSUB, 0xFFF0000000000000ULL, 0xFFF0000000000000ULL, 0x7FF8000000000000ULL, 0 }, + { O_DSUB, 0x8000000000000000ULL, 0x4000000000000000ULL, 0xC000000000000000ULL, 0 }, + { O_DSUB, 0x0000000000000000ULL, 0x0000000000000000ULL, 0x0000000000000000ULL, 0 }, + { O_DSUB, 0x3FF0000000000000ULL, 0x0000000000000000ULL, 0x3FF0000000000000ULL, 0 }, + { O_DSUB, 0xBFF0000000000000ULL, 0x0000000000000000ULL, 0xBFF0000000000000ULL, 0 }, + { O_DSUB, 0x3FF0000000000000ULL, 0x8000000000000000ULL, 0x3FF0000000000000ULL, 0 }, + { O_DSUB, 0x3FF0000000000000ULL, 0x7FF0000000000000ULL, 0xFFF0000000000000ULL, 0 }, + { O_DSUB, 0xBFF0000000000000ULL, 0x7FF0000000000000ULL, 0xFFF0000000000000ULL, 0 }, + { O_DSUB, 0x7FF0000000000000ULL, 0x4000000000000000ULL, 0x7FF0000000000000ULL, 0 }, + { O_DSUB, 0x3FF8000000000000ULL, 0x4000000000000000ULL, 0xBFE0000000000000ULL, 0 }, + { O_DMUL, 0x7FF8000000000123ULL, 0x3FF0000000000000ULL, 0x7FF8000000000123ULL, 0 }, + { O_DMUL, 0x3FF0000000000000ULL, 0x7FF0000000000005ULL, 0x7FF8000000000005ULL, 0 }, + { O_DMUL, 0x7FF0000000000005ULL, 0x7FF8000000000123ULL, 0x7FF8000000000005ULL, 0 }, + { O_DMUL, 0xFFF8000000000000ULL, 0x4000000000000000ULL, 0xFFF8000000000000ULL, 0 }, + { O_DMUL, 0x0000000000000000ULL, 0x7FF8000000000123ULL, 0x7FF8000000000123ULL, 0 }, + { O_DMUL, 0x7FF0000000000000ULL, 0x3FF0000000000000ULL, 0x7FF0000000000000ULL, 0 }, + { O_DMUL, 0x7FF0000000000000ULL, 0x7FF0000000000000ULL, 0x7FF0000000000000ULL, 0 }, + { O_DMUL, 0x7FF0000000000000ULL, 0xFFF0000000000000ULL, 0xFFF0000000000000ULL, 0 }, + { O_DMUL, 0xFFF0000000000000ULL, 0x4000000000000000ULL, 0xFFF0000000000000ULL, 0 }, + { O_DMUL, 0x4000000000000000ULL, 0xFFF0000000000000ULL, 0xFFF0000000000000ULL, 0 }, + { O_DMUL, 0x3FF0000000000000ULL, 0xBFF0000000000000ULL, 0xBFF0000000000000ULL, 0 }, + { O_DMUL, 0xBFF0000000000000ULL, 0x3FF0000000000000ULL, 0xBFF0000000000000ULL, 0 }, + { O_DMUL, 0x8000000000000000ULL, 0x8000000000000000ULL, 0x0000000000000000ULL, 0 }, + { O_DMUL, 0x8000000000000000ULL, 0x0000000000000000ULL, 0x8000000000000000ULL, 0 }, + { O_DMUL, 0x0000000000000000ULL, 0x8000000000000000ULL, 0x8000000000000000ULL, 0 }, + { O_DMUL, 0x0000000000000000ULL, 0x0000000000000000ULL, 0x0000000000000000ULL, 0 }, + { O_DMUL, 0x4002000000000000ULL, 0x3FF8000000000000ULL, 0x400B000000000000ULL, 0 }, + { O_DMUL, 0x7E37E43C8800759CULL, 0x7E37E43C8800759CULL, 0x7FF0000000000000ULL, 0 }, + { O_DMUL, 0x0000000000000000ULL, 0x7FF0000000000000ULL, 0x7FF8000000000000ULL, 0 }, + { O_DMUL, 0x7FF0000000000000ULL, 0x0000000000000000ULL, 0x7FF8000000000000ULL, 0 }, + { O_DMUL, 0xFFF0000000000000ULL, 0xC000000000000000ULL, 0x7FF0000000000000ULL, 0 }, + { O_DMUL, 0xFFF0000000000000ULL, 0xFFF0000000000000ULL, 0x7FF0000000000000ULL, 0 }, + { O_DMUL, 0x8000000000000000ULL, 0x4000000000000000ULL, 0x8000000000000000ULL, 0 }, + { O_DMUL, 0x0000000000000000ULL, 0x0000000000000000ULL, 0x0000000000000000ULL, 0 }, + { O_DMUL, 0x3FF0000000000000ULL, 0x0000000000000000ULL, 0x0000000000000000ULL, 0 }, + { O_DMUL, 0xBFF0000000000000ULL, 0x0000000000000000ULL, 0x8000000000000000ULL, 0 }, + { O_DMUL, 0x3FF0000000000000ULL, 0x8000000000000000ULL, 0x8000000000000000ULL, 0 }, + { O_DMUL, 0x3FF0000000000000ULL, 0x7FF0000000000000ULL, 0x7FF0000000000000ULL, 0 }, + { O_DMUL, 0xBFF0000000000000ULL, 0x7FF0000000000000ULL, 0xFFF0000000000000ULL, 0 }, + { O_DMUL, 0x7FF0000000000000ULL, 0x4000000000000000ULL, 0x7FF0000000000000ULL, 0 }, + { O_DMUL, 0x3FF8000000000000ULL, 0x4000000000000000ULL, 0x4008000000000000ULL, 0 }, + { O_DDIV, 0x7FF8000000000123ULL, 0x3FF0000000000000ULL, 0x7FF8000000000123ULL, 0 }, + { O_DDIV, 0x3FF0000000000000ULL, 0x7FF0000000000005ULL, 0x7FF8000000000005ULL, 0 }, + { O_DDIV, 0x7FF0000000000005ULL, 0x7FF8000000000123ULL, 0x7FF8000000000005ULL, 0 }, + { O_DDIV, 0xFFF8000000000000ULL, 0x4000000000000000ULL, 0xFFF8000000000000ULL, 0 }, + { O_DDIV, 0x0000000000000000ULL, 0x7FF8000000000123ULL, 0x7FF8000000000123ULL, 0 }, + { O_DDIV, 0x7FF0000000000000ULL, 0x3FF0000000000000ULL, 0x7FF0000000000000ULL, 0 }, + { O_DDIV, 0x7FF0000000000000ULL, 0x7FF0000000000000ULL, 0x7FF8000000000000ULL, 0 }, + { O_DDIV, 0x7FF0000000000000ULL, 0xFFF0000000000000ULL, 0x7FF8000000000000ULL, 0 }, + { O_DDIV, 0xFFF0000000000000ULL, 0x4000000000000000ULL, 0xFFF0000000000000ULL, 0 }, + { O_DDIV, 0x4000000000000000ULL, 0xFFF0000000000000ULL, 0x8000000000000000ULL, 0 }, + { O_DDIV, 0x3FF0000000000000ULL, 0xBFF0000000000000ULL, 0xBFF0000000000000ULL, 0 }, + { O_DDIV, 0xBFF0000000000000ULL, 0x3FF0000000000000ULL, 0xBFF0000000000000ULL, 0 }, + { O_DDIV, 0x8000000000000000ULL, 0x8000000000000000ULL, 0x7FF8000000000000ULL, 0 }, + { O_DDIV, 0x8000000000000000ULL, 0x0000000000000000ULL, 0x7FF8000000000000ULL, 0 }, + { O_DDIV, 0x0000000000000000ULL, 0x8000000000000000ULL, 0x7FF8000000000000ULL, 0 }, + { O_DDIV, 0x0000000000000000ULL, 0x0000000000000000ULL, 0x7FF8000000000000ULL, 0 }, + { O_DDIV, 0x4002000000000000ULL, 0x3FF8000000000000ULL, 0x3FF8000000000000ULL, 0 }, + { O_DDIV, 0x7E37E43C8800759CULL, 0x7E37E43C8800759CULL, 0x3FF0000000000000ULL, 0 }, + { O_DDIV, 0x0000000000000000ULL, 0x7FF0000000000000ULL, 0x0000000000000000ULL, 0 }, + { O_DDIV, 0x7FF0000000000000ULL, 0x0000000000000000ULL, 0x7FF0000000000000ULL, 0 }, + { O_DDIV, 0xFFF0000000000000ULL, 0xC000000000000000ULL, 0x7FF0000000000000ULL, 0 }, + { O_DDIV, 0xFFF0000000000000ULL, 0xFFF0000000000000ULL, 0x7FF8000000000000ULL, 0 }, + { O_DDIV, 0x8000000000000000ULL, 0x4000000000000000ULL, 0x8000000000000000ULL, 0 }, + { O_DDIV, 0x0000000000000000ULL, 0x0000000000000000ULL, 0x7FF8000000000000ULL, 0 }, + { O_DDIV, 0x3FF0000000000000ULL, 0x0000000000000000ULL, 0x7FF0000000000000ULL, 0 }, + { O_DDIV, 0xBFF0000000000000ULL, 0x0000000000000000ULL, 0xFFF0000000000000ULL, 0 }, + { O_DDIV, 0x3FF0000000000000ULL, 0x8000000000000000ULL, 0xFFF0000000000000ULL, 0 }, + { O_DDIV, 0x3FF0000000000000ULL, 0x7FF0000000000000ULL, 0x0000000000000000ULL, 0 }, + { O_DDIV, 0xBFF0000000000000ULL, 0x7FF0000000000000ULL, 0x8000000000000000ULL, 0 }, + { O_DDIV, 0x7FF0000000000000ULL, 0x4000000000000000ULL, 0x7FF0000000000000ULL, 0 }, + { O_DDIV, 0x3FF8000000000000ULL, 0x4000000000000000ULL, 0x3FE8000000000000ULL, 0 }, + { O_F2D, 0x000000007FC00123ULL, 0x0000000000000000ULL, 0x7FF8002460000000ULL, 0 }, + { O_F2D, 0x000000007F800005ULL, 0x0000000000000000ULL, 0x7FF80000A0000000ULL, 0 }, + { O_F2D, 0x00000000FFC00001ULL, 0x0000000000000000ULL, 0xFFF8000020000000ULL, 0 }, + { O_F2D, 0x000000007F800000ULL, 0x0000000000000000ULL, 0x7FF0000000000000ULL, 0 }, + { O_F2D, 0x00000000FF800000ULL, 0x0000000000000000ULL, 0xFFF0000000000000ULL, 0 }, + { O_F2D, 0x0000000080000000ULL, 0x0000000000000000ULL, 0x8000000000000000ULL, 0 }, + { O_F2D, 0x0000000000000000ULL, 0x0000000000000000ULL, 0x0000000000000000ULL, 0 }, + { O_F2D, 0x000000003FC00000ULL, 0x0000000000000000ULL, 0x3FF8000000000000ULL, 0 }, + { O_F2D, 0x00000000C0000000ULL, 0x0000000000000000ULL, 0xC000000000000000ULL, 0 }, + { O_D2F, 0x7FF8000000000123ULL, 0x0000000000000000ULL, 0x000000007FC00000ULL, 0 }, + { O_D2F, 0xFFF8000020000000ULL, 0x0000000000000000ULL, 0x00000000FFC00001ULL, 0 }, + { O_D2F, 0x7FF0000000000005ULL, 0x0000000000000000ULL, 0x000000007FC00000ULL, 0 }, + { O_D2F, 0x7FF0000000000000ULL, 0x0000000000000000ULL, 0x000000007F800000ULL, 0 }, + { O_D2F, 0xFFF0000000000000ULL, 0x0000000000000000ULL, 0x00000000FF800000ULL, 0 }, + { O_D2F, 0x8000000000000000ULL, 0x0000000000000000ULL, 0x0000000080000000ULL, 0 }, + { O_D2F, 0x0000000000000000ULL, 0x0000000000000000ULL, 0x0000000000000000ULL, 0 }, + { O_D2F, 0x3FF8000000000000ULL, 0x0000000000000000ULL, 0x000000003FC00000ULL, 0 }, + { O_D2F, 0xC000000000000000ULL, 0x0000000000000000ULL, 0x00000000C0000000ULL, 0 }, + { O_D2F, 0x7E37E43C8800759CULL, 0x0000000000000000ULL, 0x000000007F800000ULL, 0 }, + { O_D2F, 0xFE37E43C8800759CULL, 0x0000000000000000ULL, 0x00000000FF800000ULL, 0 }, + { O_F2I, 0x000000007FC00123ULL, 0x0000000000000000ULL, 0x0000000000000000ULL, 0 }, + { O_F2U, 0x000000007FC00123ULL, 0x0000000000000000ULL, 0x0000000000000000ULL, 0 }, + { O_F2I, 0x00000000FFC00000ULL, 0x0000000000000000ULL, 0x0000000000000000ULL, 0 }, + { O_F2U, 0x00000000FFC00000ULL, 0x0000000000000000ULL, 0x0000000000000000ULL, 0 }, + { O_F2I, 0x000000007F800005ULL, 0x0000000000000000ULL, 0x0000000000000000ULL, 0 }, + { O_F2U, 0x000000007F800005ULL, 0x0000000000000000ULL, 0x0000000000000000ULL, 0 }, + { O_F2I, 0x000000007F800000ULL, 0x0000000000000000ULL, 0x000000007FFFFFFFULL, 0 }, + { O_F2U, 0x000000007F800000ULL, 0x0000000000000000ULL, 0x00000000FFFFFFFFULL, 0 }, + { O_F2I, 0x00000000FF800000ULL, 0x0000000000000000ULL, 0x0000000080000000ULL, 0 }, + { O_F2U, 0x00000000FF800000ULL, 0x0000000000000000ULL, 0x0000000000000000ULL, 0 }, + { O_F2I, 0x000000004F32D05EULL, 0x0000000000000000ULL, 0x000000007FFFFFFFULL, 0 }, + { O_F2U, 0x000000004F32D05EULL, 0x0000000000000000ULL, 0x00000000B2D05E00ULL, 0 }, + { O_F2I, 0x00000000CF32D05EULL, 0x0000000000000000ULL, 0x0000000080000000ULL, 0 }, + { O_F2U, 0x00000000CF32D05EULL, 0x0000000000000000ULL, 0x0000000000000000ULL, 0 }, + { O_F2I, 0x000000004F9502F9ULL, 0x0000000000000000ULL, 0x000000007FFFFFFFULL, 0 }, + { O_F2U, 0x000000004F9502F9ULL, 0x0000000000000000ULL, 0x00000000FFFFFFFFULL, 0 }, + { O_F2I, 0x00000000C0200000ULL, 0x0000000000000000ULL, 0x00000000FFFFFFFEULL, 0 }, + { O_F2U, 0x00000000C0200000ULL, 0x0000000000000000ULL, 0x0000000000000000ULL, 0 }, + { O_F2I, 0x0000000040600000ULL, 0x0000000000000000ULL, 0x0000000000000003ULL, 0 }, + { O_F2U, 0x0000000040600000ULL, 0x0000000000000000ULL, 0x0000000000000003ULL, 0 }, + { O_F2I, 0x00000000BF800000ULL, 0x0000000000000000ULL, 0x00000000FFFFFFFFULL, 0 }, + { O_F2U, 0x00000000BF800000ULL, 0x0000000000000000ULL, 0x0000000000000000ULL, 0 }, + { O_F2I, 0x0000000080000000ULL, 0x0000000000000000ULL, 0x0000000000000000ULL, 0 }, + { O_F2U, 0x0000000080000000ULL, 0x0000000000000000ULL, 0x0000000000000000ULL, 0 }, + { O_D2I, 0x7FF8000000000123ULL, 0x0000000000000000ULL, 0x0000000000000000ULL, 0 }, + { O_D2U, 0x7FF8000000000123ULL, 0x0000000000000000ULL, 0x0000000000000000ULL, 0 }, + { O_D2I, 0xFFF8000000000000ULL, 0x0000000000000000ULL, 0x0000000000000000ULL, 0 }, + { O_D2U, 0xFFF8000000000000ULL, 0x0000000000000000ULL, 0x0000000000000000ULL, 0 }, + { O_D2I, 0x7FF0000000000005ULL, 0x0000000000000000ULL, 0x0000000000000000ULL, 0 }, + { O_D2U, 0x7FF0000000000005ULL, 0x0000000000000000ULL, 0x0000000000000000ULL, 0 }, + { O_D2I, 0x7FF0000000000000ULL, 0x0000000000000000ULL, 0x000000007FFFFFFFULL, 0 }, + { O_D2U, 0x7FF0000000000000ULL, 0x0000000000000000ULL, 0x00000000FFFFFFFFULL, 0 }, + { O_D2I, 0xFFF0000000000000ULL, 0x0000000000000000ULL, 0x0000000080000000ULL, 0 }, + { O_D2U, 0xFFF0000000000000ULL, 0x0000000000000000ULL, 0x0000000000000000ULL, 0 }, + { O_D2I, 0x41E65A0BC0000000ULL, 0x0000000000000000ULL, 0x000000007FFFFFFFULL, 0 }, + { O_D2U, 0x41E65A0BC0000000ULL, 0x0000000000000000ULL, 0x00000000B2D05E00ULL, 0 }, + { O_D2I, 0xC1E65A0BC0000000ULL, 0x0000000000000000ULL, 0x0000000080000000ULL, 0 }, + { O_D2U, 0xC1E65A0BC0000000ULL, 0x0000000000000000ULL, 0x0000000000000000ULL, 0 }, + { O_D2I, 0x41F2A05F20000000ULL, 0x0000000000000000ULL, 0x000000007FFFFFFFULL, 0 }, + { O_D2U, 0x41F2A05F20000000ULL, 0x0000000000000000ULL, 0x00000000FFFFFFFFULL, 0 }, + { O_D2I, 0xC004000000000000ULL, 0x0000000000000000ULL, 0x00000000FFFFFFFEULL, 0 }, + { O_D2U, 0xC004000000000000ULL, 0x0000000000000000ULL, 0x0000000000000000ULL, 0 }, + { O_D2I, 0x400C000000000000ULL, 0x0000000000000000ULL, 0x0000000000000003ULL, 0 }, + { O_D2U, 0x400C000000000000ULL, 0x0000000000000000ULL, 0x0000000000000003ULL, 0 }, + { O_D2I, 0xBFF0000000000000ULL, 0x0000000000000000ULL, 0x00000000FFFFFFFFULL, 0 }, + { O_D2U, 0xBFF0000000000000ULL, 0x0000000000000000ULL, 0x0000000000000000ULL, 0 }, + { O_D2I, 0x8000000000000000ULL, 0x0000000000000000ULL, 0x0000000000000000ULL, 0 }, + { O_D2U, 0x8000000000000000ULL, 0x0000000000000000ULL, 0x0000000000000000ULL, 0 }, + { O_FCLS, 0x000000007FC00123ULL, 0x0000000000000000ULL, 0x0000000000000001ULL, 0 }, + { O_FCLS, 0x00000000FFC00000ULL, 0x0000000000000000ULL, 0x0000000000000009ULL, 0 }, + { O_FCLS, 0x000000007F800005ULL, 0x0000000000000000ULL, 0x0000000000000001ULL, 0 }, + { O_FCLS, 0x000000007F800000ULL, 0x0000000000000000ULL, 0x0000000000000012ULL, 0 }, + { O_FCLS, 0x00000000FF800000ULL, 0x0000000000000000ULL, 0x000000000000001AULL, 0 }, + { O_FCLS, 0x0000000000000000ULL, 0x0000000000000000ULL, 0x0000000000000044ULL, 0 }, + { O_FCLS, 0x0000000080000000ULL, 0x0000000000000000ULL, 0x000000000000004CULL, 0 }, + { O_FCLS, 0x000000003FC00000ULL, 0x0000000000000000ULL, 0x0000000000000024ULL, 0 }, + { O_FCLS, 0x00000000C0000000ULL, 0x0000000000000000ULL, 0x000000000000002CULL, 0 }, + { O_DCLS, 0x7FF8000000000123ULL, 0x0000000000000000ULL, 0x0000000000000001ULL, 0 }, + { O_DCLS, 0xFFF8000000000000ULL, 0x0000000000000000ULL, 0x0000000000000009ULL, 0 }, + { O_DCLS, 0x7FF0000000000005ULL, 0x0000000000000000ULL, 0x0000000000000001ULL, 0 }, + { O_DCLS, 0x7FF0000000000000ULL, 0x0000000000000000ULL, 0x0000000000000012ULL, 0 }, + { O_DCLS, 0xFFF0000000000000ULL, 0x0000000000000000ULL, 0x000000000000001AULL, 0 }, + { O_DCLS, 0x0000000000000000ULL, 0x0000000000000000ULL, 0x0000000000000044ULL, 0 }, + { O_DCLS, 0x8000000000000000ULL, 0x0000000000000000ULL, 0x000000000000004CULL, 0 }, + { O_DCLS, 0x3FF8000000000000ULL, 0x0000000000000000ULL, 0x0000000000000024ULL, 0 }, + { O_DCLS, 0xC000000000000000ULL, 0x0000000000000000ULL, 0x000000000000002CULL, 0 }, + { O_SQRT, 0xBFF0000000000000ULL, 0x0000000000000000ULL, 0x7FF8000000000000ULL, 0 }, + { O_SQRT, 0x7FF0000000000000ULL, 0x0000000000000000ULL, 0x7FF0000000000000ULL, 0 }, + { O_SQRT, 0xFFF0000000000000ULL, 0x0000000000000000ULL, 0x7FF8000000000000ULL, 0 }, + { O_SQRT, 0x7FF8000000000123ULL, 0x0000000000000000ULL, 0x7FF8000000000000ULL, 0 }, + { O_SQRT, 0x4010000000000000ULL, 0x0000000000000000ULL, 0x4000000000000000ULL, 0 }, + { O_SQRT, 0x8000000000000000ULL, 0x0000000000000000ULL, 0x8000000000000000ULL, 0 }, + { O_LOG, 0x7FF0000000000000ULL, 0x0000000000000000ULL, 0x7FF0000000000000ULL, 0 }, + { O_LOG, 0xFFF0000000000000ULL, 0x0000000000000000ULL, 0x7FF8000000000000ULL, 0 }, + { O_LOG, 0x7FF8000000000123ULL, 0x0000000000000000ULL, 0x7FF8000000000000ULL, 0 }, + { O_LOG, 0xBFF0000000000000ULL, 0x0000000000000000ULL, 0x7FF8000000000000ULL, 0 }, + { O_LOG, 0x0000000000000000ULL, 0x0000000000000000ULL, 0xFFF0000000000000ULL, 0 }, + { O_LOG, 0x3FF0000000000000ULL, 0x0000000000000000ULL, 0x0000000000000000ULL, 0 }, + { O_ASIN, 0x4000000000000000ULL, 0x0000000000000000ULL, 0x7FF8000000000000ULL, 0 }, + { O_ASIN, 0xC000000000000000ULL, 0x0000000000000000ULL, 0x7FF8000000000000ULL, 0 }, + { O_ASIN, 0x3FF0000000000000ULL, 0x0000000000000000ULL, 0x3FF921FB54442D18ULL, 0 }, + { O_ASIN, 0xBFF0000000000000ULL, 0x0000000000000000ULL, 0xBFF921FB54442D18ULL, 0 }, + { O_ASIN, 0x7FF8000000000123ULL, 0x0000000000000000ULL, 0x7FF8000000000000ULL, 0 }, + { O_ACOS, 0x4000000000000000ULL, 0x0000000000000000ULL, 0x7FF8000000000000ULL, 0 }, + { O_ACOS, 0x3FF0000000000000ULL, 0x0000000000000000ULL, 0x0000000000000000ULL, 0 }, + { O_ATANH, 0x4000000000000000ULL, 0x0000000000000000ULL, 0x7FF8000000000000ULL, 0 }, + { O_ATANH, 0xC000000000000000ULL, 0x0000000000000000ULL, 0x7FF8000000000000ULL, 0 }, + { O_ATANH, 0x3FF0000000000000ULL, 0x0000000000000000ULL, 0x7FF0000000000000ULL, 0 }, + { O_ATANH, 0xBFF0000000000000ULL, 0x0000000000000000ULL, 0xFFF0000000000000ULL, 0 }, + { O_ATANH, 0x7FF8000000000123ULL, 0x0000000000000000ULL, 0x7FF8000000000000ULL, 0 }, +}; +#else +#define NC 199 +static const CaseT kCases[NC] = { + { O_FLOOR, 0x400599999999999AULL, 0x0000000000000000ULL, 0x4000000000000000ULL, 0 }, + { O_FLOOR, 0xC00599999999999AULL, 0x0000000000000000ULL, 0xC008000000000000ULL, 0 }, + { O_FLOOR, 0x8000000000000000ULL, 0x0000000000000000ULL, 0x8000000000000000ULL, 0 }, + { O_FLOOR, 0x0000000000000000ULL, 0x0000000000000000ULL, 0x0000000000000000ULL, 0 }, + { O_FLOOR, 0xBFD3333333333333ULL, 0x0000000000000000ULL, 0xBFF0000000000000ULL, 0 }, + { O_FLOOR, 0x3FD3333333333333ULL, 0x0000000000000000ULL, 0x0000000000000000ULL, 0 }, + { O_FLOOR, 0x7FF0000000000000ULL, 0x0000000000000000ULL, 0x7FF0000000000000ULL, 0 }, + { O_FLOOR, 0xFFF0000000000000ULL, 0x0000000000000000ULL, 0xFFF0000000000000ULL, 0 }, + { O_FLOOR, 0x7FF8000000000000ULL, 0x0000000000000000ULL, 0x7FF8000000000000ULL, 0 }, + { O_FLOOR, 0x7E37E43C8800759CULL, 0x0000000000000000ULL, 0x7E37E43C8800759CULL, 0 }, + { O_FLOOR, 0xBFE0000000000000ULL, 0x0000000000000000ULL, 0xBFF0000000000000ULL, 0 }, + { O_CEIL, 0x4002666666666666ULL, 0x0000000000000000ULL, 0x4008000000000000ULL, 0 }, + { O_CEIL, 0xC002666666666666ULL, 0x0000000000000000ULL, 0xC000000000000000ULL, 0 }, + { O_CEIL, 0x0000000000000000ULL, 0x0000000000000000ULL, 0x0000000000000000ULL, 0 }, + { O_CEIL, 0x8000000000000000ULL, 0x0000000000000000ULL, 0x8000000000000000ULL, 0 }, + { O_CEIL, 0xBFE0000000000000ULL, 0x0000000000000000ULL, 0x8000000000000000ULL, 0 }, + { O_CEIL, 0x3FE0000000000000ULL, 0x0000000000000000ULL, 0x3FF0000000000000ULL, 0 }, + { O_CEIL, 0x7FF0000000000000ULL, 0x0000000000000000ULL, 0x7FF0000000000000ULL, 0 }, + { O_CEIL, 0x7FF8000000000000ULL, 0x0000000000000000ULL, 0x7FF8000000000000ULL, 0 }, + { O_TRUNC, 0xC00599999999999AULL, 0x0000000000000000ULL, 0xC000000000000000ULL, 0 }, + { O_TRUNC, 0x400599999999999AULL, 0x0000000000000000ULL, 0x4000000000000000ULL, 0 }, + { O_TRUNC, 0xBFE0000000000000ULL, 0x0000000000000000ULL, 0x8000000000000000ULL, 0 }, + { O_TRUNC, 0x8000000000000000ULL, 0x0000000000000000ULL, 0x8000000000000000ULL, 0 }, + { O_TRUNC, 0x7FF0000000000000ULL, 0x0000000000000000ULL, 0x7FF0000000000000ULL, 0 }, + { O_ROUND, 0x4004000000000000ULL, 0x0000000000000000ULL, 0x4008000000000000ULL, 0 }, + { O_ROUND, 0xC004000000000000ULL, 0x0000000000000000ULL, 0xC008000000000000ULL, 0 }, + { O_ROUND, 0x3FDFFFFFFFFFFFFFULL, 0x0000000000000000ULL, 0x0000000000000000ULL, 0 }, + { O_ROUND, 0xBFDFFFFFFFFFFFFFULL, 0x0000000000000000ULL, 0x8000000000000000ULL, 0 }, + { O_ROUND, 0x4330000000000001ULL, 0x0000000000000000ULL, 0x4330000000000001ULL, 0 }, + { O_ROUND, 0x8000000000000000ULL, 0x0000000000000000ULL, 0x8000000000000000ULL, 0 }, + { O_ROUND, 0x3FE0000000000000ULL, 0x0000000000000000ULL, 0x3FF0000000000000ULL, 0 }, + { O_ROUND, 0x3FF8000000000000ULL, 0x0000000000000000ULL, 0x4000000000000000ULL, 0 }, + { O_ROUND, 0x7FF0000000000000ULL, 0x0000000000000000ULL, 0x7FF0000000000000ULL, 0 }, + { O_ROUND, 0x7FF8000000000000ULL, 0x0000000000000000ULL, 0x7FF8000000000000ULL, 0 }, + { O_ROUND, 0xBFD999999999999AULL, 0x0000000000000000ULL, 0x8000000000000000ULL, 0 }, + { O_RINT, 0x4004000000000000ULL, 0x0000000000000000ULL, 0x4000000000000000ULL, 0 }, + { O_RINT, 0x400C000000000000ULL, 0x0000000000000000ULL, 0x4010000000000000ULL, 0 }, + { O_RINT, 0xC004000000000000ULL, 0x0000000000000000ULL, 0xC000000000000000ULL, 0 }, + { O_RINT, 0xBFD3333333333333ULL, 0x0000000000000000ULL, 0x8000000000000000ULL, 0 }, + { O_RINT, 0x3FD3333333333333ULL, 0x0000000000000000ULL, 0x0000000000000000ULL, 0 }, + { O_RINT, 0x3FF8000000000000ULL, 0x0000000000000000ULL, 0x4000000000000000ULL, 0 }, + { O_RINT, 0x4330000000000001ULL, 0x0000000000000000ULL, 0x4330000000000001ULL, 0 }, + { O_RINT, 0x7FF0000000000000ULL, 0x0000000000000000ULL, 0x7FF0000000000000ULL, 0 }, + { O_RINT, 0x7FF8000000000000ULL, 0x0000000000000000ULL, 0x7FF8000000000000ULL, 0 }, + { O_RINT, 0x8000000000000000ULL, 0x0000000000000000ULL, 0x8000000000000000ULL, 0 }, + { O_NEARBYINT, 0x4004000000000000ULL, 0x0000000000000000ULL, 0x4000000000000000ULL, 0 }, + { O_NEARBYINT, 0xC00C000000000000ULL, 0x0000000000000000ULL, 0xC010000000000000ULL, 0 }, + { O_FLOORF, 0x0000000080000000ULL, 0x0000000000000000ULL, 0x0000000080000000ULL, 0 }, + { O_FLOORF, 0x0000000040300000ULL, 0x0000000000000000ULL, 0x0000000040000000ULL, 0 }, + { O_FLOORF, 0x00000000C0300000ULL, 0x0000000000000000ULL, 0x00000000C0400000ULL, 0 }, + { O_CEILF, 0x0000000000000000ULL, 0x0000000000000000ULL, 0x0000000000000000ULL, 0 }, + { O_CEILF, 0x00000000BF000000ULL, 0x0000000000000000ULL, 0x0000000080000000ULL, 0 }, + { O_TRUNCF, 0x00000000C0200000ULL, 0x0000000000000000ULL, 0x00000000C0000000ULL, 0 }, + { O_ROUNDF, 0x0000000040200000ULL, 0x0000000000000000ULL, 0x0000000040400000ULL, 0 }, + { O_ROUNDF, 0x0000000080000000ULL, 0x0000000000000000ULL, 0x0000000080000000ULL, 0 }, + { O_RINTF, 0x0000000040200000ULL, 0x0000000000000000ULL, 0x0000000040000000ULL, 0 }, + { O_RINTF, 0x0000000040600000ULL, 0x0000000000000000ULL, 0x0000000040800000ULL, 0 }, + { O_RINTF, 0x00000000BE800000ULL, 0x0000000000000000ULL, 0x0000000080000000ULL, 0 }, + { O_NEARBYINTF, 0x0000000040200000ULL, 0x0000000000000000ULL, 0x0000000040000000ULL, 0 }, + { O_NEARBYINTF, 0x00000000BFC00000ULL, 0x0000000000000000ULL, 0x00000000C0000000ULL, 0 }, + { O_FMAXF, 0x000000003F800000ULL, 0x000000007FC00000ULL, 0x000000003F800000ULL, 0 }, + { O_FMAXF, 0x000000007FC00000ULL, 0x00000000C0000000ULL, 0x00000000C0000000ULL, 0 }, + { O_FMAXF, 0x00000000BF000000ULL, 0x0000000040000000ULL, 0x0000000040000000ULL, 0 }, + { O_FMINF, 0x000000003F800000ULL, 0x000000007FC00000ULL, 0x000000003F800000ULL, 0 }, + { O_FMINF, 0x00000000BF000000ULL, 0x0000000040000000ULL, 0x00000000BF000000ULL, 0 }, + { O_FMINF, 0x0000000040400000ULL, 0x00000000FF800000ULL, 0x00000000FF800000ULL, 0 }, + { O_EXP, 0x7FF8000000000000ULL, 0x0000000000000000ULL, 0x7FF8000000000000ULL, 0 }, + { O_EXP, 0x7FF0000000000000ULL, 0x0000000000000000ULL, 0x7FF0000000000000ULL, 0 }, + { O_EXP, 0xFFF0000000000000ULL, 0x0000000000000000ULL, 0x0000000000000000ULL, 0 }, + { O_EXP, 0x40F86A0000000000ULL, 0x0000000000000000ULL, 0x7FF0000000000000ULL, 0 }, + { O_EXP, 0xC0F86A0000000000ULL, 0x0000000000000000ULL, 0x0000000000000000ULL, 0 }, + { O_EXP, 0x4090000000000000ULL, 0x0000000000000000ULL, 0x7FF0000000000000ULL, 0 }, + { O_EXP, 0xC090000000000000ULL, 0x0000000000000000ULL, 0x0000000000000000ULL, 0 }, + { O_EXP, 0x4089000000000000ULL, 0x0000000000000000ULL, 0x7FF0000000000000ULL, 0 }, + { O_EXP, 0xC089000000000000ULL, 0x0000000000000000ULL, 0x0000000000000000ULL, 0 }, + { O_EXP, 0x0000000000000000ULL, 0x0000000000000000ULL, 0x3FF0000000000000ULL, 0 }, + { O_EXP, 0x8000000000000000ULL, 0x0000000000000000ULL, 0x3FF0000000000000ULL, 0 }, + { O_EXP, 0x3FF8000000000000ULL, 0x0000000000000000ULL, 0x4011ED3FE64FC541ULL, 1 }, + { O_EXP, 0xC000000000000000ULL, 0x0000000000000000ULL, 0x3FC152AAA3BF81CCULL, 1 }, + { O_EXP2, 0x4008000000000000ULL, 0x0000000000000000ULL, 0x4020000000000000ULL, 1 }, + { O_EXP2, 0x7FF8000000000000ULL, 0x0000000000000000ULL, 0x7FF8000000000000ULL, 0 }, + { O_EXP2, 0x7FF0000000000000ULL, 0x0000000000000000ULL, 0x7FF0000000000000ULL, 0 }, + { O_EXP2, 0xFFF0000000000000ULL, 0x0000000000000000ULL, 0x0000000000000000ULL, 0 }, + { O_EXPM1, 0xFFF0000000000000ULL, 0x0000000000000000ULL, 0xBFF0000000000000ULL, 0 }, + { O_EXPM1, 0x7FF0000000000000ULL, 0x0000000000000000ULL, 0x7FF0000000000000ULL, 0 }, + { O_EXPM1, 0x7FF8000000000000ULL, 0x0000000000000000ULL, 0x7FF8000000000000ULL, 0 }, + { O_EXPM1, 0x8000000000000000ULL, 0x0000000000000000ULL, 0x8000000000000000ULL, 0 }, + { O_EXPM1, 0x3FE0000000000000ULL, 0x0000000000000000ULL, 0x3FE4C2531C3C0D38ULL, 1 }, + { O_SINH, 0x7FF0000000000000ULL, 0x0000000000000000ULL, 0x7FF0000000000000ULL, 0 }, + { O_SINH, 0xFFF0000000000000ULL, 0x0000000000000000ULL, 0xFFF0000000000000ULL, 0 }, + { O_SINH, 0x7FF8000000000000ULL, 0x0000000000000000ULL, 0x7FF8000000000000ULL, 0 }, + { O_SINH, 0x8000000000000000ULL, 0x0000000000000000ULL, 0x8000000000000000ULL, 0 }, + { O_SINH, 0x4089000000000000ULL, 0x0000000000000000ULL, 0x7FF0000000000000ULL, 0 }, + { O_SINH, 0xC089000000000000ULL, 0x0000000000000000ULL, 0xFFF0000000000000ULL, 0 }, + { O_SINH, 0x3FF0000000000000ULL, 0x0000000000000000ULL, 0x3FF2CD9FC44EB982ULL, 1 }, + { O_SINH, 0xBFF0000000000000ULL, 0x0000000000000000ULL, 0xBFF2CD9FC44EB982ULL, 1 }, + { O_COSH, 0x7FF0000000000000ULL, 0x0000000000000000ULL, 0x7FF0000000000000ULL, 0 }, + { O_COSH, 0xFFF0000000000000ULL, 0x0000000000000000ULL, 0x7FF0000000000000ULL, 0 }, + { O_COSH, 0x7FF8000000000000ULL, 0x0000000000000000ULL, 0x7FF8000000000000ULL, 0 }, + { O_COSH, 0x0000000000000000ULL, 0x0000000000000000ULL, 0x3FF0000000000000ULL, 0 }, + { O_COSH, 0x3FF0000000000000ULL, 0x0000000000000000ULL, 0x3FF8B07551D9F550ULL, 1 }, + { O_TANH, 0x7FF0000000000000ULL, 0x0000000000000000ULL, 0x3FF0000000000000ULL, 0 }, + { O_TANH, 0xFFF0000000000000ULL, 0x0000000000000000ULL, 0xBFF0000000000000ULL, 0 }, + { O_TANH, 0x7FF8000000000000ULL, 0x0000000000000000ULL, 0x7FF8000000000000ULL, 0 }, + { O_TANH, 0x8000000000000000ULL, 0x0000000000000000ULL, 0x8000000000000000ULL, 0 }, + { O_TANH, 0x4044000000000000ULL, 0x0000000000000000ULL, 0x3FF0000000000000ULL, 0 }, + { O_TANH, 0xC044000000000000ULL, 0x0000000000000000ULL, 0xBFF0000000000000ULL, 0 }, + { O_TANH, 0x4079000000000000ULL, 0x0000000000000000ULL, 0x3FF0000000000000ULL, 0 }, + { O_TANH, 0x3FE0000000000000ULL, 0x0000000000000000ULL, 0x3FDD9353D7568AF3ULL, 1 }, + { O_TANH, 0xBFE0000000000000ULL, 0x0000000000000000ULL, 0xBFDD9353D7568AF3ULL, 1 }, + { O_LOG1P, 0x8000000000000000ULL, 0x0000000000000000ULL, 0x8000000000000000ULL, 0 }, + { O_LOG1P, 0x0000000000000000ULL, 0x0000000000000000ULL, 0x0000000000000000ULL, 0 }, + { O_LOG1P, 0xBFF0000000000000ULL, 0x0000000000000000ULL, 0xFFF0000000000000ULL, 0 }, + { O_LOG1P, 0x7FF0000000000000ULL, 0x0000000000000000ULL, 0x7FF0000000000000ULL, 0 }, + { O_LOG1P, 0x7FF8000000000000ULL, 0x0000000000000000ULL, 0x7FF8000000000000ULL, 0 }, + { O_LOG1P, 0x3FF0000000000000ULL, 0x0000000000000000ULL, 0x3FE62E42FEFA39EFULL, 1 }, + { O_CBRT, 0x7FF0000000000000ULL, 0x0000000000000000ULL, 0x7FF0000000000000ULL, 0 }, + { O_CBRT, 0xFFF0000000000000ULL, 0x0000000000000000ULL, 0xFFF0000000000000ULL, 0 }, + { O_CBRT, 0x7FF8000000000000ULL, 0x0000000000000000ULL, 0x7FF8000000000000ULL, 0 }, + { O_CBRT, 0x8000000000000000ULL, 0x0000000000000000ULL, 0x8000000000000000ULL, 0 }, + { O_CBRT, 0x403B000000000000ULL, 0x0000000000000000ULL, 0x4008000000000000ULL, 1 }, + { O_POW, 0x7FF8000000000000ULL, 0x0000000000000000ULL, 0x3FF0000000000000ULL, 0 }, + { O_POW, 0x7FF8000000000000ULL, 0x8000000000000000ULL, 0x3FF0000000000000ULL, 0 }, + { O_POW, 0x3FF0000000000000ULL, 0x7FF8000000000000ULL, 0x3FF0000000000000ULL, 0 }, + { O_POW, 0x3FF0000000000000ULL, 0x7FF0000000000000ULL, 0x3FF0000000000000ULL, 0 }, + { O_POW, 0xBFF0000000000000ULL, 0x7FF0000000000000ULL, 0x3FF0000000000000ULL, 0 }, + { O_POW, 0xBFF0000000000000ULL, 0xFFF0000000000000ULL, 0x3FF0000000000000ULL, 0 }, + { O_POW, 0x4000000000000000ULL, 0x7FF8000000000000ULL, 0x7FF8000000000000ULL, 0 }, + { O_POW, 0x7FF8000000000000ULL, 0x4000000000000000ULL, 0x7FF8000000000000ULL, 0 }, + { O_POW, 0x0000000000000000ULL, 0xC008000000000000ULL, 0x7FF0000000000000ULL, 0 }, + { O_POW, 0x8000000000000000ULL, 0xC008000000000000ULL, 0xFFF0000000000000ULL, 0 }, + { O_POW, 0x8000000000000000ULL, 0xC000000000000000ULL, 0x7FF0000000000000ULL, 0 }, + { O_POW, 0x8000000000000000ULL, 0xFFF0000000000000ULL, 0x7FF0000000000000ULL, 0 }, + { O_POW, 0x0000000000000000ULL, 0xBFE0000000000000ULL, 0x7FF0000000000000ULL, 0 }, + { O_POW, 0x8000000000000000ULL, 0x4008000000000000ULL, 0x8000000000000000ULL, 0 }, + { O_POW, 0x8000000000000000ULL, 0x4000000000000000ULL, 0x0000000000000000ULL, 0 }, + { O_POW, 0x0000000000000000ULL, 0x3FE0000000000000ULL, 0x0000000000000000ULL, 0 }, + { O_POW, 0x3FE0000000000000ULL, 0xFFF0000000000000ULL, 0x7FF0000000000000ULL, 0 }, + { O_POW, 0x4000000000000000ULL, 0xFFF0000000000000ULL, 0x0000000000000000ULL, 0 }, + { O_POW, 0x3FE0000000000000ULL, 0x7FF0000000000000ULL, 0x0000000000000000ULL, 0 }, + { O_POW, 0x4000000000000000ULL, 0x7FF0000000000000ULL, 0x7FF0000000000000ULL, 0 }, + { O_POW, 0xFFF0000000000000ULL, 0xC008000000000000ULL, 0x8000000000000000ULL, 0 }, + { O_POW, 0xFFF0000000000000ULL, 0xC000000000000000ULL, 0x0000000000000000ULL, 0 }, + { O_POW, 0xFFF0000000000000ULL, 0x4008000000000000ULL, 0xFFF0000000000000ULL, 0 }, + { O_POW, 0xFFF0000000000000ULL, 0x4000000000000000ULL, 0x7FF0000000000000ULL, 0 }, + { O_POW, 0xFFF0000000000000ULL, 0x3FE0000000000000ULL, 0x7FF0000000000000ULL, 0 }, + { O_POW, 0x7FF0000000000000ULL, 0xBFF0000000000000ULL, 0x0000000000000000ULL, 0 }, + { O_POW, 0x7FF0000000000000ULL, 0x4000000000000000ULL, 0x7FF0000000000000ULL, 0 }, + { O_POW, 0xC020000000000000ULL, 0x3FD5555555555555ULL, 0x7FF8000000000000ULL, 0 }, + { O_POW, 0xC000000000000000ULL, 0x3FE0000000000000ULL, 0x7FF8000000000000ULL, 0 }, + { O_POW, 0x4000000000000000ULL, 0x4024000000000000ULL, 0x4090000000000000ULL, 0 }, + { O_POW, 0x4000000000000000ULL, 0xC008000000000000ULL, 0x3FC0000000000000ULL, 0 }, + { O_POW, 0xC000000000000000ULL, 0x4008000000000000ULL, 0xC020000000000000ULL, 0 }, + { O_POW, 0xC000000000000000ULL, 0x4000000000000000ULL, 0x4010000000000000ULL, 0 }, + { O_POW, 0x4024000000000000ULL, 0x4034000000000000ULL, 0x4415AF1D78B58C40ULL, 0 }, + { O_POW, 0xC000000000000000ULL, 0x4415AF1D78B58C40ULL, 0x7FF0000000000000ULL, 0 }, + { O_POW, 0xC000000000000000ULL, 0x40E3882000000000ULL, 0xFFF0000000000000ULL, 0 }, + { O_POW, 0x4000000000000000ULL, 0x3FE0000000000000ULL, 0x3FF6A09E667F3BCDULL, 1 }, + { O_POW, 0x4000000000000000ULL, 0x3FF8000000000000ULL, 0x4006A09E667F3BCDULL, 1 }, + { O_POW, 0x4024000000000000ULL, 0x3FD3333333333333ULL, 0x3FFFEC982D5BB8AFULL, 1 }, + { O_POW, 0x3FF000001AD7F29BULL, 0x40F86A0000000000ULL, 0x3FF0292A5D0CD567ULL, 1 }, + { O_FMOD, 0x401E000000000000ULL, 0x4000000000000000ULL, 0x3FF8000000000000ULL, 0 }, + { O_FMOD, 0xC01E000000000000ULL, 0x4000000000000000ULL, 0xBFF8000000000000ULL, 0 }, + { O_FMOD, 0xC010000000000000ULL, 0x4000000000000000ULL, 0x8000000000000000ULL, 0 }, + { O_FMOD, 0x4010000000000000ULL, 0x4000000000000000ULL, 0x0000000000000000ULL, 0 }, + { O_FMOD, 0x4014000000000000ULL, 0x4008000000000000ULL, 0x4000000000000000ULL, 0 }, + { O_FMOD, 0x3FF0000000000000ULL, 0x7FF0000000000000ULL, 0x3FF0000000000000ULL, 0 }, + { O_FMOD, 0xBFF0000000000000ULL, 0xFFF0000000000000ULL, 0xBFF0000000000000ULL, 0 }, + { O_FMOD, 0x3FF0000000000000ULL, 0x0000000000000000ULL, 0x7FF8000000000000ULL, 0 }, + { O_FMOD, 0x3FF0000000000000ULL, 0x8000000000000000ULL, 0x7FF8000000000000ULL, 0 }, + { O_FMOD, 0x7FF0000000000000ULL, 0x4000000000000000ULL, 0x7FF8000000000000ULL, 0 }, + { O_FMOD, 0xFFF0000000000000ULL, 0x4000000000000000ULL, 0x7FF8000000000000ULL, 0 }, + { O_FMOD, 0x7FF8000000000000ULL, 0x4000000000000000ULL, 0x7FF8000000000000ULL, 0 }, + { O_FMOD, 0x4000000000000000ULL, 0x7FF8000000000000ULL, 0x7FF8000000000000ULL, 0 }, + { O_FMOD, 0x0000000000000000ULL, 0x4014000000000000ULL, 0x0000000000000000ULL, 0 }, + { O_FMOD, 0x8000000000000000ULL, 0x4014000000000000ULL, 0x8000000000000000ULL, 0 }, + { O_FMOD, 0x0000000000000000ULL, 0x0000000000000000ULL, 0x7FF8000000000000ULL, 0 }, + { O_FMOD, 0x7FF0000000000000ULL, 0x7FF0000000000000ULL, 0x7FF8000000000000ULL, 0 }, + { O_FDIM, 0x7FF8000000000000ULL, 0x3FF0000000000000ULL, 0x7FF8000000000000ULL, 0 }, + { O_FDIM, 0x3FF0000000000000ULL, 0x7FF8000000000000ULL, 0x7FF8000000000000ULL, 0 }, + { O_FDIM, 0x4014000000000000ULL, 0x4008000000000000ULL, 0x4000000000000000ULL, 0 }, + { O_FDIM, 0x4008000000000000ULL, 0x4014000000000000ULL, 0x0000000000000000ULL, 0 }, + { O_FDIM, 0x7FF0000000000000ULL, 0x3FF0000000000000ULL, 0x7FF0000000000000ULL, 0 }, + { O_FDIM, 0x3FF0000000000000ULL, 0x7FF0000000000000ULL, 0x0000000000000000ULL, 0 }, + { O_FDIM, 0x7FF0000000000000ULL, 0x7FF0000000000000ULL, 0x0000000000000000ULL, 0 }, + { O_HYPOT, 0x7FF0000000000000ULL, 0x7FF8000000000000ULL, 0x7FF0000000000000ULL, 0 }, + { O_HYPOT, 0x7FF8000000000000ULL, 0xFFF0000000000000ULL, 0x7FF0000000000000ULL, 0 }, + { O_HYPOT, 0xFFF0000000000000ULL, 0x3FF0000000000000ULL, 0x7FF0000000000000ULL, 0 }, + { O_HYPOT, 0x7FF8000000000000ULL, 0x3FF0000000000000ULL, 0x7FF8000000000000ULL, 0 }, + { O_HYPOT, 0x4008000000000000ULL, 0x4010000000000000ULL, 0x4014000000000000ULL, 1 }, + { O_HYPOT, 0x0000000000000000ULL, 0x0000000000000000ULL, 0x0000000000000000ULL, 0 }, + { O_REMAINDER, 0x4014000000000000ULL, 0x4000000000000000ULL, 0x3FF0000000000000ULL, 0 }, + { O_REMAINDER, 0x401C000000000000ULL, 0x4000000000000000ULL, 0xBFF0000000000000ULL, 0 }, + { O_REMAINDER, 0x401C000000000000ULL, 0x4010000000000000ULL, 0xBFF0000000000000ULL, 0 }, + { O_REMAINDER, 0x3FF0000000000000ULL, 0x7FF0000000000000ULL, 0x3FF0000000000000ULL, 0 }, + { O_REMAINDER, 0xC008000000000000ULL, 0xFFF0000000000000ULL, 0xC008000000000000ULL, 0 }, + { O_REMAINDER, 0x3FF0000000000000ULL, 0x0000000000000000ULL, 0x7FF8000000000000ULL, 0 }, + { O_REMAINDER, 0x7FF0000000000000ULL, 0x4000000000000000ULL, 0x7FF8000000000000ULL, 0 }, + { O_REMAINDER, 0x7FF8000000000000ULL, 0x3FF0000000000000ULL, 0x7FF8000000000000ULL, 0 }, +}; +#endif +static float fFrom(u64 b) { + union { u32 u; float f; } c; + c.u = (u32)b; + return c.f; +} +static double dFrom(u64 b) { + union { u64 u; double d; } c; + c.u = b; + return c.d; +} +static u64 fBits(float f) { + union { u32 u; float f; } c; + c.f = f; + return c.u; +} +static u64 dBits(double d) { + union { u64 u; double d; } c; + c.d = d; + return c.u; +} +// The classification of x packed as isnan | isinf << 1 | isfinite << 2 | +// signbit << 3 | fpclassify << 4. +#define CLS(x) ((u64)((isnan(x) != 0) | ((isinf(x) != 0) << 1) | \ + ((isfinite(x) != 0) << 2) | ((signbit(x) != 0) << 3) | (fpclassify(x) << 4))) +// Every NaN a math.c function returns counts as the default NaN: how a NaN +// travels through a series (its sign, say) is not part of the contract. +static u64 mathBits(double d) { + return isnan(d) ? 0x7FF8000000000000ULL : dBits(d); +} +static u64 mathBitsF(float f) { + return isnan(f) ? 0x7FC00000ULL : fBits(f); +} +// 1 when got is within 1e-6 relative of want (both finite). +static int near(u64 got, u64 want) { + double g = dFrom(got); + double w = dFrom(want); + double d = g - w; + if (!isfinite(g) || !isfinite(w)) { + return 0; + } + if (d < 0.0) { + d = -d; + } + if (w < 0.0) { + w = -w; + } + return d <= w * 1e-6; +} +NOI u64 run(u16 op, u64 a, u64 b) { + switch (op) { + case O_FADD: return fBits(fFrom(a) + fFrom(b)); + case O_FSUB: return fBits(fFrom(a) - fFrom(b)); + case O_FMUL: return fBits(fFrom(a) * fFrom(b)); + case O_FDIV: return fBits(fFrom(a) / fFrom(b)); + case O_DADD: return dBits(dFrom(a) + dFrom(b)); + case O_DSUB: return dBits(dFrom(a) - dFrom(b)); + case O_DMUL: return dBits(dFrom(a) * dFrom(b)); + case O_DDIV: return dBits(dFrom(a) / dFrom(b)); + case O_F2D: return dBits((double)fFrom(a)); + case O_D2F: return fBits((float)dFrom(a)); + case O_F2I: return (u32)(long)fFrom(a); + case O_F2U: return (u32)fFrom(a); + case O_D2I: return (u32)(long)dFrom(a); + case O_D2U: return (u32)dFrom(a); + case O_FCLS: return CLS(fFrom(a)); + case O_DCLS: return CLS(dFrom(a)); + case O_SQRT: return mathBits(sqrt(dFrom(a))); + case O_LOG: return mathBits(log(dFrom(a))); + case O_ASIN: return mathBits(asin(dFrom(a))); + case O_ACOS: return mathBits(acos(dFrom(a))); + case O_ATANH: return mathBits(atanh(dFrom(a))); + case O_FLOOR: return mathBits(floor(dFrom(a))); + case O_CEIL: return mathBits(ceil(dFrom(a))); + case O_TRUNC: return mathBits(trunc(dFrom(a))); + case O_ROUND: return mathBits(round(dFrom(a))); + case O_RINT: return mathBits(rint(dFrom(a))); + case O_NEARBYINT: return mathBits(nearbyint(dFrom(a))); + case O_FLOORF: return mathBitsF(floorf(fFrom(a))); + case O_CEILF: return mathBitsF(ceilf(fFrom(a))); + case O_TRUNCF: return mathBitsF(truncf(fFrom(a))); + case O_ROUNDF: return mathBitsF(roundf(fFrom(a))); + case O_RINTF: return mathBitsF(rintf(fFrom(a))); + case O_EXP: return mathBits(exp(dFrom(a))); + case O_EXP2: return mathBits(exp2(dFrom(a))); + case O_EXPM1: return mathBits(expm1(dFrom(a))); + case O_SINH: return mathBits(sinh(dFrom(a))); + case O_COSH: return mathBits(cosh(dFrom(a))); + case O_TANH: return mathBits(tanh(dFrom(a))); + case O_LOG1P: return mathBits(log1p(dFrom(a))); + case O_CBRT: return mathBits(cbrt(dFrom(a))); + case O_POW: return mathBits(pow(dFrom(a), dFrom(b))); + case O_FMOD: return mathBits(fmod(dFrom(a), dFrom(b))); + case O_FDIM: return mathBits(fdim(dFrom(a), dFrom(b))); + case O_HYPOT: return mathBits(hypot(dFrom(a), dFrom(b))); + case O_REMAINDER: return mathBits(remainder(dFrom(a), dFrom(b))); + case O_NEARBYINTF: return mathBitsF(nearbyintf(fFrom(a))); + case O_FMAXF: return mathBitsF(fmaxf(fFrom(a), fFrom(b))); + case O_FMINF: return mathBitsF(fminf(fFrom(a), fFrom(b))); + } + return 0xDEADULL; +} +int main(void) { + u16 bad = 0u; + u16 first = 0xFFFFu; + u16 i; + for (i = 0u; i < NC; i++) { + u64 got = run(kCases[i].op, kCases[i].a, kCases[i].b); + if (got != kCases[i].want && + !(kCases[i].approx && near(got, kCases[i].want))) { + if (bad == 0u) { + first = i; + } + bad++; + } + } + *(volatile u16 *)0x025000UL = bad; + *(volatile u16 *)0x025002UL = first; + *(volatile u16 *)0x025004UL = 0xC0DEu; + for (;;) { + } + return 0; +} +EOF + # One source, two builds: FP_TABLE selects the case table (both at + # once, with the math functions they pull in, overflow a bank-0 test + # binary). Table 1 is the soft-float arithmetic, conversions and + # classification; table 2 is math.c. Table 2 finishes by about frame + # 1000: exp, sinh, pow and cbrt are series or iterations of soft-double + # operations. + # Explicit returns: under `if !` errexit is off, and a failed build + # would otherwise run the previous table's binary. + runFpNanTable() { + rm -f "$oFpNanFile" "$binFpNanFile" + "$CLANG" --target=w65816 -O2 -ffunction-sections -DFP_TABLE="$1" \ + -I"$PROJECT_ROOT/runtime/include" -c "$cFpNanFile" \ + -o "$oFpNanFile" || return 1 + "$PROJECT_ROOT/tools/link816" -o "$binFpNanFile" --text-base 0x1000 \ + "$oCrt0File" "$oFpNanFile" "$PROJECT_ROOT/runtime/math.o" \ + "$PROJECT_ROOT/runtime/softFloat.o" \ + "$PROJECT_ROOT/runtime/softDouble.o" "$oLibgccFile" \ + >/dev/null 2>&1 || return 1 + MAME_CHECK_FRAME="$2" MAME_SECS="$3" \ + bash "$PROJECT_ROOT/scripts/runInMame.sh" "$binFpNanFile" --check \ + 0x025000=0000 0x025002=ffff 0x025004=c0de >/dev/null 2>&1 + } + if ! runFpNanTable 1 400 10; then + die "MAME: float/double NaN / infinity handling wrong (mailbox 0x025000 = mismatches, 0x025002 = first case)" + fi + log "check: MAME math.c C99 Annex F special values (floor..remainder, pow, exp family)" + if ! runFpNanTable 2 1500 30; then + die "MAME: math.c special values wrong (mailbox 0x025000 = mismatches, 0x025002 = first case)" + fi + rm -f "$cFpNanFile" "$oFpNanFile" "$binFpNanFile" + # Regression: W65816SepRepCleanup's PHI-copy hoist must not move a # `STA slot` above an `AND/ORA/EOR/ADC/SBC/CMP_StackRel slot` read in # the gap. Before the fix (gap check only guarded LDA_StackRel), @@ -2970,9 +3803,6 @@ EOF fi rm -f "$cHtFile" "$oHtFile" "$binHtFile" - # Regression: free() coalescing must remove blocks absorbed - # into a lower-address neighbour from the free list. Old code - # extended the lower block but left the absorbed entry in # Signed compare of values near INT16_MIN/MAX: BMI/BPL alone # are not V-flag-aware, so the W65816 backend now applies an # EOR-with-sign-bit transform (a < b signed iff a^$8000 < @@ -3594,6 +4424,131 @@ EOF fi rm -f "$cLowerFile" "$oLowerFile" "$binLowerFile" + # Every constant shift is inline (2026-09-30): i32 << >>u >>s by 1..31 + # (i16 shifts of the halves; SRA by 17..30 was the last libcall) and + # i16 >>s by 1..15 (`cmp #$8000 ; ror a`, or a logical shift then + # ^ M - M), each on 8 values; and W65816I32IncFold's in-place i32 + # -= 1 / += 1 (the conditional borrow / carry into the high half) + # on loop counters across 0x0000 / 0x10000 / 0x80000000 / + # 0xFFFFFFFF. kExp comes from tests/regress/genShiftDec.py. + log "check: MAME constant shifts (i32 by 1..31, i16 >>s by 1..15) and i32 +/- 1 counters" + cShFile="$(mktemp --suffix=.c)" + oShFile="$(mktemp --suffix=.o)" + binShFile="$(mktemp --suffix=.bin)" + python3 "$PROJECT_ROOT/tests/regress/genShiftDec.py" > "$cShFile" + "$CLANG" --target=w65816 -O2 -ffunction-sections -c \ + "$cShFile" -o "$oShFile" + "$PROJECT_ROOT/tools/link816" -o "$binShFile" --text-base 0x1000 \ + "$oCrt0F" "$oLibgccFile" "$oShFile" \ + >/dev/null 2>&1 + if ! MAME_CHECK_FRAME=900 MAME_SECS=20 \ + bash "$PROJECT_ROOT/scripts/runInMame.sh" "$binShFile" --check \ + 0x025000=0000 0x025002=ffff 0x025004=c0de >/dev/null 2>&1; then + die "MAME: constant shifts / i32 +/- 1 wrong (mailbox 0x025000 = mismatches, 0x025002 = first result index)" + fi + rm -f "$cShFile" "$oShFile" "$binShFile" + + # Two-pointer copy loops read src[i], not src[i + 1]: before the + # 2026-09-30 fix, W65816StackSlotMerge renamed the bumped source + # pointer's frame slot onto the one the load still read, and a plain + # u16-counter `dst[i] = src[i] ^ 0x80` shifted every byte by one (50 + # of these 67 values wrong). It only escaped while a PHP/PLP wrap + # hid the copy's offsets from the pass. + log "check: MAME two-pointer copy loops read the right element (StackSlotMerge twin check)" + cCpFile="$(mktemp --suffix=.c)" + oCpFile="$(mktemp --suffix=.o)" + binCpFile="$(mktemp --suffix=.bin)" + cat > "$cCpFile" <<'EOF' +// Two-pointer copy loops read src[i], not src[i + 1]: W65816StackSlotMerge +// once renamed the bumped source pointer's slot onto the one the load still +// read (a u16-counter `dst[i] = src[i] ^ 0x80` shifted every byte down). +// Mailbox: 0x025000 = mismatches, 0x025002 = first failing index, +// 0x025004 = 0xC0DE done. +typedef unsigned short u16; +typedef unsigned long u32; +typedef unsigned char u8; +#define NOI __attribute__((noinline)) +NOI void xor16(u8 *dst, const u8 *src, u16 n) { + u16 i; + for (i = 0; i < n; i++) { + dst[i] = src[i] ^ 0x80u; + } +} +NOI void add32(u8 *dst, const u8 *src, u32 n) { + u32 i; + for (i = 0; i < n; i++) { + dst[i] = (u8)(src[i] + 1u); + } +} +NOI void walk(u8 *dst, const u8 *src, u16 n) { + while (n--) { + *dst++ = (u8)(*src++ ^ 0x55u); + } +} +NOI u16 sumTo(const u8 *p, const u8 *end) { + u16 s = 0; + while (p != end) { + s += *p++; + } + return s; +} +static u8 gSrc[40]; +static u8 gDst[40]; +static u16 gBad; +static u16 gFirst = 0xFFFFu; +static u16 gIdx; +static void check(u16 got, u16 want) { + if (got != want && gBad++ == 0u) { + gFirst = gIdx; + } + gIdx++; +} +int main(void) { + u16 i; + for (i = 0; i < 40u; i++) { + gSrc[i] = (u8)(i * 7u + 3u); + } + xor16(gDst, gSrc, 33); + for (i = 0; i < 40u; i++) { + check(gDst[i], i < 33u ? (u8)(gSrc[i] ^ 0x80u) : 0u); + } + add32(gDst, gSrc, 17); + for (i = 0; i < 17u; i++) { + check(gDst[i], (u8)(gSrc[i] + 1u)); + } + walk(gDst + 1, gSrc + 2, 9); + for (i = 0; i < 9u; i++) { + check(gDst[i + 1], (u8)(gSrc[i + 2] ^ 0x55u)); + } + { + u16 s = 0; + for (i = 3; i < 29u; i++) { + s = (u16)(s + gSrc[i]); + } + check(sumTo(gSrc + 3, gSrc + 29), s); + } + *(volatile u16 *)0x025000UL = gBad; + *(volatile u16 *)0x025002UL = gFirst; + *(volatile u16 *)0x025004UL = 0xC0DEu; + for (;;) { + } + return 0; +} +EOF + "$CLANG" --target=w65816 -O2 -ffunction-sections -c \ + "$cCpFile" -o "$oCpFile" + "$PROJECT_ROOT/tools/link816" -o "$binCpFile" --text-base 0x1000 \ + "$oCrt0F" "$oLibgccFile" "$oCpFile" \ + >/dev/null 2>&1 + if ! bash "$PROJECT_ROOT/scripts/runInMame.sh" "$binCpFile" --check \ + 0x025000=0000 0x025002=ffff 0x025004=c0de >/dev/null 2>&1; then + die "MAME: copy loop read the wrong element (mailbox 0x025000 = mismatches, 0x025002 = first index)" + fi + rm -f "$cCpFile" "$oCpFile" "$binCpFile" + + # Regression: free() coalescing must remove blocks absorbed + # into a lower-address neighbour from the free list. Old code + # extended the lower block but left the absorbed entry in # the list, creating an overlapping free entry. A subsequent # malloc could hand out the same memory to two callers. log "check: MAME runs malloc/free coalesce — three blocks freed in alloc order (#100)" diff --git a/src/llvm/lib/Target/W65816/CMakeLists.txt b/src/llvm/lib/Target/W65816/CMakeLists.txt index 06324ec..cbf2acf 100644 --- a/src/llvm/lib/Target/W65816/CMakeLists.txt +++ b/src/llvm/lib/Target/W65816/CMakeLists.txt @@ -39,6 +39,7 @@ add_llvm_target(W65816CodeGen W65816ImgCalleeSave.cpp W65816NarrowI32Mul.cpp W65816CmpPrepare.cpp + W65816CmpSink.cpp W65816PromoteFiToImg.cpp W65816StackRelToImg.cpp W65816StackSlotMerge.cpp diff --git a/src/llvm/lib/Target/W65816/W65816.h b/src/llvm/lib/Target/W65816/W65816.h index f8c784a..206fbbe 100644 --- a/src/llvm/lib/Target/W65816/W65816.h +++ b/src/llvm/lib/Target/W65816/W65816.h @@ -131,6 +131,14 @@ FunctionPass *createW65816NarrowI32Mul(); // one at a time. See W65816CmpPrepare.cpp. FunctionPass *createW65816CmpPrepare(); +// Pre-RA pass, inserted right after the register coalescer: moves the +// instruction that sets a conditional branch's flags down past the back- +// edge COPYs PHI elimination left between them, so the copies (and the +// allocator's spills) land above the compare instead of inside a PHP/PLP +// wrap. See W65816CmpSink.cpp. +FunctionPass *createW65816CmpSink(); +extern char &W65816CmpSinkID; + // Post-LSR IR pass: detect pointer-walking loops over GlobalAddress // bases and rewrite to forward-counter + indexed GEP form. LSR's // transform is faster for pointer-arg loops (single ptr increment per @@ -220,6 +228,7 @@ void initializeW65816LowerWide32Pass(PassRegistry &); void initializeW65816ImgCalleeSavePass(PassRegistry &); void initializeW65816NarrowI32MulPass(PassRegistry &); void initializeW65816CmpPreparePass(PassRegistry &); +void initializeW65816CmpSinkPass(PassRegistry &); void initializeW65816PromoteFiToImgPass(PassRegistry &); void initializeW65816StackSlotMergePass(PassRegistry &); void initializeW65816StackRelToImgPass(PassRegistry &); diff --git a/src/llvm/lib/Target/W65816/W65816AsmPrinter.cpp b/src/llvm/lib/Target/W65816/W65816AsmPrinter.cpp index a3224f9..0a596e4 100644 --- a/src/llvm/lib/Target/W65816/W65816AsmPrinter.cpp +++ b/src/llvm/lib/Target/W65816/W65816AsmPrinter.cpp @@ -404,6 +404,40 @@ void W65816AsmPrinter::emitInstruction(const MachineInstr *MI) { OutStreamer->emitLabel(SkipSym); return; } + case W65816::INC_HI_IF_CARRY_DP: + case W65816::DEC_HI_IF_BORROW_DP: + case W65816::DEC_HI_IF_BORROW_StackRel: { + // The carry / borrow into an i32's high half (see the .td). Z comes + // from the low half: after its increment for INC (Z=1: it wrapped to + // 0), before its decrement for DEC (Z=1: it is about to wrap). + // INC_HI_IF_CARRY_DP bne L ; inc $dp ; L: + // DEC_HI_IF_BORROW_DP bne L ; dec $dp ; L: + // DEC_HI_IF_BORROW_StackRel bne L ; lda $off,s ; dec a ; + // sta $off,s ; lda #0 ; L: + // The stack form restores A = 0, the low half it found, for the + // `dec a` that follows. + int64_t Off = MI->getOperand(0).getImm(); + MCSymbol *SkipSym = OutContext.createTempSymbol(); + EmitToStreamer(*OutStreamer, + MCInstBuilder(W65816::BNE).addExpr( + MCSymbolRefExpr::create(SkipSym, OutContext))); + switch (MI->getOpcode()) { + case W65816::INC_HI_IF_CARRY_DP: + emitOpImm(W65816::INC_DP, Off); + break; + case W65816::DEC_HI_IF_BORROW_DP: + emitOpImm(W65816::DEC_DP, Off); + break; + default: + emitOpImm(W65816::LDA_StackRel, Off); + emitOp(W65816::DEA); + emitOpImm(W65816::STA_StackRel, Off); + emitOpImm(W65816::LDA_Imm16, 0); + break; + } + OutStreamer->emitLabel(SkipSym); + return; + } case W65816::ADJCALLSTACKDOWN: { // DOWN is a no-op in our scheme — the PUSH16 sequence in LowerCall // already shifted SP incrementally as args were pushed. Nothing @@ -1095,10 +1129,11 @@ void W65816AsmPrinter::emitInstruction(const MachineInstr *MI) { return; } case W65816::ASRA16: { - // PHA ; ASL A (sets carry from sign bit) ; PLA ; ROR A - emitOp(W65816::PHA); - emitOp(W65816::ASL_A); - emitOp(W65816::PLA); + // CMP #$8000 sets C exactly when bit 15 (the sign) is set; ROR A + // shifts it back in at the top. 5 cycles, where the PHA ; ASL A ; + // PLA ; ROR A it replaced took 13 and touched the stack. The flags + // it leaves are ROR's either way. + emitOpImm(W65816::CMP_Imm16, 0x8000); emitOp(W65816::ROR_A); return; } diff --git a/src/llvm/lib/Target/W65816/W65816CmpSink.cpp b/src/llvm/lib/Target/W65816/W65816CmpSink.cpp new file mode 100644 index 0000000..2cea0da --- /dev/null +++ b/src/llvm/lib/Target/W65816/W65816CmpSink.cpp @@ -0,0 +1,170 @@ +//===-- W65816CmpSink.cpp - Keep a compare next to its branch --------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// PHI elimination puts a loop's back-edge copies before the block's first +// terminator. Here that is the conditional branch; the compare that sets +// its flags is an ordinary instruction above it, so the copies land between +// the two: +// +// CMPfi %next, %stack.5 ; p + 1 == end ? +// %s = COPY %sum ; the loop-carried values +// %p = COPY %next +// BNE %bb.loop +// +// Every copy becomes an LDA / STA / TXA after register allocation, which +// rewrites N and Z, and W65816StackSlotCleanup then has to save the flags +// around them with PHP / PLP: 7 cycles on every trip round a +// `while (p != end)` loop. +// +// Right after the register coalescer, this pass moves the flag-setting +// instruction down to a BEQ / BNE / BMI / BPL, past virtual-register COPYs +// that do not define anything it reads or read anything it defines. The allocator +// then sees the compare at the end of the block and gives the copies (and +// its own splits, which go before the compare's operands are needed) the +// room above it. LiveIntervals are updated in place. +// +//===----------------------------------------------------------------------===// + +#include "W65816.h" +#include "W65816InstrInfo.h" +#include "W65816Subtarget.h" +#include "llvm/CodeGen/LiveIntervals.h" +#include "llvm/CodeGen/MachineFunctionPass.h" +#include "llvm/CodeGen/MachineRegisterInfo.h" +#include "llvm/CodeGen/SlotIndexes.h" +#include "llvm/InitializePasses.h" +#include "llvm/Support/Debug.h" + +using namespace llvm; + +#define DEBUG_TYPE "w65816-cmp-sink" + +namespace { + +class W65816CmpSink : public MachineFunctionPass { +public: + static char ID; + W65816CmpSink() : MachineFunctionPass(ID) {} + StringRef getPassName() const override { + return "W65816 sink compare to its branch"; + } + void getAnalysisUsage(AnalysisUsage &AU) const override { + AU.setPreservesCFG(); + AU.addRequired(); + AU.addPreserved(); + AU.addPreserved(); + MachineFunctionPass::getAnalysisUsage(AU); + } + bool runOnMachineFunction(MachineFunction &MF) override; +}; + +} // namespace + +char W65816CmpSink::ID = 0; +char &llvm::W65816CmpSinkID = W65816CmpSink::ID; + +INITIALIZE_PASS_BEGIN(W65816CmpSink, DEBUG_TYPE, + "W65816 sink compare to its branch", false, false) +INITIALIZE_PASS_DEPENDENCY(LiveIntervalsWrapperPass) +INITIALIZE_PASS_END(W65816CmpSink, DEBUG_TYPE, + "W65816 sink compare to its branch", false, false) + +// A copy between two virtual registers: no flags, no memory, no physical +// register the compare could care about. +static bool isVirtualCopy(const MachineInstr &MI) { + return MI.isCopy() && MI.getOperand(0).getReg().isVirtual() && + MI.getOperand(1).getReg().isVirtual(); +} + +// True when moving Test below Copy keeps every value the same: Copy does +// not write a register Test reads or writes, and does not read one Test +// writes. +static bool canPass(const MachineInstr &Test, const MachineInstr &Copy) { + Register Dst = Copy.getOperand(0).getReg(); + Register Src = Copy.getOperand(1).getReg(); + for (const MachineOperand &MO : Test.operands()) { + if (!MO.isReg() || !MO.getReg()) + continue; + if (MO.getReg() == Dst) + return false; + if (MO.isDef() && MO.getReg() == Src) + return false; + } + return true; +} + +bool W65816CmpSink::runOnMachineFunction(MachineFunction &MF) { + if (skipFunction(MF.getFunction())) + return false; + LiveIntervals &LIS = getAnalysis().getLIS(); + MachineRegisterInfo &MRI = MF.getRegInfo(); + bool Changed = false; + + for (MachineBasicBlock &MBB : MF) { + MachineBasicBlock::iterator Br = MBB.getFirstTerminator(); + if (Br == MBB.end()) + continue; + // Only a branch on N or Z: the copies' loads and transfers leave C and + // V alone, so a BCC / BCS / BVC / BVS never needed the wrap, and moving + // the end of a CMP / SBC carry chain away from its first half only + // costs the allocator a register (a TAX / TXA round trip). + unsigned BrOpc = Br->getOpcode(); + if (BrOpc != W65816::BEQ && BrOpc != W65816::BNE && + BrOpc != W65816::BMI && BrOpc != W65816::BPL) + continue; + // Walk up over the copies to the instruction that sets the flags. + MachineInstr *Test = nullptr; + SmallVector Copies; + for (MachineBasicBlock::iterator It = Br; It != MBB.begin();) { + MachineInstr &MI = *--It; + if (MI.isDebugInstr()) + continue; + if (isVirtualCopy(MI)) { + Copies.push_back(&MI); + continue; + } + MachineOperand *PDef = + MI.findRegisterDefOperand(W65816::P, /*TRI=*/nullptr); + if (PDef && !PDef->isDead()) + Test = &MI; + break; + } + if (!Test || Copies.empty()) + continue; + // The copies touch neither memory nor physical registers, so only + // register dependencies can stop the move; a flag setter with an + // explicit physical register operand (or a call) is left alone. + if (Test->isCall()) + continue; + bool PhysReg = false; + for (const MachineOperand &MO : Test->operands()) + if (MO.isReg() && MO.getReg().isPhysical() && + MO.getReg() != W65816::P && !MO.isImplicit()) + PhysReg = true; + if (PhysReg) + continue; + bool Ok = true; + for (MachineInstr *C : Copies) + Ok = Ok && canPass(*Test, *C); + if (!Ok) + continue; + MBB.splice(Br, &MBB, Test->getIterator()); + LIS.handleMove(*Test, /*UpdateFlags=*/true); + // A copy that used to be the last reader of one of the compare's + // operands no longer is; greedy works from the intervals, so dropping + // the stale kill flags is enough. + for (const MachineOperand &MO : Test->operands()) + if (MO.isReg() && MO.isUse() && MO.getReg().isVirtual()) + MRI.clearKillFlags(MO.getReg()); + LLVM_DEBUG(dbgs() << "cmp-sink: " << *Test); + Changed = true; + } + return Changed; +} + +FunctionPass *llvm::createW65816CmpSink() { return new W65816CmpSink(); } diff --git a/src/llvm/lib/Target/W65816/W65816FarPtrStageReuse.cpp b/src/llvm/lib/Target/W65816/W65816FarPtrStageReuse.cpp index 94b566b..03d2ed4 100644 --- a/src/llvm/lib/Target/W65816/W65816FarPtrStageReuse.cpp +++ b/src/llvm/lib/Target/W65816/W65816FarPtrStageReuse.cpp @@ -79,6 +79,10 @@ enum Width : int8_t { WUnknown = -1, W8 = 0, W16 = 1 }; constexpr unsigned StageLo = 0xE0; constexpr unsigned StageBank = 0xE2; +// The IMG direct-page words ($C0..$CE, then $D0..$DE: one run of memory). +constexpr unsigned ImgLow = 0xC0; +constexpr unsigned ImgHigh = 0xDE; + // Liveness bit set: 1 A, 2 X, 4 Y, 8 N/Z. constexpr unsigned LiveAll = 15; @@ -153,6 +157,9 @@ private: // Results of the final marking walk. SmallPtrSet RedundantStores; SmallPtrSet RedundantLoads; + // [$E0],Y accesses whose pointer also sits, word for word, in an IMG + // direct-page pair: the pair's low address. + DenseMap Redirects; // M and X widths in force at each instruction (from the marking walk). DenseMap> ModeAt; Width modeM(const MachineInstr &MI) const { @@ -188,6 +195,10 @@ private: void computeLiveness(); bool deleteDead(); bool deleteDeadStackStores(); + bool redirectToHomes(); + unsigned stageReads(const MachineInstr &MI) const; + unsigned stageDefs(const MachineInstr &MI) const; + bool foldEmptyFlagSaves(); }; } // namespace @@ -243,7 +254,7 @@ static bool isRMWName(StringRef N) { if (endsWithAny(N, {"_A"})) return false; return startsWithAny(N, {"INC_", "DEC_", "ASL_", "LSR_", "ROL_", "ROR_", - "TSB_", "TRB_", "INC_HI_IF_CARRY"}); + "TSB_", "TRB_"}); } static bool isNZBranch(StringRef N) { @@ -294,7 +305,7 @@ static bool changesY(StringRef N) { static bool changesCarry(StringRef N) { return startsWithAny(N, {"ADC", "SBC", "ASL", "LSR", "ROL", "ROR", "CMP", "CPX", "CPY", "ASLA", "LSRA", "ASRA", "NEG", "SHL", - "SRL", "SRA", "INC_HI_IF_CARRY"}); + "SRL", "SRA"}); } Effects W65816FarPtrStageReuse::effects(const MachineInstr &MI, @@ -388,8 +399,10 @@ Effects W65816FarPtrStageReuse::effects(const MachineInstr &MI, E.UseA = E.UseX = E.UseY = true; E.DefA = E.DefX = E.DefY = false; } - // N/Z. - if (isNZBranch(N) || N == "PHP") + // N/Z. The i32 carry / borrow pseudos branch on Z (bne inside). + bool CarryPseudo = + startsWithAny(N, {"INC_HI_IF_CARRY", "DEC_HI_IF_BORROW"}); + if (isNZBranch(N) || N == "PHP" || CarryPseudo) E.UseNZ = true; if (N.starts_with("SELECT")) E.UseNZ = true; @@ -401,7 +414,7 @@ Effects W65816FarPtrStageReuse::effects(const MachineInstr &MI, "DEA", "INX", "DEX", "INY", "DEY", "XBA", "NEG", "SHL", "SRL", "SRA", "ADD", "SUB"}) || (N.starts_with("BIT") && N != "BIT_Imm16" && N != "BIT_Imm8"); - if (N.starts_with("INC_HI_IF_CARRY")) + if (CarryPseudo) SetsNZ = false; // conditional: may leave the flags alone if (SetsNZ) E.DefNZ = true; @@ -822,6 +835,28 @@ void W65816FarPtrStageReuse::transfer(MachineInstr &MI, FlowState &S, return; } + // A far-pointer access through the staging words, when the same two + // words also sit in an IMG direct-page pair: it can use the pair in + // place (W65816StackRelToImg puts a pointer's halves side by side for + // this). [dp],Y reads the low word at dp and the bank byte at dp+2. + if (Mark && HasImm && Imm == StageLo && N.contains("_DPIndLong")) { + auto Lo = S.DP.find(StageLo); + auto Bank = S.DP.find(StageBank); + if (Lo != S.DP.end() && Bank != S.DP.end() && Lo->second && + Bank->second) { + for (const auto &KV : S.DP) { + unsigned D = KV.first; + if (D < ImgLow || D > ImgHigh - 2 || KV.second != Lo->second) + continue; + auto Hi = S.DP.find(D + 2); + if (Hi != S.DP.end() && Hi->second == Bank->second) { + Redirects[&MI] = D; + break; + } + } + } + } + // Everything else. Anything outside the families this file knows // forgets everything; the rest loses exactly what it can change. if (!isKnownFamily(N)) { @@ -888,6 +923,7 @@ static FlowState meet(const FlowState &A, const FlowState &B) { void W65816FarPtrStageReuse::analyse() { RedundantStores.clear(); RedundantLoads.clear(); + Redirects.clear(); ModeAt.clear(); DenseMap In, Out; @@ -1212,9 +1248,135 @@ bool W65816FarPtrStageReuse::runRound() { analyse(); // ModeAt must describe the surviving instructions if (deleteDeadStackStores()) Changed = true; + if (Changed) + analyse(); // and the redirects the instructions as they now stand + if (redirectToHomes()) + Changed = true; return Changed; } +// Staging words an instruction may read: bit 0 = $E0/$E1, bit 1 = $E2/$E3. +// They are compiler scratch, never part of the ABI: a call does not read +// them (it clobbers them, stageDefs; the runtime helpers that use $E0.. +// fill it themselves from their arguments, and an indirect call's vector +// is $B8), nor does a plain return. Other ways out and anything +// unmodelled read both. +unsigned W65816FarPtrStageReuse::stageReads(const MachineInstr &MI) const { + if (MI.isCall()) + return 0; + if (MI.isInlineAsm()) + return 3; + if (MI.isReturn()) + return isPlainReturn(MI) ? 0 : 3; + StringRef N = name(MI); + // Register-only instructions (mode, stack, transfer and flag ops) are + // outside isKnownFamily because transfer() handles them first. TCD and + // PLD are not here: they move the direct page itself. + static const char *const RegOnly[] = { + "REP", "SEP", "PHP", "PLP", "PHA", "PLA", "PHX", "PLX", "PHY", "PLY", + "PHB", "PLB", "PHK", "PHD", "CLC", "SEC", "CLI", "SEI", "CLD", "SED", + "CLV", "NOP", "TAX", "TAY", "TXA", "TYA", "TXY", "TYX", "TSC", "TCS", + "TSX", "TXS", "PEA", "PER", "PUSH16", "PUSH16X"}; + for (const char *R : RegOnly) + if (N == R) + return 0; + if (!isKnownFamily(N)) + return 3; + if (stageDefs(MI) || !N.contains("_DP")) + return 0; + if (isDPIndexed(N)) + return 3; + int64_t Imm = 0; + if (!firstImm(MI, Imm)) + return 3; + int64_t Len = N.contains("_DPIndLong") ? 3 : 2; + unsigned R = 0; + if (Imm < StageLo + 2 && Imm + Len > StageLo) + R |= 1; + if (Imm < StageBank + 2 && Imm + Len > StageBank) + R |= 2; + return R; +} + +// Staging words a 16-bit store overwrites whole, or a call clobbers (same +// bits). +unsigned W65816FarPtrStageReuse::stageDefs(const MachineInstr &MI) const { + if (MI.isCall()) + return 3; + StringRef N = name(MI); + Width W = N == "STA_DP" || N == "STZ_DP" ? modeM(MI) + : N == "STX_DP" || N == "STY_DP" ? modeX(MI) + : WUnknown; + int64_t Imm = 0; + if (W != W16 || !firstImm(MI, Imm)) + return 0; + return Imm == StageLo ? 1u : Imm == StageBank ? 2u : 0u; +} + +// Point the accesses analyse() matched to their pointer's own DP pair, +// then delete the staging stores nothing reads any more. The loads that +// fed them die with them, in the next round's deleteDead. +bool W65816FarPtrStageReuse::redirectToHomes() { + if (Redirects.empty()) + return false; + for (auto &KV : Redirects) + for (MachineOperand &MO : KV.first->operands()) + if (MO.isImm()) { + MO.setImm(KV.second); + break; + } + Redirects.clear(); + + DenseMap LiveIn, Out; + auto outOf = [&](const MachineBasicBlock *MBB) { + unsigned O = 0; + if (MBB->succ_empty()) { + auto Last = MBB->getLastNonDebugInstr(); + if (Last == MBB->end() || !isPlainReturn(*Last)) + O = 3; + } + for (const MachineBasicBlock *Succ : MBB->successors()) + O |= LiveIn.lookup(Succ); + return O; + }; + bool Iterate = true; + while (Iterate) { + Iterate = false; + for (MachineBasicBlock *MBB : post_order(MF)) { + unsigned Live = outOf(MBB); + Out[MBB] = Live; + for (auto It = MBB->rbegin(); It != MBB->rend(); ++It) { + if (It->isMetaInstruction() || It->isDebugInstr()) + continue; + Live = (Live & ~stageDefs(*It)) | stageReads(*It); + } + if (LiveIn.lookup(MBB) != Live || !LiveIn.count(MBB)) { + LiveIn[MBB] = Live; + Iterate = true; + } + } + } + SmallVector Doomed; + for (MachineBasicBlock &MBB : *MF) { + auto Found = Out.find(&MBB); + unsigned Live = Found != Out.end() ? Found->second : 3u; + for (auto It = MBB.rbegin(); It != MBB.rend(); ++It) { + if (It->isMetaInstruction() || It->isDebugInstr()) + continue; + unsigned D = stageDefs(*It); + // Only the stores go; a call's clobber just ends the liveness. + if (D && !It->isCall() && !(Live & D)) { + Doomed.push_back(&*It); + continue; + } + Live = (Live & ~D) | stageReads(*It); + } + } + for (MachineInstr *MI : Doomed) + MI->eraseFromParent(); + return true; +} + bool W65816FarPtrStageReuse::runOnMachineFunction(MachineFunction &Fn) { if (skipFunction(Fn.getFunction()) || getenv("W65816_DISABLE_FARPTR_REUSE")) return false; @@ -1229,6 +1391,67 @@ bool W65816FarPtrStageReuse::runOnMachineFunction(MachineFunction &Fn) { break; Changed = true; } + Changed |= foldEmptyFlagSaves(); + return Changed; +} + +// W65816StackSlotCleanup keeps a branch's flags alive across the loads +// between its compare and the branch with PHP / PLP or, when X is free, +// with `TAX ... CPX #k` in place of the CMP #k. Once this pass (or +// StackRelToImg before it) has deleted every one of those loads, what is +// left is a no-op PHP ; PLP (7 cycles) or a TAX ; CPX #k that CMP #k does +// in 3, when nothing reads X after it. +bool W65816FarPtrStageReuse::foldEmptyFlagSaves() { + bool Changed = false; + auto nextReal = [](MachineBasicBlock &MBB, MachineBasicBlock::iterator It) { + while (It != MBB.end() && It->isDebugInstr()) + ++It; + return It; + }; + auto xDeadFrom = [&](MachineBasicBlock &MBB, MachineBasicBlock::iterator It) { + for (; It != MBB.end(); ++It) { + if (It->isDebugInstr()) + continue; + Effects E = effects(*It, W16); + if (E.UseX) + return false; + if (E.DefX) + return true; + } + for (MachineBasicBlock *Succ : MBB.successors()) + if (Succ->isLiveIn(W65816::X)) + return false; + return true; + }; + for (MachineBasicBlock &MBB : *MF) { + for (auto It = MBB.begin(); It != MBB.end();) { + MachineInstr &MI = *It++; + unsigned Opc = MI.getOpcode(); + if (Opc != W65816::PHP && Opc != W65816::TAX) + continue; + auto Next = nextReal(MBB, It); + if (Next == MBB.end()) + continue; + if (Opc == W65816::PHP && Next->getOpcode() == W65816::PLP) { + It = std::next(Next); + Next->eraseFromParent(); + MI.eraseFromParent(); + Changed = true; + continue; + } + if (Opc != W65816::TAX || Next->getOpcode() != W65816::CPX_Imm16 || + !Next->getOperand(0).isImm() || !xDeadFrom(MBB, std::next(Next))) + continue; + BuildMI(MBB, Next, Next->getDebugLoc(), TII->get(W65816::CMP_Imm16)) + .addImm(Next->getOperand(0).getImm()) + .addReg(W65816::A, RegState::Implicit) + .addReg(W65816::P, RegState::ImplicitDefine); + It = std::next(Next); + Next->eraseFromParent(); + MI.eraseFromParent(); + Changed = true; + } + } return Changed; } diff --git a/src/llvm/lib/Target/W65816/W65816I32IncFold.cpp b/src/llvm/lib/Target/W65816/W65816I32IncFold.cpp index 3b7dfdb..315f4f5 100644 --- a/src/llvm/lib/Target/W65816/W65816I32IncFold.cpp +++ b/src/llvm/lib/Target/W65816/W65816I32IncFold.cpp @@ -1,40 +1,40 @@ -//===-- W65816I32IncFold.cpp - Fold i32 += 1 into INC + conditional skip --===// +//===-- W65816I32IncFold.cpp - Fold i32 +/- 1 into INC/DEC + skip ---------===// // // Pre-emit peephole: detect the post-PEI 6-instruction sequence emitted -// for `i32 += 1` on a Wide32 vreg whose halves spilled to two stack-rel -// slots, and rewrite to a tighter form using INA + a conditional skip -// over the hi half. +// for `i32 += 1` or `i32 -= 1` on a value whose halves live in two stack- +// relative or direct-page slots, and rewrite it so the high half is only +// touched when the low half carries or borrows. // -// Original (after PEI, pseudos still un-expanded): -// $a = LDA_StackRel imm_lo ; load lo half -// $a = ADCi16imm $a, 1 ; CLC + ADC #1 (5 cyc) -// STA_StackRel $a, imm_lo ; store lo -// $a = LDA_StackRel imm_hi ; load hi half -// $a = ADCEi16imm $a, 0 ; ADC #0 (uses carry from lo) -// STA_StackRel $a, imm_hi ; store hi +// Original (after PEI, pseudos still un-expanded), K = 1 / -1: +// $a = LDA lo ; load lo half +// $a = ADCi16imm $a, K ; CLC + ADC #K +// STA lo ; store lo +// $a = LDA hi ; load hi half +// $a = ADCEi16imm $a, 0 / -1 ; ADC (carry from lo) +// STA hi ; store hi // -// Cycle cost: 5 + 2 + 3 + 5 + 5 + 3 + 5 = 28 cyc +// Cost: 28 cycles with stack-relative halves, 24 in the direct page. // -// Rewrite: -// $a = LDA_StackRel imm_lo ; load lo -// $a = INA_PSEUDO $a, $a ; lo + 1 — sets Z based on result -// STA_StackRel $a, imm_lo ; store lo (Z preserved) -// INC_HI_IF_CARRY_StackRel imm_hi ; AsmPrinter expands to: -// ; bne L_skip -// ; lda imm_hi, s -// ; inc a -// ; sta imm_hi, s -// ; L_skip: +// Increment: +// $a = LDA lo +// $a = INA_PSEUDO $a ; lo + 1 -- Z set iff it wrapped +// STA lo ; STA leaves Z alone +// INC_HI_IF_CARRY hi ; bne L ; ; L: // -// Cycle cost (no carry, common case): -// 5 + 2 + 5 + 3 (BNE taken) = 15 cyc — saves 13 cyc -// Cycle cost (with carry, rare case): -// 5 + 2 + 5 + 2 (BNE not-taken) + 5 + 2 + 5 = 26 cyc — saves 2 cyc +// Decrement: +// $a = LDA lo ; Z set iff lo is 0 ... +// DEC_HI_IF_BORROW hi ; ... so it borrows: bne L ; ; L: +// $a = DEA_PSEUDO $a +// STA lo // -// The Z flag from `INA` survives the intervening STA_StackRel because -// STA does not modify the processor status register. The BNE in the -// expansion of INC_HI_IF_CARRY_StackRel reads that Z to decide whether -// the hi half needs to be touched. +// No carry / borrow (the common case): 15 cycles on the stack, 13 in the +// direct page (where StackRelToImg also turns the increment's +// LDA / INA / STA into INC dp). +// +// The rewrite leaves A holding the low half (or, on the stack-relative +// carry path, the high half) where the original left the high half, and +// leaves different flags; it only fires when nothing reads either before +// they are rewritten. //===----------------------------------------------------------------------===// #include "W65816.h" @@ -55,92 +55,161 @@ public: W65816I32IncFold() : MachineFunctionPass(ID) {} bool runOnMachineFunction(MachineFunction &MF) override; StringRef getPassName() const override { - return "W65816 i32 += 1 → INC + conditional skip"; + return "W65816 i32 +/- 1 -> INC/DEC + conditional skip"; } void getAnalysisUsage(AnalysisUsage &AU) const override { MachineFunctionPass::getAnalysisUsage(AU); } }; + +// Where one half lives: a stack-relative offset or a direct-page address. +struct HalfLoc { + bool IsDP = false; + int64_t Addr = 0; + bool operator==(const HalfLoc &O) const { + return IsDP == O.IsDP && Addr == O.Addr; + } +}; + +// The matched sequence. +struct I32Step { + bool IsDec = false; + HalfLoc Lo, Hi; + MachineInstr *LdaLo = nullptr, *AdcLo = nullptr, *StaLo = nullptr; + MachineInstr *LdaHi = nullptr, *AdcHi = nullptr, *StaHi = nullptr; + MachineBasicBlock::iterator End; +}; } // namespace char W65816I32IncFold::ID = 0; INITIALIZE_PASS(W65816I32IncFold, DEBUG_TYPE, - "W65816 i32 += 1 fold", false, false) + "W65816 i32 +/- 1 fold", false, false) -// Match the 6-instruction sequence; returns the post-pattern iterator -// and fills in the lo/hi stack-rel offsets if the pattern matches. -// Tolerates intervening TAX/TXA pairs (which regalloc inserts as -// spurious A-save brackets around STAfi's conservative Defs=[A]). -// They're collected into `KillMe` so the rewrite can erase them too. -static bool matchI32AddOnePattern(MachineBasicBlock::iterator It, - MachineBasicBlock::iterator End, - int64_t &OffLo, int64_t &OffHi, - MachineBasicBlock::iterator &PatEnd, - SmallVectorImpl &KillMe) { +static bool loadLoc(const MachineInstr &MI, HalfLoc &L) { + if (MI.getOpcode() != W65816::LDA_StackRel && + MI.getOpcode() != W65816::LDA_DP) + return false; + if (MI.getNumOperands() < 1 || !MI.getOperand(0).isImm()) + return false; + L.IsDP = MI.getOpcode() == W65816::LDA_DP; + L.Addr = MI.getOperand(0).getImm(); + return true; +} + +static bool storesTo(const MachineInstr &MI, const HalfLoc &L) { + unsigned Want = L.IsDP ? W65816::STA_DP : W65816::STA_StackRel; + return MI.getOpcode() == Want && MI.getNumOperands() >= 1 && + MI.getOperand(0).isImm() && MI.getOperand(0).getImm() == L.Addr; +} + +static bool adcImm(const MachineInstr &MI, unsigned Opc, int64_t &K) { + if (MI.getOpcode() != Opc || MI.getNumOperands() < 3 || + !MI.getOperand(2).isImm()) + return false; + K = MI.getOperand(2).getImm() & 0xFFFF; + return true; +} + +// Match the 6-instruction sequence at It. Tolerates TAX/TXA pairs where +// the increment fold always has (regalloc inserts them as spurious A-save +// brackets around STAfi's conservative Defs=[A]); they are collected into +// KillMe so the rewrite can erase them too. +static bool matchI32Step(MachineBasicBlock::iterator It, + MachineBasicBlock::iterator End, I32Step &S, + SmallVectorImpl &KillMe) { auto skipDebug = [&]() { - while (It != End && It->isDebugInstr()) ++It; + while (It != End && It->isDebugInstr()) + ++It; }; auto skipTaxTxa = [&]() { while (It != End && (It->isDebugInstr() || It->getOpcode() == W65816::TAX || It->getOpcode() == W65816::TXA)) { - if (It->getOpcode() == W65816::TAX || It->getOpcode() == W65816::TXA) { + if (!It->isDebugInstr()) KillMe.push_back(&*It); - } ++It; } }; + int64_t K = 0; skipDebug(); - if (It == End) return false; - - // 1. LDA_StackRel imm_lo - if (It->getOpcode() != W65816::LDA_StackRel) return false; - if (It->getNumOperands() < 1 || !It->getOperand(0).isImm()) return false; - OffLo = It->getOperand(0).getImm(); - ++It; + if (It == End || !loadLoc(*It, S.Lo)) + return false; + S.LdaLo = &*It++; skipTaxTxa(); - if (It == End) return false; - - // 2. ADCi16imm with imm == 1 - if (It->getOpcode() != W65816::ADCi16imm) return false; - if (It->getNumOperands() < 3 || !It->getOperand(2).isImm()) return false; - if (It->getOperand(2).getImm() != 1) return false; - ++It; + if (It == End || !adcImm(*It, W65816::ADCi16imm, K) || + (K != 1 && K != 0xFFFF)) + return false; + S.IsDec = K == 0xFFFF; + S.AdcLo = &*It++; skipDebug(); - if (It == End) return false; - - // 3. STA_StackRel to same offset - if (It->getOpcode() != W65816::STA_StackRel) return false; - if (It->getNumOperands() < 1 || !It->getOperand(0).isImm()) return false; - if (It->getOperand(0).getImm() != OffLo) return false; - ++It; + if (It == End || !storesTo(*It, S.Lo)) + return false; + S.StaLo = &*It++; skipTaxTxa(); - if (It == End) return false; - - // 4. LDA_StackRel imm_hi (different offset) - if (It->getOpcode() != W65816::LDA_StackRel) return false; - if (It->getNumOperands() < 1 || !It->getOperand(0).isImm()) return false; - OffHi = It->getOperand(0).getImm(); - if (OffHi == OffLo) return false; - ++It; + if (It == End || !loadLoc(*It, S.Hi) || S.Hi == S.Lo) + return false; + S.LdaHi = &*It++; skipDebug(); - if (It == End) return false; - - // 5. ADCEi16imm with imm == 0 - if (It->getOpcode() != W65816::ADCEi16imm) return false; - if (It->getNumOperands() < 3 || !It->getOperand(2).isImm()) return false; - if (It->getOperand(2).getImm() != 0) return false; - ++It; + if (It == End || !adcImm(*It, W65816::ADCEi16imm, K) || + K != (S.IsDec ? 0xFFFF : 0)) + return false; + S.AdcHi = &*It++; skipDebug(); - if (It == End) return false; + if (It == End || !storesTo(*It, S.Hi)) + return false; + S.StaHi = &*It++; + S.End = It; + // The TAX/TXA brackets only ever sat around the increment's stores; a + // decrement that needs them is left alone rather than trusting that. + return !S.IsDec || KillMe.empty(); +} - // 6. STA_StackRel to hi offset - if (It->getOpcode() != W65816::STA_StackRel) return false; - if (It->getNumOperands() < 1 || !It->getOperand(0).isImm()) return false; - if (It->getOperand(0).getImm() != OffHi) return false; - ++It; - PatEnd = It; +// A full 16-bit definition of A that reads nothing: after one of these +// neither the old A nor the old N/Z can be observed. +static bool isFreshALoad(unsigned Opc) { + switch (Opc) { + case W65816::LDA_StackRel: case W65816::LDA_DP: case W65816::LDA_Abs: + case W65816::LDA_Long: case W65816::LDA_Imm16: case W65816::LDAi16imm: + case W65816::LDAabs: case W65816::LDA_DPIndLongY: + case W65816::TXA: case W65816::TYA: case W65816::PLA: + return true; + default: + return false; + } +} + +// Stores and moves the scan may step over: they neither read A nor read +// the flags. +static bool isAAndFlagNeutral(unsigned Opc) { + switch (Opc) { + case W65816::STX_DP: case W65816::STY_DP: case W65816::STZ_DP: + case W65816::STX_Abs: case W65816::STY_Abs: case W65816::LDX_Imm16: + case W65816::LDY_Imm16: case W65816::LDXi16imm: case W65816::BRA: + return true; + default: + return false; + } +} + +// True when A and the N/Z flags left by the sequence are dead: the next +// thing that touches either is a fresh load of A, in this block or, past +// its end, at the head of every successor. +static bool accAndNZDeadFrom(MachineBasicBlock &MBB, + MachineBasicBlock::iterator It, bool InSucc) { + for (; It != MBB.end(); ++It) { + if (It->isDebugInstr()) + continue; + if (isFreshALoad(It->getOpcode())) + return true; + if (!isAAndFlagNeutral(It->getOpcode())) + return false; + } + if (InSucc || MBB.succ_empty()) + return false; + for (MachineBasicBlock *Succ : MBB.successors()) + if (!accAndNZDeadFrom(*Succ, Succ->begin(), true)) + return false; return true; } @@ -153,11 +222,19 @@ bool W65816I32IncFold::runOnMachineFunction(MachineFunction &MF) { for (MachineBasicBlock &MBB : MF) { auto It = MBB.begin(); while (It != MBB.end()) { - int64_t OffLo = 0, OffHi = 0; - MachineBasicBlock::iterator PatEnd; + I32Step S; SmallVector KillMe; - auto Start = It; - if (!matchI32AddOnePattern(It, MBB.end(), OffLo, OffHi, PatEnd, KillMe)) { + if (!matchI32Step(It, MBB.end(), S, KillMe)) { + ++It; + continue; + } + // The high half's carry-out (C, V) must be dead too. Its readers + // (a following ADC/SBC, a BCC/BCS) all model their use of P, so the + // def's dead flag is the proof. + MachineOperand *PDef = + S.AdcHi->findRegisterDefOperand(W65816::P, /*TRI=*/nullptr); + if (!PDef || !PDef->isDead() || + !accAndNZDeadFrom(MBB, S.End, false)) { ++It; continue; } @@ -165,52 +242,34 @@ bool W65816I32IncFold::runOnMachineFunction(MachineFunction &MF) { // the pattern. These are dead because STAfi's Defs=[A] was // a conservative over-approximation; the A-source path preserves // A in the actual asm. - for (MachineInstr *MI : KillMe) MI->eraseFromParent(); - // Found the 6-instruction pattern, [Start, PatEnd). Rewrite - // in-place: keep the LDA_StackRel for lo, replace ADCi16imm - // with INA_PSEUDO, keep STA_StackRel for lo, then replace the - // entire LDA-ADCE-STA hi-half triple with INC_HI_IF_CARRY_StackRel. - DebugLoc DL = Start->getDebugLoc(); - - // Walk to the ADCi16imm (Start+1) and replace. Build a fresh - // INA_PSEUDO with the same tied-def shape: dst=A, src=A. - auto AdcIt = std::next(Start); - while (AdcIt != PatEnd && AdcIt->isDebugInstr()) ++AdcIt; - // INA_PSEUDO has constraint $src = $dst; emit with both as A. - // Operand layout: (outs Acc16:$dst), (ins Acc16:$src) - BuildMI(MBB, AdcIt, DL, TII->get(W65816::INA_PSEUDO), W65816::A) - .addReg(W65816::A); - auto Erased = AdcIt; - ++AdcIt; - Erased->eraseFromParent(); - - // Now find the start of the hi-half triple: it's at Start+3 (after - // skipping debug). Walk past STA_StackRel (lo) which is now at - // AdcIt's position. - while (AdcIt != PatEnd && AdcIt->isDebugInstr()) ++AdcIt; - // AdcIt should now point at STA_StackRel (lo). Skip it. - ++AdcIt; - while (AdcIt != PatEnd && AdcIt->isDebugInstr()) ++AdcIt; - // AdcIt now points at LDA_StackRel (hi) — start of the hi triple. - MachineBasicBlock::iterator HiStart = AdcIt; - - // Insert INC_HI_IF_CARRY_StackRel before the hi triple, then - // erase all three hi instructions. - BuildMI(MBB, HiStart, DL, TII->get(W65816::INC_HI_IF_CARRY_StackRel)) - .addImm(OffHi); - - // Erase the 3 hi instructions: LDA_StackRel, ADCEi16imm, STA_StackRel. - auto KillIt = HiStart; - for (int i = 0; i < 3 && KillIt != PatEnd; ) { - if (KillIt->isDebugInstr()) { ++KillIt; continue; } - auto Next = std::next(KillIt); - KillIt->eraseFromParent(); - KillIt = Next; - ++i; + for (MachineInstr *MI : KillMe) + MI->eraseFromParent(); + DebugLoc DL = S.AdcLo->getDebugLoc(); + if (S.IsDec) { + // lda lo ; DEC_HI_IF_BORROW hi ; dea ; sta lo + BuildMI(MBB, S.AdcLo->getIterator(), DL, + TII->get(S.Hi.IsDP ? W65816::DEC_HI_IF_BORROW_DP + : W65816::DEC_HI_IF_BORROW_StackRel)) + .addImm(S.Hi.Addr); + BuildMI(MBB, S.AdcLo->getIterator(), DL, TII->get(W65816::DEA_PSEUDO), + W65816::A) + .addReg(W65816::A); + } else { + // lda lo ; ina ; sta lo ; INC_HI_IF_CARRY hi + BuildMI(MBB, S.AdcLo->getIterator(), DL, TII->get(W65816::INA_PSEUDO), + W65816::A) + .addReg(W65816::A); + BuildMI(MBB, S.LdaHi->getIterator(), DL, + TII->get(S.Hi.IsDP ? W65816::INC_HI_IF_CARRY_DP + : W65816::INC_HI_IF_CARRY_StackRel)) + .addImm(S.Hi.Addr); } - + S.AdcLo->eraseFromParent(); + S.LdaHi->eraseFromParent(); + S.AdcHi->eraseFromParent(); + S.StaHi->eraseFromParent(); Changed = true; - It = PatEnd; + It = S.End; } } return Changed; diff --git a/src/llvm/lib/Target/W65816/W65816ISelLowering.cpp b/src/llvm/lib/Target/W65816/W65816ISelLowering.cpp index 8b8168a..5711dd6 100644 --- a/src/llvm/lib/Target/W65816/W65816ISelLowering.cpp +++ b/src/llvm/lib/Target/W65816/W65816ISelLowering.cpp @@ -1138,6 +1138,64 @@ SDValue W65816TargetLowering::LowerSignExtend(SDValue Op, return DAG.getNode(ISD::SUB, DL, MVT::i16, Xor, Sign); } +// The abs-16 address of the global a pointer names, plus Extra bytes: a raw +// GlobalAddress / ExternalSymbol, a Wide32 built from Wrapper + +// WrapperBank of the same symbol, or one of those plus a constant. Null for anything else (and for an +// external symbol with Extra, which cannot carry an offset). Our globals +// live in the load segment crt0 pins to DBR, so the DBR-relative abs-16 +// forms (LDA / STA abs) reach them, not the 14-byte [dp],Y path. +static SDValue foldGlobalAddr(SelectionDAG &DAG, const SDLoc &DL, SDValue Ptr, + int64_t Extra = 0) { + auto wrapGlobal = [&](const GlobalAddressSDNode *GA) { + return DAG.getNode(W65816ISD::Wrapper, DL, MVT::i16, + DAG.getTargetGlobalAddress(GA->getGlobal(), DL, + MVT::i16, + GA->getOffset() + Extra)); + }; + auto wrapExtern = [&](const ExternalSymbolSDNode *ES) { + if (Extra != 0) + return SDValue(); + return DAG.getNode(W65816ISD::Wrapper, DL, MVT::i16, + DAG.getTargetExternalSymbol(ES->getSymbol(), + MVT::i16)); + }; + if (auto *GA = dyn_cast(Ptr)) + return wrapGlobal(GA); + if (auto *ES = dyn_cast(Ptr)) + return wrapExtern(ES); + // A field or element: (add global, K), as an i32 load / store's second + // half is before the combiner folds K into the global's offset. + if (Ptr.getOpcode() == ISD::ADD) + if (auto *K = dyn_cast(Ptr.getOperand(1))) + return foldGlobalAddr(DAG, DL, Ptr.getOperand(0), + Extra + K->getSExtValue()); + if (!Ptr.getNode()->isMachineOpcode() || + Ptr.getMachineOpcode() != TargetOpcode::REG_SEQUENCE) + return SDValue(); + SDValue PLo, PHi; + for (unsigned i = 1; i + 1 < Ptr.getNumOperands(); i += 2) { + if (auto *CIdx = dyn_cast(Ptr.getOperand(i + 1))) { + if (CIdx->getZExtValue() == llvm::sub_lo) PLo = Ptr.getOperand(i); + else if (CIdx->getZExtValue() == llvm::sub_hi) PHi = Ptr.getOperand(i); + } + } + if (!PLo || !PHi || PLo.getOpcode() != W65816ISD::Wrapper || + PHi.getOpcode() != W65816ISD::WrapperBank) + return SDValue(); + SDValue WLo = PLo.getOperand(0); + SDValue WHi = PHi.getOperand(0); + auto *GLo = dyn_cast(WLo); + auto *GHi = dyn_cast(WHi); + auto *ELo = dyn_cast(WLo); + auto *EHi = dyn_cast(WHi); + if (GLo && GHi && GLo->getGlobal() == GHi->getGlobal() && + GLo->getOffset() == GHi->getOffset()) + return Extra == 0 ? PLo : wrapGlobal(GLo); + if (ELo && EHi && StringRef(ELo->getSymbol()) == EHi->getSymbol()) + return Extra == 0 ? PLo : SDValue(); + return SDValue(); +} + // ptr32 foundation hook. In ptr16 mode (PointerWidth=16, current // default) addresses are i16 and we return SDValue() so the legalizer // keeps the load and the existing LDAptr / STAptr selection patterns @@ -1214,40 +1272,7 @@ SDValue W65816TargetLowering::LowerLoad(SDValue Op, // emit an abs-16 (DBR-relative) load (LDA / LDA8abs) instead of // the slower [dp],Y indirect-long. Our globals are in the load // segment that crt0 pins to DBR. - SDValue FoldedLo; - if (auto *GA = dyn_cast(Ptr)) { - FoldedLo = DAG.getNode(W65816ISD::Wrapper, DL, MVT::i16, - DAG.getTargetGlobalAddress(GA->getGlobal(), DL, MVT::i16, - GA->getOffset())); - } else if (auto *ES = dyn_cast(Ptr)) { - FoldedLo = DAG.getNode(W65816ISD::Wrapper, DL, MVT::i16, - DAG.getTargetExternalSymbol(ES->getSymbol(), MVT::i16)); - } else if (Ptr.getNode()->isMachineOpcode() && - Ptr.getMachineOpcode() == TargetOpcode::REG_SEQUENCE) { - SDValue PLo, PHi; - for (unsigned i = 1; i + 1 < Ptr.getNumOperands(); i += 2) { - if (auto *CIdx = dyn_cast(Ptr.getOperand(i + 1))) { - if (CIdx->getZExtValue() == llvm::sub_lo) PLo = Ptr.getOperand(i); - else if (CIdx->getZExtValue() == llvm::sub_hi) PHi = Ptr.getOperand(i); - } - } - if (PLo && PHi && - PLo.getOpcode() == W65816ISD::Wrapper && - PHi.getOpcode() == W65816ISD::WrapperBank) { - SDValue WLo = PLo.getOperand(0); - SDValue WHi = PHi.getOperand(0); - auto *GLo = dyn_cast(WLo); - auto *GHi = dyn_cast(WHi); - auto *ELo = dyn_cast(WLo); - auto *EHi = dyn_cast(WHi); - bool SameGlobal = (GLo && GHi && GLo->getGlobal() == GHi->getGlobal() && - GLo->getOffset() == GHi->getOffset()); - bool SameExtern = (ELo && EHi && - StringRef(ELo->getSymbol()) == EHi->getSymbol()); - if (SameGlobal || SameExtern) - FoldedLo = PLo; - } - } + SDValue FoldedLo = foldGlobalAddr(DAG, DL, Ptr); if (FoldedLo) { EVT MemVT = Ld->getMemoryVT(); ISD::LoadExtType ExtType = Ld->getExtensionType(); @@ -1565,6 +1590,21 @@ SDValue W65816TargetLowering::LowerStore(SDValue Op, return StHi; } } + // A global's halves are two abs-16 stores, as its loads are two + // abs-16 loads. Through ST_PTR they were two far-pointer stores, + // each staging `#g` and the program bank from $BE into $E0 / $E2 + // first (~12x the cycles). Plain i16 stores are safe here since + // canMergeStoresTo caps store merging at i16. + SDValue GLo = foldGlobalAddr(DAG, DL, Ptr); + SDValue GHi = foldGlobalAddr(DAG, DL, Ptr, 2); + if (GLo && GHi) { + MachineMemOperand::Flags Fl = St->getMemOperand()->getFlags(); + SDValue StLo = DAG.getStore(Chain, DL, Lo, GLo, St->getPointerInfo(), + St->getAlign(), Fl); + return DAG.getStore(StLo, DL, Hi, GHi, + St->getPointerInfo().getWithOffset(2), + commonAlignment(St->getAlign(), 2), Fl); + } SDValue Two = DAG.getConstant(2, DL, PtrVT); SDValue Ptr2 = DAG.getNode(ISD::ADD, DL, PtrVT, Ptr, Two); if (PtrVT == MVT::i32) { @@ -1607,40 +1647,7 @@ SDValue W65816TargetLowering::LowerStore(SDValue Op, // the load segment that crt0 pins to DBR, so abs-16 reaches them. // This avoids the 14-byte [dp],y indirect-long path AND re-enables // the STZ peephole that the indirect path defeats. - SDValue FoldedLo; - if (auto *GA = dyn_cast(Ptr)) { - FoldedLo = DAG.getNode(W65816ISD::Wrapper, DL, MVT::i16, - DAG.getTargetGlobalAddress(GA->getGlobal(), DL, MVT::i16, - GA->getOffset())); - } else if (auto *ES = dyn_cast(Ptr)) { - FoldedLo = DAG.getNode(W65816ISD::Wrapper, DL, MVT::i16, - DAG.getTargetExternalSymbol(ES->getSymbol(), MVT::i16)); - } else if (Ptr.getNode()->isMachineOpcode() && - Ptr.getMachineOpcode() == TargetOpcode::REG_SEQUENCE) { - SDValue PLo, PHi; - for (unsigned i = 1; i + 1 < Ptr.getNumOperands(); i += 2) { - if (auto *CIdx = dyn_cast(Ptr.getOperand(i + 1))) { - if (CIdx->getZExtValue() == llvm::sub_lo) PLo = Ptr.getOperand(i); - else if (CIdx->getZExtValue() == llvm::sub_hi) PHi = Ptr.getOperand(i); - } - } - if (PLo && PHi && - PLo.getOpcode() == W65816ISD::Wrapper && - PHi.getOpcode() == W65816ISD::WrapperBank) { - SDValue WLo = PLo.getOperand(0); - SDValue WHi = PHi.getOperand(0); - auto *GLo = dyn_cast(WLo); - auto *GHi = dyn_cast(WHi); - auto *ELo = dyn_cast(WLo); - auto *EHi = dyn_cast(WHi); - bool SameGlobal = (GLo && GHi && GLo->getGlobal() == GHi->getGlobal() && - GLo->getOffset() == GHi->getOffset()); - bool SameExtern = (ELo && EHi && - StringRef(ELo->getSymbol()) == EHi->getSymbol()); - if (SameGlobal || SameExtern) - FoldedLo = PLo; - } - } + SDValue FoldedLo = foldGlobalAddr(DAG, DL, Ptr); if (FoldedLo) { // Preserve memVT — original may be a truncating store (e.g., // i16 value into i8 memory). getStore picks memVT from Val's @@ -1959,85 +1966,72 @@ SDValue W65816TargetLowering::LowerShift(SDValue Op, SelectionDAG &DAG) const { SDValue R16 = DAG.getNode(Op.getOpcode(), DL, MVT::i16, X16, N16); return DAG.getNode(ISD::TRUNCATE, DL, MVT::i8, R16); } - // Fast path: shift-by-{1,2,3,4} have inline tablegen patterns. Return - // Op (the unchanged node) so the legalizer leaves it alone — the - // pattern matcher catches it later. Returning SDValue() instead - // would fall through to the generic Expand path, which generates a - // BUILD_VECTOR-based magic-constant rewrite that we can't lower. - // Also allow `(srl x, 15)` through — pattern SRL15A handles it as - // `ASL A; LDA #0; ROL A` (3 bytes), much shorter than the libcall. - // The type-legalizer's i32-shift-by-1 expansion emits this exact - // node for the high-half "bit-from-low" slot. - // Everything else goes to a libcall (__ashlhi3 / __lshrhi3 / __ashrhi3). - // i16 only — i32 always routes to libcall (no inline i32 patterns). + // Constant i16 shifts are inline: tablegen has SHL/SRL patterns for + // every count 1..15 (single-bit chains, XBA-and-mask for 8 and up) and + // SRA patterns for 1, 2 and 15. Returning Op leaves the node for those + // patterns; returning SDValue() would fall into the generic Expand path, + // which builds a BUILD_VECTOR magic-constant rewrite we cannot lower. SDValue Amount = Op.getOperand(1); if (Op.getValueType() == MVT::i16) { if (auto *C = dyn_cast(Amount)) { uint64_t N = C->getZExtValue(); - if ((Op.getOpcode() == ISD::SHL || Op.getOpcode() == ISD::SRL) && - N >= 1 && N <= 14) - return Op; - if (N == 15 && - (Op.getOpcode() == ISD::SRL || Op.getOpcode() == ISD::SHL)) - return Op; - if (N == 1 && Op.getOpcode() == ISD::SRA) - return Op; - if (N == 15 && Op.getOpcode() == ISD::SRA) - return Op; + if (N >= 1 && N <= 15) { + if (Op.getOpcode() != ISD::SRA || N <= 2 || N == 15) + return Op; + // Arithmetic shift by 3..14: shift logically, then sign-extend the + // surviving 16-N bits, ((x >>u N) ^ M) - M with M the new sign + // bit. Eight cycles over the logical shift, where the one-bit + // chain (`cmp #$8000 ; ror a`) costs five per bit and __ashrhi3 + // is a call plus a loop. + SDLoc DL(Op); + SDValue Srl = DAG.getNode(ISD::SRL, DL, MVT::i16, Op.getOperand(0), + Amount); + SDValue M = DAG.getConstant(0x8000u >> N, DL, MVT::i16); + SDValue Flip = DAG.getNode(ISD::XOR, DL, MVT::i16, Srl, M); + return DAG.getNode(ISD::SUB, DL, MVT::i16, Flip, M); + } } } bool IsI32 = Op.getValueType() == MVT::i32; - // Inline i32 shift-by-small-constant. The libcall path is ~140 cyc - // (post-tightening); unrolling N i16 ops plus carry propagation runs - // in ~30-90 cyc. popcount, djb2-style hashes, BigInt-style code, and - // CRC routines all hit this. Larger N falls through to the libcall — - // the unrolled cost grows linearly while the libcall is constant. - // Cutoff at N=5 chosen empirically: djb2's `(h << 5) + h` is the - // common one that benefits. SRA needs an arithmetic-fill shift on - // the high half (i16 SRA by 1 is tablegen-supported); the low half is - // filled from the high's departing bit just like SRL. + // Constant i32 shifts are inline for every count: each is a few i16 + // shifts of the halves, all of them inline (above). The libcalls + // (__ashlsi3 & co.) shift a bit at a time, ~26 cycles a bit plus the + // call, so even the dearest inline form (a count of 7 or 15) wins by a + // wide margin. popcount, djb2-style hashes, CRCs and fixed-point + // arithmetic all hit these. if (IsI32) { if (auto *C = dyn_cast(Amount)) { uint64_t N = C->getZExtValue(); unsigned Op0 = Op.getOpcode(); - if (N >= 1 && N <= 5 && - (Op0 == ISD::SHL || Op0 == ISD::SRL || Op0 == ISD::SRA)) { - SDLoc DL(Op); - SDValue X = Op.getOperand(0); + SDLoc DL(Op); + SDValue X = Op.getOperand(0); + if (N >= 1 && N <= 15) { SDValue Lo = extractWide32Lo(DAG, DL, X); SDValue Hi = extractWide32Hi(DAG, DL, X); SDValue ShN = DAG.getConstant(N, DL, MVT::i16); SDValue ShCo = DAG.getConstant(16 - N, DL, MVT::i16); if (Op0 == ISD::SHL) { // (Hi:Lo) << N == ((Hi << N) | (Lo >> (16-N))) : (Lo << N) - // 4 SDAG ops instead of N iterations of 4 ops. Lets the - // combiner / isel produce ASLA16-cascade + SRL8A+LSRA16- - // cascade + single OR, avoiding the bit-by-bit OR cascade - // that the unrolled form produced. SDValue NewLo = DAG.getNode(ISD::SHL, DL, MVT::i16, Lo, ShN); SDValue HiTop = DAG.getNode(ISD::SRL, DL, MVT::i16, Lo, ShCo); SDValue HiShl = DAG.getNode(ISD::SHL, DL, MVT::i16, Hi, ShN); SDValue NewHi = DAG.getNode(ISD::OR, DL, MVT::i16, HiShl, HiTop); return buildWide32(DAG, DL, NewLo, NewHi); - } else { - // SRL/SRA by N: NewHi = Hi >> N (logical or arithmetic); - // NewLo = (Lo >> N) | (Hi << (16-N)). - SDValue NewHi = DAG.getNode(Op0, DL, MVT::i16, Hi, ShN); - SDValue LoTop = DAG.getNode(ISD::SHL, DL, MVT::i16, Hi, ShCo); - SDValue LoSrl = DAG.getNode(ISD::SRL, DL, MVT::i16, Lo, ShN); - SDValue NewLo = DAG.getNode(ISD::OR, DL, MVT::i16, LoSrl, LoTop); - return buildWide32(DAG, DL, NewLo, NewHi); } + // SRL/SRA by N: NewHi = Hi >> N (logical or arithmetic); + // NewLo = (Lo >> N) | (Hi << (16-N)). + SDValue NewHi = DAG.getNode(Op0, DL, MVT::i16, Hi, ShN); + SDValue LoTop = DAG.getNode(ISD::SHL, DL, MVT::i16, Hi, ShCo); + SDValue LoSrl = DAG.getNode(ISD::SRL, DL, MVT::i16, Lo, ShN); + SDValue NewLo = DAG.getNode(ISD::OR, DL, MVT::i16, LoSrl, LoTop); + return buildWide32(DAG, DL, NewLo, NewHi); } - // By 16 or more one half moves into the other and the rest is at most - // an i16 shift, which the i16 patterns do inline for 1..15 (SRA only - // for 15, its sign fill). `(u16)(x >> 16)`, the high word, was a - // __lshrsi3 call; now it is the high half itself. + // By 16 or more one half moves into the other and the rest is at + // most an i16 shift of it. `(u16)(x >> 16)`, the high word, is the + // high half itself. if (N >= 16 && N <= 31) { - SDLoc DL(Op); - SDValue X = Op.getOperand(0); SDValue Zero = DAG.getConstant(0, DL, MVT::i16); SDValue ShN = DAG.getConstant(N - 16, DL, MVT::i16); if (Op0 == ISD::SHL) { @@ -2046,18 +2040,21 @@ SDValue W65816TargetLowering::LowerShift(SDValue Op, SelectionDAG &DAG) const { N == 16 ? Lo : DAG.getNode(ISD::SHL, DL, MVT::i16, Lo, ShN); return buildWide32(DAG, DL, Zero, NewHi); } + SDValue Hi = extractWide32Hi(DAG, DL, X); if (Op0 == ISD::SRL) { - SDValue Hi = extractWide32Hi(DAG, DL, X); SDValue NewLo = N == 16 ? Hi : DAG.getNode(ISD::SRL, DL, MVT::i16, Hi, ShN); return buildWide32(DAG, DL, NewLo, Zero); } - if (Op0 == ISD::SRA && (N == 16 || N == 31)) { - SDValue Hi = extractWide32Hi(DAG, DL, X); - SDValue Fill = DAG.getNode(ISD::SRA, DL, MVT::i16, Hi, - DAG.getConstant(15, DL, MVT::i16)); - return buildWide32(DAG, DL, N == 16 ? Hi : Fill, Fill); - } + // SRA: the high half shifted arithmetically, and a high word of + // copies of the sign. + SDValue Fill = DAG.getNode(ISD::SRA, DL, MVT::i16, Hi, + DAG.getConstant(15, DL, MVT::i16)); + SDValue NewLo = + N == 16 ? Hi + : N == 31 ? Fill + : DAG.getNode(ISD::SRA, DL, MVT::i16, Hi, ShN); + return buildWide32(DAG, DL, NewLo, Fill); } } } @@ -2070,28 +2067,8 @@ SDValue W65816TargetLowering::LowerShift(SDValue Op, SelectionDAG &DAG) const { default: llvm_unreachable("not a shift"); } - SDValue Val = Op.getOperand(0); - if (IsI32 && Op.getOpcode() == ISD::SHL) { - // Force the high half of the input to be concretely zero when the - // shift count K is >= 16, so bits K..31 of the input are - // mathematically irrelevant. SDAG legalisation can mark those bits - // as `undef` to give the regalloc freedom, but our libcall (a true - // 32-bit shift-and-rotate loop in libgcc.s) reads ALL 32 input - // bits and propagates garbage into the result's low half. Caught - // by dadd via the dpack-inline `(u64 e) << 52` path which split - // into __ashlsi3(e_lo, 20) with X = undef → wrong mantissa. - // For SRL/SRA we'd zero/sign-extend the LOW half similarly when - // K >= 16, but those paths aren't exercising the bug yet. - if (auto *C = dyn_cast(Op.getOperand(1))) { - unsigned K = (unsigned)C->getZExtValue(); - if (K >= 16) { - SDValue Lo = extractWide32Lo(DAG, SDLoc(Op), Val); - SDValue Zero = DAG.getConstant(0, SDLoc(Op), MVT::i16); - Val = buildWide32(DAG, SDLoc(Op), Lo, Zero); - } - } - } - SmallVector Args = {Val, Op.getOperand(1)}; + // Variable counts only: every constant count is inline above. + SmallVector Args = {Op.getOperand(0), Op.getOperand(1)}; TargetLowering::MakeLibCallOptions Opts; Opts.setIsSigned(Op.getOpcode() == ISD::SRA); return makeLibCall(DAG, LC, Op.getValueType(), Args, Opts, SDLoc(Op)).first; diff --git a/src/llvm/lib/Target/W65816/W65816ISelLowering.h b/src/llvm/lib/Target/W65816/W65816ISelLowering.h index 6f0dc69..5cde32f 100644 --- a/src/llvm/lib/Target/W65816/W65816ISelLowering.h +++ b/src/llvm/lib/Target/W65816/W65816ISelLowering.h @@ -148,6 +148,16 @@ public: return TargetLowering::getConstraintType(Constraint); } + // clang marks every `short` / `char` / `bool` return signext or zeroext; + // the default widens such a value to i32 and fills X with its sign or + // zeros, which no caller reads: the ABI returns 16 bits in A alone + // (LLVM_65816_DESIGN.md), and a caller that needs the value wider + // extends it itself. Extend only to A's 16 bits. + EVT getTypeForExtReturn(LLVMContext &Context, EVT VT, + ISD::NodeType ExtendKind) const override { + return VT.bitsLT(MVT::i16) ? EVT(MVT::i16) : VT; + } + // Force i32 / i64 shifts through a libcall (__ashlsi3 / __lshrsi3 / // __ashrsi3) instead of LLVM's default ExpandToParts strategy, which // emits an SHL_PARTS node we have no pattern for. ExpandToParts also diff --git a/src/llvm/lib/Target/W65816/W65816InstrInfo.td b/src/llvm/lib/Target/W65816/W65816InstrInfo.td index 630fb90..7e7b67c 100644 --- a/src/llvm/lib/Target/W65816/W65816InstrInfo.td +++ b/src/llvm/lib/Target/W65816/W65816InstrInfo.td @@ -571,8 +571,7 @@ def LSRA8 : W65816Pseudo<(outs Acc8:$dst), (ins Acc8:$src), [(set Acc8:$dst, (srl Acc8:$src, (i8 1)))]>; // Signed shift right by 1: copy A's high bit into carry, then ROR // to bring it back into A's high bit while halving the rest. The -// AsmPrinter expands this to the 4-instruction PHA;ASL;PLA;ROR -// sequence. +// AsmPrinter expands this to `CMP #$8000 ; ROR A`. def ASRA16 : W65816Pseudo<(outs Acc16:$dst), (ins Acc16:$src), "# ASRA16 $dst, $src", [(set Acc16:$dst, (sra Acc16:$src, (i16 1)))]> { @@ -593,6 +592,9 @@ def : Pat<(shl Acc16:$src, (i16 4)), (ASLA16 (ASLA16 (ASLA16 (ASLA16 Acc16:$src))))>; def : Pat<(srl Acc16:$src, (i16 2)), (LSRA16 (LSRA16 Acc16:$src))>; +// Arithmetic shift by 2: two ASRA16s (10 cycles) beat LowerShift's +// logical-shift-and-sign-extend form (12) only at this count. +def : Pat<(sra Acc16:$src, (i16 2)), (ASRA16 (ASRA16 Acc16:$src))>; def : Pat<(srl Acc16:$src, (i16 3)), (LSRA16 (LSRA16 (LSRA16 Acc16:$src)))>; def : Pat<(srl Acc16:$src, (i16 4)), @@ -1870,11 +1872,37 @@ def EOR_StackRelIndY : InstStackRelIndY<0x53, "eor">; // sta $imm, s // L_skip: let mayLoad = 1, mayStore = 1, hasSideEffects = 0, - Defs = [A] in { + Defs = [A], Size = 7 in { def INC_HI_IF_CARRY_StackRel : W65816Pseudo<(outs), (ins i16imm:$off), "# INC_HI_IF_CARRY_StackRel $off", []>; } +// The same with the high half in the direct page: `bne L ; inc $dp ; L:`. +// A is not touched. +let mayLoad = 1, mayStore = 1, hasSideEffects = 0, Size = 4 in { +def INC_HI_IF_CARRY_DP : W65816Pseudo<(outs), (ins i16imm:$dp), + "# INC_HI_IF_CARRY_DP $dp", []>; +} + +// The borrow of an i32 decrement. W65816I32IncFold emits +// lda lo ; DEC_HI_IF_BORROW hi ; dec a ; sta lo +// so Z comes from the low half BEFORE the decrement: Z=1 means it is +// about to wrap 0 -> $FFFF and the high half must go down by one. +// The stack-relative form expands to +// bne L ; lda $off,s ; dec a ; sta $off,s ; lda #0 ; L: +// and leaves A = 0 on the borrow path, which is the low half it found, +// so the `dec a` that follows sees the same A on both paths. +let mayLoad = 1, mayStore = 1, hasSideEffects = 0, + Defs = [A], Uses = [A], Size = 10 in { +def DEC_HI_IF_BORROW_StackRel : W65816Pseudo<(outs), (ins i16imm:$off), + "# DEC_HI_IF_BORROW_StackRel $off", + []>; +} +// Direct-page form: `bne L ; dec $dp ; L:`, A untouched. +let mayLoad = 1, mayStore = 1, hasSideEffects = 0, Size = 4 in { +def DEC_HI_IF_BORROW_DP : W65816Pseudo<(outs), (ins i16imm:$dp), + "# DEC_HI_IF_BORROW_DP $dp", []>; +} //===----------------------------------------------------------------------===// // Branch patterns (placed after the Bxx defs). diff --git a/src/llvm/lib/Target/W65816/W65816StackRelToImg.cpp b/src/llvm/lib/Target/W65816/W65816StackRelToImg.cpp index f4f2a7b..e6e270b 100644 --- a/src/llvm/lib/Target/W65816/W65816StackRelToImg.cpp +++ b/src/llvm/lib/Target/W65816/W65816StackRelToImg.cpp @@ -53,10 +53,14 @@ #include "llvm/CodeGen/MachineFunction.h" #include "llvm/CodeGen/MachineFunctionPass.h" #include "llvm/CodeGen/MachineInstrBuilder.h" +#include "llvm/CodeGen/MachineLoopInfo.h" +#include "llvm/InitializePasses.h" #include "llvm/IR/DebugInfoMetadata.h" #include "llvm/Support/Debug.h" #include "llvm/Support/Format.h" #include +#include +#include using namespace llvm; @@ -73,6 +77,11 @@ public: StringRef getPassName() const override { return "W65816 stack-rel to IMG DP slot rewrite"; } + void getAnalysisUsage(AnalysisUsage &AU) const override { + AU.addRequired(); + AU.setPreservesCFG(); + MachineFunctionPass::getAnalysisUsage(AU); + } bool runOnMachineFunction(MachineFunction &MF) override; }; @@ -82,9 +91,11 @@ public: char W65816StackRelToImg::ID = 0; -INITIALIZE_PASS(W65816StackRelToImg, DEBUG_TYPE, - "W65816 stack-rel to IMG", false, false) - +INITIALIZE_PASS_BEGIN(W65816StackRelToImg, DEBUG_TYPE, + "W65816 stack-rel to IMG", false, false) +INITIALIZE_PASS_DEPENDENCY(MachineLoopInfoWrapperPass) +INITIALIZE_PASS_END(W65816StackRelToImg, DEBUG_TYPE, + "W65816 stack-rel to IMG", false, false) FunctionPass *llvm::createW65816StackRelToImg() { return new W65816StackRelToImg(); @@ -104,6 +115,18 @@ static bool isStackRelOp(unsigned Opc) { } +// The i32 carry / borrow pseudos W65816I32IncFold emits: their _DP form, or +// 0. Kept apart from the shared stack-rel table: they read AND write the +// slot, and the other users of that table take its entries as loads. +static unsigned carryDpOpcode(unsigned Opc) { + switch (Opc) { + case W65816::INC_HI_IF_CARRY_StackRel: return W65816::INC_HI_IF_CARRY_DP; + case W65816::DEC_HI_IF_BORROW_StackRel: return W65816::DEC_HI_IF_BORROW_DP; + default: return 0; + } +} + + // Whitelist of libgcc functions verified to not touch IMG0..IMG7 ($D0..$DE). static bool isImgSafeCall(const MachineInstr &MI) { if (!MI.isCall()) return false; @@ -700,373 +723,82 @@ static bool elideDeadStaCarry(MachineFunction &MF) { } -bool W65816StackRelToImg::runOnMachineFunction(MachineFunction &MF) { - if (skipFunction(MF.getFunction())) return false; - if (MF.getFunction().hasOptNone()) return false; - - // 1. Bail if the function has VLA / FP-rel addressing — offsets would - // be from FP not SP and the PHP-wrap +1 adjustment differs. - if (MF.getFrameInfo().hasVarSizedObjects()) return false; - - // Always-on peepholes that run even when the main IMG promotion bails. - const W65816Subtarget &STIp = MF.getSubtarget(); - // Run PLA;PHA collapse FIRST so adjacent brackets merge into a - // single multi-pair bracket — lets elidePhaBracket case (c) match - // the merged shape. - bool ChangedEarly = elidePlaPhaPair(MF); - ChangedEarly |= elidePhaBracket(MF, STIp.getInstrInfo()); - ChangedEarly |= elideCallResultSaveSPReload(MF, STIp.getInstrInfo()); - ChangedEarly |= elideDeadStaCarry(MF); - ChangedEarly |= elideRedundantLdaAfterPha(MF); - // elideStoreForwarding only when main IMG promotion would bail — - // running it early in non-bailing functions cascades into IMG-slot - // reallocation that regresses strcpy 1.63×. Gated below. - - // 2. Bail if the function has any non-IMG-safe call (would clobber - // our IMG0..7 promotions) or is recursive (same). Tried allowing - // IMG8..15 + own-pass save/restore for these cases (today, after - // landing W65816LowerWide32 + ImgCalleeSave-writes-only fixes), and - // saw: evalAt 498→500 (NET LOSS due to save/restore overhead) AND - // qsort #70 regression. The IMG8..15 path is not currently a win - // for our benchmarks; reverted. - StringRef SelfName = MF.getName(); - for (MachineBasicBlock &MBB : MF) { - for (MachineInstr &MI : MBB) { - if (!MI.isCall()) continue; - // Check: recursive self-call first. Apply stack-rel dead-store - // elim here since IMG promotion can't run (recursion clobbers - // IMG slots across the inner call). - bool isSelfCall = false; - for (const MachineOperand &MO : MI.operands()) { - StringRef Name; - if (MO.isGlobal()) Name = MO.getGlobal()->getName(); - else if (MO.isSymbol()) Name = MO.getSymbolName(); - else continue; - if (Name == SelfName) { isSelfCall = true; break; } - } - if (isSelfCall) { - ChangedEarly |= elideStackRelDeadStore(MF); - ChangedEarly |= elideStoreForwarding(MF); - return ChangedEarly; - } - if (!isImgSafeCall(MI)) { - ChangedEarly |= elideStoreForwarding(MF); - return ChangedEarly; - } - } - } - uint8_t imgBase = 0xD0u; - - // 3. Count stack-rel accesses per offset. CRITICAL: the stack - // pointer shifts during the function due to PHP/PLP (+1 byte) and - // PHA/PUSH16/PEA/PLA/PLY/PLX/PHX/PHY (±2 bytes). ADJCALLSTACK - // pseudos and JSL/RTL also shift SP. An access at numerical offset - // O when SP has shifted by K maps to LOGICAL offset (O - K). - // - // We compute SpAdj at every program point and use it to canonicalize - // offsets. If our dataflow can't determine SpAdj at some MBB entry - // (multiple predecessors with different shifts), we bail. - // - // Per-MBB analysis: walk forward, tracking running SpAdj. - // Inter-MBB: we require ALL paths to a MBB have the same SpAdj. - auto miSpDelta = [](const MachineInstr &MI) -> int { - switch (MI.getOpcode()) { - case W65816::PHA: case W65816::PHX: case W65816::PHY: - case W65816::PUSH16: case W65816::PEA: - return -2; - case W65816::PLA: case W65816::PLX: case W65816::PLY: - return 2; - case W65816::PHP: - return -1; - case W65816::PLP: - return 1; - default: - break; - } - // ADJCALLSTACK* / TCS / TSC / SEC...SBC...TCS sequences are - // unhandled — they can shift SP arbitrarily. Caller must bail. +// The stack pointer's shift across one instruction, in bytes (negative +// for a push). ADJCALLSTACK* / TCS / TSC / SEC...SBC...TCS sequences are +// not modelled -- they can shift SP arbitrarily; entrySpAdj treats TCS as +// the return to the canonical body SP. +static int spDelta(const MachineInstr &MI) { + switch (MI.getOpcode()) { + case W65816::PHA: case W65816::PHX: case W65816::PHY: + case W65816::PUSH16: case W65816::PEA: + return -2; + case W65816::PLA: case W65816::PLX: case W65816::PLY: + return 2; + case W65816::PHP: + return -1; + case W65816::PLP: + return 1; + default: return 0; - }; - // Fills Entry with every reachable MBB's SpAdj on entry; false when two - // paths reach a block with different shifts. A lambda because the dead - // INC_HI_IF_CARRY sweep near the end needs the same logical offsets - // after the rewrites in between. - auto computeEntrySpAdj = - [&](DenseMap &Entry) -> bool { - Entry.clear(); - Entry[&MF.front()] = 0; - SmallVector Worklist; - Worklist.push_back(&MF.front()); - SmallPtrSet Visited; - while (!Worklist.empty()) { - MachineBasicBlock *MBB = Worklist.pop_back_val(); - if (!Visited.insert(MBB).second) continue; - int Sp = Entry[MBB]; - for (MachineInstr &MI : *MBB) { - // TCS resets SP to the canonical value for the region. In - // well-formed code, the prologue's TCS sets SP for the body - // and the epilogue's TCS undoes it. Since stack-rel accesses - // don't span the prologue/epilogue boundaries (the SP-shifts - // they imply happen ONLY within prologue or epilogue, not - // body), treat TCS as "Sp = 0" (canonical body SP). - if (MI.getOpcode() == W65816::TCS) { - Sp = 0; - continue; - } - Sp += miSpDelta(MI); - } - for (MachineBasicBlock *Succ : MBB->successors()) { - auto It = Entry.find(Succ); - if (It == Entry.end()) { - Entry[Succ] = Sp; - Worklist.push_back(Succ); - } else if (It->second != Sp) { - // Conflicting SP at successor entry -> bail. - return false; - } - } - } - return true; - }; - DenseMap MBBEntrySpAdj; - if (!computeEntrySpAdj(MBBEntrySpAdj)) { - ChangedEarly |= elideStoreForwarding(MF); - return ChangedEarly; } +} - DenseMap AccessCount; - DenseMap ReadCount; - DenseMap WriteCount; - DenseMap> AccessSites; - // Slots that must stay on the stack. *_StackRelIndY (ptr32 Layer 2 - // deref pseudos) read the slot as a POINTER via `(d,S),Y` - the - // addressing mode REQUIRES stack-rel access. INC_HI_IF_CARRY_StackRel - // (W65816I32IncFold's carry into an i32's high half) reads and writes - // its slot but is not one of the stack-rel MC ops the rewrite below - // knows, so promoting its slot left it bumping the abandoned stack copy, - // and the dead-INC sweep further down then erased it outright: every - // `i32 += 1` whose high half got promoted silently lost its carry (a - // counter stuck below 64K, a far pointer that never crossed its bank). - DenseSet PromoteBlocked; - auto isStackRelIndY = [](unsigned Op) { - switch (Op) { - case W65816::LDA_StackRelIndY: case W65816::STA_StackRelIndY: - case W65816::ADC_StackRelIndY: case W65816::SBC_StackRelIndY: - case W65816::CMP_StackRelIndY: case W65816::AND_StackRelIndY: - case W65816::ORA_StackRelIndY: case W65816::EOR_StackRelIndY: - return true; - } - return false; - }; - // Also need to remember the SpAdj at each access site so the rewrite - // can compute the right DP address (DP doesn't shift, but we need - // to know which logical slot the access refers to). - DenseMap SiteSpAdj; - for (MachineBasicBlock &MBB : MF) { - int Sp = MBBEntrySpAdj[&MBB]; - for (MachineInstr &MI : MBB) { + +// Fills Entry with every reachable MBB's SP shift on entry; false when two +// paths reach a block with different shifts. An access at numerical +// offset O where SP has shifted by K is the LOGICAL slot O + K. +static bool entrySpAdj(MachineFunction &MF, + DenseMap &Entry) { + Entry.clear(); + Entry[&MF.front()] = 0; + SmallVector Worklist; + Worklist.push_back(&MF.front()); + SmallPtrSet Visited; + while (!Worklist.empty()) { + MachineBasicBlock *MBB = Worklist.pop_back_val(); + if (!Visited.insert(MBB).second) continue; + int Sp = Entry[MBB]; + for (MachineInstr &MI : *MBB) { + // TCS resets SP to the canonical value for the region. In + // well-formed code, the prologue's TCS sets SP for the body + // and the epilogue's TCS undoes it. Since stack-rel accesses + // don't span the prologue/epilogue boundaries (the SP-shifts + // they imply happen ONLY within prologue or epilogue, not + // body), treat TCS as "Sp = 0" (canonical body SP). if (MI.getOpcode() == W65816::TCS) { Sp = 0; continue; } - if (isStackRelOp(MI.getOpcode())) { - if (MI.getNumOperands() >= 1 && MI.getOperand(0).isImm()) { - int64_t Off = MI.getOperand(0).getImm(); - int64_t LogicalOff = Off + Sp; // canonical (SP-adj subtracted) - AccessCount[LogicalOff]++; - if (MI.getOpcode() == W65816::STA_StackRel) - WriteCount[LogicalOff]++; - else - ReadCount[LogicalOff]++; - AccessSites[LogicalOff].push_back(&MI); - SiteSpAdj[&MI] = Sp; - } - } else if (isStackRelIndY(MI.getOpcode()) || - MI.getOpcode() == W65816::INC_HI_IF_CARRY_StackRel) { - if (MI.getNumOperands() >= 1 && MI.getOperand(0).isImm()) { - int64_t Off = MI.getOperand(0).getImm(); - PromoteBlocked.insert(Off + Sp); - } + Sp += spDelta(MI); + } + for (MachineBasicBlock *Succ : MBB->successors()) { + auto It = Entry.find(Succ); + if (It == Entry.end()) { + Entry[Succ] = Sp; + Worklist.push_back(Succ); + } else if (It->second != Sp) { + // Conflicting SP at successor entry -> bail. + return false; } - Sp += miSpDelta(MI); } } + return true; +} - if (AccessCount.empty()) { - ChangedEarly |= elideStoreForwarding(MF); - return ChangedEarly; - } - // 3b. Scan for existing DP-immediate usage in [imgBase..imgBase+0xE]. - // Regalloc / backend may have already used these slots (via STX_DP / - // STA_DP / LDA_DP) for spills or COPY-via-DP patterns. Don't - // double-use those slots. - auto dpImmInRange = [&](const MachineInstr &MI) -> int { - unsigned Op = MI.getOpcode(); - switch (Op) { - case W65816::LDA_DP: case W65816::STA_DP: - case W65816::ADC_DP: case W65816::SBC_DP: - case W65816::CMP_DP: case W65816::AND_DP: - case W65816::ORA_DP: case W65816::EOR_DP: - case W65816::STX_DP: case W65816::STY_DP: - case W65816::STZ_DP: - if (MI.getNumOperands() >= 1 && MI.getOperand(0).isImm()) { - int64_t I = MI.getOperand(0).getImm(); - if (I >= imgBase && I <= imgBase + 0xE && (I & 1) == 0) - return static_cast((I - imgBase) / 2); // 0..7 - } - break; - } - return -1; - }; - BitVector UsedDp(8, false); - for (MachineBasicBlock &MBB : MF) { - for (MachineInstr &MI : MBB) { - int idx = dpImmInRange(MI); - if (idx >= 0) UsedDp.set(idx); - } - } - - // 4. Sort offsets by access count. Pick top 2. - SmallVector Ordered; - for (auto &P : AccessCount) Ordered.push_back(P.first); - std::sort(Ordered.begin(), Ordered.end(), - [&](int64_t A, int64_t B) { - return AccessCount[A] > AccessCount[B]; - }); - - // 5. Promote top 2 offsets to IMG0 / IMG1. - // - // Heuristic: only promote when the function has FEW (<=4) hot slots. - // Functions with many hot slots tend to be using stack slots as - // SCRATCH temps; promoting them doesn't help and creates many - // STA/LDA pairs to the SAME DP slot that don't optimize well - // (bsearch regression observed at 5+ hot slots). Functions with - // 1-4 hot slots tend to have a long-lived accumulator-like value - // where promotion IS a win (memcmp/strcpy/sumOfSquares observed - // wins at this threshold). - const unsigned kThreshold = 3u; - const unsigned kMaxPromote = 8u; - const unsigned kMaxHotSlots = 16u; - unsigned HotCount = 0; - for (int64_t Off : Ordered) { - if (AccessCount[Off] >= kThreshold) ++HotCount; - } - if (HotCount > kMaxHotSlots) { - ChangedEarly |= elideStoreForwarding(MF); - return ChangedEarly; - } - DenseMap OffsetToDp; // logical offset -> DP byte - unsigned NextDpIdx = 0; - // Caller-passed arg slots live ABOVE the return address on the stack; - // post-frame-alloc, they're at offsets >= frame_size + 4. Don't - // promote them — caller wrote to the stack location, not to DP. - int FrameSize = static_cast(MF.getFrameInfo().getStackSize()); - int ArgSlotMinOff = FrameSize + 4; - for (int64_t Off : Ordered) { - if (AccessCount[Off] < kThreshold) break; - if (OffsetToDp.size() >= kMaxPromote) break; - // Skip arg slots (offset >= frame_size + 4 from canonical SP). - if (Off >= ArgSlotMinOff) continue; - // Skip slots used as Layer 2 ptr32 deref source (their `(d,S),Y` - // uses require stack-rel addressing) or bumped by an - // INC_HI_IF_CARRY_StackRel (see PromoteBlocked). - if (PromoteBlocked.count(Off)) continue; - // Skip already-used DP slots ($D0..$DE). - while (NextDpIdx < 8 && UsedDp.test(NextDpIdx)) ++NextDpIdx; - if (NextDpIdx >= 8) break; - OffsetToDp[Off] = static_cast(imgBase + 2 * NextDpIdx); - ++NextDpIdx; - } - // 6. Rewrite each access site. (OffsetToDp may be empty — phases - // 2-5 still run as standalone peepholes.) - const W65816Subtarget &STI = MF.getSubtarget(); - const W65816InstrInfo *TII = STI.getInstrInfo(); - const TargetRegisterInfo *TRI = STI.getRegisterInfo(); +// After promotion, look for `LDA_DP X ; INA ; STA_DP X` +// (or with DEA) patterns and fold to `INC_DP X` (DEC_DP X). These +// are very common in loop body: `i++` and `counter--` both produce +// LDA-modify-STA round trips. INC dp = 6 cyc, 2 bytes; the chain +// = 10 cyc, 7 bytes. Saves 4 cyc and 5 bytes per occurrence. +// +// Safety: INC/DEC dp set N/Z based on the NEW memory value (same as +// INA/DEA on A). Any consumer expecting N/Z still works. The +// A-clobber DOES differ: after LDA-INA-STA, A holds the new value; +// after INC dp, A is unchanged. So only fire when A is dead after +// the STA (= next instruction redefines A or A is killed). +static bool foldDpIncDec(MachineFunction &MF, const TargetInstrInfo *TII) { bool Changed = false; - - // DBG_VALUE preservation for stack -> DP promotion. - // - // Post-PEI, DBG_VALUEs that point at a stack slot use a - // form. After we promote - // the slot to a DP address, the stack memory at that offset is no - // longer maintained; the live value lives at $00:DpAddr. - // - // The fully-correct rewrite would replace the DBG_VALUE's location - // expression with something like `DW_OP_constu , DW_OP_deref` - // (an absolute-address dereference, since DP addresses live in - // bank 0 by definition). Doing this safely requires DIExpression - // re-uniquing and is sensitive to whether the original expression - // was direct vs indirect. Until our DWARF consumer (pc2line.py - // expanded to a full DIE walker in Phase 3.2) can evaluate the - // resulting expressions, the conservative behaviour is to mark - // matching DBG_VALUEs as undef — the variable drops to - // at the affected PC range rather than reporting - // stale stack memory. - // - // We collect the renamed offsets here; the actual DBG_VALUE walk - // happens once at the end so we don't repeatedly scan for each - // promoted slot. - for (MachineBasicBlock &MBB : MF) { - for (MachineInstr &MI : MBB) { - if (!MI.isDebugValue()) continue; - const DIExpression *Expr = MI.getDebugExpression(); - if (!Expr) continue; - int64_t LeadingOff = 0; - SmallVector Remaining; - if (!Expr->extractLeadingOffset(LeadingOff, Remaining)) - continue; - if (!OffsetToDp.count(LeadingOff)) continue; - for (MachineOperand &Op : MI.debug_operands()) { - if (Op.isReg()) { - Op.setReg(0); - Op.setSubReg(0); - } - } - } - } - - for (auto &P : OffsetToDp) { - int64_t LogicalOff = P.first; - uint8_t DpAddr = P.second; - for (MachineInstr *MI : AccessSites[LogicalOff]) { - unsigned NewOpc = getDpOpcode(MI->getOpcode()); - if (!NewOpc) continue; - MachineBasicBlock *MBB = MI->getParent(); - DebugLoc DL = MI->getDebugLoc(); - MachineInstrBuilder NewMI = - BuildMI(*MBB, MI, DL, TII->get(NewOpc)).addImm(DpAddr); - // Carry implicit operands from the original MI. - for (const MachineOperand &MO : MI->operands()) { - if (MO.isImm()) continue; // the offset, already replaced - if (MO.isReg() && !MO.isImplicit()) { - // shouldn't happen for stack-rel ops (only 1 explicit operand) - continue; - } - if (MO.isReg() && MO.isImplicit()) { - RegState Flags = MO.isDef() ? RegState::ImplicitDefine - : RegState::Implicit; - if (MO.isKill()) Flags = Flags | RegState::Kill; - if (MO.isDead()) Flags = Flags | RegState::Dead; - NewMI.addReg(MO.getReg(), Flags); - } - } - MI->eraseFromParent(); - Changed = true; - } - } - - // Phase 2: after promotion, look for `LDA_DP X ; INA ; STA_DP X` - // (or with DEA) patterns and fold to `INC_DP X` (DEC_DP X). These - // are very common in loop body: `i++` and `counter--` both produce - // LDA-modify-STA round trips. INC dp = 6 cyc, 2 bytes; the chain - // = 10 cyc, 7 bytes. Saves 4 cyc and 5 bytes per occurrence. - // - // Safety: INC/DEC dp set N/Z based on the NEW memory value (same as - // INA/DEA on A). Any consumer expecting N/Z still works. The - // A-clobber DOES differ: after LDA-INA-STA, A holds the new value; - // after INC dp, A is unchanged. So only fire when A is dead after - // the STA (= next instruction redefines A or A is killed). for (MachineBasicBlock &MBB : MF) { SmallVector Dead; for (auto It = MBB.begin(); It != MBB.end(); ++It) { @@ -1160,6 +892,768 @@ bool W65816StackRelToImg::runOnMachineFunction(MachineFunction &MF) { Changed = true; } } + return Changed; +} + + +// DP words of the IMG block ($D0..$DE) anything in the function uses. +// Every direct-page addressing form counts, not just the loads and +// stores: an INC_DP / DEC_DP, a carry pseudo W65816I32IncFold left on a +// DP high half, or a `[dp],Y` pointer is as much a use of the word. A +// byte access marks the word it falls in, and a 24-bit pointer the word +// after too (its bank byte). +static BitVector usedImgWords(MachineFunction &MF, uint8_t ImgBase) { + const TargetInstrInfo *TII = MF.getSubtarget().getInstrInfo(); + BitVector Used(8, false); + auto mark = [&](int64_t I) { + if (I >= ImgBase && I <= ImgBase + 0xF) + Used.set(static_cast((I - ImgBase) / 2)); + }; + for (MachineBasicBlock &MBB : MF) { + for (MachineInstr &MI : MBB) { + StringRef N = TII->getName(MI.getOpcode()); + if (!N.contains("_DP") || MI.getNumOperands() < 1 || + !MI.getOperand(0).isImm()) + continue; + int64_t I = MI.getOperand(0).getImm(); + mark(I); + if (N.contains("_DPIndLong")) + mark(I + 2); + } + } + return Used; +} + + +// A far pointer staged into $E0 / $E2 from two places: stack slots +// (logical offsets) or IMG-range DP words. Weight = staging count. +struct FarPtrPair { + int64_t Lo = 0, Hi = 0; + bool LoIsDp = false, HiIsDp = false; + unsigned Weight = 0; +}; + + +// Scan the $E0 / $E2 staging stores for the slot or DP word the value in A +// came from. Heuristic, and only steers placement: W65816FarPtrStageReuse +// proves the words equal before it addresses a pointer in place. +static SmallVector +collectFarPtrPairs(MachineFunction &MF, + DenseMap &EntrySp, + uint8_t ImgBase, + const SmallPtrSetImpl *Only = nullptr) { + std::map, unsigned> Seen; + auto inImg = [&](int64_t D) { + return D >= ImgBase && D <= ImgBase + 0xE && (D & 1) == 0; + }; + for (MachineBasicBlock &MBB : MF) { + if (Only && !Only->count(&MBB)) + continue; + int Sp = EntrySp.lookup(&MBB); + bool AKnown = false, AIsDp = false; + int64_t AWhere = 0; + bool LoKnown = false, LoIsDp = false; + int64_t LoWhere = 0; + for (MachineInstr &MI : MBB) { + unsigned Opc = MI.getOpcode(); + int64_t Imm = MI.getNumOperands() >= 1 && MI.getOperand(0).isImm() + ? MI.getOperand(0).getImm() + : -1; + if (Opc == W65816::TCS) { + Sp = 0; + AKnown = LoKnown = false; + continue; + } + if ((Opc == W65816::LDA_StackRel || Opc == W65816::STA_StackRel) && + Imm >= 0) { + // A holds (or now also sits in) that slot's word. + AKnown = true; + AIsDp = false; + AWhere = Imm + Sp; + } else if ((Opc == W65816::LDA_DP || Opc == W65816::STA_DP) && + inImg(Imm)) { + AKnown = true; + AIsDp = true; + AWhere = Imm; + } else if (Opc == W65816::STA_DP && Imm == 0xE0) { + LoKnown = AKnown; + LoIsDp = AIsDp; + LoWhere = AWhere; + } else if (Opc == W65816::STA_DP && Imm == 0xE2) { + if (LoKnown && AKnown) + ++Seen[{LoWhere, LoIsDp, AWhere, AIsDp}]; + LoKnown = false; + } else if (Opc != W65816::STA_DP && Opc != W65816::STX_DP && + Opc != W65816::STY_DP && Opc != W65816::STZ_DP && + Opc != W65816::LDY_Imm16 && Opc != W65816::LDX_Imm16) { + // Anything else may change A (the table is short on purpose: a + // missed pair costs a staging, a wrong one nothing). + AKnown = false; + } + Sp += spDelta(MI); + } + } + SmallVector Pairs; + for (auto &KV : Seen) { + FarPtrPair P; + std::tie(P.Lo, P.LoIsDp, P.Hi, P.HiIsDp) = KV.first; + P.Weight = KV.second; + Pairs.push_back(P); + } + std::stable_sort(Pairs.begin(), Pairs.end(), + [](const FarPtrPair &A, const FarPtrPair &B) { + return A.Weight > B.Weight; + }); + return Pairs; +} + + +// Rewrite one stack-relative access (a stack-rel MC op or a carry pseudo) +// to its direct-page form at DpAddr, carrying its implicit operands. +static bool rewriteToDp(MachineInstr &MI, uint8_t DpAddr, + const TargetInstrInfo *TII) { + unsigned NewOpc = getDpOpcode(MI.getOpcode()); + if (!NewOpc) + NewOpc = carryDpOpcode(MI.getOpcode()); + if (!NewOpc) + return false; + MachineBasicBlock *MBB = MI.getParent(); + MachineInstrBuilder NewMI = + BuildMI(*MBB, MI, MI.getDebugLoc(), TII->get(NewOpc)).addImm(DpAddr); + for (const MachineOperand &MO : MI.operands()) { + // The offset is replaced by the DP address; stack-rel ops have no + // other explicit operand. + if (!MO.isReg() || !MO.isImplicit()) + continue; + RegState Flags = MO.isDef() ? RegState::ImplicitDefine + : RegState::Implicit; + if (MO.isKill()) Flags = Flags | RegState::Kill; + if (MO.isDead()) Flags = Flags | RegState::Dead; + NewMI.addReg(MO.getReg(), Flags); + } + MI.eraseFromParent(); + return true; +} + + +static bool isStackRelIndYOp(unsigned Op) { + switch (Op) { + case W65816::LDA_StackRelIndY: case W65816::STA_StackRelIndY: + case W65816::ADC_StackRelIndY: case W65816::SBC_StackRelIndY: + case W65816::CMP_StackRelIndY: case W65816::AND_StackRelIndY: + case W65816::ORA_StackRelIndY: case W65816::EOR_StackRelIndY: + return true; + default: + return false; + } +} + + +// A full 16-bit write of A that reads nothing (in M=16): after one, the +// old A and the old N / Z are gone. +static bool isFreshALoad(unsigned Op) { + switch (Op) { + case W65816::LDA_StackRel: case W65816::LDA_DP: case W65816::LDA_Abs: + case W65816::LDA_Long: case W65816::LDA_Imm16: case W65816::LDAi16imm: + case W65816::LDAabs: case W65816::LDA_DPIndLongY: + case W65816::TXA: case W65816::TYA: case W65816::TSC: case W65816::TDC: + return true; + default: + return false; + } +} + + +// Instructions that neither touch A nor read the flags. +static bool isANeutral(unsigned Op) { + switch (Op) { + case W65816::LDX_Imm16: case W65816::LDY_Imm16: case W65816::LDXi16imm: + case W65816::STX_DP: case W65816::STY_DP: case W65816::STX_Abs: + case W65816::STY_Abs: + return true; + default: + return false; + } +} + + +// True when A and the flags are dead where MBB begins: the first thing +// that touches either writes A afresh, or is a call that takes nothing in +// A (a call clobbers both; its argument registers are implicit uses). +static bool aDeadAtEntry(MachineBasicBlock &MBB) { + for (MachineInstr &MI : MBB) { + if (MI.isDebugInstr()) + continue; + if (isFreshALoad(MI.getOpcode()) || + (MI.isCall() && !MI.readsRegister(W65816::A, /*TRI=*/nullptr))) + return true; + if (!isANeutral(MI.getOpcode())) + return false; + } + return false; +} + + +// Loop-local promotion, for a function the whole-function promotion +// below cannot touch because it makes calls: any call may clobber +// IMG0..7. An innermost loop that makes none can still keep its hottest +// slots -- far pointers first, into adjacent words -- in IMG words the +// function never uses: copied in at the end of the preheader, and the +// ones the loop writes copied back at the start of each exit. The copies +// run once per entry into the loop; inside it every such access is a DP +// one (4 cycles, not 5), a counter's LDA / INA / STA folds to INC dp, and +// a far pointer in adjacent words is used in place by +// W65816FarPtrStageReuse instead of being staged into $E0 / $E2 on every +// access. (JoeyLib's sfx probe inlines its reference copy loop into +// main, which calls: the whole-function promotion never reached it.) +// +// Conservative throughout: a loop with a preheader and dedicated exits; +// nothing in it that calls, moves SP by more than a push, addresses the +// stack as a pointer, or runs in a frame reached through $F6; a slot no +// neighbour's access overlaps; the copy-in where A is dead and nothing +// later in the preheader writes the slot; exits whose A and flags are dead +// on entry. Anything else leaves the loop alone. +static bool promoteLoopLocal(MachineFunction &MF, MachineLoopInfo &MLI, + const TargetInstrInfo *TII, uint8_t ImgBase) { + // A plain slot pays its copies back at a cycle per access per trip; a + // far pointer's pair also drops its staging (about 17 cycles per + // access), so pairs go regardless. Same threshold as the + // whole-function promotion below. + const unsigned kLoopThreshold = 3u; + DenseMap EntrySp; + if (!entrySpAdj(MF, EntrySp)) + return false; + // Logical slots accessed anywhere (a candidate's neighbours must not + // be: a 16-bit access at S +- 1 overlaps S), and the frame-pointer test. + DenseSet AllSlots; + for (MachineBasicBlock &MBB : MF) { + int Sp = EntrySp.lookup(&MBB); + for (MachineInstr &MI : MBB) { + unsigned Op = MI.getOpcode(); + if (Op == W65816::TCS) { + Sp = 0; + continue; + } + bool HasImm = MI.getNumOperands() >= 1 && MI.getOperand(0).isImm(); + if (HasImm && TII->getName(Op).contains("_DPInd") && + MI.getOperand(0).getImm() >= 0xF6 && + MI.getOperand(0).getImm() <= 0xF8) + return false; // slots reached through the $F6 frame pointer + if (HasImm && (isStackRelOp(Op) || carryDpOpcode(Op) || + isStackRelIndYOp(Op))) + AllSlots.insert(MI.getOperand(0).getImm() + Sp); + Sp += spDelta(MI); + } + } + BitVector Used = usedImgWords(MF, ImgBase); + SmallVector Worklist(MLI.begin(), MLI.end()); + SmallVector Innermost; + while (!Worklist.empty()) { + MachineLoop *L = Worklist.pop_back_val(); + if (L->getSubLoops().empty()) + Innermost.push_back(L); + else + Worklist.append(L->begin(), L->end()); + } + bool Changed = false; + for (MachineLoop *L : Innermost) { + MachineBasicBlock *Pre = L->getLoopPreheader(); + if (!Pre || !EntrySp.count(Pre)) + continue; + SmallVector Exits; + L->getUniqueExitBlocks(Exits); + bool Ok = !Exits.empty(); + for (MachineBasicBlock *E : Exits) { + if (!EntrySp.count(E) || !aDeadAtEntry(*E)) + Ok = false; + for (MachineBasicBlock *P : E->predecessors()) + if (!L->contains(P)) + Ok = false; + } + // The loop's accesses, by logical slot. + DenseMap> Sites; + DenseSet Written; + for (MachineBasicBlock *B : L->blocks()) { + if (!Ok) + break; + int Sp = EntrySp.lookup(B); + for (MachineInstr &MI : *B) { + unsigned Op = MI.getOpcode(); + if (MI.isCall() || MI.isInlineAsm() || isStackRelIndYOp(Op) || + Op == W65816::TCS || Op == W65816::TSC || Op == W65816::TSX || + Op == W65816::TXS || Op == W65816::TCD || Op == W65816::PLD || + Op == W65816::ADDframe || Op == W65816::ALLOCAfi) { + Ok = false; + break; + } + if ((isStackRelOp(Op) || carryDpOpcode(Op)) && + MI.getNumOperands() >= 1 && MI.getOperand(0).isImm()) { + int64_t Slot = MI.getOperand(0).getImm() + Sp; + Sites[Slot].push_back(&MI); + if (Op == W65816::STA_StackRel || carryDpOpcode(Op)) + Written.insert(Slot); + } + Sp += spDelta(MI); + } + } + if (!Ok || Sites.empty()) + continue; + + // The copy-in point: before the preheader's terminators when A is not + // live into the header and the preheader does not end on a flag + // branch, else right before its last fresh load of A. Nothing after + // it in the preheader may write a slot (checked once homes exist). + MachineBasicBlock::iterator InsPt = Pre->getFirstTerminator(); + bool EndsOnBranch = InsPt != Pre->end() && InsPt->isConditionalBranch(); + if (EndsOnBranch || !aDeadAtEntry(*L->getHeader())) { + InsPt = Pre->end(); + for (auto It = Pre->getFirstTerminator(); It != Pre->begin();) { + --It; + // Logical offsets are measured from the body's SP, which a TCS + // (the prologue's, in the entry block) establishes: never go + // above one. + if (It->getOpcode() == W65816::TCS) + break; + if (isFreshALoad(It->getOpcode())) { + InsPt = It; + break; + } + } + if (InsPt == Pre->end()) + continue; + } + // SP shift and M width at the copy-in point. + int SpIns = EntrySp.lookup(Pre); + bool M16 = true; + for (auto It = Pre->begin(); It != InsPt; ++It) { + if ((It->getOpcode() == W65816::SEP || It->getOpcode() == W65816::REP) && + It->getOperand(0).isImm() && (It->getOperand(0).getImm() & 0x20)) + M16 = It->getOpcode() == W65816::REP; + if (It->getOpcode() == W65816::TCS) + SpIns = 0; // as entrySpAdj: the body's SP from here on + else + SpIns += spDelta(*It); + } + if (!M16) + continue; + + // Homes: far-pointer pairs first, side by side, then the hottest + // slots. A home is taken for the loop only: innermost loops never + // overlap in time and each copies its own homes in and out, so the + // next loop may use the same words (an exit's copy-out sits at the + // start of its block, before any copy-in a following loop puts there). + DenseMap Home; + BitVector Taken = Used; + auto usable = [&](int64_t S) { + if (!Sites.count(S) || Home.count(S) || AllSlots.count(S - 1) || + AllSlots.count(S + 1)) + return false; + return true; + }; + SmallPtrSet Blocks(L->block_begin(), + L->block_end()); + for (const FarPtrPair &FP : collectFarPtrPairs(MF, EntrySp, ImgBase, + &Blocks)) { + if (FP.LoIsDp || FP.HiIsDp || FP.Lo == FP.Hi || !usable(FP.Lo) || + !usable(FP.Hi)) + continue; + for (unsigned Idx = 0; Idx + 1 < 8; ++Idx) { + if (!Taken.test(Idx) && !Taken.test(Idx + 1)) { + Home[FP.Lo] = static_cast(ImgBase + 2 * Idx); + Home[FP.Hi] = static_cast(ImgBase + 2 * Idx + 2); + Taken.set(Idx); + Taken.set(Idx + 1); + break; + } + } + } + SmallVector Hot; + for (auto &KV : Sites) + if (KV.second.size() >= kLoopThreshold) + Hot.push_back(KV.first); + llvm::sort(Hot, [&](int64_t A, int64_t B) { + if (Sites[A].size() != Sites[B].size()) + return Sites[A].size() > Sites[B].size(); + return A < B; + }); + for (int64_t S : Hot) { + if (!usable(S)) + continue; + int Idx = Taken.find_first_unset(); + if (Idx < 0) + break; + Home[S] = static_cast(ImgBase + 2 * Idx); + Taken.set(Idx); + } + // Nothing after the copy-in point may write a home's slot. + int Sp = SpIns; + for (auto It = InsPt; It != Pre->end(); ++It) { + if (It->getOpcode() == W65816::STA_StackRel && + It->getOperand(0).isImm() && + Home.count(It->getOperand(0).getImm() + Sp)) + Home.clear(); + Sp = It->getOpcode() == W65816::TCS ? 0 : Sp + spDelta(*It); + } + // The copies address the slots from where they sit: the offsets must + // fit stack-relative addressing there. + for (auto &KV : Home) { + int64_t Off = KV.first - SpIns; + bool Fits = Off >= 0 && Off <= 0xFF; + for (MachineBasicBlock *E : Exits) { + int64_t OffE = KV.first - EntrySp.lookup(E); + if (Written.count(KV.first) && (OffE < 0 || OffE > 0xFF)) + Fits = false; + } + if (!Fits) + Home.clear(); + } + if (Home.empty()) + continue; + + // Copy in, rewrite the loop, copy back. + DebugLoc DL; + for (auto &KV : Home) { + BuildMI(*Pre, InsPt, DL, TII->get(W65816::LDA_StackRel)) + .addImm(KV.first - SpIns) + .addReg(W65816::A, RegState::ImplicitDefine); + BuildMI(*Pre, InsPt, DL, TII->get(W65816::STA_DP)) + .addImm(KV.second) + .addReg(W65816::A, RegState::Implicit); + } + for (auto &KV : Home) + for (MachineInstr *MI : Sites[KV.first]) + rewriteToDp(*MI, KV.second, TII); + for (MachineBasicBlock *E : Exits) { + int SpE = EntrySp.lookup(E); + auto At = E->begin(); + for (auto &KV : Home) { + if (!Written.count(KV.first)) + continue; + BuildMI(*E, At, DL, TII->get(W65816::LDA_DP)) + .addImm(KV.second) + .addReg(W65816::A, RegState::ImplicitDefine); + BuildMI(*E, At, DL, TII->get(W65816::STA_StackRel)) + .addImm(KV.first - SpE) + .addReg(W65816::A, RegState::Implicit); + } + } + // Debug values in the loop that name a promoted slot would read the + // stale stack copy: drop their location, as the whole-function + // promotion does. + for (MachineBasicBlock *B : L->blocks()) + for (MachineInstr &MI : *B) { + if (!MI.isDebugValue()) + continue; + const DIExpression *Expr = MI.getDebugExpression(); + int64_t Lead = 0; + SmallVector Rest; + if (!Expr || !Expr->extractLeadingOffset(Lead, Rest) || + !Home.count(Lead)) + continue; + for (MachineOperand &Op : MI.debug_operands()) + if (Op.isReg()) { + Op.setReg(0); + Op.setSubReg(0); + } + } + Changed = true; + } + if (Changed) + foldDpIncDec(MF, TII); + return Changed; +} + + +bool W65816StackRelToImg::runOnMachineFunction(MachineFunction &MF) { + if (skipFunction(MF.getFunction())) return false; + if (MF.getFunction().hasOptNone()) return false; + + // 1. Bail if the function has VLA / FP-rel addressing — offsets would + // be from FP not SP and the PHP-wrap +1 adjustment differs. + if (MF.getFrameInfo().hasVarSizedObjects()) return false; + + // Always-on peepholes that run even when the main IMG promotion bails. + const W65816Subtarget &STIp = MF.getSubtarget(); + // Run PLA;PHA collapse FIRST so adjacent brackets merge into a + // single multi-pair bracket — lets elidePhaBracket case (c) match + // the merged shape. + bool ChangedEarly = elidePlaPhaPair(MF); + ChangedEarly |= elidePhaBracket(MF, STIp.getInstrInfo()); + ChangedEarly |= elideCallResultSaveSPReload(MF, STIp.getInstrInfo()); + ChangedEarly |= elideDeadStaCarry(MF); + ChangedEarly |= elideRedundantLdaAfterPha(MF); + // elideStoreForwarding only when main IMG promotion would bail — + // running it early in non-bailing functions cascades into IMG-slot + // reallocation that regresses strcpy 1.63×. Gated below. + + // 2. Bail if the function has any non-IMG-safe call (would clobber + // our IMG0..7 promotions) or is recursive (same). Tried allowing + // IMG8..15 + own-pass save/restore for these cases (today, after + // landing W65816LowerWide32 + ImgCalleeSave-writes-only fixes), and + // saw: evalAt 498→500 (NET LOSS due to save/restore overhead) AND + // qsort #70 regression. The IMG8..15 path is not currently a win + // for our benchmarks; reverted. + StringRef SelfName = MF.getName(); + for (MachineBasicBlock &MBB : MF) { + for (MachineInstr &MI : MBB) { + if (!MI.isCall()) continue; + // Check: recursive self-call first. Apply stack-rel dead-store + // elim here since IMG promotion can't run (recursion clobbers + // IMG slots across the inner call). + bool isSelfCall = false; + for (const MachineOperand &MO : MI.operands()) { + StringRef Name; + if (MO.isGlobal()) Name = MO.getGlobal()->getName(); + else if (MO.isSymbol()) Name = MO.getSymbolName(); + else continue; + if (Name == SelfName) { isSelfCall = true; break; } + } + MachineLoopInfo &MLI = + getAnalysis().getLI(); + if (isSelfCall) { + ChangedEarly |= elideStackRelDeadStore(MF); + ChangedEarly |= elideStoreForwarding(MF); + ChangedEarly |= promoteLoopLocal(MF, MLI, STIp.getInstrInfo(), 0xD0u); + return ChangedEarly; + } + if (!isImgSafeCall(MI)) { + ChangedEarly |= elideStoreForwarding(MF); + ChangedEarly |= promoteLoopLocal(MF, MLI, STIp.getInstrInfo(), 0xD0u); + return ChangedEarly; + } + } + } + uint8_t imgBase = 0xD0u; + + // 3. Count stack-rel accesses per offset. CRITICAL: the stack + // pointer shifts during the function due to PHP/PLP (+1 byte) and + // PHA/PUSH16/PEA/PLA/PLY/PLX/PHX/PHY (±2 bytes). ADJCALLSTACK + // pseudos and JSL/RTL also shift SP. An access at numerical offset + // O when SP has shifted by K maps to LOGICAL offset (O - K). + // + // We compute SpAdj at every program point and use it to canonicalize + // offsets. If our dataflow can't determine SpAdj at some MBB entry + // (multiple predecessors with different shifts), we bail. + // + // Per-MBB analysis: walk forward, tracking running SpAdj. + // Inter-MBB: we require ALL paths to a MBB have the same SpAdj. + DenseMap MBBEntrySpAdj; + if (!entrySpAdj(MF, MBBEntrySpAdj)) { + ChangedEarly |= elideStoreForwarding(MF); + return ChangedEarly; + } + + DenseMap AccessCount; + DenseMap> AccessSites; + // Slots that must stay on the stack. *_StackRelIndY (ptr32 Layer 2 + // deref pseudos) read the slot as a POINTER via `(d,S),Y` - the + // addressing mode REQUIRES stack-rel access. + // + // INC_HI_IF_CARRY_StackRel / DEC_HI_IF_BORROW_StackRel (W65816I32IncFold's + // carry / borrow into an i32's high half) read and write their slot. + // They are counted as accesses like the MC ops and rewritten to their _DP + // forms with the rest of the slot's sites (carryDpOpcode below). They + // used to block promotion instead, which kept the high half of every + // incremented far pointer on the stack; before THAT, promoting the slot + // without rewriting them left them bumping the abandoned stack copy, and + // the dead-INC sweep further down then erased them outright: every `i32 + // += 1` whose high half got promoted silently lost its carry (a counter + // stuck below 64K, a far pointer that never crossed its bank). + DenseSet PromoteBlocked; + auto isStackRelIndY = [](unsigned Op) { + switch (Op) { + case W65816::LDA_StackRelIndY: case W65816::STA_StackRelIndY: + case W65816::ADC_StackRelIndY: case W65816::SBC_StackRelIndY: + case W65816::CMP_StackRelIndY: case W65816::AND_StackRelIndY: + case W65816::ORA_StackRelIndY: case W65816::EOR_StackRelIndY: + return true; + } + return false; + }; + // Also need to remember the SpAdj at each access site so the rewrite + // can compute the right DP address (DP doesn't shift, but we need + // to know which logical slot the access refers to). + DenseMap SiteSpAdj; + for (MachineBasicBlock &MBB : MF) { + int Sp = MBBEntrySpAdj[&MBB]; + for (MachineInstr &MI : MBB) { + if (MI.getOpcode() == W65816::TCS) { + Sp = 0; + continue; + } + if (isStackRelOp(MI.getOpcode()) || carryDpOpcode(MI.getOpcode())) { + if (MI.getNumOperands() >= 1 && MI.getOperand(0).isImm()) { + int64_t Off = MI.getOperand(0).getImm(); + int64_t LogicalOff = Off + Sp; // canonical (SP-adj subtracted) + AccessCount[LogicalOff]++; + AccessSites[LogicalOff].push_back(&MI); + SiteSpAdj[&MI] = Sp; + } + } else if (isStackRelIndY(MI.getOpcode())) { + if (MI.getNumOperands() >= 1 && MI.getOperand(0).isImm()) { + int64_t Off = MI.getOperand(0).getImm(); + PromoteBlocked.insert(Off + Sp); + } + } + Sp += spDelta(MI); + } + } + + if (AccessCount.empty()) { + ChangedEarly |= elideStoreForwarding(MF); + return ChangedEarly; + } + + // 3b. DP words ($D0..$DE) regalloc / the backend already use (via + // STX_DP / STA_DP / LDA_DP spills, COPY-via-DP, ...): don't double-use + // them (see usedImgWords). + BitVector UsedDp = usedImgWords(MF, imgBase); + + // 4. Sort offsets by access count. Pick top 2. + SmallVector Ordered; + for (auto &P : AccessCount) Ordered.push_back(P.first); + std::sort(Ordered.begin(), Ordered.end(), + [&](int64_t A, int64_t B) { + return AccessCount[A] > AccessCount[B]; + }); + + // 5. Promote top 2 offsets to IMG0 / IMG1. + // + // Heuristic: only promote when the function has FEW (<=4) hot slots. + // Functions with many hot slots tend to be using stack slots as + // SCRATCH temps; promoting them doesn't help and creates many + // STA/LDA pairs to the SAME DP slot that don't optimize well + // (bsearch regression observed at 5+ hot slots). Functions with + // 1-4 hot slots tend to have a long-lived accumulator-like value + // where promotion IS a win (memcmp/strcpy/sumOfSquares observed + // wins at this threshold). + const unsigned kThreshold = 3u; + const unsigned kMaxPromote = 8u; + const unsigned kMaxHotSlots = 16u; + unsigned HotCount = 0; + for (int64_t Off : Ordered) { + if (AccessCount[Off] >= kThreshold) ++HotCount; + } + if (HotCount > kMaxHotSlots) { + ChangedEarly |= elideStoreForwarding(MF); + return ChangedEarly; + } + DenseMap OffsetToDp; // logical offset -> DP byte + BitVector Taken = UsedDp; // DP words spoken for + // Caller-passed arg slots live ABOVE the return address on the stack; + // post-frame-alloc, they're at offsets >= frame_size + 4. Don't + // promote them — caller wrote to the stack location, not to DP. + int FrameSize = static_cast(MF.getFrameInfo().getStackSize()); + int ArgSlotMinOff = FrameSize + 4; + // Slots Layer 2 ptr32 derefs read as a POINTER (their `(d,S),Y` uses + // require stack-rel addressing) stay put (see PromoteBlocked). + auto promotable = [&](int64_t Off) { + return AccessCount.count(Off) && Off < ArgSlotMinOff && + !PromoteBlocked.count(Off) && !OffsetToDp.count(Off); + }; + auto assign = [&](int64_t Off, unsigned Idx) { + OffsetToDp[Off] = static_cast(imgBase + 2 * Idx); + Taken.set(Idx); + }; + + // 5a. Far pointers first, into adjacent words. The pointer inserters + // stage a far pointer's low word into $E0 and its bank word into $E2 + // before every [dp],Y access; with both words side by side in the + // direct page, W65816FarPtrStageReuse addresses the pointer where it + // lives and drops the staging (two loads and two stores per access, 17 + // cycles). The pairs come from the staging stores; a half that is + // already a DP word anchors its partner next to it. + for (const FarPtrPair &FP : collectFarPtrPairs(MF, MBBEntrySpAdj, imgBase)) { + if (OffsetToDp.size() >= kMaxPromote) + break; + if (!FP.LoIsDp && !FP.HiIsDp) { + if (FP.Lo == FP.Hi || !promotable(FP.Lo) || !promotable(FP.Hi) || + OffsetToDp.size() + 2 > kMaxPromote || + AccessCount[FP.Lo] + AccessCount[FP.Hi] < kThreshold) + continue; + for (unsigned Idx = 0; Idx + 1 < 8; ++Idx) { + if (!Taken.test(Idx) && !Taken.test(Idx + 1)) { + assign(FP.Lo, Idx); + assign(FP.Hi, Idx + 1); + break; + } + } + } else if (FP.LoIsDp && !FP.HiIsDp) { + int Idx = static_cast((FP.Lo - imgBase) / 2) + 1; + if (Idx < 8 && !Taken.test(Idx) && promotable(FP.Hi)) + assign(FP.Hi, Idx); + } else if (!FP.LoIsDp && FP.HiIsDp) { + int Idx = static_cast((FP.Hi - imgBase) / 2) - 1; + if (Idx >= 0 && !Taken.test(Idx) && promotable(FP.Lo)) + assign(FP.Lo, Idx); + } + } + // 5b. Then the hottest remaining slots, one word each. + for (int64_t Off : Ordered) { + if (AccessCount[Off] < kThreshold) break; + if (OffsetToDp.size() >= kMaxPromote) break; + if (!promotable(Off)) continue; + int Idx = Taken.find_first_unset(); + if (Idx < 0) break; + assign(Off, static_cast(Idx)); + } + // 6. Rewrite each access site. (OffsetToDp may be empty — phases + // 2-5 still run as standalone peepholes.) + const W65816Subtarget &STI = MF.getSubtarget(); + const W65816InstrInfo *TII = STI.getInstrInfo(); + const TargetRegisterInfo *TRI = STI.getRegisterInfo(); + bool Changed = false; + + // DBG_VALUE preservation for stack -> DP promotion. + // + // Post-PEI, DBG_VALUEs that point at a stack slot use a + // form. After we promote + // the slot to a DP address, the stack memory at that offset is no + // longer maintained; the live value lives at $00:DpAddr. + // + // The fully-correct rewrite would replace the DBG_VALUE's location + // expression with something like `DW_OP_constu , DW_OP_deref` + // (an absolute-address dereference, since DP addresses live in + // bank 0 by definition). Doing this safely requires DIExpression + // re-uniquing and is sensitive to whether the original expression + // was direct vs indirect. Until our DWARF consumer (pc2line.py + // expanded to a full DIE walker in Phase 3.2) can evaluate the + // resulting expressions, the conservative behaviour is to mark + // matching DBG_VALUEs as undef — the variable drops to + // at the affected PC range rather than reporting + // stale stack memory. + // + // We collect the renamed offsets here; the actual DBG_VALUE walk + // happens once at the end so we don't repeatedly scan for each + // promoted slot. + for (MachineBasicBlock &MBB : MF) { + for (MachineInstr &MI : MBB) { + if (!MI.isDebugValue()) continue; + const DIExpression *Expr = MI.getDebugExpression(); + if (!Expr) continue; + int64_t LeadingOff = 0; + SmallVector Remaining; + if (!Expr->extractLeadingOffset(LeadingOff, Remaining)) + continue; + if (!OffsetToDp.count(LeadingOff)) continue; + for (MachineOperand &Op : MI.debug_operands()) { + if (Op.isReg()) { + Op.setReg(0); + Op.setSubReg(0); + } + } + } + } + + for (auto &P : OffsetToDp) { + int64_t LogicalOff = P.first; + uint8_t DpAddr = P.second; + for (MachineInstr *MI : AccessSites[LogicalOff]) + Changed |= rewriteToDp(*MI, DpAddr, TII); + } + + // Phase 2: `LDA_DP X ; INA ; STA_DP X` -> `INC_DP X` (see foldDpIncDec). + Changed |= foldDpIncDec(MF, TII); // Phase 3: epilogue TAY/TYA bracket elimination when the bracketed // load is from DP (which survives TCS). Pattern: @@ -1589,6 +2083,7 @@ bool W65816StackRelToImg::runOnMachineFunction(MachineFunction &MF) { if (isStackRelOp(Fwd->getOpcode())) { StackRelOps.push_back(Fwd); } else if (Fwd->getOpcode() == W65816::INC_HI_IF_CARRY_StackRel || + Fwd->getOpcode() == W65816::DEC_HI_IF_BORROW_StackRel || Fwd->getOpcode() == W65816::PHA || Fwd->getOpcode() == W65816::PHX || Fwd->getOpcode() == W65816::PHY || @@ -2882,7 +3377,7 @@ bool W65816StackRelToImg::runOnMachineFunction(MachineFunction &MF) { if (TakesStackAddr) break; } DenseMap SweepEntrySp; - if (!TakesStackAddr && computeEntrySpAdj(SweepEntrySp)) { + if (!TakesStackAddr && entrySpAdj(MF, SweepEntrySp)) { DenseSet IncSlots; DenseSet ReadSlots; // INC_HI_IF_CARRY_StackRel and STA_StackRel sites, with their @@ -2903,13 +3398,16 @@ bool W65816StackRelToImg::runOnMachineFunction(MachineFunction &MF) { Writes.push_back({&MI, L}); } else if (Op == W65816::STA_StackRel) { Writes.push_back({&MI, L}); - } else if (isStackRelOp(Op) || isStackRelIndY(Op)) { + } else if (isStackRelOp(Op) || isStackRelIndY(Op) || + Op == W65816::DEC_HI_IF_BORROW_StackRel) { + // A DEC_HI_IF_BORROW_StackRel's slot counts as read: it is + // never erased by this sweep, so it must keep its stores. ReadSlots.insert(L - 1); ReadSlots.insert(L); ReadSlots.insert(L + 1); } } - Sp += miSpDelta(MI); + Sp += spDelta(MI); } } for (auto &W : Writes) { diff --git a/src/llvm/lib/Target/W65816/W65816StackSlotCleanup.cpp b/src/llvm/lib/Target/W65816/W65816StackSlotCleanup.cpp index b309a79..2b0c8d1 100644 --- a/src/llvm/lib/Target/W65816/W65816StackSlotCleanup.cpp +++ b/src/llvm/lib/Target/W65816/W65816StackSlotCleanup.cpp @@ -31,6 +31,7 @@ #include "W65816Subtarget.h" #include "llvm/ADT/BitVector.h" #include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/DenseSet.h" #include "llvm/ADT/SmallPtrSet.h" #include "llvm/CodeGen/MachineFrameInfo.h" #include "llvm/CodeGen/MachineFunction.h" @@ -1106,6 +1107,49 @@ bool W65816StackSlotCleanup::runOnMachineFunction(MachineFunction &MF) { continue; } } + // Compare through X instead of saving the flags, when X is free from + // the test to the end of the block and not live into a successor: + // cmp #k ; php ; ; plp ; bxx 10 cycles, 5 bytes + // tax ; ; cpx #k ; bxx 5 cycles, 4 bytes + // CPX sets N, Z and C from X - k exactly as CMP did from A - k, and + // the corrupters (loads and transfers into A) leave X alone. Only a + // 16-bit compare, and only with no SEP/REP of the index width in the + // way (an 8-bit X would drop the high byte). + if (TestsNZ && Test->getOpcode() == W65816::CMPi16imm && + Test->getOperand(0).isReg() && + Test->getOperand(0).getReg() == W65816::A && + Test->getOperand(1).isImm()) { + bool XFree = true; + for (auto It = Test->getIterator(); It != MBB.end() && XFree; ++It) { + if (It->isDebugInstr()) + continue; + for (const MachineOperand &MO : It->operands()) + if (MO.isReg() && MO.getReg() == W65816::X) + XFree = false; + if ((It->getOpcode() == W65816::SEP || + It->getOpcode() == W65816::REP) && + (!It->getOperand(0).isImm() || + (It->getOperand(0).getImm() & 0x10))) + XFree = false; + if (It->isCall()) + XFree = false; + } + for (MachineBasicBlock *Succ : MBB.successors()) + if (Succ->isLiveIn(W65816::X)) + XFree = false; + if (XFree) { + const TargetInstrInfo *TII = MF.getSubtarget().getInstrInfo(); + DebugLoc DL = Test->getDebugLoc(); + BuildMI(MBB, Test->getIterator(), DL, TII->get(W65816::TAX)); + BuildMI(MBB, Br->getIterator(), DL, TII->get(W65816::CPX_Imm16)) + .addImm(Test->getOperand(1).getImm()) + .addReg(W65816::X, RegState::Implicit) + .addReg(W65816::P, RegState::ImplicitDefine); + Test->eraseFromParent(); + Changed = true; + continue; + } + } // Stack-relative ops inside the wrap need one more byte of // displacement to compensate for PHP's S decrement. Without it, // `lda 5,s` between PHP and PLP reads at (orig_S-1)+5 = orig_S+4, @@ -1651,11 +1695,41 @@ bool W65816StackSlotCleanup::runOnMachineFunction(MachineFunction &MF) { // Pass 1c can only eliminate CMPi16imm $a, 0 if the preceding // A-modifier reliably sets N/Z to reflect A's final value. LDAfi // under FP-rel expansion (`sty $fa ; ldy #imm ; lda [$f6],y ; ldy $fa`) - // ends with `ldy` that clobbers N/Z based on OLD Y, not loaded A — so - // in FP-rel functions (VLA / huge frame), the CMP is load-bearing. - // Skip the whole pass for such functions (saves us from the sum_n - // VLA regression that the PHP-wrap-aware variant tripped). + // ends with `ldy` that clobbers N/Z based on OLD Y, not loaded A — and + // so do the other frame-index pseudos — so in FP-rel functions (VLA / + // huge frame) a CMP after one is load-bearing. The PHP-wrap-aware + // variant is off for such functions (the sum_n VLA regression it + // tripped), and a frame-index writer keeps its CMP there: with the + // loop's `cmp #0` moved past its back-edge copies to sit right after + // the counter's reload (W65816CmpSink), sum_n's second loop branched + // on the restored Y. bool ssCleanupSPRelOnly = !UsesFPRel; + auto hasFrameIndex = [](const MachineInstr &MI) { + for (const MachineOperand &MO : MI.operands()) + if (MO.isFI()) + return true; + return false; + }; + // Whether any frame-index access can take the $F6 form: FrameLowering + // sets $F6 up only for a VLA or a final frame over 200 bytes. The final + // frame holds at most the objects instructions still reference (plus + // alignment); estimateStackSize, behind UsesFPRel, also counts the dead + // staging slots every far-pointer access creates, and keeping `cmp #0` + // on its say-so cost JoeyLib's sprite paths 3 cycles a test. + bool FrameMayUseFP = MF.getFrameInfo().hasVarSizedObjects(); + if (!FrameMayUseFP) { + const MachineFrameInfo &MFI = MF.getFrameInfo(); + DenseSet Seen; + uint64_t Bytes = 0; + for (MachineBasicBlock &MBB : MF) + for (MachineInstr &MI : MBB) + for (const MachineOperand &MO : MI.operands()) + if (MO.isFI() && !MFI.isFixedObjectIndex(MO.getIndex()) && + Seen.insert(MO.getIndex()).second) + Bytes += MFI.getObjectSize(MO.getIndex()) + + MFI.getObjectAlign(MO.getIndex()).value() - 1; + FrameMayUseFP = Bytes > 200; + } for (MachineBasicBlock &MBB : MF) { SmallVector Cmps; for (MachineInstr &MI : MBB) @@ -1672,6 +1746,7 @@ bool W65816StackSlotCleanup::runOnMachineFunction(MachineFunction &MF) { const MachineInstr *Writer = aWriterBefore(Cmp->getIterator(), MBB.begin()); if (!Writer) continue; + if (FrameMayUseFP && hasFrameIndex(*Writer)) continue; // Only eliminate if there are NO LdaLike instructions between // this CMP and the next Bxx (or end of MBB). Otherwise the // CMP is the only flag-setting marker between the test value @@ -2000,7 +2075,6 @@ bool W65816StackSlotCleanup::runOnMachineFunction(MachineFunction &MF) { if (SlotB == NO_SLOT_MATCH || SlotB == SlotA) { continue; } - MachineInstr *Lda2 = &*It; ++It; if (!advancePastDebug(MBB, It)) { continue; @@ -2008,7 +2082,6 @@ bool W65816StackSlotCleanup::runOnMachineFunction(MachineFunction &MF) { if (It->getOpcode() != W65816::STA_DP) { continue; } - MachineInstr *Sta1 = &*It; ++It; if (!advancePastDebug(MBB, It)) { continue; diff --git a/src/llvm/lib/Target/W65816/W65816StackSlotMerge.cpp b/src/llvm/lib/Target/W65816/W65816StackSlotMerge.cpp index 789dbed..afcb9fc 100644 --- a/src/llvm/lib/Target/W65816/W65816StackSlotMerge.cpp +++ b/src/llvm/lib/Target/W65816/W65816StackSlotMerge.cpp @@ -204,6 +204,40 @@ static bool slotNotWrittenBetween(MachineBasicBlock::iterator Start, } +// True when something between Start and End (both exclusive, same MBB) +// may read the word at slot Slot. A 16-bit access one byte either side +// overlaps it, and inside a PHP/PLP wrap the same slot is spelled one +// byte higher, so the neighbours count as well. +static bool slotReadBetween(MachineBasicBlock::iterator Start, + MachineBasicBlock::iterator End, int64_t Slot) { + for (auto It = std::next(Start); It != End; ++It) { + if (It->isDebugInstr() || It->getOpcode() == W65816::STA_StackRel) + continue; + if (It->isCall() || It->isInlineAsm()) + return true; + if (It->getNumOperands() < 1 || !It->getOperand(0).isImm()) + continue; + int64_t Off = It->getOperand(0).getImm(); + bool StackUse = isStackRelOp(It->getOpcode()) || + It->getOpcode() == W65816::INC_HI_IF_CARRY_StackRel || + It->getOpcode() == W65816::DEC_HI_IF_BORROW_StackRel; + switch (It->getOpcode()) { + case W65816::LDA_StackRelIndY: case W65816::STA_StackRelIndY: + case W65816::ADC_StackRelIndY: case W65816::SBC_StackRelIndY: + case W65816::CMP_StackRelIndY: case W65816::AND_StackRelIndY: + case W65816::ORA_StackRelIndY: case W65816::EOR_StackRelIndY: + StackUse = true; + break; + default: + break; + } + if (StackUse && Off >= Slot - 1 && Off <= Slot + 1) + return true; + } + return false; +} + + // Returns true if MI clobbers P (N/Z/C/V flags). Mirrors LLVM's // operand-based check + an opcode whitelist for tablegen entries that // omit `Defs = [P]` (InstImplied, InstStackRel, etc.). @@ -292,6 +326,16 @@ static MachineInstr *findTwin(MachineInstr *StaX, } if (!Earlier || !Later) continue; int64_t EOff = Earlier->getOperand(0).getImm(); + // Merged, the two slots are one: the Earlier store then also + // overwrites the Later store's slot, early. A read of that slot in + // between would see the new value instead of the old one. The + // pointer walk `q = p + 1; ... *p ...; p = q` stores q first and reads + // p before the copy: merged, *p read q (strcpy copied every string + // one byte short). Only a PHP/PLP wrap around the copy, which bumps + // its offsets past recognition, used to keep this shape out. + if (slotReadBetween(Earlier->getIterator(), Later->getIterator(), + Later->getOperand(0).getImm())) + continue; // Case (2): if Later is preceded by `LDA_StackRel ` // (the PHI-copy reload), it's a PHI twin. Also require slot // Earlier-slot wasn't re-written between Earlier and Later. @@ -469,14 +513,16 @@ bool W65816StackSlotMerge::runOnMachineFunction(MachineFunction &MF) { DenseMap> Stas; DenseMap> Ldas; DenseMap AllRefs; // STA + LDA + ADC + ... count - // Slots an INC_HI_IF_CARRY_StackRel bumps. That pseudo writes its slot - // without being an STA (so the twin check below can't see the write) - // and is not a stack-rel MC op (so the rename would skip it): merging - // such a slot would leave the carry landing in the abandoned one. + // Slots an INC_HI_IF_CARRY_StackRel or DEC_HI_IF_BORROW_StackRel bumps. + // Those pseudos write their slot without being an STA (so the twin check + // below can't see the write) and are not stack-rel MC ops (so the rename + // would skip them): merging such a slot would leave the carry landing in + // the abandoned one. DenseSet IncSlots; for (MachineBasicBlock &MBB : MF) { for (MachineInstr &MI : MBB) { - if (MI.getOpcode() == W65816::INC_HI_IF_CARRY_StackRel && + if ((MI.getOpcode() == W65816::INC_HI_IF_CARRY_StackRel || + MI.getOpcode() == W65816::DEC_HI_IF_BORROW_StackRel) && MI.getNumOperands() >= 1 && MI.getOperand(0).isImm()) IncSlots.insert(MI.getOperand(0).getImm()); int64_t Off; diff --git a/src/llvm/lib/Target/W65816/W65816TargetMachine.cpp b/src/llvm/lib/Target/W65816/W65816TargetMachine.cpp index 39e7f98..da93672 100644 --- a/src/llvm/lib/Target/W65816/W65816TargetMachine.cpp +++ b/src/llvm/lib/Target/W65816/W65816TargetMachine.cpp @@ -60,6 +60,7 @@ LLVMInitializeW65816Target() { initializeW65816NegYIndYPass(PR); initializeW65816SjLjFinalizePass(PR); initializeW65816LowerWide32Pass(PR); + initializeW65816CmpSinkPass(PR); initializeW65816I32IncFoldPass(PR); initializeW65816ImgCalleeSavePass(PR); initializeW65816NarrowI32MulPass(PR); @@ -238,6 +239,10 @@ void W65816PassConfig::addPreRegAlloc() { // pass walks Acc16/Idx16/Img16 vregs; running this first means they // see the decomposed halves uniformly. addPass(createW65816LowerWide32()); + // Keep each compare next to its conditional branch once PHI elimination + // has put the back-edge copies between them (see W65816CmpSink.cpp). + if (getOptLevel() != CodeGenOptLevel::None) + insertPass(&RegisterCoalescerID, &W65816CmpSinkID); addPass(createW65816ABridgeViaX()); addPass(createW65816TiedDefSpill()); addPass(createW65816WidenAcc16()); diff --git a/src/llvm/test/CodeGen/W65816/addframe-flag-wrap.mir b/src/llvm/test/CodeGen/W65816/addframe-flag-wrap.mir index 0ec754e..2f839e2 100644 --- a/src/llvm/test/CodeGen/W65816/addframe-flag-wrap.mir +++ b/src/llvm/test/CodeGen/W65816/addframe-flag-wrap.mir @@ -13,6 +13,10 @@ # The BCC case pins the second half of the fix: ADDframe corrupts C, so a # carry branch needs the wrap too (LDA-like ops never did, and still don't). # +# The branch targets take X live in, so the wrap cannot be replaced by the +# TAX ... CPX #k form that StackSlotCleanup uses when X is free (pinned in +# cmp-via-x.mir); these cases keep testing the PHP / PLP wrap itself. +# # RUN: llc -mtriple=w65816 -run-pass=w65816-stack-slot-cleanup -o - %s | FileCheck %s # RUN: llc -mtriple=w65816 -run-pass=w65816-stack-slot-cleanup,prologepilog -o - %s | FileCheck %s --check-prefix=PEI --- | @@ -49,8 +53,10 @@ body: | BEQ %bb.1, implicit $p BRA %bb.2 bb.1: + liveins: $x RTL bb.2: + liveins: $x RTL ... --- @@ -74,8 +80,10 @@ body: | BCC %bb.1, implicit $p BRA %bb.2 bb.1: + liveins: $x RTL bb.2: + liveins: $x RTL ... --- @@ -97,7 +105,9 @@ body: | BCC %bb.1, implicit $p BRA %bb.2 bb.1: + liveins: $x RTL bb.2: + liveins: $x RTL ... diff --git a/src/llvm/test/CodeGen/W65816/andi8-flag-corrupt-wrap.mir b/src/llvm/test/CodeGen/W65816/andi8-flag-corrupt-wrap.mir index 87b6415..d767a1f 100644 --- a/src/llvm/test/CodeGen/W65816/andi8-flag-corrupt-wrap.mir +++ b/src/llvm/test/CodeGen/W65816/andi8-flag-corrupt-wrap.mir @@ -30,6 +30,10 @@ # The wrapped reload keeps its offset: eliminateFrameIndex adds the byte # PHP pushed (php-wrap-frame-index.mir). # +# The branch targets take X live in, so the wrap cannot be replaced by the +# TAX ... CPX #k form that StackSlotCleanup uses when X is free (pinned in +# cmp-via-x.mir); these cases keep testing the PHP / PLP wrap itself. +# # RUN: llc -mtriple=w65816 -run-pass=w65816-stack-slot-cleanup -o - %s | FileCheck %s --- | target triple = "w65816" @@ -60,8 +64,10 @@ body: | BEQ %bb.1, implicit $p BRA %bb.2 bb.1: + liveins: $x RTL bb.2: + liveins: $x RTL ... --- @@ -87,8 +93,10 @@ body: | BNE %bb.1, implicit $p BRA %bb.2 bb.1: + liveins: $x RTL bb.2: + liveins: $x RTL ... --- @@ -109,7 +117,9 @@ body: | BEQ %bb.1, implicit $p BRA %bb.2 bb.1: + liveins: $x RTL bb.2: + liveins: $x RTL ... diff --git a/src/llvm/test/CodeGen/W65816/cmp-sink.ll b/src/llvm/test/CodeGen/W65816/cmp-sink.ll new file mode 100644 index 0000000..eb8319b --- /dev/null +++ b/src/llvm/test/CodeGen/W65816/cmp-sink.ll @@ -0,0 +1,47 @@ +; Pin: W65816CmpSink moves the compare of a BEQ / BNE / BMI / BPL down past +; the back-edge COPYs that PHI elimination put between them, so the copies +; (LDA / STA after allocation) land above the compare and the flags need no +; PHP / PLP wrap. `while (p != end) s += *p++;` on 16-bit elements: before +; the pass both latch compares (the low and the high half of the pointer) +; were wrapped. A BCC / BCS keeps its place: loads leave C alone, and the +; end of a CMP / SBC carry chain is better left next to its first half +; (cmp-prepare.ll's postinc_ult pins that shape). +; +; RUN: llc -mtriple=w65816 -O2 -print-after=w65816-cmp-sink -o /dev/null < %s 2>&1 | FileCheck %s --check-prefix=MIR +; RUN: llc -mtriple=w65816 -O2 < %s | FileCheck %s + +target datalayout = "e-m:e-p:32:16-i16:16-i32:16-i64:16-f32:16-f64:16-a:8-n8:16-S16" + +define i16 @sum16(ptr %p, ptr %end) { +; MIR-LABEL: Machine code for function sum16 +; MIR: ADCfi +; MIR: COPY +; MIR-NEXT: COPY +; MIR-NEXT: COPY +; MIR-NEXT: CMPfi +; MIR-NEXT: BNE +; MIR: COPY +; MIR-NEXT: COPY +; MIR-NEXT: COPY +; MIR-NEXT: CMPfi +; MIR-NEXT: BNE +; CHECK-LABEL: sum16: +; CHECK-NOT: php +; CHECK: rtl +entry: + %cmp.not3 = icmp eq ptr %p, %end + br i1 %cmp.not3, label %while.end, label %while.body + +while.body: + %s.05 = phi i16 [ %add, %while.body ], [ 0, %entry ] + %p.addr.04 = phi ptr [ %incdec.ptr, %while.body ], [ %p, %entry ] + %incdec.ptr = getelementptr inbounds nuw i8, ptr %p.addr.04, i32 2 + %0 = load i16, ptr %p.addr.04, align 2 + %add = add i16 %0, %s.05 + %cmp.not = icmp eq ptr %incdec.ptr, %end + br i1 %cmp.not, label %while.end, label %while.body + +while.end: + %s.0.lcssa = phi i16 [ 0, %entry ], [ %add, %while.body ] + ret i16 %s.0.lcssa +} diff --git a/src/llvm/test/CodeGen/W65816/cmp-via-x.mir b/src/llvm/test/CodeGen/W65816/cmp-via-x.mir new file mode 100644 index 0000000..936dde9 --- /dev/null +++ b/src/llvm/test/CodeGen/W65816/cmp-via-x.mir @@ -0,0 +1,189 @@ +# Pin: when a load into A sits between a `cmp #k` and the BEQ / BNE / BMI / +# BPL that reads it, W65816StackSlotCleanup compares through X instead of +# saving the flags: `tax ; ; cpx #k` (5 cycles, 4 bytes) in place +# of `cmp #k ; php ; ; plp` (10 cycles, 5 bytes). CPX sets N, Z +# and C from X - k as CMP did from A - k. It needs X free from the test to +# the end of the block and not live into a successor. +# +# via_x: X free: TAX / CPX, no wrap. +# x_live_out: X live into a branch target: the wrap stays. +# x_in_region: the corrupter reads X (TXA): the wrap stays. +# stack_operand: `cmp d,s` has no CPX form: the wrap stays. +# +# W65816FarPtrStageReuse, the last pass, folds what later deletions leave +# empty (fold_*): `tax ; cpx #k` back into `cmp #k`, and `php ; plp` away. +# +# RUN: llc -mtriple=w65816 -run-pass=w65816-stack-slot-cleanup -o - %s | FileCheck %s +# RUN: llc -mtriple=w65816 -run-pass=w65816-farptr-reuse -o - %s | FileCheck %s --check-prefix=FOLD +--- | + target triple = "w65816" + define void @via_x() { ret void } + define void @x_live_out() { ret void } + define void @x_in_region() { ret void } + define void @stack_operand() { ret void } + define void @fold_tax_cpx() { ret void } + define void @fold_keeps_x_read() { ret void } + define void @fold_php_plp() { ret void } +... +--- +# CHECK-LABEL: name: via_x +# CHECK: $a = LDAfi %stack.0, 0 +# CHECK-NEXT: TAX +# CHECK-NEXT: $a = LDAi16imm 7 +# CHECK-NEXT: STAfi $a, %stack.1, 0 +# CHECK-NEXT: CPX_Imm16 3, implicit $x, implicit-def $p +# CHECK-NEXT: BNE %bb.1 +# CHECK-NOT: PHP +name: via_x +tracksRegLiveness: false +stack: + - { id: 0, size: 2, alignment: 1 } + - { id: 1, size: 2, alignment: 1 } +body: | + bb.0: + successors: %bb.1, %bb.2 + $a = LDAfi %stack.0, 0 + CMPi16imm killed $a, 3, implicit-def $p + $a = LDAi16imm 7 + STAfi $a, %stack.1, 0, implicit-def $a + BNE %bb.1, implicit $p + BRA %bb.2 + bb.1: + RTL + bb.2: + RTL +... +--- +# CHECK-LABEL: name: x_live_out +# CHECK: CMPi16imm killed $a, 3, implicit-def $p +# CHECK-NEXT: PHP +# CHECK-NEXT: $a = LDAi16imm 7 +# CHECK-NEXT: PLP +# CHECK-NEXT: BEQ %bb.1 +name: x_live_out +tracksRegLiveness: false +stack: + - { id: 0, size: 2, alignment: 1 } +body: | + bb.0: + successors: %bb.1, %bb.2 + $a = LDAfi %stack.0, 0 + CMPi16imm killed $a, 3, implicit-def $p + $a = LDAi16imm 7 + BEQ %bb.1, implicit $p + BRA %bb.2 + bb.1: + liveins: $x + RTL + bb.2: + RTL +... +--- +# CHECK-LABEL: name: x_in_region +# CHECK: CMPi16imm killed $a, 3, implicit-def $p +# CHECK-NEXT: PHP +# CHECK-NEXT: TXA +# CHECK-NEXT: PLP +# CHECK-NEXT: BEQ %bb.1 +name: x_in_region +tracksRegLiveness: false +stack: + - { id: 0, size: 2, alignment: 1 } +body: | + bb.0: + successors: %bb.1, %bb.2 + $a = LDAfi %stack.0, 0 + CMPi16imm killed $a, 3, implicit-def $p + TXA implicit-def $a, implicit $x + BEQ %bb.1, implicit $p + BRA %bb.2 + bb.1: + RTL + bb.2: + RTL +... +--- +# CHECK-LABEL: name: stack_operand +# CHECK: CMPfi $a, %stack.1, 0, implicit-def $p +# CHECK-NEXT: PHP +# CHECK-NEXT: $a = LDAi16imm 7 +# CHECK-NEXT: PLP +# CHECK-NEXT: BEQ %bb.1 +name: stack_operand +tracksRegLiveness: false +stack: + - { id: 0, size: 2, alignment: 1 } + - { id: 1, size: 2, alignment: 1 } +body: | + bb.0: + successors: %bb.1, %bb.2 + $a = LDAfi %stack.0, 0 + CMPfi $a, %stack.1, 0, implicit-def $p + $a = LDAi16imm 7 + BEQ %bb.1, implicit $p + BRA %bb.2 + bb.1: + RTL + bb.2: + RTL +... +--- +# FOLD-LABEL: name: fold_tax_cpx +# FOLD: LDA_StackRel 1 +# FOLD-NEXT: CMP_Imm16 3, implicit $a, implicit-def $p +# FOLD-NEXT: BNE %bb.1 +name: fold_tax_cpx +tracksRegLiveness: false +body: | + bb.0: + successors: %bb.1, %bb.2 + LDA_StackRel 1, implicit-def $a + TAX implicit-def $x, implicit $a + CPX_Imm16 3, implicit $x, implicit-def $p + BNE %bb.1, implicit $p + BRA %bb.2 + bb.1: + RTL + bb.2: + RTL +... +--- +# FOLD-LABEL: name: fold_keeps_x_read +# FOLD: TAX +# FOLD-NEXT: CPX_Imm16 3 +name: fold_keeps_x_read +tracksRegLiveness: false +body: | + bb.0: + successors: %bb.1, %bb.2 + LDA_StackRel 1, implicit-def $a + TAX implicit-def $x, implicit $a + CPX_Imm16 3, implicit $x, implicit-def $p + BNE %bb.1, implicit $p + BRA %bb.2 + bb.1: + liveins: $x + RTL + bb.2: + RTL +... +--- +# FOLD-LABEL: name: fold_php_plp +# FOLD: CMP_Imm16 3 +# FOLD-NEXT: BEQ %bb.1 +name: fold_php_plp +tracksRegLiveness: false +body: | + bb.0: + successors: %bb.1, %bb.2 + LDA_StackRel 1, implicit-def $a + CMP_Imm16 3, implicit-def $p + PHP + PLP + BEQ %bb.1, implicit $p + BRA %bb.2 + bb.1: + RTL + bb.2: + RTL +... diff --git a/src/llvm/test/CodeGen/W65816/cmp0-fprel-keep.mir b/src/llvm/test/CodeGen/W65816/cmp0-fprel-keep.mir new file mode 100644 index 0000000..6b92961 --- /dev/null +++ b/src/llvm/test/CodeGen/W65816/cmp0-fprel-keep.mir @@ -0,0 +1,60 @@ +# Pin: W65816StackSlotCleanup's Pass 1c keeps a `cmp #0` whose A comes +# from a frame-index load in a frame that addresses its slots through the +# $F6 frame pointer (a VLA, or a frame over 200 bytes). There LDAfi +# expands to `sty $fa ; ldy #off ; lda [$f6],y ; ldy $fa`, and the final +# LDY leaves N / Z describing Y, not the loaded value. Erased, the smoke +# run's sum_n(3) VLA loop branched on the restored Y and returned 1; the +# `cmp #0` only came to sit right after the load once W65816CmpSink moved +# the loop test past its back-edge copies. +# +# vla_keeps: variable-sized frame: the CMP stays. +# sp_drops: the same load in an ordinary frame: the CMP goes. +# +# RUN: llc -mtriple=w65816 -run-pass=w65816-stack-slot-cleanup -o - %s | FileCheck %s +--- | + target triple = "w65816" + define void @vla_keeps() { ret void } + define void @sp_drops() { ret void } +... +--- +# CHECK-LABEL: name: vla_keeps +# CHECK: $a = LDAfi %stack.1, 0 +# CHECK-NEXT: CMPi16imm $a, 0 +# CHECK-NEXT: BEQ +name: vla_keeps +tracksRegLiveness: false +stack: + - { id: 0, type: variable-sized, offset: 0, alignment: 1 } + - { id: 1, size: 2, alignment: 1 } +body: | + bb.0: + successors: %bb.1, %bb.2 + $a = LDAfi %stack.1, 0 + CMPi16imm $a, 0, implicit-def $p + BEQ %bb.2, implicit $p + BRA %bb.1 + bb.1: + RTL + bb.2: + RTL +... +--- +# CHECK-LABEL: name: sp_drops +# CHECK: $a = LDAfi %stack.0, 0 +# CHECK-NEXT: BEQ +name: sp_drops +tracksRegLiveness: false +stack: + - { id: 0, size: 2, alignment: 1 } +body: | + bb.0: + successors: %bb.1, %bb.2 + $a = LDAfi %stack.0, 0 + CMPi16imm $a, 0, implicit-def $p + BEQ %bb.2, implicit $p + BRA %bb.1 + bb.1: + RTL + bb.2: + RTL +... diff --git a/src/llvm/test/CodeGen/W65816/extract-wide32-regseq.ll b/src/llvm/test/CodeGen/W65816/extract-wide32-regseq.ll index fb0f536..0433085 100644 --- a/src/llvm/test/CodeGen/W65816/extract-wide32-regseq.ll +++ b/src/llvm/test/CodeGen/W65816/extract-wide32-regseq.ll @@ -16,8 +16,8 @@ define void @clear_i32(ptr %p) { ; CHECK-LABEL: clear_i32: -; CHECK: sta [0xe0 -; CHECK: sta [0xe0 +; CHECK: sta [0x{{[0-9a-f]+}}], y +; CHECK: sta [0x{{[0-9a-f]+}}], y ; CHECK: rtl store i32 0, ptr %p ret void @@ -25,10 +25,10 @@ define void @clear_i32(ptr %p) { define void @clear_i32_pair(ptr %p, ptr %q) { ; CHECK-LABEL: clear_i32_pair: -; CHECK: sta [0xe0 -; CHECK: sta [0xe0 -; CHECK: sta [0xe0 -; CHECK: sta [0xe0 +; CHECK: sta [0x{{[0-9a-f]+}}], y +; CHECK: sta [0x{{[0-9a-f]+}}], y +; CHECK: sta [0x{{[0-9a-f]+}}], y +; CHECK: sta [0x{{[0-9a-f]+}}], y ; CHECK: rtl store i32 0, ptr %p store i32 0, ptr %q diff --git a/src/llvm/test/CodeGen/W65816/far-ptr-home.ll b/src/llvm/test/CodeGen/W65816/far-ptr-home.ll new file mode 100644 index 0000000..de26f9d --- /dev/null +++ b/src/llvm/test/CodeGen/W65816/far-ptr-home.ll @@ -0,0 +1,46 @@ +; Pin: in a leaf function W65816StackRelToImg puts each far pointer's low +; and bank words in adjacent IMG direct-page words, and bumps an i32 high +; half there too (INC_HI_IF_CARRY_DP), so W65816FarPtrStageReuse reads and +; writes through the pointers in place: no $E0 / $E2 staging in the loop. +; JoeyLib's old sfxCopyUnsigned loop (the refCopy of the IIgs sfx probe): +; ~30 of its ~166 cycles a byte were the two pointers' restaging. +; +; RUN: llc -mtriple=w65816 -O2 < %s | FileCheck %s + +target datalayout = "e-m:e-p:32:16-i16:16-i32:16-i64:16-f32:16-f64:16-a:8-n8:16-S16" + +define void @refCopy(ptr %dst, ptr %src, i32 %length) { +; CHECK-LABEL: refCopy: +; CHECK: .LBB0_{{[0-9]+}}: ; %for.body +; CHECK-NOT: 0xe0 +; CHECK-NOT: 0xe2 +; CHECK: inc 0x[[SRCLO:d[0-9a-f]]] +; CHECK-NEXT: bne +; CHECK-NEXT: inc 0x[[SRCHI:d[0-9a-f]]] +; CHECK-NOT: 0xe0 +; CHECK: sta [0x[[DSTLO:d[0-9a-f]]]], y +; CHECK: inc 0x[[DSTLO]] +; CHECK-NEXT: bne +; CHECK-NOT: 0xe0 +; CHECK: lda [0x[[SRCLO]]], y +; CHECK: rtl +entry: + %cmp10.not = icmp eq i32 %length, 0 + br i1 %cmp10.not, label %for.end, label %for.body + +for.body: + %i.011 = phi i32 [ %inc, %for.body ], [ 0, %entry ] + %arrayidx = getelementptr inbounds nuw i8, ptr %src, i32 %i.011 + %0 = load i8, ptr %arrayidx, align 1 + %1 = xor i8 %0, -128 + %cmp3 = icmp eq i8 %0, -128 + %spec.store.select = select i1 %cmp3, i8 1, i8 %1 + %arrayidx5 = getelementptr inbounds nuw i8, ptr %dst, i32 %i.011 + store i8 %spec.store.select, ptr %arrayidx5, align 1 + %inc = add nuw i32 %i.011, 1 + %cmp = icmp ult i32 %inc, %length + br i1 %cmp, label %for.body, label %for.end + +for.end: + ret void +} diff --git a/src/llvm/test/CodeGen/W65816/far-ptr-home.mir b/src/llvm/test/CodeGen/W65816/far-ptr-home.mir new file mode 100644 index 0000000..aa1f47d --- /dev/null +++ b/src/llvm/test/CodeGen/W65816/far-ptr-home.mir @@ -0,0 +1,106 @@ +# Pin: W65816FarPtrStageReuse addresses a far pointer where it lives when +# its low and bank words sit side by side in the IMG direct page (which +# W65816StackRelToImg arranges, far-ptr-home.ll): an access through the +# $E0 / $E2 staging becomes [dp],Y on the pointer's own words, then the +# staging stores nothing reads any more go, and the loads that fed them. +# +# in_place: staged from $D2 / $D4 and used at once: [$D2],Y. +# home_moved: the low word is bumped after the staging: the staged +# copy is the only right one, [$E0],Y stays. +# not_adjacent: staged from $D2 / $D6: no pair to point at. +# stage_read_later: $E0 is also read directly after the access: the +# access moves and the $E0 store stays; the $E2 store, +# which nothing else reads, goes. +# +# RUN: llc -mtriple=w65816 -run-pass=w65816-farptr-reuse -o - %s | FileCheck %s +--- | + target triple = "w65816" + define void @in_place() { ret void } + define void @home_moved() { ret void } + define void @not_adjacent() { ret void } + define void @stage_read_later() { ret void } +... +--- +# CHECK-LABEL: name: in_place +# CHECK: bb.0: +# CHECK-NEXT: REP 48 +# CHECK-NEXT: LDY_Imm16 0 +# CHECK-NEXT: LDA_DPIndLongY 210 +# CHECK-NEXT: STA_DP 216 +# CHECK-NEXT: RTL +name: in_place +tracksRegLiveness: false +body: | + bb.0: + REP 48 + LDA_DP 210, implicit-def $a + STA_DP 224, implicit $a + LDA_DP 212, implicit-def $a + STA_DP 226, implicit $a + LDY_Imm16 0, implicit-def $y + LDA_DPIndLongY 224, implicit-def $a, implicit $y + STA_DP 216, implicit $a + RTL +... +--- +# CHECK-LABEL: name: home_moved +# CHECK: STA_DP 224 +# CHECK: STA_DP 226 +# CHECK: INC_DP 210 +# CHECK: LDA_DPIndLongY 224 +name: home_moved +tracksRegLiveness: false +body: | + bb.0: + REP 48 + LDA_DP 210, implicit-def $a + STA_DP 224, implicit $a + LDA_DP 212, implicit-def $a + STA_DP 226, implicit $a + INC_DP 210 + LDY_Imm16 0, implicit-def $y + LDA_DPIndLongY 224, implicit-def $a, implicit $y + STA_DP 216, implicit $a + RTL +... +--- +# CHECK-LABEL: name: not_adjacent +# CHECK: STA_DP 224 +# CHECK: STA_DP 226 +# CHECK: LDA_DPIndLongY 224 +name: not_adjacent +tracksRegLiveness: false +body: | + bb.0: + REP 48 + LDA_DP 210, implicit-def $a + STA_DP 224, implicit $a + LDA_DP 214, implicit-def $a + STA_DP 226, implicit $a + LDY_Imm16 0, implicit-def $y + LDA_DPIndLongY 224, implicit-def $a, implicit $y + STA_DP 216, implicit $a + RTL +... +--- +# CHECK-LABEL: name: stage_read_later +# CHECK: STA_DP 224 +# CHECK-NOT: STA_DP 226 +# CHECK: LDA_DPIndLongY 210 +# CHECK: LDA_DP 224 +name: stage_read_later +tracksRegLiveness: false +body: | + bb.0: + REP 48 + LDA_DP 210, implicit-def $a + STA_DP 224, implicit $a + LDA_DP 212, implicit-def $a + STA_DP 226, implicit $a + LDY_Imm16 0, implicit-def $y + LDA_DPIndLongY 224, implicit-def $a, implicit $y + STA_DP 216, implicit $a + LDA_DP 224, implicit-def $a + STA_DP 218, implicit $a + RTL +... diff --git a/src/llvm/test/CodeGen/W65816/i32-compare-chain.ll b/src/llvm/test/CodeGen/W65816/i32-compare-chain.ll index 2998f22..95f2c0d 100644 --- a/src/llvm/test/CodeGen/W65816/i32-compare-chain.ll +++ b/src/llvm/test/CodeGen/W65816/i32-compare-chain.ll @@ -596,24 +596,24 @@ define i32 @sel32_uge(i32 %a, i32 %b, i32 %x, i32 %y) { ; The loop shape from JoeyLib's old sfxCopyUnsigned: the latch's i < n is ; the chain and one bcc back to the top, and every i32 += 1 keeps the -; carry into its high half (the bne skips over it). +; carry into its high half (the bne skips over it). Both halves of the +; pointer and of i live in the direct page, the pointer's side by side, +; so the store goes through it in place (no $E0 / $E2 staging). define void @loop_ult(ptr %dst, i32 %n) { ; CHECK-LABEL: loop_ult: ; CHECK: [[TOP:.LBB[0-9_]+]]: ; %body ; CHECK-NOT: php +; CHECK-NOT: 0xe0 +; CHECK: sta [0xd2], y ; CHECK: inc 0xd2 ; CHECK-NEXT: bne -; CHECK-NEXT: lda 0x3, s -; CHECK-NEXT: inc a -; CHECK-NEXT: sta 0x3, s -; CHECK: inc 0xd4 +; CHECK-NEXT: inc 0xd4 +; CHECK: inc 0xd6 ; CHECK-NEXT: bne -; CHECK-NEXT: lda 0x5, s -; CHECK-NEXT: inc a -; CHECK-NEXT: sta 0x5, s -; CHECK: lda 0xd4 +; CHECK-NEXT: inc 0xd8 +; CHECK: lda 0xd6 ; CHECK-NEXT: cmp 0xc, s -; CHECK-NEXT: lda 0x5, s +; CHECK-NEXT: lda 0xd8 ; CHECK-NEXT: sbc 0xe, s ; CHECK-NEXT: bcc [[TOP]] ; CHECK: rtl diff --git a/src/llvm/test/CodeGen/W65816/i32-dec-fold.mir b/src/llvm/test/CodeGen/W65816/i32-dec-fold.mir new file mode 100644 index 0000000..d39f118 --- /dev/null +++ b/src/llvm/test/CodeGen/W65816/i32-dec-fold.mir @@ -0,0 +1,159 @@ +# Pin: W65816I32IncFold turns an in-place i32 -= 1 into +# lda lo ; DEC_HI_IF_BORROW hi ; dec a ; sta lo +# (Z from the low half before it goes down: only a low half of 0 borrows), +# and an i32 += 1 into lda lo ; inc a ; sta lo ; INC_HI_IF_CARRY hi, with +# the halves in stack slots or in the direct page. The folded form leaves +# A holding the low half where the original left the high half, and other +# flags, so it only fires when a fresh load of A comes first. +# +# dec_stack / dec_dp / inc_dp: the three fold shapes. +# dec_a_read / inc_a_read: A (the high half) is read after: no fold. +# The increment fold never checked this. +# dec_carry_read: the high half's carry-out is live: no fold. +# +# RUN: llc -mtriple=w65816 -run-pass=w65816-i32-inc-fold -o - %s | FileCheck %s +--- | + target triple = "w65816" + define void @dec_stack() { ret void } + define void @dec_dp() { ret void } + define void @inc_dp() { ret void } + define void @dec_a_read() { ret void } + define void @inc_a_read() { ret void } + define void @dec_carry_read() { ret void } +... +--- +# CHECK-LABEL: name: dec_stack +# CHECK: LDA_StackRel 5 +# CHECK-NEXT: DEC_HI_IF_BORROW_StackRel 3 +# CHECK-NEXT: $a = DEA_PSEUDO $a +# CHECK-NEXT: STA_StackRel 5 +# CHECK-NEXT: LDA_StackRel 5 +# CHECK-NEXT: ORA_StackRel 3 +name: dec_stack +tracksRegLiveness: true +body: | + bb.0: + successors: %bb.0, %bb.1 + LDA_StackRel 5, implicit-def $a + $a = ADCi16imm killed $a, -1, implicit-def $p + STA_StackRel 5, implicit $a + LDA_StackRel 3, implicit-def $a + $a = ADCEi16imm killed $a, -1, implicit-def dead $p, implicit $p + STA_StackRel 3, implicit $a + LDA_StackRel 5, implicit-def $a + ORA_StackRel 3, implicit $a, implicit-def $a + BNE %bb.0, implicit $p + + bb.1: + RTL +... +--- +# CHECK-LABEL: name: dec_dp +# CHECK: LDA_DP 210 +# CHECK-NEXT: DEC_HI_IF_BORROW_DP 214 +# CHECK-NEXT: $a = DEA_PSEUDO $a +# CHECK-NEXT: STA_DP 210 +name: dec_dp +tracksRegLiveness: true +body: | + bb.0: + successors: %bb.1 + LDA_DP 210, implicit-def $a + $a = ADCi16imm killed $a, 65535, implicit-def $p + STA_DP 210, implicit $a + LDA_DP 214, implicit-def $a + $a = ADCEi16imm killed $a, 65535, implicit-def dead $p, implicit $p + STA_DP 214, implicit $a + BRA %bb.1 + + bb.1: + LDA_DP 210, implicit-def $a + ORA_DP 214, implicit $a, implicit-def $a + RTL +... +--- +# CHECK-LABEL: name: inc_dp +# CHECK: LDA_DP 210 +# CHECK-NEXT: $a = INA_PSEUDO $a +# CHECK-NEXT: STA_DP 210 +# CHECK-NEXT: INC_HI_IF_CARRY_DP 214 +# CHECK-NEXT: STX_DP 216 +# CHECK-NEXT: LDA_StackRel 1 +name: inc_dp +tracksRegLiveness: true +body: | + bb.0: + liveins: $x + LDA_DP 210, implicit-def $a + $a = ADCi16imm killed $a, 1, implicit-def $p + STA_DP 210, implicit $a + LDA_DP 214, implicit-def $a + $a = ADCEi16imm killed $a, 0, implicit-def dead $p, implicit $p + STA_DP 214, implicit $a + STX_DP 216 + LDA_StackRel 1, implicit-def $a + RTL +... +--- +# CHECK-LABEL: name: dec_a_read +# CHECK: ADCi16imm +# CHECK: ADCEi16imm +# CHECK-NOT: DEC_HI_IF_BORROW +name: dec_a_read +tracksRegLiveness: true +body: | + bb.0: + LDA_StackRel 5, implicit-def $a + $a = ADCi16imm killed $a, -1, implicit-def $p + STA_StackRel 5, implicit $a + LDA_StackRel 3, implicit-def $a + $a = ADCEi16imm killed $a, -1, implicit-def dead $p, implicit $p + STA_StackRel 3, implicit $a + TAX implicit-def $x, implicit $a + LDA_StackRel 5, implicit-def $a + RTL implicit $a, implicit $x +... +--- +# CHECK-LABEL: name: inc_a_read +# CHECK: ADCi16imm +# CHECK: ADCEi16imm +# CHECK-NOT: INC_HI_IF_CARRY +name: inc_a_read +tracksRegLiveness: true +body: | + bb.0: + LDA_StackRel 5, implicit-def $a + $a = ADCi16imm killed $a, 1, implicit-def $p + STA_StackRel 5, implicit $a + LDA_StackRel 3, implicit-def $a + $a = ADCEi16imm killed $a, 0, implicit-def dead $p, implicit $p + STA_StackRel 3, implicit $a + TAX implicit-def $x, implicit $a + LDA_StackRel 5, implicit-def $a + RTL implicit $a, implicit $x +... +--- +# CHECK-LABEL: name: dec_carry_read +# CHECK: ADCi16imm +# CHECK: ADCEi16imm +# CHECK-NOT: DEC_HI_IF_BORROW +name: dec_carry_read +tracksRegLiveness: true +body: | + bb.0: + successors: %bb.1, %bb.2 + LDA_StackRel 5, implicit-def $a + $a = ADCi16imm killed $a, -1, implicit-def $p + STA_StackRel 5, implicit $a + LDA_StackRel 3, implicit-def $a + $a = ADCEi16imm killed $a, -1, implicit-def $p, implicit $p + STA_StackRel 3, implicit $a + LDA_StackRel 5, implicit-def $a + BCC %bb.2, implicit $p + + bb.1: + RTL + + bb.2: + RTL +... diff --git a/src/llvm/test/CodeGen/W65816/i32-inc-carry-slot.mir b/src/llvm/test/CodeGen/W65816/i32-inc-carry-slot.mir index 0c48c9f..c6bb159 100644 --- a/src/llvm/test/CodeGen/W65816/i32-inc-carry-slot.mir +++ b/src/llvm/test/CodeGen/W65816/i32-inc-carry-slot.mir @@ -1,29 +1,33 @@ # Pin: a stack slot bumped by INC_HI_IF_CARRY_StackRel (W65816I32IncFold's -# carry into the high half of an `i32 += 1`) is never promoted to a DP IMG -# slot by W65816StackRelToImg, nor renamed by W65816StackSlotMerge. +# carry into the high half of an `i32 += 1`) keeps its carry through +# W65816StackRelToImg and W65816StackSlotMerge. # -# The pseudo reads and writes its slot but is not one of the eight -# stack-rel MC ops those passes scan. StackRelToImg promoted the hot -# high-half slot anyway, rewrote every LDA of it to DP, left the INC -# bumping the abandoned stack copy, and its dead-INC sweep then saw a slot -# nothing reads via stack-rel and erased the INC outright. Every such -# `i32 += 1` lost its carry: the loop below is the latch of JoeyLib's old +# The pseudo reads and writes its slot but is not one of the stack-rel MC +# ops those passes scan. StackRelToImg once promoted the hot high-half +# slot anyway, rewrote every LDA of it to DP, left the INC bumping the +# abandoned stack copy, and its dead-INC sweep then saw a slot nothing +# reads via stack-rel and erased the INC outright. Every such `i32 += 1` +# lost its carry: the loop below is the latch of JoeyLib's old # sfxCopyUnsigned, whose far pointers stopped crossing their bank and whose # 32-bit counter stuck below 64K. StackSlotMerge had the same blind spot: -# it renamed the slot everywhere except in the INC. +# it renamed the slot everywhere except in the INC. The fix then kept such +# slots on the stack; since 2026-09-30 StackRelToImg promotes them and +# rewrites the pseudo to INC_HI_IF_CARRY_DP with the slot's other sites. # -# promote_keeps_carry: slots 7 (ptr lo) and 1 (i lo) may go to DP; slots 3 -# and 5 (the high halves) must stay on the stack with -# both INC_HI_IF_CARRY_StackRel intact. +# promote_keeps_carry: every slot goes to DP, each carry pseudo with its +# slot (3 -> $D4, 5 -> $D8), and the far pointer +# staged from slots 7 / 3 lands in adjacent words +# ($D2 / $D4) for FarPtrStageReuse to use in place. # merge_keeps_carry: `LDA 3 ; STA 5` and `LDA 5 ; STA 3` make slots 3 and # 5 look like one PHI value, except that slot 3 is # also bumped by the INC: no rename. -# sweep_keeps_wrapped: StackRelToImg's dead-INC sweep. The only read of -# slot 5 sits inside a PHP/PLP wrap, so it is spelled -# `LDA_StackRel 6`; matching raw offsets called slot 5 -# unread and erased the INC and the entry STA that -# initialised it (the runtime's strcpy, whose source -# bank word then came from stack garbage). +# sweep_keeps_wrapped: the only read of slot 5 sits inside a PHP/PLP wrap, +# spelled `LDA_StackRel 6`. StackRelToImg's dead-INC +# sweep once matched raw offsets, called slot 5 unread +# and erased the INC and the entry STA that initialised +# it (the runtime's strcpy, whose source bank word then +# came from stack garbage); promotion must likewise map +# the wrapped read to slot 5's word. # sweep_drops_unread: a slot nothing reads still loses its INC and STAs. # # RUN: llc -mtriple=w65816 -run-pass=w65816-stackrel-to-img -o - %s | FileCheck %s --check-prefix=IMG @@ -38,14 +42,18 @@ --- # IMG-LABEL: name: promote_keeps_carry # IMG: bb.2: -# IMG: LDA_StackRel 3 -# IMG: LDA_StackRel 3 -# IMG: LDA_StackRel 5 -# IMG: INC_DP -# IMG-NEXT: INC_HI_IF_CARRY_StackRel 3 -# IMG: INC_DP -# IMG-NEXT: INC_HI_IF_CARRY_StackRel 5 -# IMG: LDA_StackRel 5 +# IMG: LDA_DP 210 +# IMG-NEXT: STA_DP 224 +# IMG-NEXT: LDA_DP 212 +# IMG-NEXT: STA_DP 226 +# IMG: LDA_DP 212 +# IMG: LDA_DP 216 +# IMG: INC_DP 210 +# IMG-NEXT: INC_HI_IF_CARRY_DP 212 +# IMG-NEXT: INC_DP 214 +# IMG-NEXT: INC_HI_IF_CARRY_DP 216 +# IMG-NOT: StackRel {{[1357]}}, +# IMG: LDA_DP 216 # IMG-NEXT: SBC_StackRel 14 # IMG-NEXT: BCC %bb.2 name: promote_keeps_carry @@ -149,11 +157,11 @@ body: | ... --- # IMG-LABEL: name: sweep_keeps_wrapped -# IMG: STA_StackRel 5 +# IMG: STA_DP 212 # IMG: bb.1: -# IMG: INC_HI_IF_CARRY_StackRel 5 +# IMG: INC_HI_IF_CARRY_DP 212 # IMG: PHP -# IMG-NEXT: LDA_StackRel 6 +# IMG-NEXT: LDA_DP 212 # IMG-NEXT: PLP name: sweep_keeps_wrapped tracksRegLiveness: true diff --git a/src/llvm/test/CodeGen/W65816/i32-shift-16.ll b/src/llvm/test/CodeGen/W65816/i32-shift-16.ll index 5ae8ee3..31e292b 100644 --- a/src/llvm/test/CodeGen/W65816/i32-shift-16.ll +++ b/src/llvm/test/CodeGen/W65816/i32-shift-16.ll @@ -1,7 +1,7 @@ ; Pin: i32 shifts by 16 or more are half moves plus at most an i16 shift, ; not __lshrsi3 / __ashlsi3 / __ashrsi3 calls. `(u16)(x >> 16)`, the high -; word, is the high half itself. SRA by a count other than 16 or 31 still -; calls the helper (the i16 SRA patterns only do 1 and 15). +; word, is the high half itself. (Counts below 16 are pinned in +; shift-const-inline.ll.) ; ; RUN: llc -mtriple=w65816 -O2 < %s | FileCheck %s @@ -57,9 +57,22 @@ define i32 @sar31(i32 %x) { ret i32 %s } +; SRA by 17..30: the high half shifted arithmetically (a logical shift, +; then ^ M - M to spread the new sign bit) and a high word of sign fill. define i32 @sar20(i32 %x) { ; CHECK-LABEL: sar20: -; CHECK: jsl __ashrsi3 +; CHECK-NOT: jsl +; CHECK: asl a +; CHECK-NEXT: lda #0x0 +; CHECK-NEXT: adc #0xffff +; CHECK-NEXT: eor #0xffff +; CHECK: lsr a +; CHECK-NEXT: lsr a +; CHECK-NEXT: lsr a +; CHECK-NEXT: lsr a +; CHECK-NEXT: eor #0x800 +; CHECK-NEXT: clc +; CHECK-NEXT: adc #0xf800 ; CHECK: rtl %s = ashr i32 %x, 20 ret i32 %s diff --git a/src/llvm/test/CodeGen/W65816/ina-flag-corrupt-wrap.mir b/src/llvm/test/CodeGen/W65816/ina-flag-corrupt-wrap.mir index 79d8982..748d308 100644 --- a/src/llvm/test/CodeGen/W65816/ina-flag-corrupt-wrap.mir +++ b/src/llvm/test/CodeGen/W65816/ina-flag-corrupt-wrap.mir @@ -22,6 +22,10 @@ # no_test: an INA_PSEUDO with no test before it gets no wrap (nothing to # preserve, and the wrap costs 8 cycles). # +# The branch targets take X live in, so the wrap cannot be replaced by the +# TAX ... CPX #k form that StackSlotCleanup uses when X is free (pinned in +# cmp-via-x.mir); these cases keep testing the PHP / PLP wrap itself. +# # RUN: llc -mtriple=w65816 -run-pass=w65816-stack-slot-cleanup -o - %s | FileCheck %s --- | target triple = "w65816" @@ -49,8 +53,10 @@ body: | BNE %bb.1, implicit $p BRA %bb.2 bb.1: + liveins: $x RTL bb.2: + liveins: $x RTL ... --- @@ -73,8 +79,10 @@ body: | BEQ %bb.1, implicit $p BRA %bb.2 bb.1: + liveins: $x RTL bb.2: + liveins: $x RTL ... --- @@ -94,7 +102,9 @@ body: | BNE %bb.1, implicit $p BRA %bb.2 bb.1: + liveins: $x RTL bb.2: + liveins: $x RTL ... diff --git a/src/llvm/test/CodeGen/W65816/loop-local-dp.ll b/src/llvm/test/CodeGen/W65816/loop-local-dp.ll new file mode 100644 index 0000000..de52e61 --- /dev/null +++ b/src/llvm/test/CodeGen/W65816/loop-local-dp.ll @@ -0,0 +1,72 @@ +; Pin: in a function that makes calls (so W65816StackRelToImg cannot move +; slots into the direct page for the whole function), an innermost loop +; that makes none still gets its far pointer and hot slots in free IMG +; words for its duration: copied in at the end of the preheader, back at +; the start of the exit. The pointer then goes through [dp],Y in place +; (W65816FarPtrStageReuse) and its bump is `inc lo ; bne ; inc hi`. +; JoeyLib's sfx probe inlines its reference copy loop into main, which +; calls: before this the loop restaged both pointers into $E0 / $E2 for +; every byte. +; +; walk: calls before and after the loop: promoted. +; call_in_loop: the loop calls: left alone (still staged through $E0). +; +; RUN: llc -mtriple=w65816 -O2 < %s | FileCheck %s + +target datalayout = "e-m:e-p:32:16-i16:16-i32:16-i64:16-f32:16-f64:16-a:8-n8:16-S16" + +declare void @ext() +declare void @use(i16) + +define i16 @walk(ptr %p, i16 %n) { +; CHECK-LABEL: walk: +; CHECK: jsl ext +; CHECK: [[LOOP:.LBB[0-9_]+]]: ; %loop +; CHECK-NOT: 0xe0 +; CHECK-NOT: jsl +; CHECK: lda [0x{{d[0-9a-f]}}], y +; CHECK-NOT: 0xe0 +; CHECK: inc 0x{{d[0-9a-f]}} +; CHECK-NEXT: bne +; CHECK-NEXT: inc 0x{{d[0-9a-f]}} +; CHECK: bne [[LOOP]] +; CHECK: jsl ext +entry: + call void @ext() + br label %loop +loop: + %q = phi ptr [ %p, %entry ], [ %q1, %loop ] + %i = phi i16 [ %n, %entry ], [ %i1, %loop ] + %s = phi i16 [ 0, %entry ], [ %s1, %loop ] + %c = load i8, ptr %q, align 1 + %cw = zext i8 %c to i16 + %s1 = add i16 %s, %cw + %q1 = getelementptr inbounds i8, ptr %q, i32 1 + %i1 = add i16 %i, -1 + %done = icmp eq i16 %i1, 0 + br i1 %done, label %exit, label %loop +exit: + call void @ext() + ret i16 %s1 +} + +define void @call_in_loop(ptr %p, i16 %n) { +; CHECK-LABEL: call_in_loop: +; CHECK: sta 0xe0 +; CHECK: lda [0xe0], y +; CHECK: jsl use +entry: + br label %loop +loop: + %q = phi ptr [ %p, %entry ], [ %q1, %loop ] + %i = phi i16 [ %n, %entry ], [ %i1, %loop ] + %c = load i8, ptr %q, align 1 + %cw = zext i8 %c to i16 + call void @use(i16 %cw) + %q1 = getelementptr inbounds i8, ptr %q, i32 1 + %i1 = add i16 %i, -1 + %done = icmp eq i16 %i1, 0 + br i1 %done, label %exit, label %loop +exit: + ret void +} diff --git a/src/llvm/test/CodeGen/W65816/php-wrap-frame-index.mir b/src/llvm/test/CodeGen/W65816/php-wrap-frame-index.mir index 287a2d6..0b5ed14 100644 --- a/src/llvm/test/CodeGen/W65816/php-wrap-frame-index.mir +++ b/src/llvm/test/CodeGen/W65816/php-wrap-frame-index.mir @@ -17,6 +17,10 @@ # $F6+266 inside the wrap too; the local stack.1 moves from # `1,s` to `2,s`. # +# The branch targets take X live in, so the wrap cannot be replaced by the +# TAX ... CPX #k form that StackSlotCleanup uses when X is free (pinned in +# cmp-via-x.mir); these cases keep testing the PHP / PLP wrap itself. +# # RUN: llc -mtriple=w65816 -run-pass=w65816-stack-slot-cleanup,prologepilog -o - %s | FileCheck %s --- | target triple = "w65816" @@ -53,8 +57,10 @@ body: | BEQ %bb.1, implicit $p BRA %bb.2 bb.1: + liveins: $x RTL bb.2: + liveins: $x $a = LDAfi %stack.0, 0 RTL ... diff --git a/src/llvm/test/CodeGen/W65816/ret-ext-i16.ll b/src/llvm/test/CodeGen/W65816/ret-ext-i16.ll new file mode 100644 index 0000000..57ec1f1 --- /dev/null +++ b/src/llvm/test/CodeGen/W65816/ret-ext-i16.ll @@ -0,0 +1,56 @@ +; Pin: a signext / zeroext return (clang puts one on every short, char and +; bool return) is extended to A's 16 bits only. The default widened it to +; i32 and filled X with the sign (`asl a ; lda #0 ; adc #$ffff ; eor +; #$ffff ; tax`, plus a stack slot to keep A) or with zero (`ldx #0`), +; which no caller reads: the ABI returns 16 bits in A, and a caller that +; wants the value wider extends it itself (see `widen`). +; +; RUN: llc -mtriple=w65816 -O2 < %s | FileCheck %s + +target datalayout = "e-m:e-p:32:16-i16:16-i32:16-i64:16-f32:16-f64:16-a:8-n8:16-S16" + +define signext i16 @sret(i16 signext %x) { +; CHECK-LABEL: sret: +; CHECK: rep #0x30 +; CHECK-NEXT: lsr a +; CHECK-NEXT: lsr a +; CHECK-NEXT: lsr a +; CHECK-NEXT: eor #0x1000 +; CHECK-NEXT: clc +; CHECK-NEXT: adc #0xf000 +; CHECK-NEXT: rtl + %s = ashr i16 %x, 3 + ret i16 %s +} + +define zeroext i16 @zret(i16 zeroext %x) { +; CHECK-LABEL: zret: +; CHECK: rep #0x30 +; CHECK-NEXT: lsr a +; CHECK-NEXT: rtl + %s = lshr i16 %x, 1 + ret i16 %s +} + +define signext i8 @cret(i8 signext %x) { +; CHECK-LABEL: cret: +; CHECK-NOT: tax +; CHECK-NOT: ldx +; CHECK: rtl + %s = add i8 %x, 1 + ret i8 %s +} + +declare signext i16 @sext16(i16 signext) + +; The caller widens the result itself; nothing reads X after the call. +define i32 @widen(i16 %x) { +; CHECK-LABEL: widen: +; CHECK: jsl sext16 +; CHECK-NOT: txa +; CHECK-NOT: stx +; CHECK: rtl + %r = call signext i16 @sext16(i16 signext %x) + %w = sext i16 %r to i32 + ret i32 %w +} diff --git a/src/llvm/test/CodeGen/W65816/seprep-ldy-elision-kill-flag.ll b/src/llvm/test/CodeGen/W65816/seprep-ldy-elision-kill-flag.ll index ee0d05a..e7a211c 100644 --- a/src/llvm/test/CodeGen/W65816/seprep-ldy-elision-kill-flag.ll +++ b/src/llvm/test/CodeGen/W65816/seprep-ldy-elision-kill-flag.ll @@ -21,8 +21,8 @@ define void @two_i32_stores_share_y(ptr %p) { ; or without the kill-flag fix. The pin is `-verify-machineinstrs` ; not aborting. Match a minimal shape so the test still has structure. ; CHECK: ldy #0x0 -; CHECK: sta [0xe0 -; CHECK: sta [0xe0 +; CHECK: sta [0x{{[0-9a-f]+}}], y +; CHECK: sta [0x{{[0-9a-f]+}}], y ; CHECK: rtl store i32 0, ptr %p ret void diff --git a/src/llvm/test/CodeGen/W65816/shift-const-inline.ll b/src/llvm/test/CodeGen/W65816/shift-const-inline.ll new file mode 100644 index 0000000..8df6bdc --- /dev/null +++ b/src/llvm/test/CodeGen/W65816/shift-const-inline.ll @@ -0,0 +1,98 @@ +; Pin: every constant shift is inline. i16 SRA by 1 and 2 is the +; `cmp #$8000 ; ror a` pair (CMP puts the sign in C), by 3..14 a logical +; shift and then ^ M - M with M the new sign bit (the combiner turns the +; subtract into `clc ; adc #-M`), by 15 the sign fill. i32 shifts by +; 1..15 are i16 shifts of the halves and an OR; the __ashlsi3 / __lshrsi3 +; / __ashrsi3 loops (~26 cycles a bit plus the call) only serve variable +; counts, and __ashrhi3 likewise for i16. +; +; RUN: llc -mtriple=w65816 -O2 < %s | FileCheck %s + +target datalayout = "e-m:e-p:32:16-i16:16-i32:16-i64:16-f32:16-f64:16-a:8-n8:16-S16" + +define i16 @sra1(i16 %x) { +; CHECK-LABEL: sra1: +; CHECK: cmp #0x8000 +; CHECK-NEXT: ror a +; CHECK-NEXT: rtl + %s = ashr i16 %x, 1 + ret i16 %s +} + +define i16 @sra2(i16 %x) { +; CHECK-LABEL: sra2: +; CHECK: cmp #0x8000 +; CHECK-NEXT: ror a +; CHECK-NEXT: cmp #0x8000 +; CHECK-NEXT: ror a +; CHECK-NEXT: rtl + %s = ashr i16 %x, 2 + ret i16 %s +} + +define i16 @sra3(i16 %x) { +; CHECK-LABEL: sra3: +; CHECK: lsr a +; CHECK-NEXT: lsr a +; CHECK-NEXT: lsr a +; CHECK-NEXT: eor #0x1000 +; CHECK-NEXT: clc +; CHECK-NEXT: adc #0xf000 +; CHECK-NEXT: rtl + %s = ashr i16 %x, 3 + ret i16 %s +} + +define i16 @sra9(i16 %x) { +; CHECK-LABEL: sra9: +; CHECK: xba +; CHECK-NEXT: and #0xff +; CHECK-NEXT: lsr a +; CHECK-NEXT: eor #0x40 +; CHECK-NEXT: clc +; CHECK-NEXT: adc #0xffc0 +; CHECK-NEXT: rtl + %s = ashr i16 %x, 9 + ret i16 %s +} + +; A variable count still calls the helper. +define i16 @srav(i16 %x, i16 %n) { +; CHECK-LABEL: srav: +; CHECK: jsl __ashrhi3 + %s = ashr i16 %x, %n + ret i16 %s +} + +define i32 @sar9(i32 %x) { +; CHECK-LABEL: sar9: +; CHECK-NOT: jsl +; CHECK: eor #0x40 +; CHECK-NEXT: clc +; CHECK-NEXT: adc #0xffc0 +; CHECK-NEXT: tax +; CHECK: rtl + %s = ashr i32 %x, 9 + ret i32 %s +} + +define i32 @shl11(i32 %x) { +; CHECK-LABEL: shl11: +; CHECK-NOT: jsl +; CHECK: xba +; CHECK-NEXT: and #0xff00 +; CHECK-NEXT: asl a +; CHECK-NEXT: asl a +; CHECK-NEXT: asl a +; CHECK: rtl + %s = shl i32 %x, 11 + ret i32 %s +} + +define i32 @shr7(i32 %x) { +; CHECK-LABEL: shr7: +; CHECK-NOT: jsl +; CHECK: rtl + %s = lshr i32 %x, 7 + ret i32 %s +} diff --git a/src/llvm/test/CodeGen/W65816/sign-test-flags.mir b/src/llvm/test/CodeGen/W65816/sign-test-flags.mir index 71af233..23dd0dc 100644 --- a/src/llvm/test/CodeGen/W65816/sign-test-flags.mir +++ b/src/llvm/test/CodeGen/W65816/sign-test-flags.mir @@ -11,6 +11,10 @@ # 16-bit LDA already set N from bit 15. # keep_after_8bit: an 8-bit op sets N from bit 7, so the `cmp #0` stays. # +# The branch targets take X live in, so the wrap cannot be replaced by the +# TAX ... CPX #k form that StackSlotCleanup uses when X is free (pinned in +# cmp-via-x.mir); these cases keep testing the PHP / PLP wrap itself. +# # RUN: llc -mtriple=w65816 -run-pass=w65816-stack-slot-cleanup -o - %s | FileCheck %s --- | target triple = "w65816" @@ -39,8 +43,10 @@ body: | BMI %bb.1, implicit $p BRA %bb.2 bb.1: + liveins: $x RTL bb.2: + liveins: $x RTL ... --- @@ -62,8 +68,10 @@ body: | BPL %bb.1, implicit $p BRA %bb.2 bb.1: + liveins: $x RTL bb.2: + liveins: $x RTL ... --- @@ -86,8 +94,10 @@ body: | BMI %bb.1, implicit $p BRA %bb.2 bb.1: + liveins: $x RTL bb.2: + liveins: $x RTL ... --- @@ -106,8 +116,10 @@ body: | BMI %bb.1, implicit $p BRA %bb.2 bb.1: + liveins: $x RTL bb.2: + liveins: $x RTL ... --- @@ -125,7 +137,9 @@ body: | BMI %bb.1, implicit $p BRA %bb.2 bb.1: + liveins: $x RTL bb.2: + liveins: $x RTL ... diff --git a/src/llvm/test/CodeGen/W65816/srl8a-flag-corrupt-wrap.mir b/src/llvm/test/CodeGen/W65816/srl8a-flag-corrupt-wrap.mir index 994db80..2f11547 100644 --- a/src/llvm/test/CodeGen/W65816/srl8a-flag-corrupt-wrap.mir +++ b/src/llvm/test/CodeGen/W65816/srl8a-flag-corrupt-wrap.mir @@ -23,6 +23,10 @@ # The wrapped reload keeps its offset: eliminateFrameIndex adds the byte # PHP pushed (php-wrap-frame-index.mir). # +# The branch targets take X live in, so the wrap cannot be replaced by the +# TAX ... CPX #k form that StackSlotCleanup uses when X is free (pinned in +# cmp-via-x.mir); these cases keep testing the PHP / PLP wrap itself. +# # RUN: llc -mtriple=w65816 -run-pass=w65816-stack-slot-cleanup -o - %s | FileCheck %s --- | target triple = "w65816" @@ -56,8 +60,10 @@ body: | BNE %bb.1, implicit $p BRA %bb.2 bb.1: + liveins: $x RTL bb.2: + liveins: $x RTL ... --- @@ -83,8 +89,10 @@ body: | BEQ %bb.1, implicit $p BRA %bb.2 bb.1: + liveins: $x RTL bb.2: + liveins: $x RTL ... --- @@ -109,7 +117,9 @@ body: | BEQ %bb.1, implicit $p BRA %bb.2 bb.1: + liveins: $x RTL bb.2: + liveins: $x RTL ... diff --git a/src/llvm/test/CodeGen/W65816/stack-slot-merge-read-between.mir b/src/llvm/test/CodeGen/W65816/stack-slot-merge-read-between.mir new file mode 100644 index 0000000..9ec3177 --- /dev/null +++ b/src/llvm/test/CodeGen/W65816/stack-slot-merge-read-between.mir @@ -0,0 +1,94 @@ +# Pin: W65816StackSlotMerge does not merge a PHI-copy pair when the copy's +# destination slot is read between the source slot's store and the copy. +# Merged, the source's store would overwrite the destination early and +# that read would see the new value. The shape is strcpy's source walk, +# `q = p + 1; c = *p; ...; p = q`: slot 9 is q's low word (bumped in +# place), slot 3 p's, and p is staged into $E0 after q was bumped. The +# merge renamed 9 -> 3, so the load read q and every string was copied one +# byte short (the smoke run's hash-table check). The copy used to sit +# inside a PHP/PLP wrap, whose offset bump kept the pass from matching it. +# +# keep_read_between: slot 3 is read after STA 9: no rename. +# merge_no_read: the same loop without that read still merges. +# +# RUN: llc -mtriple=w65816 -run-pass=w65816-stack-slot-merge -o - %s | FileCheck %s +--- | + target triple = "w65816" + define void @keep_read_between() { ret void } + define void @merge_no_read() { ret void } +... +--- +# CHECK-LABEL: name: keep_read_between +# CHECK: bb.1: +# CHECK: LDA_StackRel 9 +# CHECK-NEXT: $a = INA_PSEUDO $a +# CHECK-NEXT: STA_StackRel 9 +# CHECK: LDA_StackRel 3 +# CHECK-NEXT: STA_DP 224 +# CHECK: LDA_StackRel 9 +# CHECK-NEXT: STA_StackRel 3 +name: keep_read_between +tracksRegLiveness: false +frameInfo: + stackSize: 16 +body: | + bb.0: + successors: %bb.1 + LDA_StackRel 20, implicit-def $a + STA_StackRel 3, implicit $a + STA_StackRel 9, implicit $a + + bb.1: + successors: %bb.1, %bb.2 + LDA_StackRel 9, implicit-def $a + $a = INA_PSEUDO $a + STA_StackRel 9, implicit $a + LDA_StackRel 3, implicit-def $a + STA_DP 224, implicit $a + LDA_StackRel 5, implicit-def $a + STA_DP 226, implicit $a + LDY_Imm16 0, implicit-def $y + LDA_DPIndLongY 224, implicit-def $a, implicit $y + STA_StackRel 15, implicit $a + LDA_StackRel 9, implicit-def $a + STA_StackRel 3, implicit $a + LDA_StackRel 15, implicit-def $a + BNE %bb.1, implicit $p + + bb.2: + RTL +... +--- +# CHECK-LABEL: name: merge_no_read +# CHECK: bb.1: +# CHECK: LDA_StackRel 3 +# CHECK-NEXT: $a = INA_PSEUDO $a +# CHECK-NEXT: STA_StackRel 3 +# CHECK-NOT: StackRel 9 +# CHECK: RTL +name: merge_no_read +tracksRegLiveness: false +frameInfo: + stackSize: 16 +body: | + bb.0: + successors: %bb.1 + LDA_StackRel 20, implicit-def $a + STA_StackRel 3, implicit $a + STA_StackRel 9, implicit $a + + bb.1: + successors: %bb.1, %bb.2 + LDA_StackRel 9, implicit-def $a + $a = INA_PSEUDO $a + STA_StackRel 9, implicit $a + LDA_StackRel 5, implicit-def $a + STA_DP 226, implicit $a + LDA_StackRel 9, implicit-def $a + STA_StackRel 3, implicit $a + LDA_StackRel 15, implicit-def $a + BNE %bb.1, implicit $p + + bb.2: + RTL +... diff --git a/src/llvm/test/CodeGen/W65816/store-i32-global.ll b/src/llvm/test/CodeGen/W65816/store-i32-global.ll new file mode 100644 index 0000000..2d660b3 --- /dev/null +++ b/src/llvm/test/CodeGen/W65816/store-i32-global.ll @@ -0,0 +1,51 @@ +; Pin: an i32 store to a global is two abs-16 stores (`sta g`, `sta g+2`), +; as an i32 load from one is two abs-16 loads. It went through the +; far-pointer ST_PTR path instead: `#g` and the program bank from $BE +; staged into $E0 / $E2 before each half, about 12x the cycles. A field +; or element (`add global, K`) folds the same way, loads included (the +; high half of an i32 load used to take the far path when the combiner +; had not yet folded its +2 into the global). +; +; RUN: llc -mtriple=w65816 -O2 < %s | FileCheck %s + +target datalayout = "e-m:e-p:32:16-i16:16-i32:16-i64:16-f32:16-f64:16-a:8-n8:16-S16" + +@gCount = global i32 0 +@gPair = global [2 x i32] zeroinitializer + +define void @setCount(i32 %v) { +; CHECK-LABEL: setCount: +; CHECK-NOT: 0xe0 +; CHECK: sta gCount +; CHECK-NEXT: lda 0xd0 +; CHECK-NEXT: sta gCount+2 +; CHECK-NEXT: rtl + store i32 %v, ptr @gCount + ret void +} + +define void @setSecond(i32 %v) { +; CHECK-LABEL: setSecond: +; CHECK-NOT: 0xe0 +; CHECK: sta gPair+4 +; CHECK-NEXT: lda 0xd0 +; CHECK-NEXT: sta gPair+6 +; CHECK-NEXT: rtl + %p = getelementptr inbounds [2 x i32], ptr @gPair, i32 0, i32 1 + store i32 %v, ptr %p + ret void +} + +define void @bump() { +; CHECK-LABEL: bump: +; CHECK-NOT: 0xe0 +; CHECK: lda gCount+2 +; CHECK: lda gCount +; CHECK: sta gCount +; CHECK: sta gCount+2 +; CHECK: rtl + %v = load i32, ptr @gCount + %w = add i32 %v, 1 + store i32 %w, ptr @gCount + ret void +} diff --git a/tests/coremark/coreMark.manifest.json b/tests/coremark/coreMark.manifest.json index 617c7e3..bca31e9 100644 --- a/tests/coremark/coreMark.manifest.json +++ b/tests/coremark/coreMark.manifest.json @@ -7,8 +7,8 @@ "num": 1, "name": "SEG1", "base": "0x001000", - "size": 40728, - "bss_start": "0x00b700", + "size": 44791, + "bss_start": "0x00d800", "bss_size": 2034, "image": "coreMark.bin", "entry_offset": "0x0000" @@ -17,7 +17,7 @@ "num": 2, "name": "SEG2", "base": "0x040000", - "size": 16480, + "size": 10368, "bss_start": "0x000000", "bss_size": 0, "image": "coreMark.seg2.bin", diff --git a/tests/fpNan/genFpTables.py b/tests/fpNan/genFpTables.py new file mode 100644 index 0000000..d11ab4f --- /dev/null +++ b/tests/fpNan/genFpTables.py @@ -0,0 +1,589 @@ +#!/usr/bin/env python3 +# genFpTables.py - expected-value tables for the soft-float smoke checks in +# scripts/smokeTest.sh ("float/double compares" and "float/double NaN and +# infinity"). Paste the output over the matching table in smokeTest.sh. +# +# python3 tests/fpNan/genFpTables.py cmp # kFBits / kDBits / kExp +# python3 tests/fpNan/genFpTables.py special # kCases +# +# Comparisons come from Python's own float compares, which are IEEE 754 +# (every ordered compare with a NaN is false). Float operands are widened +# to double exactly, so float and double share one truth table per value. +# +# The special-value table is a model of the documented runtime rules, not +# of the host FPU: a NaN operand comes back quieted with its payload kept +# (the first NaN operand when both are), an invalid operation gives the +# default NaN 0x7FC00000 / 0x7FF8000000000000 (x86 would give the negative +# one), and float/double to integer saturates with NaN converting to 0. +# Finite cases use operands whose results are exact, because softFloat.c +# truncates instead of rounding. +# +# The math.c cases follow C99 Annex F's special values (modelled here where +# Python raises instead of answering). The series-based functions (exp, +# pow with a non-integer exponent, sinh, cbrt, ...) are only ~1e-6 +# accurate, so their ordinary-value cases carry an `approx` flag: the check +# accepts a result within 1e-6 relative of the host's value. + +import math +import struct +import sys +from fractions import Fraction + +F_QUIET = 0x00400000 +D_QUIET = 0x0008000000000000 +F_DNAN = 0x7FC00000 +D_DNAN = 0x7FF8000000000000 +F_INF = 0x7F800000 +D_INF = 0x7FF0000000000000 +F_SIGN = 0x80000000 +D_SIGN = 0x8000000000000000 + +# The compare operand set: signed infinities and zeros, normals of both +# signs, a quiet NaN, a negative quiet NaN and a signaling-pattern NaN. +CMP_F = [0xFF800000, 0xC0200000, 0xBF800000, 0x80000000, 0x00000000, + 0x3F800000, 0x41200000, 0x7149F2CA, 0x7F800000, 0x7FC00000, + 0xFFC00000, 0x7F800001] +# The same values as doubles (1e30f widened exactly, so both tables share +# one truth table). +CMP_D = [0xFFF0000000000000, 0xC004000000000000, 0xBFF0000000000000, + 0x8000000000000000, 0x0000000000000000, 0x3FF0000000000000, + 0x4024000000000000, 0x46293E5940000000, 0x7FF0000000000000, + 0x7FF8000000000000, 0xFFF8000000000000, 0x7FF0000000000001] + +# Predicate order matches ALL() in the check: the six C operators, their +# negations (the unordered LLVM predicates uge/ugt/ule/ult), then the C99 +# isunordered / !isunordered / islessgreater / !islessgreater. +PREDS = [ + lambda a, b: a < b, + lambda a, b: a <= b, + lambda a, b: a > b, + lambda a, b: a >= b, + lambda a, b: a == b, + lambda a, b: a != b, + lambda a, b: not (a < b), + lambda a, b: not (a <= b), + lambda a, b: not (a > b), + lambda a, b: not (a >= b), + lambda a, b: math.isnan(a) or math.isnan(b), + lambda a, b: not (math.isnan(a) or math.isnan(b)), + lambda a, b: a < b or a > b, + lambda a, b: not (a < b or a > b), +] + + +def fval(bits): + return struct.unpack('> 23) & 0xFF + m = b & 0x7FFFFF + if e == 0: + return 'zero' + if e == 0xFF: + return 'inf' if m == 0 else 'nan' + return 'normal' + + +def dclass(b): + e = (b >> 52) & 0x7FF + m = b & 0xFFFFFFFFFFFFF + if e == 0: + return 'zero' + if e == 0x7FF: + return 'inf' if m == 0 else 'nan' + return 'normal' + + +def binop(isD, op, a, b): + cls = dclass if isD else fclass + quiet, dnan, inf, sign = ((D_QUIET, D_DNAN, D_INF, D_SIGN) if isD else + (F_QUIET, F_DNAN, F_INF, F_SIGN)) + val, tobits = (dval, dbits) if isD else (fval, fbits) + if op == 'sub': + # __subXf3(a, b) is __addXf3(a, b with its sign flipped), as in + # compiler-rt, so a NaN b comes back with its sign flipped too. + b ^= sign + op = 'add' + ca, cb = cls(a), cls(b) + if ca == 'nan': + return a | quiet + if cb == 'nan': + return b | quiet + s = (a ^ b) & sign + if op == 'add': + if ca == 'inf': + return dnan if cb == 'inf' and s else a + if cb == 'inf': + return b + if ca == 'zero' and cb == 'zero': + return a & b & sign + elif op == 'mul': + if ca == 'inf' or cb == 'inf': + return dnan if 'zero' in (ca, cb) else s | inf + if ca == 'zero' or cb == 'zero': + return s + elif op == 'div': + if ca == 'inf': + return dnan if cb == 'inf' else s | inf + if cb == 'inf': + return s + if cb == 'zero': + return dnan if ca == 'zero' else s | inf + if ca == 'zero': + return s + x, y = val(a), val(b) + if op == 'add': + r = x + y + elif op == 'mul': + r = x * y + else: + r = x / y + if not isD and abs(r) > 3.4028234663852886e38: + r = math.copysign(math.inf, r) + got = tobits(r) + assert val(got) == r or math.isinf(r), (op, hex(a), hex(b), r) + return got + + +def f2d(a): + if fclass(a) == 'nan': + return ((a & F_SIGN) << 32) | D_INF | ((a & 0x7FFFFF) << 29) | D_QUIET + return dbits(fval(a)) + + +def d2f(a): + if dclass(a) == 'nan': + return ((a >> 32) & F_SIGN) | F_DNAN | (((a & 0xFFFFFFFFFFFFF) >> 29) & 0x7FFFFF) + v = dval(a) + if abs(v) > 3.4028234663852886e38: + return fbits(math.copysign(math.inf, v)) + got = fbits(v) + assert fval(got) == v + return got + + +def toInt(v, unsigned): + if math.isnan(v): + return 0 + lo, hi = (0, 0xFFFFFFFF) if unsigned else (-0x80000000, 0x7FFFFFFF) + if math.isinf(v): + r = hi if v > 0 else lo + else: + r = max(lo, min(hi, int(v))) + return r & 0xFFFFFFFF + + +def classify(v, bitsSign): + nan = 1 if math.isnan(v) else 0 + inf = 1 if math.isinf(v) else 0 + fin = 1 if not (nan or inf) else 0 + cls = 0 if nan else 1 if inf else 4 if v == 0 else 2 + return nan | (inf << 1) | (fin << 2) | (bitsSign << 3) | (cls << 4) + + +# Operation codes, matching the switch in the check's run(). +OPS = ['fadd', 'fsub', 'fmul', 'fdiv', 'dadd', 'dsub', 'dmul', 'ddiv', + 'f2d', 'd2f', 'f2i', 'f2u', 'd2i', 'd2u', 'fcls', 'dcls', + 'sqrt', 'log', 'asin', 'acos', 'atanh', + 'floor', 'ceil', 'trunc', 'round', 'rint', 'nearbyint', + 'floorf', 'ceilf', 'truncf', 'roundf', 'rintf', + 'exp', 'exp2', 'expm1', 'sinh', 'cosh', 'tanh', 'log1p', 'cbrt', + 'pow', 'fmod', 'fdim', 'hypot', 'remainder', + 'nearbyintf', 'fmaxf', 'fminf'] + +FQ = 0x7FC00123 # quiet NaN with a payload +FS = 0x7F800005 # signaling-pattern NaN +FNQ = 0xFFC00000 # negative quiet NaN +DQ = 0x7FF8000000000123 +DS = 0x7FF0000000000005 +DNQ = 0xFFF8000000000000 +F1, FM1, F15, F2, F225, FM2, F3E38 = (fbits(1.0), fbits(-1.0), fbits(1.5), + fbits(2.0), fbits(2.25), fbits(-2.0), + fbits(3e38)) +D1, DM1, D15, D2, D225, DM2, D1E300 = (dbits(1.0), dbits(-1.0), dbits(1.5), + dbits(2.0), dbits(2.25), dbits(-2.0), + dbits(1e300)) +FZ, FNZ, FI, FNI = 0, F_SIGN, F_INF, F_INF | F_SIGN +DZ, DNZ, DI, DNI = 0, D_SIGN, D_INF, D_INF | D_SIGN + + +def fpairs(): + return [(FQ, F1), (F1, FS), (FS, FQ), (FNQ, F2), (FZ, FQ), (FI, F1), + (FI, FI), (FI, FNI), (FNI, F2), (F2, FNI), (F1, FM1), (FM1, F1), + (FNZ, FNZ), (FNZ, FZ), (FZ, FNZ), (FZ, FZ), (F225, F15), + (F3E38, F3E38), (FZ, FI), (FI, FZ), (FNI, FM2), (FNI, FNI), + (FNZ, F2), (FZ, FZ), (F1, FZ), (FM1, FZ), (F1, FNZ), (F1, FI), + (FM1, FI), (FI, F2), (F15, F2)] + + +def dpairs(): + return [(DQ, D1), (D1, DS), (DS, DQ), (DNQ, D2), (DZ, DQ), (DI, D1), + (DI, DI), (DI, DNI), (DNI, D2), (D2, DNI), (D1, DM1), (DM1, D1), + (DNZ, DNZ), (DNZ, DZ), (DZ, DNZ), (DZ, DZ), (D225, D15), + (D1E300, D1E300), (DZ, DI), (DI, DZ), (DNI, DM2), (DNI, DNI), + (DNZ, D2), (DZ, DZ), (D1, DZ), (DM1, DZ), (D1, DNZ), (D1, DI), + (DM1, DI), (DI, D2), (D15, D2)] + + +def emitSpecial(): + cases = [] + for isD, prefix, pairs in ((False, 'f', fpairs()), (True, 'd', dpairs())): + for op in ('add', 'sub', 'mul', 'div'): + for a, b in pairs: + cases.append((OPS.index(prefix + op), a, b, binop(isD, op, a, b))) + for a in (FQ, FS, 0xFFC00001, FI, FNI, FNZ, FZ, F15, FM2): + cases.append((OPS.index('f2d'), a, 0, f2d(a))) + for a in (DQ, 0xFFF8000020000000, DS, DI, DNI, DNZ, DZ, D15, DM2, D1E300, + dbits(-1e300)): + cases.append((OPS.index('d2f'), a, 0, d2f(a))) + fconv = (FQ, FNQ, FS, FI, FNI, fbits(3e9), fbits(-3e9), fbits(5e9), + fbits(-2.5), fbits(3.5), FM1, FNZ) + dconv = (DQ, DNQ, DS, DI, DNI, dbits(3e9), dbits(-3e9), dbits(5e9), + dbits(-2.5), dbits(3.5), DM1, DNZ) + for a in fconv: + cases.append((OPS.index('f2i'), a, 0, toInt(fval(a), False))) + cases.append((OPS.index('f2u'), a, 0, toInt(fval(a), True))) + for a in dconv: + cases.append((OPS.index('d2i'), a, 0, toInt(dval(a), False))) + cases.append((OPS.index('d2u'), a, 0, toInt(dval(a), True))) + for a in (FQ, FNQ, FS, FI, FNI, FZ, FNZ, F15, FM2): + cases.append((OPS.index('fcls'), a, 0, classify(fval(a), a >> 31))) + for a in (DQ, DNQ, DS, DI, DNI, DZ, DNZ, D15, DM2): + cases.append((OPS.index('dcls'), a, 0, classify(dval(a), a >> 63))) + # math.c's domain edges. The check folds every NaN result to the + # default NaN, because how a NaN travels through a series (and so its + # sign) is not part of these functions' contract. + halfPi = dbits(math.pi / 2) + mathCases = [ + ('sqrt', dbits(-1.0), D_DNAN), ('sqrt', DI, DI), ('sqrt', DNI, D_DNAN), + ('sqrt', DQ, D_DNAN), ('sqrt', dbits(4.0), dbits(2.0)), + ('sqrt', DNZ, DNZ), + ('log', DI, DI), ('log', DNI, D_DNAN), ('log', DQ, D_DNAN), + ('log', DM1, D_DNAN), ('log', DZ, DNI), ('log', D1, DZ), + ('asin', D2, D_DNAN), ('asin', DM2, D_DNAN), ('asin', D1, halfPi), + ('asin', DM1, halfPi | D_SIGN), ('asin', DQ, D_DNAN), + ('acos', D2, D_DNAN), ('acos', D1, DZ), + ('atanh', D2, D_DNAN), ('atanh', DM2, D_DNAN), ('atanh', D1, DI), + ('atanh', DM1, DNI), ('atanh', DQ, D_DNAN), + ] + cases = [c + (0,) for c in cases] + for name, a, want in mathCases: + cases.append((OPS.index(name), a, 0, want, 0)) + # Two tables, one per build of the check: both together (with the math + # functions they pull in) overflow a bank-0 test binary. + for part, table in ((1, cases), (2, annexFCases())): + print('#if FP_TABLE == 1' if part == 1 else '#else') + print('#define NC %d' % len(table)) + print('static const CaseT kCases[NC] = {') + for op, a, b, want, approx in table: + print(' { %-12s 0x%016XULL, 0x%016XULL, 0x%016XULL, %d },' % + ('O_' + OPS[op].upper() + ',', a, b, want, approx)) + print('};') + print('#endif') + + +# ---- math.c Annex F cases ---------------------------------------------- + +INF = math.inf +NAN = math.nan + + +def signedZero(r, x): + # An integral result of zero keeps x's sign (floor(-0.0) is -0.0, + # ceil(-0.5) is -0.0). + return math.copysign(0.0, x) if r == 0 else r + + +def cfloor(x): + return x if not math.isfinite(x) else signedZero(float(math.floor(x)), x) + + +def cceil(x): + return x if not math.isfinite(x) else signedZero(float(math.ceil(x)), x) + + +def ctrunc(x): + return x if not math.isfinite(x) else signedZero(float(math.trunc(x)), x) + + +def cround(x): + # C round: half away from zero, computed exactly. + if not math.isfinite(x): + return x + r = float(math.floor(Fraction(abs(x)) + Fraction(1, 2))) + return math.copysign(r, x) + + +def crint(x): + # Round to nearest, ties to even (Python's round() on a float is exact). + if not math.isfinite(x): + return x + return math.copysign(float(round(x)), x) + + +def guard(f, *args): + # Python raises where C returns: a domain error is NaN, an overflow is + # an infinity of the true result's sign. + try: + return f(*args) + except ValueError: + return NAN + except OverflowError: + return INF + + +def isOddInt(y): + return math.isfinite(y) and y == math.floor(y) and abs(y) < 2.0 ** 53 and int(y) % 2 == 1 + + +def cpow(x, y): + # C99 F.9.4.4. + if y == 0 or x == 1.0: + return 1.0 + if math.isnan(x) or math.isnan(y): + return NAN + if math.isinf(y): + if abs(x) == 1: + return 1.0 + return INF if (abs(x) > 1) == (y > 0) else 0.0 + if x == 0: + r = INF if y < 0 else 0.0 + return math.copysign(r, x) if isOddInt(y) else r + if math.isinf(x): + r = INF if y > 0 else 0.0 + return -r if (x < 0 and isOddInt(y)) else r + if x < 0 and y != math.floor(y): + return NAN + r = guard(math.pow, x, y) + # guard() cannot know an overflow's sign: a negative x to an odd power. + return -r if (math.isinf(r) and x < 0 and isOddInt(y)) else r + + +def cfmod(x, y): + # C99 F.9.7.1. + if math.isnan(x) or math.isnan(y) or math.isinf(x) or y == 0: + return NAN + if math.isinf(y): + return x + return math.fmod(x, y) + + +def cfdim(x, y): + if math.isnan(x) or math.isnan(y): + return NAN + return x - y if x > y else 0.0 + + +def cremainder(x, y): + # C99 F.9.7.2 (Python's math.remainder raises for the NaN cases). + if math.isnan(x) or math.isnan(y) or math.isinf(x) or y == 0: + return NAN + return math.remainder(x, y) + + +def ccbrt(x): + if not math.isfinite(x) or x == 0: + return x + return math.copysign(abs(x) ** (1.0 / 3.0), x) + + +def fround32(v): + return fval(fbits(v)) + + +MATH1 = { + 'floor': cfloor, 'ceil': cceil, 'trunc': ctrunc, 'round': cround, + 'rint': crint, 'nearbyint': crint, + 'exp': lambda x: guard(math.exp, x), + 'exp2': lambda x: guard(math.pow, 2.0, x), + 'expm1': lambda x: guard(math.expm1, x), + 'sinh': lambda x: guard(math.sinh, x) if not (math.isinf(x) or abs(x) > 700) else math.copysign(INF, x), + 'cosh': lambda x: guard(math.cosh, x), + 'tanh': math.tanh, + 'log1p': lambda x: -INF if x == -1 else guard(math.log1p, x), + 'cbrt': ccbrt, +} +MATH2 = {'pow': cpow, 'fmod': cfmod, 'fdim': cfdim, 'hypot': math.hypot, + 'remainder': cremainder} +FLOATOPS = {'floorf': cfloor, 'ceilf': cceil, 'truncf': ctrunc, + 'roundf': cround, 'rintf': crint, 'nearbyintf': crint} + + +def cfmax(x, y): + # C99 F.9.9.2: a NaN operand is ignored in favour of the other. + if math.isnan(x): + return y + if math.isnan(y): + return x + return max(x, y) + + +def cfmin(x, y): + if math.isnan(x): + return y + if math.isnan(y): + return x + return min(x, y) + + +FLOAT2 = {'fmaxf': cfmax, 'fminf': cfmin} + +# (op, x[, y], approx). Every function gets its special values and a few +# ordinary ones, so a normal-path regression shows too. +MATH_CASES = [ + ('floor', 2.7, 0), ('floor', -2.7, 0), ('floor', -0.0, 0), ('floor', 0.0, 0), + ('floor', -0.3, 0), ('floor', 0.3, 0), ('floor', INF, 0), ('floor', -INF, 0), + ('floor', NAN, 0), ('floor', 1e300, 0), ('floor', -0.5, 0), + ('ceil', 2.3, 0), ('ceil', -2.3, 0), ('ceil', 0.0, 0), ('ceil', -0.0, 0), + ('ceil', -0.5, 0), ('ceil', 0.5, 0), ('ceil', INF, 0), ('ceil', NAN, 0), + ('trunc', -2.7, 0), ('trunc', 2.7, 0), ('trunc', -0.5, 0), ('trunc', -0.0, 0), + ('trunc', INF, 0), + ('round', 2.5, 0), ('round', -2.5, 0), ('round', 0.49999999999999994, 0), + ('round', -0.49999999999999994, 0), ('round', 4503599627370497.0, 0), + ('round', -0.0, 0), ('round', 0.5, 0), ('round', 1.5, 0), ('round', INF, 0), + ('round', NAN, 0), ('round', -0.4, 0), + ('rint', 2.5, 0), ('rint', 3.5, 0), ('rint', -2.5, 0), ('rint', -0.3, 0), + ('rint', 0.3, 0), ('rint', 1.5, 0), ('rint', 4503599627370497.0, 0), + ('rint', INF, 0), ('rint', NAN, 0), ('rint', -0.0, 0), + ('nearbyint', 2.5, 0), ('nearbyint', -3.5, 0), + ('floorf', -0.0, 0), ('floorf', 2.75, 0), ('floorf', -2.75, 0), + ('ceilf', 0.0, 0), ('ceilf', -0.5, 0), ('truncf', -2.5, 0), + ('roundf', 2.5, 0), ('roundf', -0.0, 0), + ('rintf', 2.5, 0), ('rintf', 3.5, 0), ('rintf', -0.25, 0), + ('nearbyintf', 2.5, 0), ('nearbyintf', -1.5, 0), + ('fmaxf', 1.0, NAN, 0), ('fmaxf', NAN, -2.0, 0), ('fmaxf', -0.5, 2.0, 0), + ('fminf', 1.0, NAN, 0), ('fminf', -0.5, 2.0, 0), ('fminf', 3.0, -INF, 0), + ('exp', NAN, 0), ('exp', INF, 0), ('exp', -INF, 0), ('exp', 1e5, 0), + ('exp', -1e5, 0), ('exp', 1024.0, 0), ('exp', -1024.0, 0), ('exp', 800.0, 0), + ('exp', -800.0, 0), ('exp', 0.0, 0), ('exp', -0.0, 0), ('exp', 1.5, 1), + ('exp', -2.0, 1), + ('exp2', 3.0, 1), ('exp2', NAN, 0), ('exp2', INF, 0), ('exp2', -INF, 0), + ('expm1', -INF, 0), ('expm1', INF, 0), ('expm1', NAN, 0), ('expm1', -0.0, 0), + ('expm1', 0.5, 1), + ('sinh', INF, 0), ('sinh', -INF, 0), ('sinh', NAN, 0), ('sinh', -0.0, 0), + ('sinh', 800.0, 0), ('sinh', -800.0, 0), ('sinh', 1.0, 1), ('sinh', -1.0, 1), + ('cosh', INF, 0), ('cosh', -INF, 0), ('cosh', NAN, 0), ('cosh', 0.0, 0), + ('cosh', 1.0, 1), + ('tanh', INF, 0), ('tanh', -INF, 0), ('tanh', NAN, 0), ('tanh', -0.0, 0), + ('tanh', 40.0, 0), ('tanh', -40.0, 0), ('tanh', 400.0, 0), ('tanh', 0.5, 1), + ('tanh', -0.5, 1), + ('log1p', -0.0, 0), ('log1p', 0.0, 0), ('log1p', -1.0, 0), ('log1p', INF, 0), + ('log1p', NAN, 0), ('log1p', 1.0, 1), + ('cbrt', INF, 0), ('cbrt', -INF, 0), ('cbrt', NAN, 0), ('cbrt', -0.0, 0), + ('cbrt', 27.0, 1), + ('pow', NAN, 0.0, 0), ('pow', NAN, -0.0, 0), ('pow', 1.0, NAN, 0), + ('pow', 1.0, INF, 0), ('pow', -1.0, INF, 0), ('pow', -1.0, -INF, 0), + ('pow', 2.0, NAN, 0), ('pow', NAN, 2.0, 0), ('pow', 0.0, -3.0, 0), + ('pow', -0.0, -3.0, 0), ('pow', -0.0, -2.0, 0), ('pow', -0.0, -INF, 0), + ('pow', 0.0, -0.5, 0), ('pow', -0.0, 3.0, 0), ('pow', -0.0, 2.0, 0), + ('pow', 0.0, 0.5, 0), ('pow', 0.5, -INF, 0), ('pow', 2.0, -INF, 0), + ('pow', 0.5, INF, 0), ('pow', 2.0, INF, 0), ('pow', -INF, -3.0, 0), + ('pow', -INF, -2.0, 0), ('pow', -INF, 3.0, 0), ('pow', -INF, 2.0, 0), + ('pow', -INF, 0.5, 0), ('pow', INF, -1.0, 0), ('pow', INF, 2.0, 0), + ('pow', -8.0, 1.0 / 3.0, 0), ('pow', -2.0, 0.5, 0), ('pow', 2.0, 10.0, 0), + ('pow', 2.0, -3.0, 0), ('pow', -2.0, 3.0, 0), ('pow', -2.0, 2.0, 0), + ('pow', 10.0, 20.0, 0), ('pow', -2.0, 1e20, 0), ('pow', -2.0, 40001.0, 0), + ('pow', 2.0, 0.5, 1), ('pow', 2.0, 1.5, 1), ('pow', 10.0, 0.3, 1), + ('pow', 1.0000001, 100000.0, 1), + ('fmod', 7.5, 2.0, 0), ('fmod', -7.5, 2.0, 0), ('fmod', -4.0, 2.0, 0), + ('fmod', 4.0, 2.0, 0), ('fmod', 5.0, 3.0, 0), ('fmod', 1.0, INF, 0), + ('fmod', -1.0, -INF, 0), ('fmod', 1.0, 0.0, 0), ('fmod', 1.0, -0.0, 0), + ('fmod', INF, 2.0, 0), ('fmod', -INF, 2.0, 0), ('fmod', NAN, 2.0, 0), + ('fmod', 2.0, NAN, 0), ('fmod', 0.0, 5.0, 0), ('fmod', -0.0, 5.0, 0), + ('fmod', 0.0, 0.0, 0), ('fmod', INF, INF, 0), + ('fdim', NAN, 1.0, 0), ('fdim', 1.0, NAN, 0), ('fdim', 5.0, 3.0, 0), + ('fdim', 3.0, 5.0, 0), ('fdim', INF, 1.0, 0), ('fdim', 1.0, INF, 0), + ('fdim', INF, INF, 0), + ('hypot', INF, NAN, 0), ('hypot', NAN, -INF, 0), ('hypot', -INF, 1.0, 0), + ('hypot', NAN, 1.0, 0), ('hypot', 3.0, 4.0, 1), ('hypot', 0.0, 0.0, 0), + ('remainder', 5.0, 2.0, 0), ('remainder', 7.0, 2.0, 0), + ('remainder', 7.0, 4.0, 0), ('remainder', 1.0, INF, 0), + ('remainder', -3.0, -INF, 0), ('remainder', 1.0, 0.0, 0), + ('remainder', INF, 2.0, 0), ('remainder', NAN, 1.0, 0), +] + + +def nanFold(bits, isF): + # The check folds every NaN a math function returns to the default NaN. + v = fval(bits) if isF else dval(bits) + if math.isnan(v): + return F_DNAN if isF else D_DNAN + return bits + + +def annexFCases(): + out = [] + for c in MATH_CASES: + name = c[0] + if name in FLOAT2: + x, y, approx = fround32(c[1]), fround32(c[2]), c[3] + want = fround32(FLOAT2[name](x, y)) + out.append((OPS.index(name), fbits(x), fbits(y), nanFold(fbits(want), True), approx)) + continue + if name in MATH2: + x, y, approx = c[1], c[2], c[3] + want = MATH2[name](x, y) + a, b = dbits(x), dbits(y) + else: + x, approx = c[1], c[2] + y = 0.0 + if name in FLOATOPS: + x = fround32(x) + want = fround32(FLOATOPS[name](x)) + out.append((OPS.index(name), fbits(x), 0, nanFold(fbits(want), True), approx)) + continue + want = MATH1[name](x) + a, b = dbits(x), 0 + out.append((OPS.index(name), a, b, nanFold(dbits(want), False), approx)) + return out + + +if __name__ == '__main__': + if len(sys.argv) != 2 or sys.argv[1] not in ('cmp', 'special'): + sys.exit('usage: genFpTables.py cmp|special') + emitCmp() if sys.argv[1] == 'cmp' else emitSpecial() diff --git a/tests/lua/luaTest.manifest.json b/tests/lua/luaTest.manifest.json index b42592b..4d6e11e 100644 --- a/tests/lua/luaTest.manifest.json +++ b/tests/lua/luaTest.manifest.json @@ -7,8 +7,8 @@ "num": 1, "name": "SEG1", "base": "0x001000", - "size": 43790, - "bss_start": "0x00e200", + "size": 44531, + "bss_start": "0x00e300", "bss_size": 4248, "image": "luaTest.bin", "entry_offset": "0x0000" @@ -17,7 +17,7 @@ "num": 2, "name": "SEG2", "base": "0x040000", - "size": 43204, + "size": 44940, "bss_start": "0x000000", "bss_size": 0, "image": "luaTest.seg2.bin", @@ -27,7 +27,7 @@ "num": 3, "name": "SEG3", "base": "0x050000", - "size": 41056, + "size": 41839, "bss_start": "0x000000", "bss_size": 0, "image": "luaTest.seg3.bin", @@ -37,7 +37,7 @@ "num": 4, "name": "SEG4", "base": "0x060000", - "size": 28810, + "size": 28814, "bss_start": "0x000000", "bss_size": 0, "image": "luaTest.seg4.bin", diff --git a/tests/regress/genShiftDec.py b/tests/regress/genShiftDec.py new file mode 100644 index 0000000..11360ba --- /dev/null +++ b/tests/regress/genShiftDec.py @@ -0,0 +1,160 @@ +#!/usr/bin/env python3 +# Emits the C source of the smoke check "constant shifts and i32 +/- 1", +# with kExp holding the expected results computed here. +V32 = [0x00000000, 0xFFFFFFFF, 0x80000000, 0x7FFFFFFF, 0x12345678, 0xFEDCBA98, 0x00018001, 0x8000FFFF] +V16 = [0x0000, 0xFFFF, 0x8000, 0x7FFF, 0x1234, 0xFEDC, 0x8001, 0x4000] +# (start, count) for the down-counting loops: each decrements a u32 `count` +# times from `start` and returns the final value's halves and a checksum. +DOWN = [(0x00010002, 5), (0x00000002, 4), (0x00020000, 1), (0x80000001, 3), (0x00000000, 1), (0x00030001, 2)] +UP = [(0x0000FFFE, 5), (0xFFFFFFFE, 4), (0x0001FFFF, 1), (0x7FFFFFFF, 2)] + +def s32(x): + x &= 0xFFFFFFFF + return x - (1 << 32) if x & 0x80000000 else x + +def s16(x): + x &= 0xFFFF + return x - (1 << 16) if x & 0x8000 else x + +exp = [] +for op in ("shl", "shr", "sar"): + for n in range(1, 32): + for v in V32: + if op == "shl": + r = (v << n) & 0xFFFFFFFF + elif op == "shr": + r = v >> n + else: + r = (s32(v) >> n) & 0xFFFFFFFF + exp += [r & 0xFFFF, r >> 16] +for n in range(1, 16): + for v in V16: + exp.append((s16(v) >> n) & 0xFFFF) +# Down loops: final value and the sum (mod 2^16) of every low half seen. +for start, cnt in DOWN: + v = start + s = 0 + for _ in range(cnt): + v = (v - 1) & 0xFFFFFFFF + s = (s + (v & 0xFFFF) + (v >> 16)) & 0xFFFF + exp += [v & 0xFFFF, v >> 16, s] +for start, cnt in UP: + v = start + s = 0 + for _ in range(cnt): + v = (v + 1) & 0xFFFFFFFF + s = (s + (v & 0xFFFF) + (v >> 16)) & 0xFFFF + exp += [v & 0xFFFF, v >> 16, s] + +def rows(vals, per, fmt): + out = [] + for i in range(0, len(vals), per): + out.append(" " + ", ".join(fmt % x for x in vals[i:i + per]) + ",") + return "\n".join(out) + +print("""// Every constant i32 shift (<< >>u >>s by 1..31) on 8 values, every +// constant i16 >>s (1..15) on 8 values, and in-place i32 -= 1 / += 1 loop +// counters across 0x0000 / 0x10000 / 0x80000000 / 0xFFFFFFFF. kExp is +// computed by genShiftDec.py. Mailbox: 0x025000 = mismatches, 0x025002 = +// first failing index, 0x025004 = 0xC0DE done. int is 16 bits, long 32. +typedef unsigned short u16; +typedef short s16; +typedef unsigned long u32; +typedef long s32; +#define NOI __attribute__((noinline)) static +#define NV 8 +static const u32 kV32[NV] = { %s }; +static const u16 kV16[NV] = { %s };""" % (", ".join("0x%08XUL" % v for v in V32), ", ".join("0x%04Xu" % v for v in V16))) +print("static const u16 kExp[%d] = {\n%s\n};" % (len(exp), rows(exp, 12, "0x%04X"))) +print(r"""#define SH(n) \ + NOI u32 shl##n(u32 x) { return x << n; } \ + NOI u32 shr##n(u32 x) { return x >> n; } \ + NOI u32 sar##n(u32 x) { return (u32)((s32)x >> n); } +#define L7(M, a, b, c, d, e, f, g) M(a) M(b) M(c) M(d) M(e) M(f) M(g) +#define L8(M, a, b, c, d, e, f, g, h) L7(M, a, b, c, d, e, f, g) M(h) +L8(SH, 1, 2, 3, 4, 5, 6, 7, 8) L8(SH, 9, 10, 11, 12, 13, 14, 15, 16) +L8(SH, 17, 18, 19, 20, 21, 22, 23, 24) L7(SH, 25, 26, 27, 28, 29, 30, 31) +#define S16(n) NOI u16 sa16_##n(u16 x) { return (u16)((s16)x >> n); } +L8(S16, 1, 2, 3, 4, 5, 6, 7, 8) L7(S16, 9, 10, 11, 12, 13, 14, 15) +typedef u32 (*Fn32)(u32); +typedef u16 (*Fn16)(u16); +#define P31(p) p##1, p##2, p##3, p##4, p##5, p##6, p##7, p##8, p##9, p##10, \ + p##11, p##12, p##13, p##14, p##15, p##16, p##17, p##18, p##19, p##20, \ + p##21, p##22, p##23, p##24, p##25, p##26, p##27, p##28, p##29, p##30, p##31 +static Fn32 const kFn32[93] = { P31(shl), P31(shr), P31(sar) }; +static Fn16 const kFn16[15] = { sa16_1, sa16_2, sa16_3, sa16_4, sa16_5, sa16_6, sa16_7, + sa16_8, sa16_9, sa16_10, sa16_11, sa16_12, sa16_13, sa16_14, sa16_15 }; +// The counter lives in a frame / direct-page slot across the loop, where +// W65816I32IncFold turns its -= 1 / += 1 into the conditional-carry form. +static volatile u16 gSink; +NOI u32 down(u32 v, u16 n, u16 *sum) { + u16 s = 0; + while (n != 0u) { + v--; + s = (u16)(s + (u16)v + (u16)(v >> 16)); + gSink = s; + n--; + } + *sum = s; + return v; +} +NOI u32 up(u32 v, u16 n, u16 *sum) { + u16 s = 0; + while (n != 0u) { + v++; + s = (u16)(s + (u16)v + (u16)(v >> 16)); + gSink = s; + n--; + } + *sum = s; + return v; +}""") +print("static const u32 kDownStart[%d] = { %s };" % (len(DOWN), ", ".join("0x%08XUL" % d[0] for d in DOWN))) +print("static const u16 kDownCount[%d] = { %s };" % (len(DOWN), ", ".join("%du" % d[1] for d in DOWN))) +print("static const u32 kUpStart[%d] = { %s };" % (len(UP), ", ".join("0x%08XUL" % d[0] for d in UP))) +print("static const u16 kUpCount[%d] = { %s };" % (len(UP), ", ".join("%du" % d[1] for d in UP))) +print(r"""static u16 gBad; +static u16 gFirst = 0xFFFFu; +static u16 gIdx; +static void check(u16 got) { + if (got != kExp[gIdx] && gBad++ == 0u) { + gFirst = gIdx; + } + gIdx++; +} +int main(void) { + u16 f; + u16 i; + u16 s; + u32 r; + for (f = 0u; f < 93u; f++) { + for (i = 0u; i < NV; i++) { + r = kFn32[f](kV32[i]); + check((u16)r); + check((u16)(r >> 16)); + } + } + for (f = 0u; f < 15u; f++) { + for (i = 0u; i < NV; i++) { + check(kFn16[f](kV16[i])); + } + }""") +print(" for (i = 0u; i < %du; i++) {" % len(DOWN)) +print(r""" r = down(kDownStart[i], kDownCount[i], &s); + check((u16)r); + check((u16)(r >> 16)); + check(s); + }""") +print(" for (i = 0u; i < %du; i++) {" % len(UP)) +print(r""" r = up(kUpStart[i], kUpCount[i], &s); + check((u16)r); + check((u16)(r >> 16)); + check(s); + } + *(volatile u16 *)0x025000UL = gBad; + *(volatile u16 *)0x025002UL = gFirst; + *(volatile u16 *)0x025004UL = 0xC0DEu; + for (;;) { + } + return 0; +}""")