ia64.S revision 89837
1.explicit 2.text 3.ident "ia64.S, Version 1.1" 4.ident "IA-64 ISA artwork by Andy Polyakov <appro@fy.chalmers.se>" 5 6// 7// ==================================================================== 8// Written by Andy Polyakov <appro@fy.chalmers.se> for the OpenSSL 9// project. 10// 11// Rights for redistribution and usage in source and binary forms are 12// granted according to the OpenSSL license. Warranty of any kind is 13// disclaimed. 14// ==================================================================== 15// 16 17// Q. How much faster does it get? 18// A. Here is the output from 'openssl speed rsa dsa' for vanilla 19// 0.9.6a compiled with gcc version 2.96 20000731 (Red Hat 20// Linux 7.1 2.96-81): 21// 22// sign verify sign/s verify/s 23// rsa 512 bits 0.0036s 0.0003s 275.3 2999.2 24// rsa 1024 bits 0.0203s 0.0011s 49.3 894.1 25// rsa 2048 bits 0.1331s 0.0040s 7.5 250.9 26// rsa 4096 bits 0.9270s 0.0147s 1.1 68.1 27// sign verify sign/s verify/s 28// dsa 512 bits 0.0035s 0.0043s 288.3 234.8 29// dsa 1024 bits 0.0111s 0.0135s 90.0 74.2 30// 31// And here is similar output but for this assembler 32// implementation:-) 33// 34// sign verify sign/s verify/s 35// rsa 512 bits 0.0021s 0.0001s 549.4 9638.5 36// rsa 1024 bits 0.0055s 0.0002s 183.8 4481.1 37// rsa 2048 bits 0.0244s 0.0006s 41.4 1726.3 38// rsa 4096 bits 0.1295s 0.0018s 7.7 561.5 39// sign verify sign/s verify/s 40// dsa 512 bits 0.0012s 0.0013s 891.9 756.6 41// dsa 1024 bits 0.0023s 0.0028s 440.4 376.2 42// 43// Yes, you may argue that it's not fair comparison as it's 44// possible to craft the C implementation with BN_UMULT_HIGH 45// inline assembler macro. But of course! Here is the output 46// with the macro: 47// 48// sign verify sign/s verify/s 49// rsa 512 bits 0.0020s 0.0002s 495.0 6561.0 50// rsa 1024 bits 0.0086s 0.0004s 116.2 2235.7 51// rsa 2048 bits 0.0519s 0.0015s 19.3 667.3 52// rsa 4096 bits 0.3464s 0.0053s 2.9 187.7 53// sign verify sign/s verify/s 54// dsa 512 bits 0.0016s 0.0020s 613.1 510.5 55// dsa 1024 bits 0.0045s 0.0054s 221.0 183.9 56// 57// My code is still way faster, huh:-) And I believe that even 58// higher performance can be achieved. Note that as keys get 59// longer, performance gain is larger. Why? According to the 60// profiler there is another player in the field, namely 61// BN_from_montgomery consuming larger and larger portion of CPU 62// time as keysize decreases. I therefore consider putting effort 63// to assembler implementation of the following routine: 64// 65// void bn_mul_add_mont (BN_ULONG *rp,BN_ULONG *np,int nl,BN_ULONG n0) 66// { 67// int i,j; 68// BN_ULONG v; 69// 70// for (i=0; i<nl; i++) 71// { 72// v=bn_mul_add_words(rp,np,nl,(rp[0]*n0)&BN_MASK2); 73// nrp++; 74// rp++; 75// if (((nrp[-1]+=v)&BN_MASK2) < v) 76// for (j=0; ((++nrp[j])&BN_MASK2) == 0; j++) ; 77// } 78// } 79// 80// It might as well be beneficial to implement even combaX 81// variants, as it appears as it can literally unleash the 82// performance (see comment section to bn_mul_comba8 below). 83// 84// And finally for your reference the output for 0.9.6a compiled 85// with SGIcc version 0.01.0-12 (keep in mind that for the moment 86// of this writing it's not possible to convince SGIcc to use 87// BN_UMULT_HIGH inline assembler macro, yet the code is fast, 88// i.e. for a compiler generated one:-): 89// 90// sign verify sign/s verify/s 91// rsa 512 bits 0.0022s 0.0002s 452.7 5894.3 92// rsa 1024 bits 0.0097s 0.0005s 102.7 2002.9 93// rsa 2048 bits 0.0578s 0.0017s 17.3 600.2 94// rsa 4096 bits 0.3838s 0.0061s 2.6 164.5 95// sign verify sign/s verify/s 96// dsa 512 bits 0.0018s 0.0022s 547.3 459.6 97// dsa 1024 bits 0.0051s 0.0062s 196.6 161.3 98// 99// Oh! Benchmarks were performed on 733MHz Lion-class Itanium 100// system running Redhat Linux 7.1 (very special thanks to Ray 101// McCaffity of Williams Communications for providing an account). 102// 103// Q. What's the heck with 'rum 1<<5' at the end of every function? 104// A. Well, by clearing the "upper FP registers written" bit of the 105// User Mask I want to excuse the kernel from preserving upper 106// (f32-f128) FP register bank over process context switch, thus 107// minimizing bus bandwidth consumption during the switch (i.e. 108// after PKI opration completes and the program is off doing 109// something else like bulk symmetric encryption). Having said 110// this, I also want to point out that it might be good idea 111// to compile the whole toolkit (as well as majority of the 112// programs for that matter) with -mfixed-range=f32-f127 command 113// line option. No, it doesn't prevent the compiler from writing 114// to upper bank, but at least discourages to do so. If you don't 115// like the idea you have the option to compile the module with 116// -Drum=nop.m in command line. 117// 118 119#if 1 120// 121// bn_[add|sub]_words routines. 122// 123// Loops are spinning in 2*(n+5) ticks on Itanuim (provided that the 124// data reside in L1 cache, i.e. 2 ticks away). It's possible to 125// compress the epilogue and get down to 2*n+6, but at the cost of 126// scalability (the neat feature of this implementation is that it 127// shall automagically spin in n+5 on "wider" IA-64 implementations:-) 128// I consider that the epilogue is short enough as it is to trade tiny 129// performance loss on Itanium for scalability. 130// 131// BN_ULONG bn_add_words(BN_ULONG *rp, BN_ULONG *ap, BN_ULONG *bp,int num) 132// 133.global bn_add_words# 134.proc bn_add_words# 135.align 64 136.skip 32 // makes the loop body aligned at 64-byte boundary 137bn_add_words: 138 .prologue 139 .fframe 0 140 .save ar.pfs,r2 141{ .mii; alloc r2=ar.pfs,4,12,0,16 142 cmp4.le p6,p0=r35,r0 };; 143{ .mfb; mov r8=r0 // return value 144(p6) br.ret.spnt.many b0 };; 145 146 .save ar.lc,r3 147{ .mib; sub r10=r35,r0,1 148 mov r3=ar.lc 149 brp.loop.imp .L_bn_add_words_ctop,.L_bn_add_words_cend-16 150 } 151 .body 152{ .mib; mov r14=r32 // rp 153 mov r9=pr };; 154{ .mii; mov r15=r33 // ap 155 mov ar.lc=r10 156 mov ar.ec=6 } 157{ .mib; mov r16=r34 // bp 158 mov pr.rot=1<<16 };; 159 160.L_bn_add_words_ctop: 161{ .mii; (p16) ld8 r32=[r16],8 // b=*(bp++) 162 (p18) add r39=r37,r34 163 (p19) cmp.ltu.unc p56,p0=r40,r38 } 164{ .mfb; (p0) nop.m 0x0 165 (p0) nop.f 0x0 166 (p0) nop.b 0x0 } 167{ .mii; (p16) ld8 r35=[r15],8 // a=*(ap++) 168 (p58) cmp.eq.or p57,p0=-1,r41 // (p20) 169 (p58) add r41=1,r41 } // (p20) 170{ .mfb; (p21) st8 [r14]=r42,8 // *(rp++)=r 171 (p0) nop.f 0x0 172 br.ctop.sptk .L_bn_add_words_ctop };; 173.L_bn_add_words_cend: 174 175{ .mii; 176(p59) add r8=1,r8 // return value 177 mov pr=r9,-1 178 mov ar.lc=r3 } 179{ .mbb; nop.b 0x0 180 br.ret.sptk.many b0 };; 181.endp bn_add_words# 182 183// 184// BN_ULONG bn_sub_words(BN_ULONG *rp, BN_ULONG *ap, BN_ULONG *bp,int num) 185// 186.global bn_sub_words# 187.proc bn_sub_words# 188.align 64 189.skip 32 // makes the loop body aligned at 64-byte boundary 190bn_sub_words: 191 .prologue 192 .fframe 0 193 .save ar.pfs,r2 194{ .mii; alloc r2=ar.pfs,4,12,0,16 195 cmp4.le p6,p0=r35,r0 };; 196{ .mfb; mov r8=r0 // return value 197(p6) br.ret.spnt.many b0 };; 198 199 .save ar.lc,r3 200{ .mib; sub r10=r35,r0,1 201 mov r3=ar.lc 202 brp.loop.imp .L_bn_sub_words_ctop,.L_bn_sub_words_cend-16 203 } 204 .body 205{ .mib; mov r14=r32 // rp 206 mov r9=pr };; 207{ .mii; mov r15=r33 // ap 208 mov ar.lc=r10 209 mov ar.ec=6 } 210{ .mib; mov r16=r34 // bp 211 mov pr.rot=1<<16 };; 212 213.L_bn_sub_words_ctop: 214{ .mii; (p16) ld8 r32=[r16],8 // b=*(bp++) 215 (p18) sub r39=r37,r34 216 (p19) cmp.gtu.unc p56,p0=r40,r38 } 217{ .mfb; (p0) nop.m 0x0 218 (p0) nop.f 0x0 219 (p0) nop.b 0x0 } 220{ .mii; (p16) ld8 r35=[r15],8 // a=*(ap++) 221 (p58) cmp.eq.or p57,p0=0,r41 // (p20) 222 (p58) add r41=-1,r41 } // (p20) 223{ .mbb; (p21) st8 [r14]=r42,8 // *(rp++)=r 224 (p0) nop.b 0x0 225 br.ctop.sptk .L_bn_sub_words_ctop };; 226.L_bn_sub_words_cend: 227 228{ .mii; 229(p59) add r8=1,r8 // return value 230 mov pr=r9,-1 231 mov ar.lc=r3 } 232{ .mbb; nop.b 0x0 233 br.ret.sptk.many b0 };; 234.endp bn_sub_words# 235#endif 236 237#if 0 238#define XMA_TEMPTATION 239#endif 240 241#if 1 242// 243// BN_ULONG bn_mul_words(BN_ULONG *rp, BN_ULONG *ap, int num, BN_ULONG w) 244// 245.global bn_mul_words# 246.proc bn_mul_words# 247.align 64 248.skip 32 // makes the loop body aligned at 64-byte boundary 249bn_mul_words: 250 .prologue 251 .fframe 0 252 .save ar.pfs,r2 253#ifdef XMA_TEMPTATION 254{ .mfi; alloc r2=ar.pfs,4,0,0,0 };; 255#else 256{ .mfi; alloc r2=ar.pfs,4,4,0,8 };; 257#endif 258{ .mib; mov r8=r0 // return value 259 cmp4.le p6,p0=r34,r0 260(p6) br.ret.spnt.many b0 };; 261 262 .save ar.lc,r3 263{ .mii; sub r10=r34,r0,1 264 mov r3=ar.lc 265 mov r9=pr };; 266 267 .body 268{ .mib; setf.sig f8=r35 // w 269 mov pr.rot=0x400001<<16 270 // ------^----- serves as (p48) at first (p26) 271 brp.loop.imp .L_bn_mul_words_ctop,.L_bn_mul_words_cend-16 272 } 273 274#ifndef XMA_TEMPTATION 275 276{ .mii; mov r14=r32 // rp 277 mov r15=r33 // ap 278 mov ar.lc=r10 } 279{ .mii; mov r39=0 // serves as r33 at first (p26) 280 mov ar.ec=12 };; 281 282// This loop spins in 2*(n+11) ticks. It's scheduled for data in L2 283// cache (i.e. 9 ticks away) as floating point load/store instructions 284// bypass L1 cache and L2 latency is actually best-case scenario for 285// ldf8. The loop is not scalable and shall run in 2*(n+11) even on 286// "wider" IA-64 implementations. It's a trade-off here. n+22 loop 287// would give us ~5% in *overall* performance improvement on "wider" 288// IA-64, but would hurt Itanium for about same because of longer 289// epilogue. As it's a matter of few percents in either case I've 290// chosen to trade the scalability for development time (you can see 291// this very instruction sequence in bn_mul_add_words loop which in 292// turn is scalable). 293.L_bn_mul_words_ctop: 294{ .mfi; (p25) getf.sig r36=f49 // low 295 (p21) xmpy.lu f45=f37,f8 296 (p27) cmp.ltu p52,p48=r39,r38 } 297{ .mfi; (p16) ldf8 f32=[r15],8 298 (p21) xmpy.hu f38=f37,f8 299 (p0) nop.i 0x0 };; 300{ .mii; (p26) getf.sig r32=f43 // high 301 .pred.rel "mutex",p48,p52 302 (p48) add r38=r37,r33 // (p26) 303 (p52) add r38=r37,r33,1 } // (p26) 304{ .mfb; (p27) st8 [r14]=r39,8 305 (p0) nop.f 0x0 306 br.ctop.sptk .L_bn_mul_words_ctop };; 307.L_bn_mul_words_cend: 308 309{ .mii; nop.m 0x0 310.pred.rel "mutex",p49,p53 311(p49) add r8=r34,r0 312(p53) add r8=r34,r0,1 } 313{ .mfb; nop.m 0x0 314 nop.f 0x0 315 nop.b 0x0 } 316 317#else // XMA_TEMPTATION 318 319 setf.sig f37=r0 // serves as carry at (p18) tick 320 mov ar.lc=r10 321 mov ar.ec=5;; 322 323// Most of you examining this code very likely wonder why in the name 324// of Intel the following loop is commented out? Indeed, it looks so 325// neat that you find it hard to believe that it's something wrong 326// with it, right? The catch is that every iteration depends on the 327// result from previous one and the latter isn't available instantly. 328// The loop therefore spins at the latency of xma minus 1, or in other 329// words at 6*(n+4) ticks:-( Compare to the "production" loop above 330// that runs in 2*(n+11) where the low latency problem is worked around 331// by moving the dependency to one-tick latent interger ALU. Note that 332// "distance" between ldf8 and xma is not latency of ldf8, but the 333// *difference* between xma and ldf8 latencies. 334.L_bn_mul_words_ctop: 335{ .mfi; (p16) ldf8 f32=[r33],8 336 (p18) xma.hu f38=f34,f8,f39 } 337{ .mfb; (p20) stf8 [r32]=f37,8 338 (p18) xma.lu f35=f34,f8,f39 339 br.ctop.sptk .L_bn_mul_words_ctop };; 340.L_bn_mul_words_cend: 341 342 getf.sig r8=f41 // the return value 343 344#endif // XMA_TEMPTATION 345 346{ .mii; nop.m 0x0 347 mov pr=r9,-1 348 mov ar.lc=r3 } 349{ .mfb; rum 1<<5 // clear um.mfh 350 nop.f 0x0 351 br.ret.sptk.many b0 };; 352.endp bn_mul_words# 353#endif 354 355#if 1 356// 357// BN_ULONG bn_mul_add_words(BN_ULONG *rp, BN_ULONG *ap, int num, BN_ULONG w) 358// 359.global bn_mul_add_words# 360.proc bn_mul_add_words# 361.align 64 362//.skip 0 // makes the loop split at 64-byte boundary 363bn_mul_add_words: 364 .prologue 365 .fframe 0 366 .save ar.pfs,r2 367{ .mii; alloc r2=ar.pfs,4,12,0,16 368 cmp4.le p6,p0=r34,r0 };; 369{ .mfb; mov r8=r0 // return value 370(p6) br.ret.spnt.many b0 };; 371 372 .save ar.lc,r3 373{ .mii; sub r10=r34,r0,1 374 mov r3=ar.lc 375 mov r9=pr };; 376 377 .body 378{ .mib; setf.sig f8=r35 // w 379 mov pr.rot=0x400001<<16 380 // ------^----- serves as (p48) at first (p26) 381 brp.loop.imp .L_bn_mul_add_words_ctop,.L_bn_mul_add_words_cend-16 382 } 383{ .mii; mov r14=r32 // rp 384 mov r15=r33 // ap 385 mov ar.lc=r10 } 386{ .mii; mov r39=0 // serves as r33 at first (p26) 387 mov r18=r32 // rp copy 388 mov ar.ec=14 };; 389 390// This loop spins in 3*(n+13) ticks on Itanium and should spin in 391// 2*(n+13) on "wider" IA-64 implementations (to be verified with new 392// �-architecture manuals as they become available). As usual it's 393// possible to compress the epilogue, down to 10 in this case, at the 394// cost of scalability. Compressed (and therefore non-scalable) loop 395// running at 3*(n+10) would buy you ~10% on Itanium but take ~35% 396// from "wider" IA-64 so let it be scalable! Special attention was 397// paid for having the loop body split at 64-byte boundary. ld8 is 398// scheduled for L1 cache as the data is more than likely there. 399// Indeed, bn_mul_words has put it there a moment ago:-) 400.L_bn_mul_add_words_ctop: 401{ .mfi; (p25) getf.sig r36=f49 // low 402 (p21) xmpy.lu f45=f37,f8 403 (p27) cmp.ltu p52,p48=r39,r38 } 404{ .mfi; (p16) ldf8 f32=[r15],8 405 (p21) xmpy.hu f38=f37,f8 406 (p27) add r43=r43,r39 };; 407{ .mii; (p26) getf.sig r32=f43 // high 408 .pred.rel "mutex",p48,p52 409 (p48) add r38=r37,r33 // (p26) 410 (p52) add r38=r37,r33,1 } // (p26) 411{ .mfb; (p27) cmp.ltu.unc p56,p0=r43,r39 412 (p0) nop.f 0x0 413 (p0) nop.b 0x0 } 414{ .mii; (p26) ld8 r42=[r18],8 415 (p58) cmp.eq.or p57,p0=-1,r44 416 (p58) add r44=1,r44 } 417{ .mfb; (p29) st8 [r14]=r45,8 418 (p0) nop.f 0x0 419 br.ctop.sptk .L_bn_mul_add_words_ctop};; 420.L_bn_mul_add_words_cend: 421 422{ .mii; nop.m 0x0 423.pred.rel "mutex",p51,p55 424(p51) add r8=r36,r0 425(p55) add r8=r36,r0,1 } 426{ .mfb; nop.m 0x0 427 nop.f 0x0 428 nop.b 0x0 };; 429{ .mii; 430(p59) add r8=1,r8 431 mov pr=r9,-1 432 mov ar.lc=r3 } 433{ .mfb; rum 1<<5 // clear um.mfh 434 nop.f 0x0 435 br.ret.sptk.many b0 };; 436.endp bn_mul_add_words# 437#endif 438 439#if 1 440// 441// void bn_sqr_words(BN_ULONG *rp, BN_ULONG *ap, int num) 442// 443.global bn_sqr_words# 444.proc bn_sqr_words# 445.align 64 446.skip 32 // makes the loop body aligned at 64-byte boundary 447bn_sqr_words: 448 .prologue 449 .fframe 0 450 .save ar.pfs,r2 451{ .mii; alloc r2=ar.pfs,3,0,0,0 452 sxt4 r34=r34 };; 453{ .mii; cmp.le p6,p0=r34,r0 454 mov r8=r0 } // return value 455{ .mfb; nop.f 0x0 456(p6) br.ret.spnt.many b0 };; 457 458 .save ar.lc,r3 459{ .mii; sub r10=r34,r0,1 460 mov r3=ar.lc 461 mov r9=pr };; 462 463 .body 464{ .mib; 465 mov pr.rot=1<<16 466 brp.loop.imp .L_bn_sqr_words_ctop,.L_bn_sqr_words_cend-16 467 } 468{ .mii; add r34=8,r32 469 mov ar.lc=r10 470 mov ar.ec=18 };; 471 472// 2*(n+17) on Itanium, (n+17) on "wider" IA-64 implementations. It's 473// possible to compress the epilogue (I'm getting tired to write this 474// comment over and over) and get down to 2*n+16 at the cost of 475// scalability. The decision will very likely be reconsidered after the 476// benchmark program is profiled. I.e. if perfomance gain on Itanium 477// will appear larger than loss on "wider" IA-64, then the loop should 478// be explicitely split and the epilogue compressed. 479.L_bn_sqr_words_ctop: 480{ .mfi; (p16) ldf8 f32=[r33],8 481 (p25) xmpy.lu f42=f41,f41 482 (p0) nop.i 0x0 } 483{ .mib; (p33) stf8 [r32]=f50,16 484 (p0) nop.i 0x0 485 (p0) nop.b 0x0 } 486{ .mfi; (p0) nop.m 0x0 487 (p25) xmpy.hu f52=f41,f41 488 (p0) nop.i 0x0 } 489{ .mib; (p33) stf8 [r34]=f60,16 490 (p0) nop.i 0x0 491 br.ctop.sptk .L_bn_sqr_words_ctop };; 492.L_bn_sqr_words_cend: 493 494{ .mii; nop.m 0x0 495 mov pr=r9,-1 496 mov ar.lc=r3 } 497{ .mfb; rum 1<<5 // clear um.mfh 498 nop.f 0x0 499 br.ret.sptk.many b0 };; 500.endp bn_sqr_words# 501#endif 502 503#if 1 504// Apparently we win nothing by implementing special bn_sqr_comba8. 505// Yes, it is possible to reduce the number of multiplications by 506// almost factor of two, but then the amount of additions would 507// increase by factor of two (as we would have to perform those 508// otherwise performed by xma ourselves). Normally we would trade 509// anyway as multiplications are way more expensive, but not this 510// time... Multiplication kernel is fully pipelined and as we drain 511// one 128-bit multiplication result per clock cycle multiplications 512// are effectively as inexpensive as additions. Special implementation 513// might become of interest for "wider" IA-64 implementation as you'll 514// be able to get through the multiplication phase faster (there won't 515// be any stall issues as discussed in the commentary section below and 516// you therefore will be able to employ all 4 FP units)... But these 517// Itanium days it's simply too hard to justify the effort so I just 518// drop down to bn_mul_comba8 code:-) 519// 520// void bn_sqr_comba8(BN_ULONG *r, BN_ULONG *a) 521// 522.global bn_sqr_comba8# 523.proc bn_sqr_comba8# 524.align 64 525bn_sqr_comba8: 526 .prologue 527 .fframe 0 528 .save ar.pfs,r2 529{ .mii; alloc r2=ar.pfs,2,1,0,0 530 mov r34=r33 531 add r14=8,r33 };; 532 .body 533{ .mii; add r17=8,r34 534 add r15=16,r33 535 add r18=16,r34 } 536{ .mfb; add r16=24,r33 537 br .L_cheat_entry_point8 };; 538.endp bn_sqr_comba8# 539#endif 540 541#if 1 542// I've estimated this routine to run in ~120 ticks, but in reality 543// (i.e. according to ar.itc) it takes ~160 ticks. Are those extra 544// cycles consumed for instructions fetch? Or did I misinterpret some 545// clause in Itanium �-architecture manual? Comments are welcomed and 546// highly appreciated. 547// 548// However! It should be noted that even 160 ticks is darn good result 549// as it's over 10 (yes, ten, spelled as t-e-n) times faster than the 550// C version (compiled with gcc with inline assembler). I really 551// kicked compiler's butt here, didn't I? Yeah! This brings us to the 552// following statement. It's damn shame that this routine isn't called 553// very often nowadays! According to the profiler most CPU time is 554// consumed by bn_mul_add_words called from BN_from_montgomery. In 555// order to estimate what we're missing, I've compared the performance 556// of this routine against "traditional" implementation, i.e. against 557// following routine: 558// 559// void bn_mul_comba8(BN_ULONG *r, BN_ULONG *a, BN_ULONG *b) 560// { r[ 8]=bn_mul_words( &(r[0]),a,8,b[0]); 561// r[ 9]=bn_mul_add_words(&(r[1]),a,8,b[1]); 562// r[10]=bn_mul_add_words(&(r[2]),a,8,b[2]); 563// r[11]=bn_mul_add_words(&(r[3]),a,8,b[3]); 564// r[12]=bn_mul_add_words(&(r[4]),a,8,b[4]); 565// r[13]=bn_mul_add_words(&(r[5]),a,8,b[5]); 566// r[14]=bn_mul_add_words(&(r[6]),a,8,b[6]); 567// r[15]=bn_mul_add_words(&(r[7]),a,8,b[7]); 568// } 569// 570// The one below is over 8 times faster than the one above:-( Even 571// more reasons to "combafy" bn_mul_add_mont... 572// 573// And yes, this routine really made me wish there were an optimizing 574// assembler! It also feels like it deserves a dedication. 575// 576// To my wife for being there and to my kids... 577// 578// void bn_mul_comba8(BN_ULONG *r, BN_ULONG *a, BN_ULONG *b) 579// 580#define carry1 r14 581#define carry2 r15 582#define carry3 r34 583.global bn_mul_comba8# 584.proc bn_mul_comba8# 585.align 64 586bn_mul_comba8: 587 .prologue 588 .fframe 0 589 .save ar.pfs,r2 590{ .mii; alloc r2=ar.pfs,3,0,0,0 591 add r14=8,r33 592 add r17=8,r34 } 593 .body 594{ .mii; add r15=16,r33 595 add r18=16,r34 596 add r16=24,r33 } 597.L_cheat_entry_point8: 598{ .mmi; add r19=24,r34 599 600 ldf8 f32=[r33],32 };; 601 602{ .mmi; ldf8 f120=[r34],32 603 ldf8 f121=[r17],32 } 604{ .mmi; ldf8 f122=[r18],32 605 ldf8 f123=[r19],32 };; 606{ .mmi; ldf8 f124=[r34] 607 ldf8 f125=[r17] } 608{ .mmi; ldf8 f126=[r18] 609 ldf8 f127=[r19] } 610 611{ .mmi; ldf8 f33=[r14],32 612 ldf8 f34=[r15],32 } 613{ .mmi; ldf8 f35=[r16],32;; 614 ldf8 f36=[r33] } 615{ .mmi; ldf8 f37=[r14] 616 ldf8 f38=[r15] } 617{ .mfi; ldf8 f39=[r16] 618// -------\ Entering multiplier's heaven /------- 619// ------------\ /------------ 620// -----------------\ /----------------- 621// ----------------------\/---------------------- 622 xma.hu f41=f32,f120,f0 } 623{ .mfi; xma.lu f40=f32,f120,f0 };; // (*) 624{ .mfi; xma.hu f51=f32,f121,f0 } 625{ .mfi; xma.lu f50=f32,f121,f0 };; 626{ .mfi; xma.hu f61=f32,f122,f0 } 627{ .mfi; xma.lu f60=f32,f122,f0 };; 628{ .mfi; xma.hu f71=f32,f123,f0 } 629{ .mfi; xma.lu f70=f32,f123,f0 };; 630{ .mfi; xma.hu f81=f32,f124,f0 } 631{ .mfi; xma.lu f80=f32,f124,f0 };; 632{ .mfi; xma.hu f91=f32,f125,f0 } 633{ .mfi; xma.lu f90=f32,f125,f0 };; 634{ .mfi; xma.hu f101=f32,f126,f0 } 635{ .mfi; xma.lu f100=f32,f126,f0 };; 636{ .mfi; xma.hu f111=f32,f127,f0 } 637{ .mfi; xma.lu f110=f32,f127,f0 };;// 638// (*) You can argue that splitting at every second bundle would 639// prevent "wider" IA-64 implementations from achieving the peak 640// performance. Well, not really... The catch is that if you 641// intend to keep 4 FP units busy by splitting at every fourth 642// bundle and thus perform these 16 multiplications in 4 ticks, 643// the first bundle *below* would stall because the result from 644// the first xma bundle *above* won't be available for another 3 645// ticks (if not more, being an optimist, I assume that "wider" 646// implementation will have same latency:-). This stall will hold 647// you back and the performance would be as if every second bundle 648// were split *anyway*... 649{ .mfi; getf.sig r16=f40 650 xma.hu f42=f33,f120,f41 651 add r33=8,r32 } 652{ .mfi; xma.lu f41=f33,f120,f41 };; 653{ .mfi; getf.sig r24=f50 654 xma.hu f52=f33,f121,f51 } 655{ .mfi; xma.lu f51=f33,f121,f51 };; 656{ .mfi; st8 [r32]=r16,16 657 xma.hu f62=f33,f122,f61 } 658{ .mfi; xma.lu f61=f33,f122,f61 };; 659{ .mfi; xma.hu f72=f33,f123,f71 } 660{ .mfi; xma.lu f71=f33,f123,f71 };; 661{ .mfi; xma.hu f82=f33,f124,f81 } 662{ .mfi; xma.lu f81=f33,f124,f81 };; 663{ .mfi; xma.hu f92=f33,f125,f91 } 664{ .mfi; xma.lu f91=f33,f125,f91 };; 665{ .mfi; xma.hu f102=f33,f126,f101 } 666{ .mfi; xma.lu f101=f33,f126,f101 };; 667{ .mfi; xma.hu f112=f33,f127,f111 } 668{ .mfi; xma.lu f111=f33,f127,f111 };;// 669//-------------------------------------------------// 670{ .mfi; getf.sig r25=f41 671 xma.hu f43=f34,f120,f42 } 672{ .mfi; xma.lu f42=f34,f120,f42 };; 673{ .mfi; getf.sig r16=f60 674 xma.hu f53=f34,f121,f52 } 675{ .mfi; xma.lu f52=f34,f121,f52 };; 676{ .mfi; getf.sig r17=f51 677 xma.hu f63=f34,f122,f62 678 add r25=r25,r24 } 679{ .mfi; xma.lu f62=f34,f122,f62 680 mov carry1=0 };; 681{ .mfi; cmp.ltu p6,p0=r25,r24 682 xma.hu f73=f34,f123,f72 } 683{ .mfi; xma.lu f72=f34,f123,f72 };; 684{ .mfi; st8 [r33]=r25,16 685 xma.hu f83=f34,f124,f82 686(p6) add carry1=1,carry1 } 687{ .mfi; xma.lu f82=f34,f124,f82 };; 688{ .mfi; xma.hu f93=f34,f125,f92 } 689{ .mfi; xma.lu f92=f34,f125,f92 };; 690{ .mfi; xma.hu f103=f34,f126,f102 } 691{ .mfi; xma.lu f102=f34,f126,f102 };; 692{ .mfi; xma.hu f113=f34,f127,f112 } 693{ .mfi; xma.lu f112=f34,f127,f112 };;// 694//-------------------------------------------------// 695{ .mfi; getf.sig r18=f42 696 xma.hu f44=f35,f120,f43 697 add r17=r17,r16 } 698{ .mfi; xma.lu f43=f35,f120,f43 };; 699{ .mfi; getf.sig r24=f70 700 xma.hu f54=f35,f121,f53 } 701{ .mfi; mov carry2=0 702 xma.lu f53=f35,f121,f53 };; 703{ .mfi; getf.sig r25=f61 704 xma.hu f64=f35,f122,f63 705 cmp.ltu p7,p0=r17,r16 } 706{ .mfi; add r18=r18,r17 707 xma.lu f63=f35,f122,f63 };; 708{ .mfi; getf.sig r26=f52 709 xma.hu f74=f35,f123,f73 710(p7) add carry2=1,carry2 } 711{ .mfi; cmp.ltu p7,p0=r18,r17 712 xma.lu f73=f35,f123,f73 713 add r18=r18,carry1 };; 714{ .mfi; 715 xma.hu f84=f35,f124,f83 716(p7) add carry2=1,carry2 } 717{ .mfi; cmp.ltu p7,p0=r18,carry1 718 xma.lu f83=f35,f124,f83 };; 719{ .mfi; st8 [r32]=r18,16 720 xma.hu f94=f35,f125,f93 721(p7) add carry2=1,carry2 } 722{ .mfi; xma.lu f93=f35,f125,f93 };; 723{ .mfi; xma.hu f104=f35,f126,f103 } 724{ .mfi; xma.lu f103=f35,f126,f103 };; 725{ .mfi; xma.hu f114=f35,f127,f113 } 726{ .mfi; mov carry1=0 727 xma.lu f113=f35,f127,f113 728 add r25=r25,r24 };;// 729//-------------------------------------------------// 730{ .mfi; getf.sig r27=f43 731 xma.hu f45=f36,f120,f44 732 cmp.ltu p6,p0=r25,r24 } 733{ .mfi; xma.lu f44=f36,f120,f44 734 add r26=r26,r25 };; 735{ .mfi; getf.sig r16=f80 736 xma.hu f55=f36,f121,f54 737(p6) add carry1=1,carry1 } 738{ .mfi; xma.lu f54=f36,f121,f54 };; 739{ .mfi; getf.sig r17=f71 740 xma.hu f65=f36,f122,f64 741 cmp.ltu p6,p0=r26,r25 } 742{ .mfi; xma.lu f64=f36,f122,f64 743 add r27=r27,r26 };; 744{ .mfi; getf.sig r18=f62 745 xma.hu f75=f36,f123,f74 746(p6) add carry1=1,carry1 } 747{ .mfi; cmp.ltu p6,p0=r27,r26 748 xma.lu f74=f36,f123,f74 749 add r27=r27,carry2 };; 750{ .mfi; getf.sig r19=f53 751 xma.hu f85=f36,f124,f84 752(p6) add carry1=1,carry1 } 753{ .mfi; xma.lu f84=f36,f124,f84 754 cmp.ltu p6,p0=r27,carry2 };; 755{ .mfi; st8 [r33]=r27,16 756 xma.hu f95=f36,f125,f94 757(p6) add carry1=1,carry1 } 758{ .mfi; xma.lu f94=f36,f125,f94 };; 759{ .mfi; xma.hu f105=f36,f126,f104 } 760{ .mfi; mov carry2=0 761 xma.lu f104=f36,f126,f104 762 add r17=r17,r16 };; 763{ .mfi; xma.hu f115=f36,f127,f114 764 cmp.ltu p7,p0=r17,r16 } 765{ .mfi; xma.lu f114=f36,f127,f114 766 add r18=r18,r17 };;// 767//-------------------------------------------------// 768{ .mfi; getf.sig r20=f44 769 xma.hu f46=f37,f120,f45 770(p7) add carry2=1,carry2 } 771{ .mfi; cmp.ltu p7,p0=r18,r17 772 xma.lu f45=f37,f120,f45 773 add r19=r19,r18 };; 774{ .mfi; getf.sig r24=f90 775 xma.hu f56=f37,f121,f55 } 776{ .mfi; xma.lu f55=f37,f121,f55 };; 777{ .mfi; getf.sig r25=f81 778 xma.hu f66=f37,f122,f65 779(p7) add carry2=1,carry2 } 780{ .mfi; cmp.ltu p7,p0=r19,r18 781 xma.lu f65=f37,f122,f65 782 add r20=r20,r19 };; 783{ .mfi; getf.sig r26=f72 784 xma.hu f76=f37,f123,f75 785(p7) add carry2=1,carry2 } 786{ .mfi; cmp.ltu p7,p0=r20,r19 787 xma.lu f75=f37,f123,f75 788 add r20=r20,carry1 };; 789{ .mfi; getf.sig r27=f63 790 xma.hu f86=f37,f124,f85 791(p7) add carry2=1,carry2 } 792{ .mfi; xma.lu f85=f37,f124,f85 793 cmp.ltu p7,p0=r20,carry1 };; 794{ .mfi; getf.sig r28=f54 795 xma.hu f96=f37,f125,f95 796(p7) add carry2=1,carry2 } 797{ .mfi; st8 [r32]=r20,16 798 xma.lu f95=f37,f125,f95 };; 799{ .mfi; xma.hu f106=f37,f126,f105 } 800{ .mfi; mov carry1=0 801 xma.lu f105=f37,f126,f105 802 add r25=r25,r24 };; 803{ .mfi; xma.hu f116=f37,f127,f115 804 cmp.ltu p6,p0=r25,r24 } 805{ .mfi; xma.lu f115=f37,f127,f115 806 add r26=r26,r25 };;// 807//-------------------------------------------------// 808{ .mfi; getf.sig r29=f45 809 xma.hu f47=f38,f120,f46 810(p6) add carry1=1,carry1 } 811{ .mfi; cmp.ltu p6,p0=r26,r25 812 xma.lu f46=f38,f120,f46 813 add r27=r27,r26 };; 814{ .mfi; getf.sig r16=f100 815 xma.hu f57=f38,f121,f56 816(p6) add carry1=1,carry1 } 817{ .mfi; cmp.ltu p6,p0=r27,r26 818 xma.lu f56=f38,f121,f56 819 add r28=r28,r27 };; 820{ .mfi; getf.sig r17=f91 821 xma.hu f67=f38,f122,f66 822(p6) add carry1=1,carry1 } 823{ .mfi; cmp.ltu p6,p0=r28,r27 824 xma.lu f66=f38,f122,f66 825 add r29=r29,r28 };; 826{ .mfi; getf.sig r18=f82 827 xma.hu f77=f38,f123,f76 828(p6) add carry1=1,carry1 } 829{ .mfi; cmp.ltu p6,p0=r29,r28 830 xma.lu f76=f38,f123,f76 831 add r29=r29,carry2 };; 832{ .mfi; getf.sig r19=f73 833 xma.hu f87=f38,f124,f86 834(p6) add carry1=1,carry1 } 835{ .mfi; xma.lu f86=f38,f124,f86 836 cmp.ltu p6,p0=r29,carry2 };; 837{ .mfi; getf.sig r20=f64 838 xma.hu f97=f38,f125,f96 839(p6) add carry1=1,carry1 } 840{ .mfi; st8 [r33]=r29,16 841 xma.lu f96=f38,f125,f96 };; 842{ .mfi; getf.sig r21=f55 843 xma.hu f107=f38,f126,f106 } 844{ .mfi; mov carry2=0 845 xma.lu f106=f38,f126,f106 846 add r17=r17,r16 };; 847{ .mfi; xma.hu f117=f38,f127,f116 848 cmp.ltu p7,p0=r17,r16 } 849{ .mfi; xma.lu f116=f38,f127,f116 850 add r18=r18,r17 };;// 851//-------------------------------------------------// 852{ .mfi; getf.sig r22=f46 853 xma.hu f48=f39,f120,f47 854(p7) add carry2=1,carry2 } 855{ .mfi; cmp.ltu p7,p0=r18,r17 856 xma.lu f47=f39,f120,f47 857 add r19=r19,r18 };; 858{ .mfi; getf.sig r24=f110 859 xma.hu f58=f39,f121,f57 860(p7) add carry2=1,carry2 } 861{ .mfi; cmp.ltu p7,p0=r19,r18 862 xma.lu f57=f39,f121,f57 863 add r20=r20,r19 };; 864{ .mfi; getf.sig r25=f101 865 xma.hu f68=f39,f122,f67 866(p7) add carry2=1,carry2 } 867{ .mfi; cmp.ltu p7,p0=r20,r19 868 xma.lu f67=f39,f122,f67 869 add r21=r21,r20 };; 870{ .mfi; getf.sig r26=f92 871 xma.hu f78=f39,f123,f77 872(p7) add carry2=1,carry2 } 873{ .mfi; cmp.ltu p7,p0=r21,r20 874 xma.lu f77=f39,f123,f77 875 add r22=r22,r21 };; 876{ .mfi; getf.sig r27=f83 877 xma.hu f88=f39,f124,f87 878(p7) add carry2=1,carry2 } 879{ .mfi; cmp.ltu p7,p0=r22,r21 880 xma.lu f87=f39,f124,f87 881 add r22=r22,carry1 };; 882{ .mfi; getf.sig r28=f74 883 xma.hu f98=f39,f125,f97 884(p7) add carry2=1,carry2 } 885{ .mfi; xma.lu f97=f39,f125,f97 886 cmp.ltu p7,p0=r22,carry1 };; 887{ .mfi; getf.sig r29=f65 888 xma.hu f108=f39,f126,f107 889(p7) add carry2=1,carry2 } 890{ .mfi; st8 [r32]=r22,16 891 xma.lu f107=f39,f126,f107 };; 892{ .mfi; getf.sig r30=f56 893 xma.hu f118=f39,f127,f117 } 894{ .mfi; xma.lu f117=f39,f127,f117 };;// 895//-------------------------------------------------// 896// Leaving muliplier's heaven... Quite a ride, huh? 897 898{ .mii; getf.sig r31=f47 899 add r25=r25,r24 900 mov carry1=0 };; 901{ .mii; getf.sig r16=f111 902 cmp.ltu p6,p0=r25,r24 903 add r26=r26,r25 };; 904{ .mfb; getf.sig r17=f102 } 905{ .mii; 906(p6) add carry1=1,carry1 907 cmp.ltu p6,p0=r26,r25 908 add r27=r27,r26 };; 909{ .mfb; nop.m 0x0 } 910{ .mii; 911(p6) add carry1=1,carry1 912 cmp.ltu p6,p0=r27,r26 913 add r28=r28,r27 };; 914{ .mii; getf.sig r18=f93 915 add r17=r17,r16 916 mov carry3=0 } 917{ .mii; 918(p6) add carry1=1,carry1 919 cmp.ltu p6,p0=r28,r27 920 add r29=r29,r28 };; 921{ .mii; getf.sig r19=f84 922 cmp.ltu p7,p0=r17,r16 } 923{ .mii; 924(p6) add carry1=1,carry1 925 cmp.ltu p6,p0=r29,r28 926 add r30=r30,r29 };; 927{ .mii; getf.sig r20=f75 928 add r18=r18,r17 } 929{ .mii; 930(p6) add carry1=1,carry1 931 cmp.ltu p6,p0=r30,r29 932 add r31=r31,r30 };; 933{ .mfb; getf.sig r21=f66 } 934{ .mii; (p7) add carry3=1,carry3 935 cmp.ltu p7,p0=r18,r17 936 add r19=r19,r18 } 937{ .mfb; nop.m 0x0 } 938{ .mii; 939(p6) add carry1=1,carry1 940 cmp.ltu p6,p0=r31,r30 941 add r31=r31,carry2 };; 942{ .mfb; getf.sig r22=f57 } 943{ .mii; (p7) add carry3=1,carry3 944 cmp.ltu p7,p0=r19,r18 945 add r20=r20,r19 } 946{ .mfb; nop.m 0x0 } 947{ .mii; 948(p6) add carry1=1,carry1 949 cmp.ltu p6,p0=r31,carry2 };; 950{ .mfb; getf.sig r23=f48 } 951{ .mii; (p7) add carry3=1,carry3 952 cmp.ltu p7,p0=r20,r19 953 add r21=r21,r20 } 954{ .mii; 955(p6) add carry1=1,carry1 } 956{ .mfb; st8 [r33]=r31,16 };; 957 958{ .mfb; getf.sig r24=f112 } 959{ .mii; (p7) add carry3=1,carry3 960 cmp.ltu p7,p0=r21,r20 961 add r22=r22,r21 };; 962{ .mfb; getf.sig r25=f103 } 963{ .mii; (p7) add carry3=1,carry3 964 cmp.ltu p7,p0=r22,r21 965 add r23=r23,r22 };; 966{ .mfb; getf.sig r26=f94 } 967{ .mii; (p7) add carry3=1,carry3 968 cmp.ltu p7,p0=r23,r22 969 add r23=r23,carry1 };; 970{ .mfb; getf.sig r27=f85 } 971{ .mii; (p7) add carry3=1,carry3 972 cmp.ltu p7,p8=r23,carry1};; 973{ .mii; getf.sig r28=f76 974 add r25=r25,r24 975 mov carry1=0 } 976{ .mii; st8 [r32]=r23,16 977 (p7) add carry2=1,carry3 978 (p8) add carry2=0,carry3 };; 979 980{ .mfb; nop.m 0x0 } 981{ .mii; getf.sig r29=f67 982 cmp.ltu p6,p0=r25,r24 983 add r26=r26,r25 };; 984{ .mfb; getf.sig r30=f58 } 985{ .mii; 986(p6) add carry1=1,carry1 987 cmp.ltu p6,p0=r26,r25 988 add r27=r27,r26 };; 989{ .mfb; getf.sig r16=f113 } 990{ .mii; 991(p6) add carry1=1,carry1 992 cmp.ltu p6,p0=r27,r26 993 add r28=r28,r27 };; 994{ .mfb; getf.sig r17=f104 } 995{ .mii; 996(p6) add carry1=1,carry1 997 cmp.ltu p6,p0=r28,r27 998 add r29=r29,r28 };; 999{ .mfb; getf.sig r18=f95 } 1000{ .mii; 1001(p6) add carry1=1,carry1 1002 cmp.ltu p6,p0=r29,r28 1003 add r30=r30,r29 };; 1004{ .mii; getf.sig r19=f86 1005 add r17=r17,r16 1006 mov carry3=0 } 1007{ .mii; 1008(p6) add carry1=1,carry1 1009 cmp.ltu p6,p0=r30,r29 1010 add r30=r30,carry2 };; 1011{ .mii; getf.sig r20=f77 1012 cmp.ltu p7,p0=r17,r16 1013 add r18=r18,r17 } 1014{ .mii; 1015(p6) add carry1=1,carry1 1016 cmp.ltu p6,p0=r30,carry2 };; 1017{ .mfb; getf.sig r21=f68 } 1018{ .mii; st8 [r33]=r30,16 1019(p6) add carry1=1,carry1 };; 1020 1021{ .mfb; getf.sig r24=f114 } 1022{ .mii; (p7) add carry3=1,carry3 1023 cmp.ltu p7,p0=r18,r17 1024 add r19=r19,r18 };; 1025{ .mfb; getf.sig r25=f105 } 1026{ .mii; (p7) add carry3=1,carry3 1027 cmp.ltu p7,p0=r19,r18 1028 add r20=r20,r19 };; 1029{ .mfb; getf.sig r26=f96 } 1030{ .mii; (p7) add carry3=1,carry3 1031 cmp.ltu p7,p0=r20,r19 1032 add r21=r21,r20 };; 1033{ .mfb; getf.sig r27=f87 } 1034{ .mii; (p7) add carry3=1,carry3 1035 cmp.ltu p7,p0=r21,r20 1036 add r21=r21,carry1 };; 1037{ .mib; getf.sig r28=f78 1038 add r25=r25,r24 } 1039{ .mib; (p7) add carry3=1,carry3 1040 cmp.ltu p7,p8=r21,carry1};; 1041{ .mii; st8 [r32]=r21,16 1042 (p7) add carry2=1,carry3 1043 (p8) add carry2=0,carry3 } 1044 1045{ .mii; mov carry1=0 1046 cmp.ltu p6,p0=r25,r24 1047 add r26=r26,r25 };; 1048{ .mfb; getf.sig r16=f115 } 1049{ .mii; 1050(p6) add carry1=1,carry1 1051 cmp.ltu p6,p0=r26,r25 1052 add r27=r27,r26 };; 1053{ .mfb; getf.sig r17=f106 } 1054{ .mii; 1055(p6) add carry1=1,carry1 1056 cmp.ltu p6,p0=r27,r26 1057 add r28=r28,r27 };; 1058{ .mfb; getf.sig r18=f97 } 1059{ .mii; 1060(p6) add carry1=1,carry1 1061 cmp.ltu p6,p0=r28,r27 1062 add r28=r28,carry2 };; 1063{ .mib; getf.sig r19=f88 1064 add r17=r17,r16 } 1065{ .mib; 1066(p6) add carry1=1,carry1 1067 cmp.ltu p6,p0=r28,carry2 };; 1068{ .mii; st8 [r33]=r28,16 1069(p6) add carry1=1,carry1 } 1070 1071{ .mii; mov carry2=0 1072 cmp.ltu p7,p0=r17,r16 1073 add r18=r18,r17 };; 1074{ .mfb; getf.sig r24=f116 } 1075{ .mii; (p7) add carry2=1,carry2 1076 cmp.ltu p7,p0=r18,r17 1077 add r19=r19,r18 };; 1078{ .mfb; getf.sig r25=f107 } 1079{ .mii; (p7) add carry2=1,carry2 1080 cmp.ltu p7,p0=r19,r18 1081 add r19=r19,carry1 };; 1082{ .mfb; getf.sig r26=f98 } 1083{ .mii; (p7) add carry2=1,carry2 1084 cmp.ltu p7,p0=r19,carry1};; 1085{ .mii; st8 [r32]=r19,16 1086 (p7) add carry2=1,carry2 } 1087 1088{ .mfb; add r25=r25,r24 };; 1089 1090{ .mfb; getf.sig r16=f117 } 1091{ .mii; mov carry1=0 1092 cmp.ltu p6,p0=r25,r24 1093 add r26=r26,r25 };; 1094{ .mfb; getf.sig r17=f108 } 1095{ .mii; 1096(p6) add carry1=1,carry1 1097 cmp.ltu p6,p0=r26,r25 1098 add r26=r26,carry2 };; 1099{ .mfb; nop.m 0x0 } 1100{ .mii; 1101(p6) add carry1=1,carry1 1102 cmp.ltu p6,p0=r26,carry2 };; 1103{ .mii; st8 [r33]=r26,16 1104(p6) add carry1=1,carry1 } 1105 1106{ .mfb; add r17=r17,r16 };; 1107{ .mfb; getf.sig r24=f118 } 1108{ .mii; mov carry2=0 1109 cmp.ltu p7,p0=r17,r16 1110 add r17=r17,carry1 };; 1111{ .mii; (p7) add carry2=1,carry2 1112 cmp.ltu p7,p0=r17,carry1};; 1113{ .mii; st8 [r32]=r17 1114 (p7) add carry2=1,carry2 };; 1115{ .mfb; add r24=r24,carry2 };; 1116{ .mib; st8 [r33]=r24 } 1117 1118{ .mib; rum 1<<5 // clear um.mfh 1119 br.ret.sptk.many b0 };; 1120.endp bn_mul_comba8# 1121#undef carry3 1122#undef carry2 1123#undef carry1 1124#endif 1125 1126#if 1 1127// It's possible to make it faster (see comment to bn_sqr_comba8), but 1128// I reckon it doesn't worth the effort. Basically because the routine 1129// (actually both of them) practically never called... So I just play 1130// same trick as with bn_sqr_comba8. 1131// 1132// void bn_sqr_comba4(BN_ULONG *r, BN_ULONG *a) 1133// 1134.global bn_sqr_comba4# 1135.proc bn_sqr_comba4# 1136.align 64 1137bn_sqr_comba4: 1138 .prologue 1139 .fframe 0 1140 .save ar.pfs,r2 1141{ .mii; alloc r2=ar.pfs,2,1,0,0 1142 mov r34=r33 1143 add r14=8,r33 };; 1144 .body 1145{ .mii; add r17=8,r34 1146 add r15=16,r33 1147 add r18=16,r34 } 1148{ .mfb; add r16=24,r33 1149 br .L_cheat_entry_point4 };; 1150.endp bn_sqr_comba4# 1151#endif 1152 1153#if 1 1154// Runs in ~115 cycles and ~4.5 times faster than C. Well, whatever... 1155// 1156// void bn_mul_comba4(BN_ULONG *r, BN_ULONG *a, BN_ULONG *b) 1157// 1158#define carry1 r14 1159#define carry2 r15 1160.global bn_mul_comba4# 1161.proc bn_mul_comba4# 1162.align 64 1163bn_mul_comba4: 1164 .prologue 1165 .fframe 0 1166 .save ar.pfs,r2 1167{ .mii; alloc r2=ar.pfs,3,0,0,0 1168 add r14=8,r33 1169 add r17=8,r34 } 1170 .body 1171{ .mii; add r15=16,r33 1172 add r18=16,r34 1173 add r16=24,r33 };; 1174.L_cheat_entry_point4: 1175{ .mmi; add r19=24,r34 1176 1177 ldf8 f32=[r33] } 1178 1179{ .mmi; ldf8 f120=[r34] 1180 ldf8 f121=[r17] };; 1181{ .mmi; ldf8 f122=[r18] 1182 ldf8 f123=[r19] } 1183 1184{ .mmi; ldf8 f33=[r14] 1185 ldf8 f34=[r15] } 1186{ .mfi; ldf8 f35=[r16] 1187 1188 xma.hu f41=f32,f120,f0 } 1189{ .mfi; xma.lu f40=f32,f120,f0 };; 1190{ .mfi; xma.hu f51=f32,f121,f0 } 1191{ .mfi; xma.lu f50=f32,f121,f0 };; 1192{ .mfi; xma.hu f61=f32,f122,f0 } 1193{ .mfi; xma.lu f60=f32,f122,f0 };; 1194{ .mfi; xma.hu f71=f32,f123,f0 } 1195{ .mfi; xma.lu f70=f32,f123,f0 };;// 1196// Major stall takes place here, and 3 more places below. Result from 1197// first xma is not available for another 3 ticks. 1198{ .mfi; getf.sig r16=f40 1199 xma.hu f42=f33,f120,f41 1200 add r33=8,r32 } 1201{ .mfi; xma.lu f41=f33,f120,f41 };; 1202{ .mfi; getf.sig r24=f50 1203 xma.hu f52=f33,f121,f51 } 1204{ .mfi; xma.lu f51=f33,f121,f51 };; 1205{ .mfi; st8 [r32]=r16,16 1206 xma.hu f62=f33,f122,f61 } 1207{ .mfi; xma.lu f61=f33,f122,f61 };; 1208{ .mfi; xma.hu f72=f33,f123,f71 } 1209{ .mfi; xma.lu f71=f33,f123,f71 };;// 1210//-------------------------------------------------// 1211{ .mfi; getf.sig r25=f41 1212 xma.hu f43=f34,f120,f42 } 1213{ .mfi; xma.lu f42=f34,f120,f42 };; 1214{ .mfi; getf.sig r16=f60 1215 xma.hu f53=f34,f121,f52 } 1216{ .mfi; xma.lu f52=f34,f121,f52 };; 1217{ .mfi; getf.sig r17=f51 1218 xma.hu f63=f34,f122,f62 1219 add r25=r25,r24 } 1220{ .mfi; mov carry1=0 1221 xma.lu f62=f34,f122,f62 };; 1222{ .mfi; st8 [r33]=r25,16 1223 xma.hu f73=f34,f123,f72 1224 cmp.ltu p6,p0=r25,r24 } 1225{ .mfi; xma.lu f72=f34,f123,f72 };;// 1226//-------------------------------------------------// 1227{ .mfi; getf.sig r18=f42 1228 xma.hu f44=f35,f120,f43 1229(p6) add carry1=1,carry1 } 1230{ .mfi; add r17=r17,r16 1231 xma.lu f43=f35,f120,f43 1232 mov carry2=0 };; 1233{ .mfi; getf.sig r24=f70 1234 xma.hu f54=f35,f121,f53 1235 cmp.ltu p7,p0=r17,r16 } 1236{ .mfi; xma.lu f53=f35,f121,f53 };; 1237{ .mfi; getf.sig r25=f61 1238 xma.hu f64=f35,f122,f63 1239 add r18=r18,r17 } 1240{ .mfi; xma.lu f63=f35,f122,f63 1241(p7) add carry2=1,carry2 };; 1242{ .mfi; getf.sig r26=f52 1243 xma.hu f74=f35,f123,f73 1244 cmp.ltu p7,p0=r18,r17 } 1245{ .mfi; xma.lu f73=f35,f123,f73 1246 add r18=r18,carry1 };; 1247//-------------------------------------------------// 1248{ .mii; st8 [r32]=r18,16 1249(p7) add carry2=1,carry2 1250 cmp.ltu p7,p0=r18,carry1 };; 1251 1252{ .mfi; getf.sig r27=f43 // last major stall 1253(p7) add carry2=1,carry2 };; 1254{ .mii; getf.sig r16=f71 1255 add r25=r25,r24 1256 mov carry1=0 };; 1257{ .mii; getf.sig r17=f62 1258 cmp.ltu p6,p0=r25,r24 1259 add r26=r26,r25 };; 1260{ .mii; 1261(p6) add carry1=1,carry1 1262 cmp.ltu p6,p0=r26,r25 1263 add r27=r27,r26 };; 1264{ .mii; 1265(p6) add carry1=1,carry1 1266 cmp.ltu p6,p0=r27,r26 1267 add r27=r27,carry2 };; 1268{ .mii; getf.sig r18=f53 1269(p6) add carry1=1,carry1 1270 cmp.ltu p6,p0=r27,carry2 };; 1271{ .mfi; st8 [r33]=r27,16 1272(p6) add carry1=1,carry1 } 1273 1274{ .mii; getf.sig r19=f44 1275 add r17=r17,r16 1276 mov carry2=0 };; 1277{ .mii; getf.sig r24=f72 1278 cmp.ltu p7,p0=r17,r16 1279 add r18=r18,r17 };; 1280{ .mii; (p7) add carry2=1,carry2 1281 cmp.ltu p7,p0=r18,r17 1282 add r19=r19,r18 };; 1283{ .mii; (p7) add carry2=1,carry2 1284 cmp.ltu p7,p0=r19,r18 1285 add r19=r19,carry1 };; 1286{ .mii; getf.sig r25=f63 1287 (p7) add carry2=1,carry2 1288 cmp.ltu p7,p0=r19,carry1};; 1289{ .mii; st8 [r32]=r19,16 1290 (p7) add carry2=1,carry2 } 1291 1292{ .mii; getf.sig r26=f54 1293 add r25=r25,r24 1294 mov carry1=0 };; 1295{ .mii; getf.sig r16=f73 1296 cmp.ltu p6,p0=r25,r24 1297 add r26=r26,r25 };; 1298{ .mii; 1299(p6) add carry1=1,carry1 1300 cmp.ltu p6,p0=r26,r25 1301 add r26=r26,carry2 };; 1302{ .mii; getf.sig r17=f64 1303(p6) add carry1=1,carry1 1304 cmp.ltu p6,p0=r26,carry2 };; 1305{ .mii; st8 [r33]=r26,16 1306(p6) add carry1=1,carry1 } 1307 1308{ .mii; getf.sig r24=f74 1309 add r17=r17,r16 1310 mov carry2=0 };; 1311{ .mii; cmp.ltu p7,p0=r17,r16 1312 add r17=r17,carry1 };; 1313 1314{ .mii; (p7) add carry2=1,carry2 1315 cmp.ltu p7,p0=r17,carry1};; 1316{ .mii; st8 [r32]=r17,16 1317 (p7) add carry2=1,carry2 };; 1318 1319{ .mii; add r24=r24,carry2 };; 1320{ .mii; st8 [r33]=r24 } 1321 1322{ .mib; rum 1<<5 // clear um.mfh 1323 br.ret.sptk.many b0 };; 1324.endp bn_mul_comba4# 1325#undef carry2 1326#undef carry1 1327#endif 1328 1329#if 1 1330// 1331// BN_ULONG bn_div_words(BN_ULONG h, BN_ULONG l, BN_ULONG d) 1332// 1333// In the nutshell it's a port of my MIPS III/IV implementation. 1334// 1335#define AT r14 1336#define H r16 1337#define HH r20 1338#define L r17 1339#define D r18 1340#define DH r22 1341#define I r21 1342 1343#if 0 1344// Some preprocessors (most notably HP-UX) apper to be allergic to 1345// macros enclosed to parenthesis as these three will be. 1346#define cont p16 1347#define break p0 // p20 1348#define equ p24 1349#else 1350cont=p16 1351break=p0 1352equ=p24 1353#endif 1354 1355.global abort# 1356.global bn_div_words# 1357.proc bn_div_words# 1358.align 64 1359bn_div_words: 1360 .prologue 1361 .fframe 0 1362 .save ar.pfs,r2 1363 .save b0,r3 1364{ .mii; alloc r2=ar.pfs,3,5,0,8 1365 mov r3=b0 1366 mov r10=pr };; 1367{ .mmb; cmp.eq p6,p0=r34,r0 1368 mov r8=-1 1369(p6) br.ret.spnt.many b0 };; 1370 1371 .body 1372{ .mii; mov H=r32 // save h 1373 mov ar.ec=0 // don't rotate at exit 1374 mov pr.rot=0 } 1375{ .mii; mov L=r33 // save l 1376 mov r36=r0 };; 1377 1378.L_divw_shift: // -vv- note signed comparison 1379{ .mfi; (p0) cmp.lt p16,p0=r0,r34 // d 1380 (p0) shladd r33=r34,1,r0 } 1381{ .mfb; (p0) add r35=1,r36 1382 (p0) nop.f 0x0 1383(p16) br.wtop.dpnt .L_divw_shift };; 1384 1385{ .mii; mov D=r34 1386 shr.u DH=r34,32 1387 sub r35=64,r36 };; 1388{ .mii; setf.sig f7=DH 1389 shr.u AT=H,r35 1390 mov I=r36 };; 1391{ .mib; cmp.ne p6,p0=r0,AT 1392 shl H=H,r36 1393(p6) br.call.spnt.clr b0=abort };; // overflow, die... 1394 1395{ .mfi; fcvt.xuf.s1 f7=f7 1396 shr.u AT=L,r35 };; 1397{ .mii; shl L=L,r36 1398 or H=H,AT };; 1399 1400{ .mii; nop.m 0x0 1401 cmp.leu p6,p0=D,H;; 1402(p6) sub H=H,D } 1403 1404{ .mlx; setf.sig f14=D 1405 movl AT=0xffffffff };; 1406/////////////////////////////////////////////////////////// 1407{ .mii; setf.sig f6=H 1408 shr.u HH=H,32;; 1409 cmp.eq p6,p7=HH,DH };; 1410{ .mfb; 1411(p6) setf.sig f8=AT 1412(p7) fcvt.xuf.s1 f6=f6 1413(p7) br.call.sptk b6=.L_udiv64_32_b6 };; 1414 1415{ .mfi; getf.sig r33=f8 // q 1416 xmpy.lu f9=f8,f14 } 1417{ .mfi; xmpy.hu f10=f8,f14 1418 shrp H=H,L,32 };; 1419 1420{ .mmi; getf.sig r35=f9 // tl 1421 getf.sig r31=f10 };; // th 1422 1423.L_divw_1st_iter: 1424{ .mii; (p0) add r32=-1,r33 1425 (p0) cmp.eq equ,cont=HH,r31 };; 1426{ .mii; (p0) cmp.ltu p8,p0=r35,D 1427 (p0) sub r34=r35,D 1428 (equ) cmp.leu break,cont=r35,H };; 1429{ .mib; (cont) cmp.leu cont,break=HH,r31 1430 (p8) add r31=-1,r31 1431(cont) br.wtop.spnt .L_divw_1st_iter };; 1432/////////////////////////////////////////////////////////// 1433{ .mii; sub H=H,r35 1434 shl r8=r33,32 1435 shl L=L,32 };; 1436/////////////////////////////////////////////////////////// 1437{ .mii; setf.sig f6=H 1438 shr.u HH=H,32;; 1439 cmp.eq p6,p7=HH,DH };; 1440{ .mfb; 1441(p6) setf.sig f8=AT 1442(p7) fcvt.xuf.s1 f6=f6 1443(p7) br.call.sptk b6=.L_udiv64_32_b6 };; 1444 1445{ .mfi; getf.sig r33=f8 // q 1446 xmpy.lu f9=f8,f14 } 1447{ .mfi; xmpy.hu f10=f8,f14 1448 shrp H=H,L,32 };; 1449 1450{ .mmi; getf.sig r35=f9 // tl 1451 getf.sig r31=f10 };; // th 1452 1453.L_divw_2nd_iter: 1454{ .mii; (p0) add r32=-1,r33 1455 (p0) cmp.eq equ,cont=HH,r31 };; 1456{ .mii; (p0) cmp.ltu p8,p0=r35,D 1457 (p0) sub r34=r35,D 1458 (equ) cmp.leu break,cont=r35,H };; 1459{ .mib; (cont) cmp.leu cont,break=HH,r31 1460 (p8) add r31=-1,r31 1461(cont) br.wtop.spnt .L_divw_2nd_iter };; 1462/////////////////////////////////////////////////////////// 1463{ .mii; sub H=H,r35 1464 or r8=r8,r33 1465 mov ar.pfs=r2 };; 1466{ .mii; shr.u r9=H,I // remainder if anybody wants it 1467 mov pr=r10,-1 } 1468{ .mfb; br.ret.sptk.many b0 };; 1469 1470// Unsigned 64 by 32 (well, by 64 for the moment) bit integer division 1471// procedure. 1472// 1473// inputs: f6 = (double)a, f7 = (double)b 1474// output: f8 = (int)(a/b) 1475// clobbered: f8,f9,f10,f11,pred 1476pred=p15 1477// This procedure is essentially Intel code and therefore is 1478// copyrighted to Intel Corporation (I suppose...). It's sligtly 1479// modified for specific needs. 1480.align 32 1481.skip 16 1482.L_udiv64_32_b6: 1483 frcpa.s1 f8,pred=f6,f7;; // [0] y0 = 1 / b 1484 1485(pred) fnma.s1 f9=f7,f8,f1 // [5] e0 = 1 - b * y0 1486(pred) fmpy.s1 f10=f6,f8;; // [5] q0 = a * y0 1487(pred) fmpy.s1 f11=f9,f9 // [10] e1 = e0 * e0 1488(pred) fma.s1 f10=f9,f10,f10;; // [10] q1 = q0 + e0 * q0 1489(pred) fma.s1 f8=f9,f8,f8 //;; // [15] y1 = y0 + e0 * y0 1490(pred) fma.s1 f9=f11,f10,f10;; // [15] q2 = q1 + e1 * q1 1491(pred) fma.s1 f8=f11,f8,f8 //;; // [20] y2 = y1 + e1 * y1 1492(pred) fnma.s1 f10=f7,f9,f6;; // [20] r2 = a - b * q2 1493(pred) fma.s1 f8=f10,f8,f9;; // [25] q3 = q2 + r2 * y2 1494 1495 fcvt.fxu.trunc.s1 f8=f8 // [30] q = trunc(q3) 1496 br.ret.sptk.many b6;; 1497.endp bn_div_words# 1498#endif 1499