1/* 2 * arch/alpha/lib/ev6-stxncpy.S 3 * 21264 version contributed by Rick Gorton <rick.gorton@api-networks.com> 4 * 5 * Copy no more than COUNT bytes of the null-terminated string from 6 * SRC to DST. 7 * 8 * This is an internal routine used by strncpy, stpncpy, and strncat. 9 * As such, it uses special linkage conventions to make implementation 10 * of these public functions more efficient. 11 * 12 * On input: 13 * t9 = return address 14 * a0 = DST 15 * a1 = SRC 16 * a2 = COUNT 17 * 18 * Furthermore, COUNT may not be zero. 19 * 20 * On output: 21 * t0 = last word written 22 * t10 = bitmask (with one bit set) indicating the byte position of 23 * the end of the range specified by COUNT 24 * t12 = bitmask (with one bit set) indicating the last byte written 25 * a0 = unaligned address of the last *word* written 26 * a2 = the number of full words left in COUNT 27 * 28 * Furthermore, v0, a3-a5, t11, and $at are untouched. 29 * 30 * Much of the information about 21264 scheduling/coding comes from: 31 * Compiler Writer's Guide for the Alpha 21264 32 * abbreviated as 'CWG' in other comments here 33 * ftp.digital.com/pub/Digital/info/semiconductor/literature/dsc-library.html 34 * Scheduling notation: 35 * E - either cluster 36 * U - upper subcluster; U0 - subcluster U0; U1 - subcluster U1 37 * L - lower subcluster; L0 - subcluster L0; L1 - subcluster L1 38 * Try not to change the actual algorithm if possible for consistency. 39 */ 40 41#include <asm/regdef.h> 42 43 .set noat 44 .set noreorder 45 46 .text 47 48 49 50 .ent stxncpy_aligned 51 .align 4 52stxncpy_aligned: 53 .frame sp, 0, t9, 0 54 .prologue 0 55 56 /* On entry to this basic block: 57 t0 == the first destination word for masking back in 58 t1 == the first source word. */ 59 60 /* Create the 1st output word and detect 0's in the 1st input word. */ 61 lda t2, -1 # E : build a mask against false zero 62 mskqh t2, a1, t2 # U : detection in the src word (stall) 63 mskqh t1, a1, t3 # U : 64 ornot t1, t2, t2 # E : (stall) 65 66 mskql t0, a1, t0 # U : assemble the first output word 67 cmpbge zero, t2, t8 # E : bits set iff null found 68 or t0, t3, t0 # E : (stall) 69 beq a2, $a_eoc # U : 70 71 bne t8, $a_eos # U : 72 nop 73 nop 74 nop 75 76 /* On entry to this basic block: 77 t0 == a source word not containing a null. */ 78 79 /* 80 * nops here to: 81 * separate store quads from load quads 82 * limit of 1 bcond/quad to permit training 83 */ 84$a_loop: 85 stq_u t0, 0(a0) # L : 86 addq a0, 8, a0 # E : 87 subq a2, 1, a2 # E : 88 nop 89 90 ldq_u t0, 0(a1) # L : 91 addq a1, 8, a1 # E : 92 cmpbge zero, t0, t8 # E : 93 beq a2, $a_eoc # U : 94 95 beq t8, $a_loop # U : 96 nop 97 nop 98 nop 99 100 /* Take care of the final (partial) word store. At this point 101 the end-of-count bit is set in t8 iff it applies. 102 103 On entry to this basic block we have: 104 t0 == the source word containing the null 105 t8 == the cmpbge mask that found it. */ 106 107$a_eos: 108 negq t8, t12 # E : find low bit set 109 and t8, t12, t12 # E : (stall) 110 /* For the sake of the cache, don't read a destination word 111 if we're not going to need it. */ 112 and t12, 0x80, t6 # E : (stall) 113 bne t6, 1f # U : (stall) 114 115 /* We're doing a partial word store and so need to combine 116 our source and original destination words. */ 117 ldq_u t1, 0(a0) # L : 118 subq t12, 1, t6 # E : 119 or t12, t6, t8 # E : (stall) 120 zapnot t0, t8, t0 # U : clear src bytes > null (stall) 121 122 zap t1, t8, t1 # .. e1 : clear dst bytes <= null 123 or t0, t1, t0 # e1 : (stall) 124 nop 125 nop 126 1271: stq_u t0, 0(a0) # L : 128 ret (t9) # L0 : Latency=3 129 nop 130 nop 131 132 /* Add the end-of-count bit to the eos detection bitmask. */ 133$a_eoc: 134 or t10, t8, t8 # E : 135 br $a_eos # L0 : Latency=3 136 nop 137 nop 138 139 .end stxncpy_aligned 140 141 .align 4 142 .ent __stxncpy 143 .globl __stxncpy 144__stxncpy: 145 .frame sp, 0, t9, 0 146 .prologue 0 147 148 /* Are source and destination co-aligned? */ 149 xor a0, a1, t1 # E : 150 and a0, 7, t0 # E : find dest misalignment 151 and t1, 7, t1 # E : (stall) 152 addq a2, t0, a2 # E : bias count by dest misalignment (stall) 153 154 subq a2, 1, a2 # E : 155 and a2, 7, t2 # E : (stall) 156 srl a2, 3, a2 # U : a2 = loop counter = (count - 1)/8 (stall) 157 addq zero, 1, t10 # E : 158 159 sll t10, t2, t10 # U : t10 = bitmask of last count byte 160 bne t1, $unaligned # U : 161 /* We are co-aligned; take care of a partial first word. */ 162 ldq_u t1, 0(a1) # L : load first src word 163 addq a1, 8, a1 # E : 164 165 beq t0, stxncpy_aligned # U : avoid loading dest word if not needed 166 ldq_u t0, 0(a0) # L : 167 nop 168 nop 169 170 br stxncpy_aligned # .. e1 : 171 nop 172 nop 173 nop 174 175 176 177/* The source and destination are not co-aligned. Align the destination 178 and cope. We have to be very careful about not reading too much and 179 causing a SEGV. */ 180 181 .align 4 182$u_head: 183 /* We know just enough now to be able to assemble the first 184 full source word. We can still find a zero at the end of it 185 that prevents us from outputting the whole thing. 186 187 On entry to this basic block: 188 t0 == the first dest word, unmasked 189 t1 == the shifted low bits of the first source word 190 t6 == bytemask that is -1 in dest word bytes */ 191 192 ldq_u t2, 8(a1) # L : Latency=3 load second src word 193 addq a1, 8, a1 # E : 194 mskql t0, a0, t0 # U : mask trailing garbage in dst 195 extqh t2, a1, t4 # U : (3 cycle stall on t2) 196 197 or t1, t4, t1 # E : first aligned src word complete (stall) 198 mskqh t1, a0, t1 # U : mask leading garbage in src (stall) 199 or t0, t1, t0 # E : first output word complete (stall) 200 or t0, t6, t6 # E : mask original data for zero test (stall) 201 202 cmpbge zero, t6, t8 # E : 203 beq a2, $u_eocfin # U : 204 lda t6, -1 # E : 205 nop 206 207 bne t8, $u_final # U : 208 mskql t6, a1, t6 # U : mask out bits already seen 209 stq_u t0, 0(a0) # L : store first output word 210 or t6, t2, t2 # E : (stall) 211 212 cmpbge zero, t2, t8 # E : find nulls in second partial 213 addq a0, 8, a0 # E : 214 subq a2, 1, a2 # E : 215 bne t8, $u_late_head_exit # U : 216 217 /* Finally, we've got all the stupid leading edge cases taken care 218 of and we can set up to enter the main loop. */ 219 extql t2, a1, t1 # U : position hi-bits of lo word 220 beq a2, $u_eoc # U : 221 ldq_u t2, 8(a1) # L : read next high-order source word 222 addq a1, 8, a1 # E : 223 224 extqh t2, a1, t0 # U : position lo-bits of hi word (stall) 225 cmpbge zero, t2, t8 # E : 226 nop 227 bne t8, $u_eos # U : 228 229 /* Unaligned copy main loop. In order to avoid reading too much, 230 the loop is structured to detect zeros in aligned source words. 231 This has, unfortunately, effectively pulled half of a loop 232 iteration out into the head and half into the tail, but it does 233 prevent nastiness from accumulating in the very thing we want 234 to run as fast as possible. 235 236 On entry to this basic block: 237 t0 == the shifted low-order bits from the current source word 238 t1 == the shifted high-order bits from the previous source word 239 t2 == the unshifted current source word 240 241 We further know that t2 does not contain a null terminator. */ 242 243 .align 4 244$u_loop: 245 or t0, t1, t0 # E : current dst word now complete 246 subq a2, 1, a2 # E : decrement word count 247 extql t2, a1, t1 # U : extract low bits for next time 248 addq a0, 8, a0 # E : 249 250 stq_u t0, -8(a0) # U : save the current word 251 beq a2, $u_eoc # U : 252 ldq_u t2, 8(a1) # U : Latency=3 load high word for next time 253 addq a1, 8, a1 # E : 254 255 extqh t2, a1, t0 # U : extract low bits (2 cycle stall) 256 cmpbge zero, t2, t8 # E : test new word for eos 257 nop 258 beq t8, $u_loop # U : 259 260 /* We've found a zero somewhere in the source word we just read. 261 If it resides in the lower half, we have one (probably partial) 262 word to write out, and if it resides in the upper half, we 263 have one full and one partial word left to write out. 264 265 On entry to this basic block: 266 t0 == the shifted low-order bits from the current source word 267 t1 == the shifted high-order bits from the previous source word 268 t2 == the unshifted current source word. */ 269$u_eos: 270 or t0, t1, t0 # E : first (partial) source word complete 271 nop 272 cmpbge zero, t0, t8 # E : is the null in this first bit? (stall) 273 bne t8, $u_final # U : (stall) 274 275 stq_u t0, 0(a0) # L : the null was in the high-order bits 276 addq a0, 8, a0 # E : 277 subq a2, 1, a2 # E : 278 nop 279 280$u_late_head_exit: 281 extql t2, a1, t0 # U : 282 cmpbge zero, t0, t8 # E : 283 or t8, t10, t6 # E : (stall) 284 cmoveq a2, t6, t8 # E : Latency=2, extra map slot (stall) 285 286 /* Take care of a final (probably partial) result word. 287 On entry to this basic block: 288 t0 == assembled source word 289 t8 == cmpbge mask that found the null. */ 290$u_final: 291 negq t8, t6 # E : isolate low bit set 292 and t6, t8, t12 # E : (stall) 293 and t12, 0x80, t6 # E : avoid dest word load if we can (stall) 294 bne t6, 1f # U : (stall) 295 296 ldq_u t1, 0(a0) # L : 297 subq t12, 1, t6 # E : 298 or t6, t12, t8 # E : (stall) 299 zapnot t0, t8, t0 # U : kill source bytes > null 300 301 zap t1, t8, t1 # U : kill dest bytes <= null 302 or t0, t1, t0 # E : (stall) 303 nop 304 nop 305 3061: stq_u t0, 0(a0) # L : 307 ret (t9) # L0 : Latency=3 308 309 /* Got to end-of-count before end of string. 310 On entry to this basic block: 311 t1 == the shifted high-order bits from the previous source word */ 312$u_eoc: 313 and a1, 7, t6 # E : avoid final load if possible 314 sll t10, t6, t6 # U : (stall) 315 and t6, 0xff, t6 # E : (stall) 316 bne t6, 1f # U : (stall) 317 318 ldq_u t2, 8(a1) # L : load final src word 319 nop 320 extqh t2, a1, t0 # U : extract low bits for last word (stall) 321 or t1, t0, t1 # E : (stall) 322 3231: cmpbge zero, t1, t8 # E : 324 mov t1, t0 # E : 325 326$u_eocfin: # end-of-count, final word 327 or t10, t8, t8 # E : 328 br $u_final # L0 : Latency=3 329 330 /* Unaligned copy entry point. */ 331 .align 4 332$unaligned: 333 334 ldq_u t1, 0(a1) # L : load first source word 335 and a0, 7, t4 # E : find dest misalignment 336 and a1, 7, t5 # E : find src misalignment 337 /* Conditionally load the first destination word and a bytemask 338 with 0xff indicating that the destination byte is sacrosanct. */ 339 mov zero, t0 # E : 340 341 mov zero, t6 # E : 342 beq t4, 1f # U : 343 ldq_u t0, 0(a0) # L : 344 lda t6, -1 # E : 345 346 mskql t6, a0, t6 # U : 347 nop 348 nop 349 subq a1, t4, a1 # E : sub dest misalignment from src addr 350 351 /* If source misalignment is larger than dest misalignment, we need 352 extra startup checks to avoid SEGV. */ 353 3541: cmplt t4, t5, t12 # E : 355 extql t1, a1, t1 # U : shift src into place 356 lda t2, -1 # E : for creating masks later 357 beq t12, $u_head # U : (stall) 358 359 extql t2, a1, t2 # U : 360 cmpbge zero, t1, t8 # E : is there a zero? 361 andnot t2, t6, t2 # E : dest mask for a single word copy 362 or t8, t10, t5 # E : test for end-of-count too 363 364 cmpbge zero, t2, t3 # E : 365 cmoveq a2, t5, t8 # E : Latency=2, extra map slot 366 nop # E : keep with cmoveq 367 andnot t8, t3, t8 # E : (stall) 368 369 beq t8, $u_head # U : 370 /* At this point we've found a zero in the first partial word of 371 the source. We need to isolate the valid source data and mask 372 it into the original destination data. (Incidentally, we know 373 that we'll need at least one byte of that original dest word.) */ 374 ldq_u t0, 0(a0) # L : 375 negq t8, t6 # E : build bitmask of bytes <= zero 376 mskqh t1, t4, t1 # U : 377 378 and t6, t8, t12 # E : 379 subq t12, 1, t6 # E : (stall) 380 or t6, t12, t8 # E : (stall) 381 zapnot t2, t8, t2 # U : prepare source word; mirror changes (stall) 382 383 zapnot t1, t8, t1 # U : to source validity mask 384 andnot t0, t2, t0 # E : zero place for source to reside 385 or t0, t1, t0 # E : and put it there (stall both t0, t1) 386 stq_u t0, 0(a0) # L : (stall) 387 388 ret (t9) # L0 : Latency=3 389 nop 390 nop 391 nop 392 393 .end __stxncpy 394