1/* 2 * Copyright (c) 2008, 2016, Oracle and/or its affiliates. All rights reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. 8 * 9 * This code is distributed in the hope that it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 12 * version 2 for more details (a copy is included in the LICENSE file that 13 * accompanied this code). 14 * 15 * You should have received a copy of the GNU General Public License version 16 * 2 along with this work; if not, write to the Free Software Foundation, 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 18 * 19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 20 * or visit www.oracle.com if you need additional information or have any 21 * questions. 22 * 23 */ 24 25#include "precompiled.hpp" 26#include "asm/assembler.hpp" 27#include "assembler_arm.inline.hpp" 28#include "interpreter/interpreter.hpp" 29#include "nativeInst_arm.hpp" 30#include "oops/instanceOop.hpp" 31#include "oops/method.hpp" 32#include "oops/objArrayKlass.hpp" 33#include "oops/oop.inline.hpp" 34#include "prims/methodHandles.hpp" 35#include "runtime/frame.inline.hpp" 36#include "runtime/handles.inline.hpp" 37#include "runtime/sharedRuntime.hpp" 38#include "runtime/stubCodeGenerator.hpp" 39#include "runtime/stubRoutines.hpp" 40#ifdef COMPILER2 41#include "opto/runtime.hpp" 42#endif 43 44// Declaration and definition of StubGenerator (no .hpp file). 45// For a more detailed description of the stub routine structure 46// see the comment in stubRoutines.hpp 47 48#define __ _masm-> 49 50#ifdef PRODUCT 51#define BLOCK_COMMENT(str) /* nothing */ 52#else 53#define BLOCK_COMMENT(str) __ block_comment(str) 54#endif 55 56#define BIND(label) bind(label); BLOCK_COMMENT(#label ":") 57 58// ------------------------------------------------------------------------------------------------------------------------- 59// Stub Code definitions 60 61// Platform dependent parameters for array copy stubs 62 63// Note: we have noticed a huge change in behavior on a microbenchmark 64// from platform to platform depending on the configuration. 65 66// Instead of adding a series of command line options (which 67// unfortunately have to be done in the shared file and cannot appear 68// only in the ARM port), the tested result are hard-coded here in a set 69// of options, selected by specifying 'ArmCopyPlatform' 70 71// Currently, this 'platform' is hardcoded to a value that is a good 72// enough trade-off. However, one can easily modify this file to test 73// the hard-coded configurations or create new ones. If the gain is 74// significant, we could decide to either add command line options or 75// add code to automatically choose a configuration. 76 77// see comments below for the various configurations created 78#define DEFAULT_ARRAYCOPY_CONFIG 0 79#define TEGRA2_ARRAYCOPY_CONFIG 1 80#define IMX515_ARRAYCOPY_CONFIG 2 81 82// Hard coded choices (XXX: could be changed to a command line option) 83#define ArmCopyPlatform DEFAULT_ARRAYCOPY_CONFIG 84 85#ifdef AARCH64 86#define ArmCopyCacheLineSize 64 87#else 88#define ArmCopyCacheLineSize 32 // not worth optimizing to 64 according to measured gains 89#endif // AARCH64 90 91// TODO-AARCH64: tune and revise AArch64 arraycopy optimizations 92 93// configuration for each kind of loop 94typedef struct { 95 int pld_distance; // prefetch distance (0 => no prefetch, <0: prefetch_before); 96#ifndef AARCH64 97 bool split_ldm; // if true, split each STM in STMs with fewer registers 98 bool split_stm; // if true, split each LTM in LTMs with fewer registers 99#endif // !AARCH64 100} arraycopy_loop_config; 101 102// configuration for all loops 103typedef struct { 104 // const char *description; 105 arraycopy_loop_config forward_aligned; 106 arraycopy_loop_config backward_aligned; 107 arraycopy_loop_config forward_shifted; 108 arraycopy_loop_config backward_shifted; 109} arraycopy_platform_config; 110 111// configured platforms 112static arraycopy_platform_config arraycopy_configurations[] = { 113 // configuration parameters for arraycopy loops 114#ifdef AARCH64 115 { 116 {-256 }, // forward aligned 117 {-128 }, // backward aligned 118 {-256 }, // forward shifted 119 {-128 } // backward shifted 120 } 121#else 122 123 // Configurations were chosen based on manual analysis of benchmark 124 // results, minimizing overhead with respect to best results on the 125 // different test cases. 126 127 // Prefetch before is always favored since it avoids dirtying the 128 // cache uselessly for small copies. Code for prefetch after has 129 // been kept in case the difference is significant for some 130 // platforms but we might consider dropping it. 131 132 // distance, ldm, stm 133 { 134 // default: tradeoff tegra2/imx515/nv-tegra2, 135 // Notes on benchmarking: 136 // - not far from optimal configuration on nv-tegra2 137 // - within 5% of optimal configuration except for backward aligned on IMX 138 // - up to 40% from optimal configuration for backward shifted and backward align for tegra2 139 // but still on par with the operating system copy 140 {-256, true, true }, // forward aligned 141 {-256, true, true }, // backward aligned 142 {-256, false, false }, // forward shifted 143 {-256, true, true } // backward shifted 144 }, 145 { 146 // configuration tuned on tegra2-4. 147 // Warning: should not be used on nv-tegra2 ! 148 // Notes: 149 // - prefetch after gives 40% gain on backward copies on tegra2-4, 150 // resulting in better number than the operating system 151 // copy. However, this can lead to a 300% loss on nv-tegra and has 152 // more impact on the cache (fetches futher than what is 153 // copied). Use this configuration with care, in case it improves 154 // reference benchmarks. 155 {-256, true, true }, // forward aligned 156 {96, false, false }, // backward aligned 157 {-256, false, false }, // forward shifted 158 {96, false, false } // backward shifted 159 }, 160 { 161 // configuration tuned on imx515 162 // Notes: 163 // - smaller prefetch distance is sufficient to get good result and might be more stable 164 // - refined backward aligned options within 5% of optimal configuration except for 165 // tests were the arrays fit in the cache 166 {-160, false, false }, // forward aligned 167 {-160, false, false }, // backward aligned 168 {-160, false, false }, // forward shifted 169 {-160, true, true } // backward shifted 170 } 171#endif // AARCH64 172}; 173 174class StubGenerator: public StubCodeGenerator { 175 176#ifdef PRODUCT 177#define inc_counter_np(a,b,c) ((void)0) 178#else 179#define inc_counter_np(counter, t1, t2) \ 180 BLOCK_COMMENT("inc_counter " #counter); \ 181 __ inc_counter(&counter, t1, t2); 182#endif 183 184 private: 185 186 address generate_call_stub(address& return_address) { 187 StubCodeMark mark(this, "StubRoutines", "call_stub"); 188 address start = __ pc(); 189 190#ifdef AARCH64 191 const int saved_regs_size = 192; 192 193 __ stp(FP, LR, Address(SP, -saved_regs_size, pre_indexed)); 194 __ mov(FP, SP); 195 196 int sp_offset = 16; 197 assert(frame::entry_frame_call_wrapper_offset * wordSize == sp_offset, "adjust this code"); 198 __ stp(R0, ZR, Address(SP, sp_offset)); sp_offset += 16; 199 200 const int saved_result_and_result_type_offset = sp_offset; 201 __ stp(R1, R2, Address(SP, sp_offset)); sp_offset += 16; 202 __ stp(R19, R20, Address(SP, sp_offset)); sp_offset += 16; 203 __ stp(R21, R22, Address(SP, sp_offset)); sp_offset += 16; 204 __ stp(R23, R24, Address(SP, sp_offset)); sp_offset += 16; 205 __ stp(R25, R26, Address(SP, sp_offset)); sp_offset += 16; 206 __ stp(R27, R28, Address(SP, sp_offset)); sp_offset += 16; 207 208 __ stp_d(V8, V9, Address(SP, sp_offset)); sp_offset += 16; 209 __ stp_d(V10, V11, Address(SP, sp_offset)); sp_offset += 16; 210 __ stp_d(V12, V13, Address(SP, sp_offset)); sp_offset += 16; 211 __ stp_d(V14, V15, Address(SP, sp_offset)); sp_offset += 16; 212 assert (sp_offset == saved_regs_size, "adjust this code"); 213 214 __ mov(Rmethod, R3); 215 __ mov(Rthread, R7); 216 __ reinit_heapbase(); 217 218 { // Pass parameters 219 Label done_parameters, pass_parameters; 220 221 __ mov(Rparams, SP); 222 __ cbz_w(R6, done_parameters); 223 224 __ sub(Rtemp, SP, R6, ex_uxtw, LogBytesPerWord); 225 __ align_reg(SP, Rtemp, StackAlignmentInBytes); 226 __ add(Rparams, SP, R6, ex_uxtw, LogBytesPerWord); 227 228 __ bind(pass_parameters); 229 __ subs_w(R6, R6, 1); 230 __ ldr(Rtemp, Address(R5, wordSize, post_indexed)); 231 __ str(Rtemp, Address(Rparams, -wordSize, pre_indexed)); 232 __ b(pass_parameters, ne); 233 234 __ bind(done_parameters); 235 236#ifdef ASSERT 237 { 238 Label L; 239 __ cmp(SP, Rparams); 240 __ b(L, eq); 241 __ stop("SP does not match Rparams"); 242 __ bind(L); 243 } 244#endif 245 } 246 247 __ mov(Rsender_sp, SP); 248 __ blr(R4); 249 return_address = __ pc(); 250 251 __ mov(SP, FP); 252 253 __ ldp(R1, R2, Address(SP, saved_result_and_result_type_offset)); 254 255 { // Handle return value 256 Label cont; 257 __ str(R0, Address(R1)); 258 259 __ cmp_w(R2, T_DOUBLE); 260 __ ccmp_w(R2, T_FLOAT, Assembler::flags_for_condition(eq), ne); 261 __ b(cont, ne); 262 263 __ str_d(V0, Address(R1)); 264 __ bind(cont); 265 } 266 267 sp_offset = saved_result_and_result_type_offset + 16; 268 __ ldp(R19, R20, Address(SP, sp_offset)); sp_offset += 16; 269 __ ldp(R21, R22, Address(SP, sp_offset)); sp_offset += 16; 270 __ ldp(R23, R24, Address(SP, sp_offset)); sp_offset += 16; 271 __ ldp(R25, R26, Address(SP, sp_offset)); sp_offset += 16; 272 __ ldp(R27, R28, Address(SP, sp_offset)); sp_offset += 16; 273 274 __ ldp_d(V8, V9, Address(SP, sp_offset)); sp_offset += 16; 275 __ ldp_d(V10, V11, Address(SP, sp_offset)); sp_offset += 16; 276 __ ldp_d(V12, V13, Address(SP, sp_offset)); sp_offset += 16; 277 __ ldp_d(V14, V15, Address(SP, sp_offset)); sp_offset += 16; 278 assert (sp_offset == saved_regs_size, "adjust this code"); 279 280 __ ldp(FP, LR, Address(SP, saved_regs_size, post_indexed)); 281 __ ret(); 282 283#else // AARCH64 284 285 assert(frame::entry_frame_call_wrapper_offset == 0, "adjust this code"); 286 287 __ mov(Rtemp, SP); 288 __ push(RegisterSet(FP) | RegisterSet(LR)); 289#ifndef __SOFTFP__ 290 __ fstmdbd(SP, FloatRegisterSet(D8, 8), writeback); 291#endif 292 __ stmdb(SP, RegisterSet(R0, R2) | RegisterSet(R4, R6) | RegisterSet(R8, R10) | altFP_7_11, writeback); 293 __ mov(Rmethod, R3); 294 __ ldmia(Rtemp, RegisterSet(R1, R3) | Rthread); // stacked arguments 295 296 // XXX: TODO 297 // Would be better with respect to native tools if the following 298 // setting of FP was changed to conform to the native ABI, with FP 299 // pointing to the saved FP slot (and the corresponding modifications 300 // for entry_frame_call_wrapper_offset and frame::real_fp). 301 __ mov(FP, SP); 302 303 { 304 Label no_parameters, pass_parameters; 305 __ cmp(R3, 0); 306 __ b(no_parameters, eq); 307 308 __ bind(pass_parameters); 309 __ ldr(Rtemp, Address(R2, wordSize, post_indexed)); // Rtemp OK, unused and scratchable 310 __ subs(R3, R3, 1); 311 __ push(Rtemp); 312 __ b(pass_parameters, ne); 313 __ bind(no_parameters); 314 } 315 316 __ mov(Rsender_sp, SP); 317 __ blx(R1); 318 return_address = __ pc(); 319 320 __ add(SP, FP, wordSize); // Skip link to JavaCallWrapper 321 __ pop(RegisterSet(R2, R3)); 322#ifndef __ABI_HARD__ 323 __ cmp(R3, T_LONG); 324 __ cmp(R3, T_DOUBLE, ne); 325 __ str(R0, Address(R2)); 326 __ str(R1, Address(R2, wordSize), eq); 327#else 328 Label cont, l_float, l_double; 329 330 __ cmp(R3, T_DOUBLE); 331 __ b(l_double, eq); 332 333 __ cmp(R3, T_FLOAT); 334 __ b(l_float, eq); 335 336 __ cmp(R3, T_LONG); 337 __ str(R0, Address(R2)); 338 __ str(R1, Address(R2, wordSize), eq); 339 __ b(cont); 340 341 342 __ bind(l_double); 343 __ fstd(D0, Address(R2)); 344 __ b(cont); 345 346 __ bind(l_float); 347 __ fsts(S0, Address(R2)); 348 349 __ bind(cont); 350#endif 351 352 __ pop(RegisterSet(R4, R6) | RegisterSet(R8, R10) | altFP_7_11); 353#ifndef __SOFTFP__ 354 __ fldmiad(SP, FloatRegisterSet(D8, 8), writeback); 355#endif 356 __ pop(RegisterSet(FP) | RegisterSet(PC)); 357 358#endif // AARCH64 359 return start; 360 } 361 362 363 // (in) Rexception_obj: exception oop 364 address generate_catch_exception() { 365 StubCodeMark mark(this, "StubRoutines", "catch_exception"); 366 address start = __ pc(); 367 368 __ str(Rexception_obj, Address(Rthread, Thread::pending_exception_offset())); 369 __ b(StubRoutines::_call_stub_return_address); 370 371 return start; 372 } 373 374 375 // (in) Rexception_pc: return address 376 address generate_forward_exception() { 377 StubCodeMark mark(this, "StubRoutines", "forward exception"); 378 address start = __ pc(); 379 380 __ mov(c_rarg0, Rthread); 381 __ mov(c_rarg1, Rexception_pc); 382 __ call_VM_leaf(CAST_FROM_FN_PTR(address, 383 SharedRuntime::exception_handler_for_return_address), 384 c_rarg0, c_rarg1); 385 __ ldr(Rexception_obj, Address(Rthread, Thread::pending_exception_offset())); 386 const Register Rzero = __ zero_register(Rtemp); // Rtemp OK (cleared by above call) 387 __ str(Rzero, Address(Rthread, Thread::pending_exception_offset())); 388 389#ifdef ASSERT 390 // make sure exception is set 391 { Label L; 392 __ cbnz(Rexception_obj, L); 393 __ stop("StubRoutines::forward exception: no pending exception (2)"); 394 __ bind(L); 395 } 396#endif 397 398 // Verify that there is really a valid exception in RAX. 399 __ verify_oop(Rexception_obj); 400 401 __ jump(R0); // handler is returned in R0 by runtime function 402 return start; 403 } 404 405 406#ifndef AARCH64 407 408 // Integer division shared routine 409 // Input: 410 // R0 - dividend 411 // R2 - divisor 412 // Output: 413 // R0 - remainder 414 // R1 - quotient 415 // Destroys: 416 // R2 417 // LR 418 address generate_idiv_irem() { 419 Label positive_arguments, negative_or_zero, call_slow_path; 420 Register dividend = R0; 421 Register divisor = R2; 422 Register remainder = R0; 423 Register quotient = R1; 424 Register tmp = LR; 425 assert(dividend == remainder, "must be"); 426 427 address start = __ pc(); 428 429 // Check for special cases: divisor <= 0 or dividend < 0 430 __ cmp(divisor, 0); 431 __ orrs(quotient, dividend, divisor, ne); 432 __ b(negative_or_zero, le); 433 434 __ bind(positive_arguments); 435 // Save return address on stack to free one extra register 436 __ push(LR); 437 // Approximate the mamximum order of the quotient 438 __ clz(tmp, dividend); 439 __ clz(quotient, divisor); 440 __ subs(tmp, quotient, tmp); 441 __ mov(quotient, 0); 442 // Jump to the appropriate place in the unrolled loop below 443 __ ldr(PC, Address(PC, tmp, lsl, 2), pl); 444 // If divisor is greater than dividend, return immediately 445 __ pop(PC); 446 447 // Offset table 448 Label offset_table[32]; 449 int i; 450 for (i = 0; i <= 31; i++) { 451 __ emit_address(offset_table[i]); 452 } 453 454 // Unrolled loop of 32 division steps 455 for (i = 31; i >= 0; i--) { 456 __ bind(offset_table[i]); 457 __ cmp(remainder, AsmOperand(divisor, lsl, i)); 458 __ sub(remainder, remainder, AsmOperand(divisor, lsl, i), hs); 459 __ add(quotient, quotient, 1 << i, hs); 460 } 461 __ pop(PC); 462 463 __ bind(negative_or_zero); 464 // Find the combination of argument signs and jump to corresponding handler 465 __ andr(quotient, dividend, 0x80000000, ne); 466 __ orr(quotient, quotient, AsmOperand(divisor, lsr, 31), ne); 467 __ add(PC, PC, AsmOperand(quotient, ror, 26), ne); 468 __ str(LR, Address(Rthread, JavaThread::saved_exception_pc_offset())); 469 470 // The leaf runtime function can destroy R0-R3 and R12 registers which are still alive 471 RegisterSet saved_registers = RegisterSet(R3) | RegisterSet(R12); 472#if R9_IS_SCRATCHED 473 // Safer to save R9 here since callers may have been written 474 // assuming R9 survives. This is suboptimal but may not be worth 475 // revisiting for this slow case. 476 477 // save also R10 for alignment 478 saved_registers = saved_registers | RegisterSet(R9, R10); 479#endif 480 { 481 // divisor == 0 482 FixedSizeCodeBlock zero_divisor(_masm, 8, true); 483 __ push(saved_registers); 484 __ mov(R0, Rthread); 485 __ mov(R1, LR); 486 __ mov(R2, SharedRuntime::IMPLICIT_DIVIDE_BY_ZERO); 487 __ b(call_slow_path); 488 } 489 490 { 491 // divisor > 0 && dividend < 0 492 FixedSizeCodeBlock positive_divisor_negative_dividend(_masm, 8, true); 493 __ push(LR); 494 __ rsb(dividend, dividend, 0); 495 __ bl(positive_arguments); 496 __ rsb(remainder, remainder, 0); 497 __ rsb(quotient, quotient, 0); 498 __ pop(PC); 499 } 500 501 { 502 // divisor < 0 && dividend > 0 503 FixedSizeCodeBlock negative_divisor_positive_dividend(_masm, 8, true); 504 __ push(LR); 505 __ rsb(divisor, divisor, 0); 506 __ bl(positive_arguments); 507 __ rsb(quotient, quotient, 0); 508 __ pop(PC); 509 } 510 511 { 512 // divisor < 0 && dividend < 0 513 FixedSizeCodeBlock negative_divisor_negative_dividend(_masm, 8, true); 514 __ push(LR); 515 __ rsb(dividend, dividend, 0); 516 __ rsb(divisor, divisor, 0); 517 __ bl(positive_arguments); 518 __ rsb(remainder, remainder, 0); 519 __ pop(PC); 520 } 521 522 __ bind(call_slow_path); 523 __ call(CAST_FROM_FN_PTR(address, SharedRuntime::continuation_for_implicit_exception)); 524 __ pop(saved_registers); 525 __ bx(R0); 526 527 return start; 528 } 529 530 531 // As per atomic.hpp the Atomic read-modify-write operations must be logically implemented as: 532 // <fence>; <op>; <membar StoreLoad|StoreStore> 533 // But for load-linked/store-conditional based systems a fence here simply means 534 // no load/store can be reordered with respect to the initial load-linked, so we have: 535 // <membar storeload|loadload> ; load-linked; <op>; store-conditional; <membar storeload|storestore> 536 // There are no memory actions in <op> so nothing further is needed. 537 // 538 // So we define the following for convenience: 539#define MEMBAR_ATOMIC_OP_PRE \ 540 MacroAssembler::Membar_mask_bits(MacroAssembler::StoreLoad|MacroAssembler::LoadLoad) 541#define MEMBAR_ATOMIC_OP_POST \ 542 MacroAssembler::Membar_mask_bits(MacroAssembler::StoreLoad|MacroAssembler::StoreStore) 543 544 // Note: JDK 9 only supports ARMv7+ so we always have ldrexd available even though the 545 // code below allows for it to be otherwise. The else clause indicates an ARMv5 system 546 // for which we do not support MP and so membars are not necessary. This ARMv5 code will 547 // be removed in the future. 548 549 // Support for jint Atomic::add(jint add_value, volatile jint *dest) 550 // 551 // Arguments : 552 // 553 // add_value: R0 554 // dest: R1 555 // 556 // Results: 557 // 558 // R0: the new stored in dest 559 // 560 // Overwrites: 561 // 562 // R1, R2, R3 563 // 564 address generate_atomic_add() { 565 address start; 566 567 StubCodeMark mark(this, "StubRoutines", "atomic_add"); 568 Label retry; 569 start = __ pc(); 570 Register addval = R0; 571 Register dest = R1; 572 Register prev = R2; 573 Register ok = R2; 574 Register newval = R3; 575 576 if (VM_Version::supports_ldrex()) { 577 __ membar(MEMBAR_ATOMIC_OP_PRE, prev); 578 __ bind(retry); 579 __ ldrex(newval, Address(dest)); 580 __ add(newval, addval, newval); 581 __ strex(ok, newval, Address(dest)); 582 __ cmp(ok, 0); 583 __ b(retry, ne); 584 __ mov (R0, newval); 585 __ membar(MEMBAR_ATOMIC_OP_POST, prev); 586 } else { 587 __ bind(retry); 588 __ ldr (prev, Address(dest)); 589 __ add(newval, addval, prev); 590 __ atomic_cas_bool(prev, newval, dest, 0, noreg/*ignored*/); 591 __ b(retry, ne); 592 __ mov (R0, newval); 593 } 594 __ bx(LR); 595 596 return start; 597 } 598 599 // Support for jint Atomic::xchg(jint exchange_value, volatile jint *dest) 600 // 601 // Arguments : 602 // 603 // exchange_value: R0 604 // dest: R1 605 // 606 // Results: 607 // 608 // R0: the value previously stored in dest 609 // 610 // Overwrites: 611 // 612 // R1, R2, R3 613 // 614 address generate_atomic_xchg() { 615 address start; 616 617 StubCodeMark mark(this, "StubRoutines", "atomic_xchg"); 618 start = __ pc(); 619 Register newval = R0; 620 Register dest = R1; 621 Register prev = R2; 622 623 Label retry; 624 625 if (VM_Version::supports_ldrex()) { 626 Register ok=R3; 627 __ membar(MEMBAR_ATOMIC_OP_PRE, prev); 628 __ bind(retry); 629 __ ldrex(prev, Address(dest)); 630 __ strex(ok, newval, Address(dest)); 631 __ cmp(ok, 0); 632 __ b(retry, ne); 633 __ mov (R0, prev); 634 __ membar(MEMBAR_ATOMIC_OP_POST, prev); 635 } else { 636 __ bind(retry); 637 __ ldr (prev, Address(dest)); 638 __ atomic_cas_bool(prev, newval, dest, 0, noreg/*ignored*/); 639 __ b(retry, ne); 640 __ mov (R0, prev); 641 } 642 __ bx(LR); 643 644 return start; 645 } 646 647 // Support for jint Atomic::cmpxchg(jint exchange_value, volatile jint *dest, jint compare_value) 648 // 649 // Arguments : 650 // 651 // compare_value: R0 652 // exchange_value: R1 653 // dest: R2 654 // 655 // Results: 656 // 657 // R0: the value previously stored in dest 658 // 659 // Overwrites: 660 // 661 // R0, R1, R2, R3, Rtemp 662 // 663 address generate_atomic_cmpxchg() { 664 address start; 665 666 StubCodeMark mark(this, "StubRoutines", "atomic_cmpxchg"); 667 start = __ pc(); 668 Register cmp = R0; 669 Register newval = R1; 670 Register dest = R2; 671 Register temp1 = R3; 672 Register temp2 = Rtemp; // Rtemp free (native ABI) 673 674 __ membar(MEMBAR_ATOMIC_OP_PRE, temp1); 675 676 // atomic_cas returns previous value in R0 677 __ atomic_cas(temp1, temp2, cmp, newval, dest, 0); 678 679 __ membar(MEMBAR_ATOMIC_OP_POST, temp1); 680 681 __ bx(LR); 682 683 return start; 684 } 685 686 // Support for jlong Atomic::cmpxchg(jlong exchange_value, volatile jlong *dest, jlong compare_value) 687 // reordered before by a wrapper to (jlong compare_value, jlong exchange_value, volatile jlong *dest) 688 // 689 // Arguments : 690 // 691 // compare_value: R1 (High), R0 (Low) 692 // exchange_value: R3 (High), R2 (Low) 693 // dest: SP+0 694 // 695 // Results: 696 // 697 // R0:R1: the value previously stored in dest 698 // 699 // Overwrites: 700 // 701 address generate_atomic_cmpxchg_long() { 702 address start; 703 704 StubCodeMark mark(this, "StubRoutines", "atomic_cmpxchg_long"); 705 start = __ pc(); 706 Register cmp_lo = R0; 707 Register cmp_hi = R1; 708 Register newval_lo = R2; 709 Register newval_hi = R3; 710 Register addr = Rtemp; /* After load from stack */ 711 Register temp_lo = R4; 712 Register temp_hi = R5; 713 Register temp_result = R8; 714 assert_different_registers(cmp_lo, newval_lo, temp_lo, addr, temp_result, R7); 715 assert_different_registers(cmp_hi, newval_hi, temp_hi, addr, temp_result, R7); 716 717 __ membar(MEMBAR_ATOMIC_OP_PRE, Rtemp); // Rtemp free (native ABI) 718 719 // Stack is unaligned, maintain double word alignment by pushing 720 // odd number of regs. 721 __ push(RegisterSet(temp_result) | RegisterSet(temp_lo, temp_hi)); 722 __ ldr(addr, Address(SP, 12)); 723 724 // atomic_cas64 returns previous value in temp_lo, temp_hi 725 __ atomic_cas64(temp_lo, temp_hi, temp_result, cmp_lo, cmp_hi, 726 newval_lo, newval_hi, addr, 0); 727 __ mov(R0, temp_lo); 728 __ mov(R1, temp_hi); 729 730 __ pop(RegisterSet(temp_result) | RegisterSet(temp_lo, temp_hi)); 731 732 __ membar(MEMBAR_ATOMIC_OP_POST, Rtemp); // Rtemp free (native ABI) 733 __ bx(LR); 734 735 return start; 736 } 737 738 address generate_atomic_load_long() { 739 address start; 740 741 StubCodeMark mark(this, "StubRoutines", "atomic_load_long"); 742 start = __ pc(); 743 Register result_lo = R0; 744 Register result_hi = R1; 745 Register src = R0; 746 747 if (!os::is_MP()) { 748 __ ldmia(src, RegisterSet(result_lo, result_hi)); 749 __ bx(LR); 750 } else if (VM_Version::supports_ldrexd()) { 751 __ ldrexd(result_lo, Address(src)); 752 __ clrex(); // FIXME: safe to remove? 753 __ bx(LR); 754 } else { 755 __ stop("Atomic load(jlong) unsupported on this platform"); 756 __ bx(LR); 757 } 758 759 return start; 760 } 761 762 address generate_atomic_store_long() { 763 address start; 764 765 StubCodeMark mark(this, "StubRoutines", "atomic_store_long"); 766 start = __ pc(); 767 Register newval_lo = R0; 768 Register newval_hi = R1; 769 Register dest = R2; 770 Register scratch_lo = R2; 771 Register scratch_hi = R3; /* After load from stack */ 772 Register result = R3; 773 774 if (!os::is_MP()) { 775 __ stmia(dest, RegisterSet(newval_lo, newval_hi)); 776 __ bx(LR); 777 } else if (VM_Version::supports_ldrexd()) { 778 __ mov(Rtemp, dest); // get dest to Rtemp 779 Label retry; 780 __ bind(retry); 781 __ ldrexd(scratch_lo, Address(Rtemp)); 782 __ strexd(result, R0, Address(Rtemp)); 783 __ rsbs(result, result, 1); 784 __ b(retry, eq); 785 __ bx(LR); 786 } else { 787 __ stop("Atomic store(jlong) unsupported on this platform"); 788 __ bx(LR); 789 } 790 791 return start; 792 } 793 794 795#endif // AARCH64 796 797#ifdef COMPILER2 798 // Support for uint StubRoutine::Arm::partial_subtype_check( Klass sub, Klass super ); 799 // Arguments : 800 // 801 // ret : R0, returned 802 // icc/xcc: set as R0 (depending on wordSize) 803 // sub : R1, argument, not changed 804 // super: R2, argument, not changed 805 // raddr: LR, blown by call 806 address generate_partial_subtype_check() { 807 __ align(CodeEntryAlignment); 808 StubCodeMark mark(this, "StubRoutines", "partial_subtype_check"); 809 address start = __ pc(); 810 811 // based on SPARC check_klass_subtype_[fast|slow]_path (without CompressedOops) 812 813 // R0 used as tmp_reg (in addition to return reg) 814 Register sub_klass = R1; 815 Register super_klass = R2; 816 Register tmp_reg2 = R3; 817 Register tmp_reg3 = R4; 818#define saved_set tmp_reg2, tmp_reg3 819 820 Label L_loop, L_fail; 821 822 int sc_offset = in_bytes(Klass::secondary_super_cache_offset()); 823 824 // fast check should be redundant 825 826 // slow check 827 { 828 __ raw_push(saved_set); 829 830 // a couple of useful fields in sub_klass: 831 int ss_offset = in_bytes(Klass::secondary_supers_offset()); 832 833 // Do a linear scan of the secondary super-klass chain. 834 // This code is rarely used, so simplicity is a virtue here. 835 836 inc_counter_np(SharedRuntime::_partial_subtype_ctr, tmp_reg2, tmp_reg3); 837 838 Register scan_temp = tmp_reg2; 839 Register count_temp = tmp_reg3; 840 841 // We will consult the secondary-super array. 842 __ ldr(scan_temp, Address(sub_klass, ss_offset)); 843 844 Register search_key = super_klass; 845 846 // Load the array length. 847 __ ldr_s32(count_temp, Address(scan_temp, Array<Klass*>::length_offset_in_bytes())); 848 __ add(scan_temp, scan_temp, Array<Klass*>::base_offset_in_bytes()); 849 850 __ add(count_temp, count_temp, 1); 851 852 // Top of search loop 853 __ bind(L_loop); 854 // Notes: 855 // scan_temp starts at the array elements 856 // count_temp is 1+size 857 __ subs(count_temp, count_temp, 1); 858 __ b(L_fail, eq); // not found in the array 859 860 // Load next super to check 861 // In the array of super classes elements are pointer sized. 862 int element_size = wordSize; 863 __ ldr(R0, Address(scan_temp, element_size, post_indexed)); 864 865 // Look for Rsuper_klass on Rsub_klass's secondary super-class-overflow list 866 __ subs(R0, R0, search_key); // set R0 to 0 on success (and flags to eq) 867 868 // A miss means we are NOT a subtype and need to keep looping 869 __ b(L_loop, ne); 870 871 // Falling out the bottom means we found a hit; we ARE a subtype 872 873 // Success. Cache the super we found and proceed in triumph. 874 __ str(super_klass, Address(sub_klass, sc_offset)); 875 876 // Return success 877 // R0 is already 0 and flags are already set to eq 878 __ raw_pop(saved_set); 879 __ ret(); 880 881 // Return failure 882 __ bind(L_fail); 883#ifdef AARCH64 884 // count_temp is 0, can't use ZR here 885 __ adds(R0, count_temp, 1); // sets the flags 886#else 887 __ movs(R0, 1); // sets the flags 888#endif 889 __ raw_pop(saved_set); 890 __ ret(); 891 } 892 return start; 893 } 894#undef saved_set 895#endif // COMPILER2 896 897 898 //---------------------------------------------------------------------------------------------------- 899 // Non-destructive plausibility checks for oops 900 901 address generate_verify_oop() { 902 StubCodeMark mark(this, "StubRoutines", "verify_oop"); 903 address start = __ pc(); 904 905 // Incoming arguments: 906 // 907 // R0: error message (char* ) 908 // R1: address of register save area 909 // R2: oop to verify 910 // 911 // All registers are saved before calling this stub. However, condition flags should be saved here. 912 913 const Register oop = R2; 914 const Register klass = R3; 915 const Register tmp1 = R6; 916 const Register tmp2 = R8; 917 918 const Register flags = Rtmp_save0; // R4/R19 919 const Register ret_addr = Rtmp_save1; // R5/R20 920 assert_different_registers(oop, klass, tmp1, tmp2, flags, ret_addr, R7); 921 922 Label exit, error; 923 InlinedAddress verify_oop_count((address) StubRoutines::verify_oop_count_addr()); 924 925#ifdef AARCH64 926 __ mrs(flags, Assembler::SysReg_NZCV); 927#else 928 __ mrs(Assembler::CPSR, flags); 929#endif // AARCH64 930 931 __ ldr_literal(tmp1, verify_oop_count); 932 __ ldr_s32(tmp2, Address(tmp1)); 933 __ add(tmp2, tmp2, 1); 934 __ str_32(tmp2, Address(tmp1)); 935 936 // make sure object is 'reasonable' 937 __ cbz(oop, exit); // if obj is NULL it is ok 938 939 // Check if the oop is in the right area of memory 940 // Note: oop_mask and oop_bits must be updated if the code is saved/reused 941 const address oop_mask = (address) Universe::verify_oop_mask(); 942 const address oop_bits = (address) Universe::verify_oop_bits(); 943 __ mov_address(tmp1, oop_mask, symbolic_Relocation::oop_mask_reference); 944 __ andr(tmp2, oop, tmp1); 945 __ mov_address(tmp1, oop_bits, symbolic_Relocation::oop_bits_reference); 946 __ cmp(tmp2, tmp1); 947 __ b(error, ne); 948 949 // make sure klass is 'reasonable' 950 __ load_klass(klass, oop); // get klass 951 __ cbz(klass, error); // if klass is NULL it is broken 952 953 // return if everything seems ok 954 __ bind(exit); 955 956#ifdef AARCH64 957 __ msr(Assembler::SysReg_NZCV, flags); 958#else 959 __ msr(Assembler::CPSR_f, flags); 960#endif // AARCH64 961 962 __ ret(); 963 964 // handle errors 965 __ bind(error); 966 967 __ mov(ret_addr, LR); // save return address 968 969 // R0: error message 970 // R1: register save area 971 __ call(CAST_FROM_FN_PTR(address, MacroAssembler::debug)); 972 973 __ mov(LR, ret_addr); 974 __ b(exit); 975 976 __ bind_literal(verify_oop_count); 977 978 return start; 979 } 980 981 //---------------------------------------------------------------------------------------------------- 982 // Array copy stubs 983 984 // 985 // Generate overlap test for array copy stubs 986 // 987 // Input: 988 // R0 - array1 989 // R1 - array2 990 // R2 - element count, 32-bit int 991 // 992 // input registers are preserved 993 // 994 void array_overlap_test(address no_overlap_target, int log2_elem_size, Register tmp1, Register tmp2) { 995 assert(no_overlap_target != NULL, "must be generated"); 996 array_overlap_test(no_overlap_target, NULL, log2_elem_size, tmp1, tmp2); 997 } 998 void array_overlap_test(Label& L_no_overlap, int log2_elem_size, Register tmp1, Register tmp2) { 999 array_overlap_test(NULL, &L_no_overlap, log2_elem_size, tmp1, tmp2); 1000 } 1001 void array_overlap_test(address no_overlap_target, Label* NOLp, int log2_elem_size, Register tmp1, Register tmp2) { 1002 const Register from = R0; 1003 const Register to = R1; 1004 const Register count = R2; 1005 const Register to_from = tmp1; // to - from 1006#ifndef AARCH64 1007 const Register byte_count = (log2_elem_size == 0) ? count : tmp2; // count << log2_elem_size 1008#endif // AARCH64 1009 assert_different_registers(from, to, count, tmp1, tmp2); 1010 1011 // no_overlap version works if 'to' lower (unsigned) than 'from' 1012 // and or 'to' more than (count*size) from 'from' 1013 1014 BLOCK_COMMENT("Array Overlap Test:"); 1015 __ subs(to_from, to, from); 1016#ifndef AARCH64 1017 if (log2_elem_size != 0) { 1018 __ mov(byte_count, AsmOperand(count, lsl, log2_elem_size)); 1019 } 1020#endif // !AARCH64 1021 if (NOLp == NULL) 1022 __ b(no_overlap_target,lo); 1023 else 1024 __ b((*NOLp), lo); 1025#ifdef AARCH64 1026 __ subs(ZR, to_from, count, ex_sxtw, log2_elem_size); 1027#else 1028 __ cmp(to_from, byte_count); 1029#endif // AARCH64 1030 if (NOLp == NULL) 1031 __ b(no_overlap_target, ge); 1032 else 1033 __ b((*NOLp), ge); 1034 } 1035 1036#ifdef AARCH64 1037 // TODO-AARCH64: revise usages of bulk_* methods (probably ldp`s and stp`s should interlace) 1038 1039 // Loads [from, from + count*wordSize) into regs[0], regs[1], ..., regs[count-1] 1040 // and increases 'from' by count*wordSize. 1041 void bulk_load_forward(Register from, const Register regs[], int count) { 1042 assert (count > 0 && count % 2 == 0, "count must be positive even number"); 1043 int bytes = count * wordSize; 1044 1045 int offset = 0; 1046 __ ldp(regs[0], regs[1], Address(from, bytes, post_indexed)); 1047 offset += 2*wordSize; 1048 1049 for (int i = 2; i < count; i += 2) { 1050 __ ldp(regs[i], regs[i+1], Address(from, -bytes + offset)); 1051 offset += 2*wordSize; 1052 } 1053 1054 assert (offset == bytes, "must be"); 1055 } 1056 1057 // Stores regs[0], regs[1], ..., regs[count-1] to [to, to + count*wordSize) 1058 // and increases 'to' by count*wordSize. 1059 void bulk_store_forward(Register to, const Register regs[], int count) { 1060 assert (count > 0 && count % 2 == 0, "count must be positive even number"); 1061 int bytes = count * wordSize; 1062 1063 int offset = 0; 1064 __ stp(regs[0], regs[1], Address(to, bytes, post_indexed)); 1065 offset += 2*wordSize; 1066 1067 for (int i = 2; i < count; i += 2) { 1068 __ stp(regs[i], regs[i+1], Address(to, -bytes + offset)); 1069 offset += 2*wordSize; 1070 } 1071 1072 assert (offset == bytes, "must be"); 1073 } 1074 1075 // Loads [from - count*wordSize, from) into regs[0], regs[1], ..., regs[count-1] 1076 // and decreases 'from' by count*wordSize. 1077 // Note that the word with lowest address goes to regs[0]. 1078 void bulk_load_backward(Register from, const Register regs[], int count) { 1079 assert (count > 0 && count % 2 == 0, "count must be positive even number"); 1080 int bytes = count * wordSize; 1081 1082 int offset = 0; 1083 1084 for (int i = count - 2; i > 0; i -= 2) { 1085 offset += 2*wordSize; 1086 __ ldp(regs[i], regs[i+1], Address(from, -offset)); 1087 } 1088 1089 offset += 2*wordSize; 1090 __ ldp(regs[0], regs[1], Address(from, -bytes, pre_indexed)); 1091 1092 assert (offset == bytes, "must be"); 1093 } 1094 1095 // Stores regs[0], regs[1], ..., regs[count-1] into [to - count*wordSize, to) 1096 // and decreases 'to' by count*wordSize. 1097 // Note that regs[0] value goes into the memory with lowest address. 1098 void bulk_store_backward(Register to, const Register regs[], int count) { 1099 assert (count > 0 && count % 2 == 0, "count must be positive even number"); 1100 int bytes = count * wordSize; 1101 1102 int offset = 0; 1103 1104 for (int i = count - 2; i > 0; i -= 2) { 1105 offset += 2*wordSize; 1106 __ stp(regs[i], regs[i+1], Address(to, -offset)); 1107 } 1108 1109 offset += 2*wordSize; 1110 __ stp(regs[0], regs[1], Address(to, -bytes, pre_indexed)); 1111 1112 assert (offset == bytes, "must be"); 1113 } 1114#endif // AARCH64 1115 1116 // TODO-AARCH64: rearrange in-loop prefetches: 1117 // probably we should choose between "prefetch-store before or after store", not "before or after load". 1118 void prefetch(Register from, Register to, int offset, int to_delta = 0) { 1119 __ prefetch_read(Address(from, offset)); 1120#ifdef AARCH64 1121 // Next line commented out to avoid significant loss of performance in memory copy - JDK-8078120 1122 // __ prfm(pstl1keep, Address(to, offset + to_delta)); 1123#endif // AARCH64 1124 } 1125 1126 // Generate the inner loop for forward aligned array copy 1127 // 1128 // Arguments 1129 // from: src address, 64 bits aligned 1130 // to: dst address, wordSize aligned 1131 // count: number of elements (32-bit int) 1132 // bytes_per_count: number of bytes for each unit of 'count' 1133 // 1134 // Return the minimum initial value for count 1135 // 1136 // Notes: 1137 // - 'from' aligned on 64-bit (recommended for 32-bit ARM in case this speeds up LDMIA, required for AArch64) 1138 // - 'to' aligned on wordSize 1139 // - 'count' must be greater or equal than the returned value 1140 // 1141 // Increases 'from' and 'to' by count*bytes_per_count. 1142 // 1143 // Scratches 'count', R3. 1144 // On AArch64 also scratches R4-R10; on 32-bit ARM R4-R10 are preserved (saved/restored). 1145 // 1146 int generate_forward_aligned_copy_loop(Register from, Register to, Register count, int bytes_per_count) { 1147 assert (from == R0 && to == R1 && count == R2, "adjust the implementation below"); 1148 1149 const int bytes_per_loop = 8*wordSize; // 8 registers are read and written on every loop iteration 1150 arraycopy_loop_config *config=&arraycopy_configurations[ArmCopyPlatform].forward_aligned; 1151 int pld_offset = config->pld_distance; 1152 const int count_per_loop = bytes_per_loop / bytes_per_count; 1153 1154#ifndef AARCH64 1155 bool split_read= config->split_ldm; 1156 bool split_write= config->split_stm; 1157 1158 // XXX optim: use VLDM/VSTM when available (Neon) with PLD 1159 // NEONCopyPLD 1160 // PLD [r1, #0xC0] 1161 // VLDM r1!,{d0-d7} 1162 // VSTM r0!,{d0-d7} 1163 // SUBS r2,r2,#0x40 1164 // BGE NEONCopyPLD 1165 1166 __ push(RegisterSet(R4,R10)); 1167#endif // !AARCH64 1168 1169 const bool prefetch_before = pld_offset < 0; 1170 const bool prefetch_after = pld_offset > 0; 1171 1172 Label L_skip_pld; 1173 1174 // predecrease to exit when there is less than count_per_loop 1175 __ sub_32(count, count, count_per_loop); 1176 1177 if (pld_offset != 0) { 1178 pld_offset = (pld_offset < 0) ? -pld_offset : pld_offset; 1179 1180 prefetch(from, to, 0); 1181 1182 if (prefetch_before) { 1183 // If prefetch is done ahead, final PLDs that overflow the 1184 // copied area can be easily avoided. 'count' is predecreased 1185 // by the prefetch distance to optimize the inner loop and the 1186 // outer loop skips the PLD. 1187 __ subs_32(count, count, (bytes_per_loop+pld_offset)/bytes_per_count); 1188 1189 // skip prefetch for small copies 1190 __ b(L_skip_pld, lt); 1191 } 1192 1193 int offset = ArmCopyCacheLineSize; 1194 while (offset <= pld_offset) { 1195 prefetch(from, to, offset); 1196 offset += ArmCopyCacheLineSize; 1197 }; 1198 } 1199 1200#ifdef AARCH64 1201 const Register data_regs[8] = {R3, R4, R5, R6, R7, R8, R9, R10}; 1202#endif // AARCH64 1203 { 1204 // LDM (32-bit ARM) / LDP (AArch64) copy of 'bytes_per_loop' bytes 1205 1206 // 32-bit ARM note: we have tried implementing loop unrolling to skip one 1207 // PLD with 64 bytes cache line but the gain was not significant. 1208 1209 Label L_copy_loop; 1210 __ align(OptoLoopAlignment); 1211 __ BIND(L_copy_loop); 1212 1213 if (prefetch_before) { 1214 prefetch(from, to, bytes_per_loop + pld_offset); 1215 __ BIND(L_skip_pld); 1216 } 1217 1218#ifdef AARCH64 1219 bulk_load_forward(from, data_regs, 8); 1220#else 1221 if (split_read) { 1222 // Split the register set in two sets so that there is less 1223 // latency between LDM and STM (R3-R6 available while R7-R10 1224 // still loading) and less register locking issue when iterating 1225 // on the first LDM. 1226 __ ldmia(from, RegisterSet(R3, R6), writeback); 1227 __ ldmia(from, RegisterSet(R7, R10), writeback); 1228 } else { 1229 __ ldmia(from, RegisterSet(R3, R10), writeback); 1230 } 1231#endif // AARCH64 1232 1233 __ subs_32(count, count, count_per_loop); 1234 1235 if (prefetch_after) { 1236 prefetch(from, to, pld_offset, bytes_per_loop); 1237 } 1238 1239#ifdef AARCH64 1240 bulk_store_forward(to, data_regs, 8); 1241#else 1242 if (split_write) { 1243 __ stmia(to, RegisterSet(R3, R6), writeback); 1244 __ stmia(to, RegisterSet(R7, R10), writeback); 1245 } else { 1246 __ stmia(to, RegisterSet(R3, R10), writeback); 1247 } 1248#endif // AARCH64 1249 1250 __ b(L_copy_loop, ge); 1251 1252 if (prefetch_before) { 1253 // the inner loop may end earlier, allowing to skip PLD for the last iterations 1254 __ cmn_32(count, (bytes_per_loop + pld_offset)/bytes_per_count); 1255 __ b(L_skip_pld, ge); 1256 } 1257 } 1258 BLOCK_COMMENT("Remaining bytes:"); 1259 // still 0..bytes_per_loop-1 aligned bytes to copy, count already decreased by (at least) bytes_per_loop bytes 1260 1261 // __ add(count, count, ...); // addition useless for the bit tests 1262 assert (pld_offset % bytes_per_loop == 0, "decreasing count by pld_offset before loop must not change tested bits"); 1263 1264#ifdef AARCH64 1265 assert (bytes_per_loop == 64, "adjust the code below"); 1266 assert (bytes_per_count <= 8, "adjust the code below"); 1267 1268 { 1269 Label L; 1270 __ tbz(count, exact_log2(32/bytes_per_count), L); 1271 1272 bulk_load_forward(from, data_regs, 4); 1273 bulk_store_forward(to, data_regs, 4); 1274 1275 __ bind(L); 1276 } 1277 1278 { 1279 Label L; 1280 __ tbz(count, exact_log2(16/bytes_per_count), L); 1281 1282 bulk_load_forward(from, data_regs, 2); 1283 bulk_store_forward(to, data_regs, 2); 1284 1285 __ bind(L); 1286 } 1287 1288 { 1289 Label L; 1290 __ tbz(count, exact_log2(8/bytes_per_count), L); 1291 1292 __ ldr(R3, Address(from, 8, post_indexed)); 1293 __ str(R3, Address(to, 8, post_indexed)); 1294 1295 __ bind(L); 1296 } 1297 1298 if (bytes_per_count <= 4) { 1299 Label L; 1300 __ tbz(count, exact_log2(4/bytes_per_count), L); 1301 1302 __ ldr_w(R3, Address(from, 4, post_indexed)); 1303 __ str_w(R3, Address(to, 4, post_indexed)); 1304 1305 __ bind(L); 1306 } 1307 1308 if (bytes_per_count <= 2) { 1309 Label L; 1310 __ tbz(count, exact_log2(2/bytes_per_count), L); 1311 1312 __ ldrh(R3, Address(from, 2, post_indexed)); 1313 __ strh(R3, Address(to, 2, post_indexed)); 1314 1315 __ bind(L); 1316 } 1317 1318 if (bytes_per_count <= 1) { 1319 Label L; 1320 __ tbz(count, 0, L); 1321 1322 __ ldrb(R3, Address(from, 1, post_indexed)); 1323 __ strb(R3, Address(to, 1, post_indexed)); 1324 1325 __ bind(L); 1326 } 1327#else 1328 __ tst(count, 16 / bytes_per_count); 1329 __ ldmia(from, RegisterSet(R3, R6), writeback, ne); // copy 16 bytes 1330 __ stmia(to, RegisterSet(R3, R6), writeback, ne); 1331 1332 __ tst(count, 8 / bytes_per_count); 1333 __ ldmia(from, RegisterSet(R3, R4), writeback, ne); // copy 8 bytes 1334 __ stmia(to, RegisterSet(R3, R4), writeback, ne); 1335 1336 if (bytes_per_count <= 4) { 1337 __ tst(count, 4 / bytes_per_count); 1338 __ ldr(R3, Address(from, 4, post_indexed), ne); // copy 4 bytes 1339 __ str(R3, Address(to, 4, post_indexed), ne); 1340 } 1341 1342 if (bytes_per_count <= 2) { 1343 __ tst(count, 2 / bytes_per_count); 1344 __ ldrh(R3, Address(from, 2, post_indexed), ne); // copy 2 bytes 1345 __ strh(R3, Address(to, 2, post_indexed), ne); 1346 } 1347 1348 if (bytes_per_count == 1) { 1349 __ tst(count, 1); 1350 __ ldrb(R3, Address(from, 1, post_indexed), ne); 1351 __ strb(R3, Address(to, 1, post_indexed), ne); 1352 } 1353 1354 __ pop(RegisterSet(R4,R10)); 1355#endif // AARCH64 1356 1357 return count_per_loop; 1358 } 1359 1360 1361 // Generate the inner loop for backward aligned array copy 1362 // 1363 // Arguments 1364 // end_from: src end address, 64 bits aligned 1365 // end_to: dst end address, wordSize aligned 1366 // count: number of elements (32-bit int) 1367 // bytes_per_count: number of bytes for each unit of 'count' 1368 // 1369 // Return the minimum initial value for count 1370 // 1371 // Notes: 1372 // - 'end_from' aligned on 64-bit (recommended for 32-bit ARM in case this speeds up LDMIA, required for AArch64) 1373 // - 'end_to' aligned on wordSize 1374 // - 'count' must be greater or equal than the returned value 1375 // 1376 // Decreases 'end_from' and 'end_to' by count*bytes_per_count. 1377 // 1378 // Scratches 'count', R3. 1379 // On AArch64 also scratches R4-R10; on 32-bit ARM R4-R10 are preserved (saved/restored). 1380 // 1381 int generate_backward_aligned_copy_loop(Register end_from, Register end_to, Register count, int bytes_per_count) { 1382 assert (end_from == R0 && end_to == R1 && count == R2, "adjust the implementation below"); 1383 1384 const int bytes_per_loop = 8*wordSize; // 8 registers are read and written on every loop iteration 1385 const int count_per_loop = bytes_per_loop / bytes_per_count; 1386 1387 arraycopy_loop_config *config=&arraycopy_configurations[ArmCopyPlatform].backward_aligned; 1388 int pld_offset = config->pld_distance; 1389 1390#ifndef AARCH64 1391 bool split_read= config->split_ldm; 1392 bool split_write= config->split_stm; 1393 1394 // See the forward copy variant for additional comments. 1395 1396 __ push(RegisterSet(R4,R10)); 1397#endif // !AARCH64 1398 1399 __ sub_32(count, count, count_per_loop); 1400 1401 const bool prefetch_before = pld_offset < 0; 1402 const bool prefetch_after = pld_offset > 0; 1403 1404 Label L_skip_pld; 1405 1406 if (pld_offset != 0) { 1407 pld_offset = (pld_offset < 0) ? -pld_offset : pld_offset; 1408 1409 prefetch(end_from, end_to, -wordSize); 1410 1411 if (prefetch_before) { 1412 __ subs_32(count, count, (bytes_per_loop + pld_offset) / bytes_per_count); 1413 __ b(L_skip_pld, lt); 1414 } 1415 1416 int offset = ArmCopyCacheLineSize; 1417 while (offset <= pld_offset) { 1418 prefetch(end_from, end_to, -(wordSize + offset)); 1419 offset += ArmCopyCacheLineSize; 1420 }; 1421 } 1422 1423#ifdef AARCH64 1424 const Register data_regs[8] = {R3, R4, R5, R6, R7, R8, R9, R10}; 1425#endif // AARCH64 1426 { 1427 // LDM (32-bit ARM) / LDP (AArch64) copy of 'bytes_per_loop' bytes 1428 1429 // 32-bit ARM note: we have tried implementing loop unrolling to skip one 1430 // PLD with 64 bytes cache line but the gain was not significant. 1431 1432 Label L_copy_loop; 1433 __ align(OptoLoopAlignment); 1434 __ BIND(L_copy_loop); 1435 1436 if (prefetch_before) { 1437 prefetch(end_from, end_to, -(wordSize + bytes_per_loop + pld_offset)); 1438 __ BIND(L_skip_pld); 1439 } 1440 1441#ifdef AARCH64 1442 bulk_load_backward(end_from, data_regs, 8); 1443#else 1444 if (split_read) { 1445 __ ldmdb(end_from, RegisterSet(R7, R10), writeback); 1446 __ ldmdb(end_from, RegisterSet(R3, R6), writeback); 1447 } else { 1448 __ ldmdb(end_from, RegisterSet(R3, R10), writeback); 1449 } 1450#endif // AARCH64 1451 1452 __ subs_32(count, count, count_per_loop); 1453 1454 if (prefetch_after) { 1455 prefetch(end_from, end_to, -(wordSize + pld_offset), -bytes_per_loop); 1456 } 1457 1458#ifdef AARCH64 1459 bulk_store_backward(end_to, data_regs, 8); 1460#else 1461 if (split_write) { 1462 __ stmdb(end_to, RegisterSet(R7, R10), writeback); 1463 __ stmdb(end_to, RegisterSet(R3, R6), writeback); 1464 } else { 1465 __ stmdb(end_to, RegisterSet(R3, R10), writeback); 1466 } 1467#endif // AARCH64 1468 1469 __ b(L_copy_loop, ge); 1470 1471 if (prefetch_before) { 1472 __ cmn_32(count, (bytes_per_loop + pld_offset)/bytes_per_count); 1473 __ b(L_skip_pld, ge); 1474 } 1475 } 1476 BLOCK_COMMENT("Remaining bytes:"); 1477 // still 0..bytes_per_loop-1 aligned bytes to copy, count already decreased by (at least) bytes_per_loop bytes 1478 1479 // __ add(count, count, ...); // addition useless for the bit tests 1480 assert (pld_offset % bytes_per_loop == 0, "decreasing count by pld_offset before loop must not change tested bits"); 1481 1482#ifdef AARCH64 1483 assert (bytes_per_loop == 64, "adjust the code below"); 1484 assert (bytes_per_count <= 8, "adjust the code below"); 1485 1486 { 1487 Label L; 1488 __ tbz(count, exact_log2(32/bytes_per_count), L); 1489 1490 bulk_load_backward(end_from, data_regs, 4); 1491 bulk_store_backward(end_to, data_regs, 4); 1492 1493 __ bind(L); 1494 } 1495 1496 { 1497 Label L; 1498 __ tbz(count, exact_log2(16/bytes_per_count), L); 1499 1500 bulk_load_backward(end_from, data_regs, 2); 1501 bulk_store_backward(end_to, data_regs, 2); 1502 1503 __ bind(L); 1504 } 1505 1506 { 1507 Label L; 1508 __ tbz(count, exact_log2(8/bytes_per_count), L); 1509 1510 __ ldr(R3, Address(end_from, -8, pre_indexed)); 1511 __ str(R3, Address(end_to, -8, pre_indexed)); 1512 1513 __ bind(L); 1514 } 1515 1516 if (bytes_per_count <= 4) { 1517 Label L; 1518 __ tbz(count, exact_log2(4/bytes_per_count), L); 1519 1520 __ ldr_w(R3, Address(end_from, -4, pre_indexed)); 1521 __ str_w(R3, Address(end_to, -4, pre_indexed)); 1522 1523 __ bind(L); 1524 } 1525 1526 if (bytes_per_count <= 2) { 1527 Label L; 1528 __ tbz(count, exact_log2(2/bytes_per_count), L); 1529 1530 __ ldrh(R3, Address(end_from, -2, pre_indexed)); 1531 __ strh(R3, Address(end_to, -2, pre_indexed)); 1532 1533 __ bind(L); 1534 } 1535 1536 if (bytes_per_count <= 1) { 1537 Label L; 1538 __ tbz(count, 0, L); 1539 1540 __ ldrb(R3, Address(end_from, -1, pre_indexed)); 1541 __ strb(R3, Address(end_to, -1, pre_indexed)); 1542 1543 __ bind(L); 1544 } 1545#else 1546 __ tst(count, 16 / bytes_per_count); 1547 __ ldmdb(end_from, RegisterSet(R3, R6), writeback, ne); // copy 16 bytes 1548 __ stmdb(end_to, RegisterSet(R3, R6), writeback, ne); 1549 1550 __ tst(count, 8 / bytes_per_count); 1551 __ ldmdb(end_from, RegisterSet(R3, R4), writeback, ne); // copy 8 bytes 1552 __ stmdb(end_to, RegisterSet(R3, R4), writeback, ne); 1553 1554 if (bytes_per_count <= 4) { 1555 __ tst(count, 4 / bytes_per_count); 1556 __ ldr(R3, Address(end_from, -4, pre_indexed), ne); // copy 4 bytes 1557 __ str(R3, Address(end_to, -4, pre_indexed), ne); 1558 } 1559 1560 if (bytes_per_count <= 2) { 1561 __ tst(count, 2 / bytes_per_count); 1562 __ ldrh(R3, Address(end_from, -2, pre_indexed), ne); // copy 2 bytes 1563 __ strh(R3, Address(end_to, -2, pre_indexed), ne); 1564 } 1565 1566 if (bytes_per_count == 1) { 1567 __ tst(count, 1); 1568 __ ldrb(R3, Address(end_from, -1, pre_indexed), ne); 1569 __ strb(R3, Address(end_to, -1, pre_indexed), ne); 1570 } 1571 1572 __ pop(RegisterSet(R4,R10)); 1573#endif // AARCH64 1574 1575 return count_per_loop; 1576 } 1577 1578 1579 // Generate the inner loop for shifted forward array copy (unaligned copy). 1580 // It can be used when bytes_per_count < wordSize, i.e. 1581 // byte/short copy on 32-bit ARM, byte/short/int/compressed-oop copy on AArch64. 1582 // 1583 // Arguments 1584 // from: start src address, 64 bits aligned 1585 // to: start dst address, (now) wordSize aligned 1586 // count: number of elements (32-bit int) 1587 // bytes_per_count: number of bytes for each unit of 'count' 1588 // lsr_shift: shift applied to 'old' value to skipped already written bytes 1589 // lsl_shift: shift applied to 'new' value to set the high bytes of the next write 1590 // 1591 // Return the minimum initial value for count 1592 // 1593 // Notes: 1594 // - 'from' aligned on 64-bit (recommended for 32-bit ARM in case this speeds up LDMIA, required for AArch64) 1595 // - 'to' aligned on wordSize 1596 // - 'count' must be greater or equal than the returned value 1597 // - 'lsr_shift' + 'lsl_shift' = BitsPerWord 1598 // - 'bytes_per_count' is 1 or 2 on 32-bit ARM; 1, 2 or 4 on AArch64 1599 // 1600 // Increases 'to' by count*bytes_per_count. 1601 // 1602 // Scratches 'from' and 'count', R3-R10, R12 1603 // 1604 // On entry: 1605 // - R12 is preloaded with the first 'BitsPerWord' bits read just before 'from' 1606 // - (R12 >> lsr_shift) is the part not yet written (just before 'to') 1607 // --> (*to) = (R12 >> lsr_shift) | (*from) << lsl_shift); ... 1608 // 1609 // This implementation may read more bytes than required. 1610 // Actually, it always reads exactly all data from the copied region with upper bound aligned up by wordSize, 1611 // so excessive read do not cross a word bound and is thus harmless. 1612 // 1613 int generate_forward_shifted_copy_loop(Register from, Register to, Register count, int bytes_per_count, int lsr_shift, int lsl_shift) { 1614 assert (from == R0 && to == R1 && count == R2, "adjust the implementation below"); 1615 1616 const int bytes_per_loop = 8*wordSize; // 8 registers are read and written on every loop iter 1617 const int count_per_loop = bytes_per_loop / bytes_per_count; 1618 1619 arraycopy_loop_config *config=&arraycopy_configurations[ArmCopyPlatform].forward_shifted; 1620 int pld_offset = config->pld_distance; 1621 1622#ifndef AARCH64 1623 bool split_read= config->split_ldm; 1624 bool split_write= config->split_stm; 1625#endif // !AARCH64 1626 1627 const bool prefetch_before = pld_offset < 0; 1628 const bool prefetch_after = pld_offset > 0; 1629 Label L_skip_pld, L_last_read, L_done; 1630 if (pld_offset != 0) { 1631 1632 pld_offset = (pld_offset < 0) ? -pld_offset : pld_offset; 1633 1634 prefetch(from, to, 0); 1635 1636 if (prefetch_before) { 1637 __ cmp_32(count, count_per_loop); 1638 __ b(L_last_read, lt); 1639 // skip prefetch for small copies 1640 // warning: count is predecreased by the prefetch distance to optimize the inner loop 1641 __ subs_32(count, count, ((bytes_per_loop + pld_offset) / bytes_per_count) + count_per_loop); 1642 __ b(L_skip_pld, lt); 1643 } 1644 1645 int offset = ArmCopyCacheLineSize; 1646 while (offset <= pld_offset) { 1647 prefetch(from, to, offset); 1648 offset += ArmCopyCacheLineSize; 1649 }; 1650 } 1651 1652 Label L_shifted_loop; 1653 1654 __ align(OptoLoopAlignment); 1655 __ BIND(L_shifted_loop); 1656 1657 if (prefetch_before) { 1658 // do it early if there might be register locking issues 1659 prefetch(from, to, bytes_per_loop + pld_offset); 1660 __ BIND(L_skip_pld); 1661 } else { 1662 __ cmp_32(count, count_per_loop); 1663 __ b(L_last_read, lt); 1664 } 1665 1666#ifdef AARCH64 1667 const Register data_regs[9] = {R3, R4, R5, R6, R7, R8, R9, R10, R12}; 1668 __ logical_shift_right(R3, R12, lsr_shift); // part of R12 not yet written 1669 __ subs_32(count, count, count_per_loop); 1670 bulk_load_forward(from, &data_regs[1], 8); 1671#else 1672 // read 32 bytes 1673 if (split_read) { 1674 // if write is not split, use less registers in first set to reduce locking 1675 RegisterSet set1 = split_write ? RegisterSet(R4, R7) : RegisterSet(R4, R5); 1676 RegisterSet set2 = (split_write ? RegisterSet(R8, R10) : RegisterSet(R6, R10)) | R12; 1677 __ ldmia(from, set1, writeback); 1678 __ mov(R3, AsmOperand(R12, lsr, lsr_shift)); // part of R12 not yet written 1679 __ ldmia(from, set2, writeback); 1680 __ subs(count, count, count_per_loop); // XXX: should it be before the 2nd LDM ? (latency vs locking) 1681 } else { 1682 __ mov(R3, AsmOperand(R12, lsr, lsr_shift)); // part of R12 not yet written 1683 __ ldmia(from, RegisterSet(R4, R10) | R12, writeback); // Note: small latency on R4 1684 __ subs(count, count, count_per_loop); 1685 } 1686#endif // AARCH64 1687 1688 if (prefetch_after) { 1689 // do it after the 1st ldm/ldp anyway (no locking issues with early STM/STP) 1690 prefetch(from, to, pld_offset, bytes_per_loop); 1691 } 1692 1693 // prepare (shift) the values in R3..R10 1694 __ orr(R3, R3, AsmOperand(R4, lsl, lsl_shift)); // merged below low bytes of next val 1695 __ logical_shift_right(R4, R4, lsr_shift); // unused part of next val 1696 __ orr(R4, R4, AsmOperand(R5, lsl, lsl_shift)); // ... 1697 __ logical_shift_right(R5, R5, lsr_shift); 1698 __ orr(R5, R5, AsmOperand(R6, lsl, lsl_shift)); 1699 __ logical_shift_right(R6, R6, lsr_shift); 1700 __ orr(R6, R6, AsmOperand(R7, lsl, lsl_shift)); 1701#ifndef AARCH64 1702 if (split_write) { 1703 // write the first half as soon as possible to reduce stm locking 1704 __ stmia(to, RegisterSet(R3, R6), writeback, prefetch_before ? gt : ge); 1705 } 1706#endif // !AARCH64 1707 __ logical_shift_right(R7, R7, lsr_shift); 1708 __ orr(R7, R7, AsmOperand(R8, lsl, lsl_shift)); 1709 __ logical_shift_right(R8, R8, lsr_shift); 1710 __ orr(R8, R8, AsmOperand(R9, lsl, lsl_shift)); 1711 __ logical_shift_right(R9, R9, lsr_shift); 1712 __ orr(R9, R9, AsmOperand(R10, lsl, lsl_shift)); 1713 __ logical_shift_right(R10, R10, lsr_shift); 1714 __ orr(R10, R10, AsmOperand(R12, lsl, lsl_shift)); 1715 1716#ifdef AARCH64 1717 bulk_store_forward(to, data_regs, 8); 1718#else 1719 if (split_write) { 1720 __ stmia(to, RegisterSet(R7, R10), writeback, prefetch_before ? gt : ge); 1721 } else { 1722 __ stmia(to, RegisterSet(R3, R10), writeback, prefetch_before ? gt : ge); 1723 } 1724#endif // AARCH64 1725 __ b(L_shifted_loop, gt); // no need to loop if 0 (when count need not be precise modulo bytes_per_loop) 1726 1727 if (prefetch_before) { 1728 // the first loop may end earlier, allowing to skip pld at the end 1729 __ cmn_32(count, (bytes_per_loop + pld_offset)/bytes_per_count); 1730#ifndef AARCH64 1731 __ stmia(to, RegisterSet(R3, R10), writeback); // stmia was skipped 1732#endif // !AARCH64 1733 __ b(L_skip_pld, ge); 1734 __ adds_32(count, count, ((bytes_per_loop + pld_offset) / bytes_per_count) + count_per_loop); 1735 } 1736 1737 __ BIND(L_last_read); 1738 __ b(L_done, eq); 1739 1740#ifdef AARCH64 1741 assert(bytes_per_count < 8, "adjust the code below"); 1742 1743 __ logical_shift_right(R3, R12, lsr_shift); 1744 1745 { 1746 Label L; 1747 __ tbz(count, exact_log2(32/bytes_per_count), L); 1748 bulk_load_forward(from, &data_regs[1], 4); 1749 __ orr(R3, R3, AsmOperand(R4, lsl, lsl_shift)); 1750 __ logical_shift_right(R4, R4, lsr_shift); 1751 __ orr(R4, R4, AsmOperand(R5, lsl, lsl_shift)); 1752 __ logical_shift_right(R5, R5, lsr_shift); 1753 __ orr(R5, R5, AsmOperand(R6, lsl, lsl_shift)); 1754 __ logical_shift_right(R6, R6, lsr_shift); 1755 __ orr(R6, R6, AsmOperand(R7, lsl, lsl_shift)); 1756 bulk_store_forward(to, data_regs, 4); 1757 __ logical_shift_right(R3, R7, lsr_shift); 1758 __ bind(L); 1759 } 1760 1761 { 1762 Label L; 1763 __ tbz(count, exact_log2(16/bytes_per_count), L); 1764 bulk_load_forward(from, &data_regs[1], 2); 1765 __ orr(R3, R3, AsmOperand(R4, lsl, lsl_shift)); 1766 __ logical_shift_right(R4, R4, lsr_shift); 1767 __ orr(R4, R4, AsmOperand(R5, lsl, lsl_shift)); 1768 bulk_store_forward(to, data_regs, 2); 1769 __ logical_shift_right(R3, R5, lsr_shift); 1770 __ bind(L); 1771 } 1772 1773 { 1774 Label L; 1775 __ tbz(count, exact_log2(8/bytes_per_count), L); 1776 __ ldr(R4, Address(from, 8, post_indexed)); 1777 __ orr(R3, R3, AsmOperand(R4, lsl, lsl_shift)); 1778 __ str(R3, Address(to, 8, post_indexed)); 1779 __ logical_shift_right(R3, R4, lsr_shift); 1780 __ bind(L); 1781 } 1782 1783 const int have_bytes = lsl_shift/BitsPerByte; // number of already read bytes in R3 1784 1785 // It remains less than wordSize to write. 1786 // Do not check count if R3 already has maximal number of loaded elements (one less than wordSize). 1787 if (have_bytes < wordSize - bytes_per_count) { 1788 Label L; 1789 __ andr(count, count, (uintx)(8/bytes_per_count-1)); // make count exact 1790 __ cmp_32(count, have_bytes/bytes_per_count); // do we have enough bytes to store? 1791 __ b(L, le); 1792 __ ldr(R4, Address(from, 8, post_indexed)); 1793 __ orr(R3, R3, AsmOperand(R4, lsl, lsl_shift)); 1794 __ bind(L); 1795 } 1796 1797 { 1798 Label L; 1799 __ tbz(count, exact_log2(4/bytes_per_count), L); 1800 __ str_w(R3, Address(to, 4, post_indexed)); 1801 if (bytes_per_count < 4) { 1802 __ logical_shift_right(R3, R3, 4*BitsPerByte); 1803 } 1804 __ bind(L); 1805 } 1806 1807 if (bytes_per_count <= 2) { 1808 Label L; 1809 __ tbz(count, exact_log2(2/bytes_per_count), L); 1810 __ strh(R3, Address(to, 2, post_indexed)); 1811 if (bytes_per_count < 2) { 1812 __ logical_shift_right(R3, R3, 2*BitsPerByte); 1813 } 1814 __ bind(L); 1815 } 1816 1817 if (bytes_per_count <= 1) { 1818 Label L; 1819 __ tbz(count, exact_log2(1/bytes_per_count), L); 1820 __ strb(R3, Address(to, 1, post_indexed)); 1821 __ bind(L); 1822 } 1823#else 1824 switch (bytes_per_count) { 1825 case 2: 1826 __ mov(R3, AsmOperand(R12, lsr, lsr_shift)); 1827 __ tst(count, 8); 1828 __ ldmia(from, RegisterSet(R4, R7), writeback, ne); 1829 __ orr(R3, R3, AsmOperand(R4, lsl, lsl_shift), ne); // merged below low bytes of next val 1830 __ mov(R4, AsmOperand(R4, lsr, lsr_shift), ne); // unused part of next val 1831 __ orr(R4, R4, AsmOperand(R5, lsl, lsl_shift), ne); // ... 1832 __ mov(R5, AsmOperand(R5, lsr, lsr_shift), ne); 1833 __ orr(R5, R5, AsmOperand(R6, lsl, lsl_shift), ne); 1834 __ mov(R6, AsmOperand(R6, lsr, lsr_shift), ne); 1835 __ orr(R6, R6, AsmOperand(R7, lsl, lsl_shift), ne); 1836 __ stmia(to, RegisterSet(R3, R6), writeback, ne); 1837 __ mov(R3, AsmOperand(R7, lsr, lsr_shift), ne); 1838 1839 __ tst(count, 4); 1840 __ ldmia(from, RegisterSet(R4, R5), writeback, ne); 1841 __ orr(R3, R3, AsmOperand(R4, lsl, lsl_shift), ne); // merged below low bytes of next val 1842 __ mov(R4, AsmOperand(R4, lsr, lsr_shift), ne); // unused part of next val 1843 __ orr(R4, R4, AsmOperand(R5, lsl, lsl_shift), ne); // ... 1844 __ stmia(to, RegisterSet(R3, R4), writeback, ne); 1845 __ mov(R3, AsmOperand(R5, lsr, lsr_shift), ne); 1846 1847 __ tst(count, 2); 1848 __ ldr(R4, Address(from, 4, post_indexed), ne); 1849 __ orr(R3, R3, AsmOperand(R4, lsl, lsl_shift), ne); 1850 __ str(R3, Address(to, 4, post_indexed), ne); 1851 __ mov(R3, AsmOperand(R4, lsr, lsr_shift), ne); 1852 1853 __ tst(count, 1); 1854 __ strh(R3, Address(to, 2, post_indexed), ne); // one last short 1855 break; 1856 1857 case 1: 1858 __ mov(R3, AsmOperand(R12, lsr, lsr_shift)); 1859 __ tst(count, 16); 1860 __ ldmia(from, RegisterSet(R4, R7), writeback, ne); 1861 __ orr(R3, R3, AsmOperand(R4, lsl, lsl_shift), ne); // merged below low bytes of next val 1862 __ mov(R4, AsmOperand(R4, lsr, lsr_shift), ne); // unused part of next val 1863 __ orr(R4, R4, AsmOperand(R5, lsl, lsl_shift), ne); // ... 1864 __ mov(R5, AsmOperand(R5, lsr, lsr_shift), ne); 1865 __ orr(R5, R5, AsmOperand(R6, lsl, lsl_shift), ne); 1866 __ mov(R6, AsmOperand(R6, lsr, lsr_shift), ne); 1867 __ orr(R6, R6, AsmOperand(R7, lsl, lsl_shift), ne); 1868 __ stmia(to, RegisterSet(R3, R6), writeback, ne); 1869 __ mov(R3, AsmOperand(R7, lsr, lsr_shift), ne); 1870 1871 __ tst(count, 8); 1872 __ ldmia(from, RegisterSet(R4, R5), writeback, ne); 1873 __ orr(R3, R3, AsmOperand(R4, lsl, lsl_shift), ne); // merged below low bytes of next val 1874 __ mov(R4, AsmOperand(R4, lsr, lsr_shift), ne); // unused part of next val 1875 __ orr(R4, R4, AsmOperand(R5, lsl, lsl_shift), ne); // ... 1876 __ stmia(to, RegisterSet(R3, R4), writeback, ne); 1877 __ mov(R3, AsmOperand(R5, lsr, lsr_shift), ne); 1878 1879 __ tst(count, 4); 1880 __ ldr(R4, Address(from, 4, post_indexed), ne); 1881 __ orr(R3, R3, AsmOperand(R4, lsl, lsl_shift), ne); 1882 __ str(R3, Address(to, 4, post_indexed), ne); 1883 __ mov(R3, AsmOperand(R4, lsr, lsr_shift), ne); 1884 1885 __ andr(count, count, 3); 1886 __ cmp(count, 2); 1887 1888 // Note: R3 might contain enough bytes ready to write (3 needed at most), 1889 // thus load on lsl_shift==24 is not needed (in fact forces reading 1890 // beyond source buffer end boundary) 1891 if (lsl_shift == 8) { 1892 __ ldr(R4, Address(from, 4, post_indexed), ge); 1893 __ orr(R3, R3, AsmOperand(R4, lsl, lsl_shift), ge); 1894 } else if (lsl_shift == 16) { 1895 __ ldr(R4, Address(from, 4, post_indexed), gt); 1896 __ orr(R3, R3, AsmOperand(R4, lsl, lsl_shift), gt); 1897 } 1898 1899 __ strh(R3, Address(to, 2, post_indexed), ge); // two last bytes 1900 __ mov(R3, AsmOperand(R3, lsr, 16), gt); 1901 1902 __ tst(count, 1); 1903 __ strb(R3, Address(to, 1, post_indexed), ne); // one last byte 1904 break; 1905 } 1906#endif // AARCH64 1907 1908 __ BIND(L_done); 1909 return 0; // no minimum 1910 } 1911 1912 // Generate the inner loop for shifted backward array copy (unaligned copy). 1913 // It can be used when bytes_per_count < wordSize, i.e. 1914 // byte/short copy on 32-bit ARM, byte/short/int/compressed-oop copy on AArch64. 1915 // 1916 // Arguments 1917 // end_from: end src address, 64 bits aligned 1918 // end_to: end dst address, (now) wordSize aligned 1919 // count: number of elements (32-bit int) 1920 // bytes_per_count: number of bytes for each unit of 'count' 1921 // lsl_shift: shift applied to 'old' value to skipped already written bytes 1922 // lsr_shift: shift applied to 'new' value to set the low bytes of the next write 1923 // 1924 // Return the minimum initial value for count 1925 // 1926 // Notes: 1927 // - 'end_from' aligned on 64-bit (recommended for 32-bit ARM in case this speeds up LDMIA, required for AArch64) 1928 // - 'end_to' aligned on wordSize 1929 // - 'count' must be greater or equal than the returned value 1930 // - 'lsr_shift' + 'lsl_shift' = 'BitsPerWord' 1931 // - 'bytes_per_count' is 1 or 2 on 32-bit ARM; 1, 2 or 4 on AArch64 1932 // 1933 // Decreases 'end_to' by count*bytes_per_count. 1934 // 1935 // Scratches 'end_from', 'count', R3-R10, R12 1936 // 1937 // On entry: 1938 // - R3 is preloaded with the first 'BitsPerWord' bits read just after 'from' 1939 // - (R3 << lsl_shift) is the part not yet written 1940 // --> (*--to) = (R3 << lsl_shift) | (*--from) >> lsr_shift); ... 1941 // 1942 // This implementation may read more bytes than required. 1943 // Actually, it always reads exactly all data from the copied region with beginning aligned down by wordSize, 1944 // so excessive read do not cross a word bound and is thus harmless. 1945 // 1946 int generate_backward_shifted_copy_loop(Register end_from, Register end_to, Register count, int bytes_per_count, int lsr_shift, int lsl_shift) { 1947 assert (end_from == R0 && end_to == R1 && count == R2, "adjust the implementation below"); 1948 1949 const int bytes_per_loop = 8*wordSize; // 8 registers are read and written on every loop iter 1950 const int count_per_loop = bytes_per_loop / bytes_per_count; 1951 1952 arraycopy_loop_config *config=&arraycopy_configurations[ArmCopyPlatform].backward_shifted; 1953 int pld_offset = config->pld_distance; 1954 1955#ifndef AARCH64 1956 bool split_read= config->split_ldm; 1957 bool split_write= config->split_stm; 1958#endif // !AARCH64 1959 1960 1961 const bool prefetch_before = pld_offset < 0; 1962 const bool prefetch_after = pld_offset > 0; 1963 1964 Label L_skip_pld, L_done, L_last_read; 1965 if (pld_offset != 0) { 1966 1967 pld_offset = (pld_offset < 0) ? -pld_offset : pld_offset; 1968 1969 prefetch(end_from, end_to, -wordSize); 1970 1971 if (prefetch_before) { 1972 __ cmp_32(count, count_per_loop); 1973 __ b(L_last_read, lt); 1974 1975 // skip prefetch for small copies 1976 // warning: count is predecreased by the prefetch distance to optimize the inner loop 1977 __ subs_32(count, count, ((bytes_per_loop + pld_offset)/bytes_per_count) + count_per_loop); 1978 __ b(L_skip_pld, lt); 1979 } 1980 1981 int offset = ArmCopyCacheLineSize; 1982 while (offset <= pld_offset) { 1983 prefetch(end_from, end_to, -(wordSize + offset)); 1984 offset += ArmCopyCacheLineSize; 1985 }; 1986 } 1987 1988 Label L_shifted_loop; 1989 __ align(OptoLoopAlignment); 1990 __ BIND(L_shifted_loop); 1991 1992 if (prefetch_before) { 1993 // do the 1st ldm/ldp first anyway (no locking issues with early STM/STP) 1994 prefetch(end_from, end_to, -(wordSize + bytes_per_loop + pld_offset)); 1995 __ BIND(L_skip_pld); 1996 } else { 1997 __ cmp_32(count, count_per_loop); 1998 __ b(L_last_read, lt); 1999 } 2000 2001#ifdef AARCH64 2002 __ logical_shift_left(R12, R3, lsl_shift); 2003 const Register data_regs[9] = {R3, R4, R5, R6, R7, R8, R9, R10, R12}; 2004 bulk_load_backward(end_from, data_regs, 8); 2005#else 2006 if (split_read) { 2007 __ ldmdb(end_from, RegisterSet(R7, R10), writeback); 2008 __ mov(R12, AsmOperand(R3, lsl, lsl_shift)); // part of R3 not yet written 2009 __ ldmdb(end_from, RegisterSet(R3, R6), writeback); 2010 } else { 2011 __ mov(R12, AsmOperand(R3, lsl, lsl_shift)); // part of R3 not yet written 2012 __ ldmdb(end_from, RegisterSet(R3, R10), writeback); 2013 } 2014#endif // AARCH64 2015 2016 __ subs_32(count, count, count_per_loop); 2017 2018 if (prefetch_after) { // do prefetch during ldm/ldp latency 2019 prefetch(end_from, end_to, -(wordSize + pld_offset), -bytes_per_loop); 2020 } 2021 2022 // prepare the values in R4..R10,R12 2023 __ orr(R12, R12, AsmOperand(R10, lsr, lsr_shift)); // merged above high bytes of prev val 2024 __ logical_shift_left(R10, R10, lsl_shift); // unused part of prev val 2025 __ orr(R10, R10, AsmOperand(R9, lsr, lsr_shift)); // ... 2026 __ logical_shift_left(R9, R9, lsl_shift); 2027 __ orr(R9, R9, AsmOperand(R8, lsr, lsr_shift)); 2028 __ logical_shift_left(R8, R8, lsl_shift); 2029 __ orr(R8, R8, AsmOperand(R7, lsr, lsr_shift)); 2030 __ logical_shift_left(R7, R7, lsl_shift); 2031 __ orr(R7, R7, AsmOperand(R6, lsr, lsr_shift)); 2032 __ logical_shift_left(R6, R6, lsl_shift); 2033 __ orr(R6, R6, AsmOperand(R5, lsr, lsr_shift)); 2034#ifndef AARCH64 2035 if (split_write) { 2036 // store early to reduce locking issues 2037 __ stmdb(end_to, RegisterSet(R6, R10) | R12, writeback, prefetch_before ? gt : ge); 2038 } 2039#endif // !AARCH64 2040 __ logical_shift_left(R5, R5, lsl_shift); 2041 __ orr(R5, R5, AsmOperand(R4, lsr, lsr_shift)); 2042 __ logical_shift_left(R4, R4, lsl_shift); 2043 __ orr(R4, R4, AsmOperand(R3, lsr, lsr_shift)); 2044 2045#ifdef AARCH64 2046 bulk_store_backward(end_to, &data_regs[1], 8); 2047#else 2048 if (split_write) { 2049 __ stmdb(end_to, RegisterSet(R4, R5), writeback, prefetch_before ? gt : ge); 2050 } else { 2051 __ stmdb(end_to, RegisterSet(R4, R10) | R12, writeback, prefetch_before ? gt : ge); 2052 } 2053#endif // AARCH64 2054 2055 __ b(L_shifted_loop, gt); // no need to loop if 0 (when count need not be precise modulo bytes_per_loop) 2056 2057 if (prefetch_before) { 2058 // the first loop may end earlier, allowing to skip pld at the end 2059 __ cmn_32(count, ((bytes_per_loop + pld_offset)/bytes_per_count)); 2060#ifndef AARCH64 2061 __ stmdb(end_to, RegisterSet(R4, R10) | R12, writeback); // stmdb was skipped 2062#endif // !AARCH64 2063 __ b(L_skip_pld, ge); 2064 __ adds_32(count, count, ((bytes_per_loop + pld_offset) / bytes_per_count) + count_per_loop); 2065 } 2066 2067 __ BIND(L_last_read); 2068 __ b(L_done, eq); 2069 2070#ifdef AARCH64 2071 assert(bytes_per_count < 8, "adjust the code below"); 2072 2073 __ logical_shift_left(R12, R3, lsl_shift); 2074 2075 { 2076 Label L; 2077 __ tbz(count, exact_log2(32/bytes_per_count), L); 2078 bulk_load_backward(end_from, &data_regs[4], 4); 2079 2080 __ orr(R12, R12, AsmOperand(R10, lsr, lsr_shift)); 2081 __ logical_shift_left(R10, R10, lsl_shift); 2082 __ orr(R10, R10, AsmOperand(R9, lsr, lsr_shift)); 2083 __ logical_shift_left(R9, R9, lsl_shift); 2084 __ orr(R9, R9, AsmOperand(R8, lsr, lsr_shift)); 2085 __ logical_shift_left(R8, R8, lsl_shift); 2086 __ orr(R8, R8, AsmOperand(R7, lsr, lsr_shift)); 2087 2088 bulk_store_backward(end_to, &data_regs[5], 4); 2089 __ logical_shift_left(R12, R7, lsl_shift); 2090 __ bind(L); 2091 } 2092 2093 { 2094 Label L; 2095 __ tbz(count, exact_log2(16/bytes_per_count), L); 2096 bulk_load_backward(end_from, &data_regs[6], 2); 2097 2098 __ orr(R12, R12, AsmOperand(R10, lsr, lsr_shift)); 2099 __ logical_shift_left(R10, R10, lsl_shift); 2100 __ orr(R10, R10, AsmOperand(R9, lsr, lsr_shift)); 2101 2102 bulk_store_backward(end_to, &data_regs[7], 2); 2103 __ logical_shift_left(R12, R9, lsl_shift); 2104 __ bind(L); 2105 } 2106 2107 { 2108 Label L; 2109 __ tbz(count, exact_log2(8/bytes_per_count), L); 2110 __ ldr(R10, Address(end_from, -8, pre_indexed)); 2111 __ orr(R12, R12, AsmOperand(R10, lsr, lsr_shift)); 2112 __ str(R12, Address(end_to, -8, pre_indexed)); 2113 __ logical_shift_left(R12, R10, lsl_shift); 2114 __ bind(L); 2115 } 2116 2117 const int have_bytes = lsr_shift/BitsPerByte; // number of already read bytes in R12 2118 2119 // It remains less than wordSize to write. 2120 // Do not check count if R12 already has maximal number of loaded elements (one less than wordSize). 2121 if (have_bytes < wordSize - bytes_per_count) { 2122 Label L; 2123 __ andr(count, count, (uintx)(8/bytes_per_count-1)); // make count exact 2124 __ cmp_32(count, have_bytes/bytes_per_count); // do we have enough bytes to store? 2125 __ b(L, le); 2126 __ ldr(R10, Address(end_from, -8, pre_indexed)); 2127 __ orr(R12, R12, AsmOperand(R10, lsr, lsr_shift)); 2128 __ bind(L); 2129 } 2130 2131 assert (bytes_per_count <= 4, "must be"); 2132 2133 { 2134 Label L; 2135 __ tbz(count, exact_log2(4/bytes_per_count), L); 2136 __ logical_shift_right(R9, R12, (wordSize-4)*BitsPerByte); 2137 __ str_w(R9, Address(end_to, -4, pre_indexed)); // Write 4 MSB 2138 if (bytes_per_count < 4) { 2139 __ logical_shift_left(R12, R12, 4*BitsPerByte); // Promote remaining bytes to MSB 2140 } 2141 __ bind(L); 2142 } 2143 2144 if (bytes_per_count <= 2) { 2145 Label L; 2146 __ tbz(count, exact_log2(2/bytes_per_count), L); 2147 __ logical_shift_right(R9, R12, (wordSize-2)*BitsPerByte); 2148 __ strh(R9, Address(end_to, -2, pre_indexed)); // Write 2 MSB 2149 if (bytes_per_count < 2) { 2150 __ logical_shift_left(R12, R12, 2*BitsPerByte); // Promote remaining bytes to MSB 2151 } 2152 __ bind(L); 2153 } 2154 2155 if (bytes_per_count <= 1) { 2156 Label L; 2157 __ tbz(count, exact_log2(1/bytes_per_count), L); 2158 __ logical_shift_right(R9, R12, (wordSize-1)*BitsPerByte); 2159 __ strb(R9, Address(end_to, -1, pre_indexed)); // Write 1 MSB 2160 __ bind(L); 2161 } 2162#else 2163 switch(bytes_per_count) { 2164 case 2: 2165 __ mov(R12, AsmOperand(R3, lsl, lsl_shift)); // part of R3 not yet written 2166 __ tst(count, 8); 2167 __ ldmdb(end_from, RegisterSet(R7,R10), writeback, ne); 2168 __ orr(R12, R12, AsmOperand(R10, lsr, lsr_shift), ne); 2169 __ mov(R10, AsmOperand(R10, lsl, lsl_shift),ne); // unused part of prev val 2170 __ orr(R10, R10, AsmOperand(R9, lsr, lsr_shift),ne); // ... 2171 __ mov(R9, AsmOperand(R9, lsl, lsl_shift),ne); 2172 __ orr(R9, R9, AsmOperand(R8, lsr, lsr_shift),ne); 2173 __ mov(R8, AsmOperand(R8, lsl, lsl_shift),ne); 2174 __ orr(R8, R8, AsmOperand(R7, lsr, lsr_shift),ne); 2175 __ stmdb(end_to, RegisterSet(R8,R10)|R12, writeback, ne); 2176 __ mov(R12, AsmOperand(R7, lsl, lsl_shift), ne); 2177 2178 __ tst(count, 4); 2179 __ ldmdb(end_from, RegisterSet(R9, R10), writeback, ne); 2180 __ orr(R12, R12, AsmOperand(R10, lsr, lsr_shift), ne); 2181 __ mov(R10, AsmOperand(R10, lsl, lsl_shift),ne); // unused part of prev val 2182 __ orr(R10, R10, AsmOperand(R9, lsr,lsr_shift),ne); // ... 2183 __ stmdb(end_to, RegisterSet(R10)|R12, writeback, ne); 2184 __ mov(R12, AsmOperand(R9, lsl, lsl_shift), ne); 2185 2186 __ tst(count, 2); 2187 __ ldr(R10, Address(end_from, -4, pre_indexed), ne); 2188 __ orr(R12, R12, AsmOperand(R10, lsr, lsr_shift), ne); 2189 __ str(R12, Address(end_to, -4, pre_indexed), ne); 2190 __ mov(R12, AsmOperand(R10, lsl, lsl_shift), ne); 2191 2192 __ tst(count, 1); 2193 __ mov(R12, AsmOperand(R12, lsr, lsr_shift),ne); 2194 __ strh(R12, Address(end_to, -2, pre_indexed), ne); // one last short 2195 break; 2196 2197 case 1: 2198 __ mov(R12, AsmOperand(R3, lsl, lsl_shift)); // part of R3 not yet written 2199 __ tst(count, 16); 2200 __ ldmdb(end_from, RegisterSet(R7,R10), writeback, ne); 2201 __ orr(R12, R12, AsmOperand(R10, lsr, lsr_shift), ne); 2202 __ mov(R10, AsmOperand(R10, lsl, lsl_shift),ne); // unused part of prev val 2203 __ orr(R10, R10, AsmOperand(R9, lsr, lsr_shift),ne); // ... 2204 __ mov(R9, AsmOperand(R9, lsl, lsl_shift),ne); 2205 __ orr(R9, R9, AsmOperand(R8, lsr, lsr_shift),ne); 2206 __ mov(R8, AsmOperand(R8, lsl, lsl_shift),ne); 2207 __ orr(R8, R8, AsmOperand(R7, lsr, lsr_shift),ne); 2208 __ stmdb(end_to, RegisterSet(R8,R10)|R12, writeback, ne); 2209 __ mov(R12, AsmOperand(R7, lsl, lsl_shift), ne); 2210 2211 __ tst(count, 8); 2212 __ ldmdb(end_from, RegisterSet(R9,R10), writeback, ne); 2213 __ orr(R12, R12, AsmOperand(R10, lsr, lsr_shift), ne); 2214 __ mov(R10, AsmOperand(R10, lsl, lsl_shift),ne); // unused part of prev val 2215 __ orr(R10, R10, AsmOperand(R9, lsr, lsr_shift),ne); // ... 2216 __ stmdb(end_to, RegisterSet(R10)|R12, writeback, ne); 2217 __ mov(R12, AsmOperand(R9, lsl, lsl_shift), ne); 2218 2219 __ tst(count, 4); 2220 __ ldr(R10, Address(end_from, -4, pre_indexed), ne); 2221 __ orr(R12, R12, AsmOperand(R10, lsr, lsr_shift), ne); 2222 __ str(R12, Address(end_to, -4, pre_indexed), ne); 2223 __ mov(R12, AsmOperand(R10, lsl, lsl_shift), ne); 2224 2225 __ tst(count, 2); 2226 if (lsr_shift != 24) { 2227 // avoid useless reading R10 when we already have 3 bytes ready in R12 2228 __ ldr(R10, Address(end_from, -4, pre_indexed), ne); 2229 __ orr(R12, R12, AsmOperand(R10, lsr,lsr_shift), ne); 2230 } 2231 2232 // Note: R12 contains enough bytes ready to write (3 needed at most) 2233 // write the 2 MSBs 2234 __ mov(R9, AsmOperand(R12, lsr, 16), ne); 2235 __ strh(R9, Address(end_to, -2, pre_indexed), ne); 2236 // promote remaining to MSB 2237 __ mov(R12, AsmOperand(R12, lsl, 16), ne); 2238 2239 __ tst(count, 1); 2240 // write the MSB of R12 2241 __ mov(R12, AsmOperand(R12, lsr, 24), ne); 2242 __ strb(R12, Address(end_to, -1, pre_indexed), ne); 2243 2244 break; 2245 } 2246#endif // AARCH64 2247 2248 __ BIND(L_done); 2249 return 0; // no minimum 2250 } 2251 2252 // This method is very useful for merging forward/backward implementations 2253 Address get_addr_with_indexing(Register base, int delta, bool forward) { 2254 if (forward) { 2255 return Address(base, delta, post_indexed); 2256 } else { 2257 return Address(base, -delta, pre_indexed); 2258 } 2259 } 2260 2261#ifdef AARCH64 2262 // Loads one 'size_in_bytes'-sized value from 'from' in given direction, i.e. 2263 // if forward: loads value at from and increases from by size 2264 // if !forward: loads value at from-size_in_bytes and decreases from by size 2265 void load_one(Register rd, Register from, int size_in_bytes, bool forward) { 2266 assert_different_registers(from, rd); 2267 Address addr = get_addr_with_indexing(from, size_in_bytes, forward); 2268 __ load_sized_value(rd, addr, size_in_bytes, false); 2269 } 2270 2271 // Stores one 'size_in_bytes'-sized value to 'to' in given direction (see load_one) 2272 void store_one(Register rd, Register to, int size_in_bytes, bool forward) { 2273 assert_different_registers(to, rd); 2274 Address addr = get_addr_with_indexing(to, size_in_bytes, forward); 2275 __ store_sized_value(rd, addr, size_in_bytes); 2276 } 2277#else 2278 // load_one and store_one are the same as for AArch64 except for 2279 // *) Support for condition execution 2280 // *) Second value register argument for 8-byte values 2281 2282 void load_one(Register rd, Register from, int size_in_bytes, bool forward, AsmCondition cond = al, Register rd2 = noreg) { 2283 assert_different_registers(from, rd, rd2); 2284 if (size_in_bytes < 8) { 2285 Address addr = get_addr_with_indexing(from, size_in_bytes, forward); 2286 __ load_sized_value(rd, addr, size_in_bytes, false, cond); 2287 } else { 2288 assert (rd2 != noreg, "second value register must be specified"); 2289 assert (rd->encoding() < rd2->encoding(), "wrong value register set"); 2290 2291 if (forward) { 2292 __ ldmia(from, RegisterSet(rd) | rd2, writeback, cond); 2293 } else { 2294 __ ldmdb(from, RegisterSet(rd) | rd2, writeback, cond); 2295 } 2296 } 2297 } 2298 2299 void store_one(Register rd, Register to, int size_in_bytes, bool forward, AsmCondition cond = al, Register rd2 = noreg) { 2300 assert_different_registers(to, rd, rd2); 2301 if (size_in_bytes < 8) { 2302 Address addr = get_addr_with_indexing(to, size_in_bytes, forward); 2303 __ store_sized_value(rd, addr, size_in_bytes, cond); 2304 } else { 2305 assert (rd2 != noreg, "second value register must be specified"); 2306 assert (rd->encoding() < rd2->encoding(), "wrong value register set"); 2307 2308 if (forward) { 2309 __ stmia(to, RegisterSet(rd) | rd2, writeback, cond); 2310 } else { 2311 __ stmdb(to, RegisterSet(rd) | rd2, writeback, cond); 2312 } 2313 } 2314 } 2315#endif // AARCH64 2316 2317 // Copies data from 'from' to 'to' in specified direction to align 'from' by 64 bits. 2318 // (on 32-bit ARM 64-bit alignment is better for LDM). 2319 // 2320 // Arguments: 2321 // from: beginning (if forward) or upper bound (if !forward) of the region to be read 2322 // to: beginning (if forward) or upper bound (if !forward) of the region to be written 2323 // count: 32-bit int, maximum number of elements which can be copied 2324 // bytes_per_count: size of an element 2325 // forward: specifies copy direction 2326 // 2327 // Notes: 2328 // 'from' and 'to' must be aligned by 'bytes_per_count' 2329 // 'count' must not be less than the returned value 2330 // shifts 'from' and 'to' by the number of copied bytes in corresponding direction 2331 // decreases 'count' by the number of elements copied 2332 // 2333 // Returns maximum number of bytes which may be copied. 2334 int align_src(Register from, Register to, Register count, Register tmp, int bytes_per_count, bool forward) { 2335 assert_different_registers(from, to, count, tmp); 2336#ifdef AARCH64 2337 // TODO-AARCH64: replace by simple loop? 2338 Label Laligned_by_2, Laligned_by_4, Laligned_by_8; 2339 2340 if (bytes_per_count == 1) { 2341 __ tbz(from, 0, Laligned_by_2); 2342 __ sub_32(count, count, 1); 2343 load_one(tmp, from, 1, forward); 2344 store_one(tmp, to, 1, forward); 2345 } 2346 2347 __ BIND(Laligned_by_2); 2348 2349 if (bytes_per_count <= 2) { 2350 __ tbz(from, 1, Laligned_by_4); 2351 __ sub_32(count, count, 2/bytes_per_count); 2352 load_one(tmp, from, 2, forward); 2353 store_one(tmp, to, 2, forward); 2354 } 2355 2356 __ BIND(Laligned_by_4); 2357 2358 if (bytes_per_count <= 4) { 2359 __ tbz(from, 2, Laligned_by_8); 2360 __ sub_32(count, count, 4/bytes_per_count); 2361 load_one(tmp, from, 4, forward); 2362 store_one(tmp, to, 4, forward); 2363 } 2364 __ BIND(Laligned_by_8); 2365#else // AARCH64 2366 if (bytes_per_count < 8) { 2367 Label L_align_src; 2368 __ BIND(L_align_src); 2369 __ tst(from, 7); 2370 // ne => not aligned: copy one element and (if bytes_per_count < 4) loop 2371 __ sub(count, count, 1, ne); 2372 load_one(tmp, from, bytes_per_count, forward, ne); 2373 store_one(tmp, to, bytes_per_count, forward, ne); 2374 if (bytes_per_count < 4) { 2375 __ b(L_align_src, ne); // if bytes_per_count == 4, then 0 or 1 loop iterations are enough 2376 } 2377 } 2378#endif // AARCH64 2379 return 7/bytes_per_count; 2380 } 2381 2382 // Copies 'count' of 'bytes_per_count'-sized elements in the specified direction. 2383 // 2384 // Arguments: 2385 // from: beginning (if forward) or upper bound (if !forward) of the region to be read 2386 // to: beginning (if forward) or upper bound (if !forward) of the region to be written 2387 // count: 32-bit int, number of elements to be copied 2388 // entry: copy loop entry point 2389 // bytes_per_count: size of an element 2390 // forward: specifies copy direction 2391 // 2392 // Notes: 2393 // shifts 'from' and 'to' 2394 void copy_small_array(Register from, Register to, Register count, Register tmp, Register tmp2, int bytes_per_count, bool forward, Label & entry) { 2395 assert_different_registers(from, to, count, tmp); 2396 2397 __ align(OptoLoopAlignment); 2398#ifdef AARCH64 2399 Label L_small_array_done, L_small_array_loop; 2400 __ BIND(entry); 2401 __ cbz_32(count, L_small_array_done); 2402 2403 __ BIND(L_small_array_loop); 2404 __ subs_32(count, count, 1); 2405 load_one(tmp, from, bytes_per_count, forward); 2406 store_one(tmp, to, bytes_per_count, forward); 2407 __ b(L_small_array_loop, gt); 2408 2409 __ BIND(L_small_array_done); 2410#else 2411 Label L_small_loop; 2412 __ BIND(L_small_loop); 2413 store_one(tmp, to, bytes_per_count, forward, al, tmp2); 2414 __ BIND(entry); // entry point 2415 __ subs(count, count, 1); 2416 load_one(tmp, from, bytes_per_count, forward, ge, tmp2); 2417 __ b(L_small_loop, ge); 2418#endif // AARCH64 2419 } 2420 2421 // Aligns 'to' by reading one word from 'from' and writting its part to 'to'. 2422 // 2423 // Arguments: 2424 // to: beginning (if forward) or upper bound (if !forward) of the region to be written 2425 // count: 32-bit int, number of elements allowed to be copied 2426 // to_remainder: remainder of dividing 'to' by wordSize 2427 // bytes_per_count: size of an element 2428 // forward: specifies copy direction 2429 // Rval: contains an already read but not yet written word; 2430 // its' LSBs (if forward) or MSBs (if !forward) are to be written to align 'to'. 2431 // 2432 // Notes: 2433 // 'count' must not be less then the returned value 2434 // 'to' must be aligned by bytes_per_count but must not be aligned by wordSize 2435 // shifts 'to' by the number of written bytes (so that it becomes the bound of memory to be written) 2436 // decreases 'count' by the the number of elements written 2437 // Rval's MSBs or LSBs remain to be written further by generate_{forward,backward}_shifted_copy_loop 2438 int align_dst(Register to, Register count, Register Rval, Register tmp, 2439 int to_remainder, int bytes_per_count, bool forward) { 2440 assert_different_registers(to, count, tmp, Rval); 2441 2442 assert (0 < to_remainder && to_remainder < wordSize, "to_remainder is not valid"); 2443 assert (to_remainder % bytes_per_count == 0, "to must be aligned by bytes_per_count"); 2444 2445 int bytes_to_write = forward ? (wordSize - to_remainder) : to_remainder; 2446 2447 int offset = 0; 2448 2449 for (int l = 0; l < LogBytesPerWord; ++l) { 2450 int s = (1 << l); 2451 if (bytes_to_write & s) { 2452 int new_offset = offset + s*BitsPerByte; 2453 if (forward) { 2454 if (offset == 0) { 2455 store_one(Rval, to, s, forward); 2456 } else { 2457 __ logical_shift_right(tmp, Rval, offset); 2458 store_one(tmp, to, s, forward); 2459 } 2460 } else { 2461 __ logical_shift_right(tmp, Rval, BitsPerWord - new_offset); 2462 store_one(tmp, to, s, forward); 2463 } 2464 2465 offset = new_offset; 2466 } 2467 } 2468 2469 assert (offset == bytes_to_write * BitsPerByte, "all bytes must be copied"); 2470 2471 __ sub_32(count, count, bytes_to_write/bytes_per_count); 2472 2473 return bytes_to_write / bytes_per_count; 2474 } 2475 2476 // Copies 'count' of elements using shifted copy loop 2477 // 2478 // Arguments: 2479 // from: beginning (if forward) or upper bound (if !forward) of the region to be read 2480 // to: beginning (if forward) or upper bound (if !forward) of the region to be written 2481 // count: 32-bit int, number of elements to be copied 2482 // to_remainder: remainder of dividing 'to' by wordSize 2483 // bytes_per_count: size of an element 2484 // forward: specifies copy direction 2485 // Rval: contains an already read but not yet written word 2486 // 2487 // 2488 // Notes: 2489 // 'count' must not be less then the returned value 2490 // 'from' must be aligned by wordSize 2491 // 'to' must be aligned by bytes_per_count but must not be aligned by wordSize 2492 // shifts 'to' by the number of copied bytes 2493 // 2494 // Scratches R3-R10, R12 2495 int align_dst_and_generate_shifted_copy_loop(Register from, Register to, Register count, Register Rval, 2496 int to_remainder, int bytes_per_count, bool forward) { 2497 2498 assert (0 < to_remainder && to_remainder < wordSize, "to_remainder is invalid"); 2499 2500 const Register tmp = forward ? R3 : R12; // TODO-AARCH64: on cojoint_short R4 was used for tmp 2501 assert_different_registers(from, to, count, Rval, tmp); 2502 2503 int required_to_align = align_dst(to, count, Rval, tmp, to_remainder, bytes_per_count, forward); 2504 2505 int lsr_shift = (wordSize - to_remainder) * BitsPerByte; 2506 int lsl_shift = to_remainder * BitsPerByte; 2507 2508 int min_copy; 2509 if (forward) { 2510 min_copy = generate_forward_shifted_copy_loop(from, to, count, bytes_per_count, lsr_shift, lsl_shift); 2511 } else { 2512 min_copy = generate_backward_shifted_copy_loop(from, to, count, bytes_per_count, lsr_shift, lsl_shift); 2513 } 2514 2515 return min_copy + required_to_align; 2516 } 2517 2518 // Copies 'count' of elements using shifted copy loop 2519 // 2520 // Arguments: 2521 // from: beginning (if forward) or upper bound (if !forward) of the region to be read 2522 // to: beginning (if forward) or upper bound (if !forward) of the region to be written 2523 // count: 32-bit int, number of elements to be copied 2524 // bytes_per_count: size of an element 2525 // forward: specifies copy direction 2526 // 2527 // Notes: 2528 // 'count' must not be less then the returned value 2529 // 'from' must be aligned by wordSize 2530 // 'to' must be aligned by bytes_per_count but must not be aligned by wordSize 2531 // shifts 'to' by the number of copied bytes 2532 // 2533 // Scratches 'from', 'count', R3 and R12. 2534 // On AArch64 also scratches R4-R10, on 32-bit ARM saves them to use. 2535 int align_dst_and_generate_shifted_copy_loop(Register from, Register to, Register count, int bytes_per_count, bool forward) { 2536 2537 const Register Rval = forward ? R12 : R3; // as generate_{forward,backward}_shifted_copy_loop expect 2538 2539 int min_copy = 0; 2540 2541 // Note: if {seq} is a sequence of numbers, L{seq} means that if the execution reaches this point, 2542 // then the remainder of 'to' divided by wordSize is one of elements of {seq}. 2543 2544#ifdef AARCH64 2545 // TODO-AARCH64: simplify, tune 2546 2547 load_one(Rval, from, wordSize, forward); 2548 2549 Label L_loop_finished; 2550 2551 switch (bytes_per_count) { 2552 case 4: 2553 min_copy = align_dst_and_generate_shifted_copy_loop(from, to, count, Rval, 4, bytes_per_count, forward); 2554 break; 2555 case 2: 2556 { 2557 Label L2, L4, L6; 2558 2559 __ tbz(to, 1, L4); 2560 __ tbz(to, 2, L2); 2561 2562 __ BIND(L6); 2563 int min_copy6 = align_dst_and_generate_shifted_copy_loop(from, to, count, Rval, 6, bytes_per_count, forward); 2564 __ b(L_loop_finished); 2565 2566 __ BIND(L2); 2567 int min_copy2 = align_dst_and_generate_shifted_copy_loop(from, to, count, Rval, 2, bytes_per_count, forward); 2568 __ b(L_loop_finished); 2569 2570 __ BIND(L4); 2571 int min_copy4 = align_dst_and_generate_shifted_copy_loop(from, to, count, Rval, 4, bytes_per_count, forward); 2572 2573 min_copy = MAX2(MAX2(min_copy2, min_copy4), min_copy6); 2574 break; 2575 } 2576 case 1: 2577 { 2578 Label L1, L2, L3, L4, L5, L6, L7; 2579 Label L15, L26; 2580 Label L246; 2581 2582 __ tbz(to, 0, L246); 2583 __ tbz(to, 1, L15); 2584 __ tbz(to, 2, L3); 2585 2586 __ BIND(L7); 2587 int min_copy7 = align_dst_and_generate_shifted_copy_loop(from, to, count, Rval, 7, bytes_per_count, forward); 2588 __ b(L_loop_finished); 2589 2590 __ BIND(L246); 2591 __ tbnz(to, 1, L26); 2592 2593 __ BIND(L4); 2594 int min_copy4 = align_dst_and_generate_shifted_copy_loop(from, to, count, Rval, 4, bytes_per_count, forward); 2595 __ b(L_loop_finished); 2596 2597 __ BIND(L15); 2598 __ tbz(to, 2, L1); 2599 2600 __ BIND(L5); 2601 int min_copy5 = align_dst_and_generate_shifted_copy_loop(from, to, count, Rval, 5, bytes_per_count, forward); 2602 __ b(L_loop_finished); 2603 2604 __ BIND(L3); 2605 int min_copy3 = align_dst_and_generate_shifted_copy_loop(from, to, count, Rval, 3, bytes_per_count, forward); 2606 __ b(L_loop_finished); 2607 2608 __ BIND(L26); 2609 __ tbz(to, 2, L2); 2610 2611 __ BIND(L6); 2612 int min_copy6 = align_dst_and_generate_shifted_copy_loop(from, to, count, Rval, 6, bytes_per_count, forward); 2613 __ b(L_loop_finished); 2614 2615 __ BIND(L1); 2616 int min_copy1 = align_dst_and_generate_shifted_copy_loop(from, to, count, Rval, 1, bytes_per_count, forward); 2617 __ b(L_loop_finished); 2618 2619 __ BIND(L2); 2620 int min_copy2 = align_dst_and_generate_shifted_copy_loop(from, to, count, Rval, 2, bytes_per_count, forward); 2621 2622 2623 min_copy = MAX2(min_copy1, min_copy2); 2624 min_copy = MAX2(min_copy, min_copy3); 2625 min_copy = MAX2(min_copy, min_copy4); 2626 min_copy = MAX2(min_copy, min_copy5); 2627 min_copy = MAX2(min_copy, min_copy6); 2628 min_copy = MAX2(min_copy, min_copy7); 2629 break; 2630 } 2631 default: 2632 ShouldNotReachHere(); 2633 break; 2634 } 2635 __ BIND(L_loop_finished); 2636 2637#else 2638 __ push(RegisterSet(R4,R10)); 2639 load_one(Rval, from, wordSize, forward); 2640 2641 switch (bytes_per_count) { 2642 case 2: 2643 min_copy = align_dst_and_generate_shifted_copy_loop(from, to, count, Rval, 2, bytes_per_count, forward); 2644 break; 2645 case 1: 2646 { 2647 Label L1, L2, L3; 2648 int min_copy1, min_copy2, min_copy3; 2649 2650 Label L_loop_finished; 2651 2652 if (forward) { 2653 __ tbz(to, 0, L2); 2654 __ tbz(to, 1, L1); 2655 2656 __ BIND(L3); 2657 min_copy3 = align_dst_and_generate_shifted_copy_loop(from, to, count, Rval, 3, bytes_per_count, forward); 2658 __ b(L_loop_finished); 2659 2660 __ BIND(L1); 2661 min_copy1 = align_dst_and_generate_shifted_copy_loop(from, to, count, Rval, 1, bytes_per_count, forward); 2662 __ b(L_loop_finished); 2663 2664 __ BIND(L2); 2665 min_copy2 = align_dst_and_generate_shifted_copy_loop(from, to, count, Rval, 2, bytes_per_count, forward); 2666 } else { 2667 __ tbz(to, 0, L2); 2668 __ tbnz(to, 1, L3); 2669 2670 __ BIND(L1); 2671 min_copy1 = align_dst_and_generate_shifted_copy_loop(from, to, count, Rval, 1, bytes_per_count, forward); 2672 __ b(L_loop_finished); 2673 2674 __ BIND(L3); 2675 min_copy3 = align_dst_and_generate_shifted_copy_loop(from, to, count, Rval, 3, bytes_per_count, forward); 2676 __ b(L_loop_finished); 2677 2678 __ BIND(L2); 2679 min_copy2 = align_dst_and_generate_shifted_copy_loop(from, to, count, Rval, 2, bytes_per_count, forward); 2680 } 2681 2682 min_copy = MAX2(MAX2(min_copy1, min_copy2), min_copy3); 2683 2684 __ BIND(L_loop_finished); 2685 2686 break; 2687 } 2688 default: 2689 ShouldNotReachHere(); 2690 break; 2691 } 2692 2693 __ pop(RegisterSet(R4,R10)); 2694#endif // AARCH64 2695 2696 return min_copy; 2697 } 2698 2699#ifndef PRODUCT 2700 int * get_arraycopy_counter(int bytes_per_count) { 2701 switch (bytes_per_count) { 2702 case 1: 2703 return &SharedRuntime::_jbyte_array_copy_ctr; 2704 case 2: 2705 return &SharedRuntime::_jshort_array_copy_ctr; 2706 case 4: 2707 return &SharedRuntime::_jint_array_copy_ctr; 2708 case 8: 2709 return &SharedRuntime::_jlong_array_copy_ctr; 2710 default: 2711 ShouldNotReachHere(); 2712 return NULL; 2713 } 2714 } 2715#endif // !PRODUCT 2716 2717 // 2718 // Generate stub for primitive array copy. If "aligned" is true, the 2719 // "from" and "to" addresses are assumed to be heapword aligned. 2720 // 2721 // If "disjoint" is true, arrays are assumed to be disjoint, otherwise they may overlap and 2722 // "nooverlap_target" must be specified as the address to jump if they don't. 2723 // 2724 // Arguments for generated stub: 2725 // from: R0 2726 // to: R1 2727 // count: R2 treated as signed 32-bit int 2728 // 2729 address generate_primitive_copy(bool aligned, const char * name, bool status, int bytes_per_count, bool disjoint, address nooverlap_target = NULL) { 2730 __ align(CodeEntryAlignment); 2731 StubCodeMark mark(this, "StubRoutines", name); 2732 address start = __ pc(); 2733 2734 const Register from = R0; // source array address 2735 const Register to = R1; // destination array address 2736 const Register count = R2; // elements count 2737 const Register tmp1 = R3; 2738 const Register tmp2 = R12; 2739 2740 if (!aligned) { 2741 BLOCK_COMMENT("Entry:"); 2742 } 2743 2744 __ zap_high_non_significant_bits(R2); 2745 2746 if (!disjoint) { 2747 assert (nooverlap_target != NULL, "must be specified for conjoint case"); 2748 array_overlap_test(nooverlap_target, exact_log2(bytes_per_count), tmp1, tmp2); 2749 } 2750 2751 inc_counter_np(*get_arraycopy_counter(bytes_per_count), tmp1, tmp2); 2752 2753 // Conjoint case: since execution reaches this point, the arrays overlap, so performing backward copy 2754 // Disjoint case: perform forward copy 2755 bool forward = disjoint; 2756 2757 2758 if (!forward) { 2759 // Set 'from' and 'to' to upper bounds 2760 int log_bytes_per_count = exact_log2(bytes_per_count); 2761 __ add_ptr_scaled_int32(to, to, count, log_bytes_per_count); 2762 __ add_ptr_scaled_int32(from, from, count, log_bytes_per_count); 2763 } 2764 2765 // There are two main copy loop implementations: 2766 // *) The huge and complex one applicable only for large enough arrays 2767 // *) The small and simple one applicable for any array (but not efficient for large arrays). 2768 // Currently "small" implementation is used if and only if the "large" one could not be used. 2769 // XXX optim: tune the limit higher ? 2770 // Large implementation lower applicability bound is actually determined by 2771 // aligned copy loop which require <=7 bytes for src alignment, and 8 words for aligned copy loop. 2772 const int small_copy_limit = (8*wordSize + 7) / bytes_per_count; 2773 2774 Label L_small_array; 2775 __ cmp_32(count, small_copy_limit); 2776 __ b(L_small_array, le); // TODO-AARCH64: le vs lt 2777 2778 // Otherwise proceed with large implementation. 2779 2780 bool from_is_aligned = (bytes_per_count >= 8); 2781 if (aligned && forward && (HeapWordSize % 8 == 0)) { 2782 // if 'from' is heapword aligned and HeapWordSize is divisible by 8, 2783 // then from is aligned by 8 2784 from_is_aligned = true; 2785 } 2786 2787 int count_required_to_align = from_is_aligned ? 0 : align_src(from, to, count, tmp1, bytes_per_count, forward); 2788 assert (small_copy_limit >= count_required_to_align, "alignment could exhaust count"); 2789 2790 // now 'from' is aligned 2791 2792 bool to_is_aligned = false; 2793 2794 if (bytes_per_count >= wordSize) { 2795 // 'to' is aligned by bytes_per_count, so it is aligned by wordSize 2796 to_is_aligned = true; 2797 } else { 2798 if (aligned && (8 % HeapWordSize == 0) && (HeapWordSize % wordSize == 0)) { 2799 // Originally 'from' and 'to' were heapword aligned; 2800 // (from - to) has not been changed, so since now 'from' is 8-byte aligned, then it is also heapword aligned, 2801 // so 'to' is also heapword aligned and thus aligned by wordSize. 2802 to_is_aligned = true; 2803 } 2804 } 2805 2806 Label L_unaligned_dst; 2807 2808 if (!to_is_aligned) { 2809 BLOCK_COMMENT("Check dst alignment:"); 2810 __ tst(to, wordSize - 1); 2811 __ b(L_unaligned_dst, ne); // 'to' is not aligned 2812 } 2813 2814 // 'from' and 'to' are properly aligned 2815 2816 int min_copy; 2817 if (forward) { 2818 min_copy = generate_forward_aligned_copy_loop (from, to, count, bytes_per_count); 2819 } else { 2820 min_copy = generate_backward_aligned_copy_loop(from, to, count, bytes_per_count); 2821 } 2822 assert(small_copy_limit >= count_required_to_align + min_copy, "first loop might exhaust count"); 2823 2824 if (status) { 2825 __ mov(R0, 0); // OK 2826 } 2827 2828 __ ret(); 2829 2830 { 2831 copy_small_array(from, to, count, tmp1, tmp2, bytes_per_count, forward, L_small_array /* entry */); 2832 2833 if (status) { 2834 __ mov(R0, 0); // OK 2835 } 2836 2837 __ ret(); 2838 } 2839 2840 if (! to_is_aligned) { 2841 __ BIND(L_unaligned_dst); 2842 int min_copy_shifted = align_dst_and_generate_shifted_copy_loop(from, to, count, bytes_per_count, forward); 2843 assert (small_copy_limit >= count_required_to_align + min_copy_shifted, "first loop might exhaust count"); 2844 2845 if (status) { 2846 __ mov(R0, 0); // OK 2847 } 2848 2849 __ ret(); 2850 } 2851 2852 return start; 2853 } 2854 2855#if INCLUDE_ALL_GCS 2856 // 2857 // Generate pre-write barrier for array. 2858 // 2859 // Input: 2860 // addr - register containing starting address 2861 // count - register containing element count, 32-bit int 2862 // callee_saved_regs - 2863 // the call must preserve this number of registers: R0, R1, ..., R[callee_saved_regs-1] 2864 // 2865 // callee_saved_regs must include addr and count 2866 // Blows all volatile registers (R0-R3 on 32-bit ARM, R0-R18 on AArch64, Rtemp, LR) except for callee_saved_regs. 2867 void gen_write_ref_array_pre_barrier(Register addr, Register count, int callee_saved_regs) { 2868 BarrierSet* bs = Universe::heap()->barrier_set(); 2869 if (bs->has_write_ref_pre_barrier()) { 2870 assert(bs->has_write_ref_array_pre_opt(), 2871 "Else unsupported barrier set."); 2872 2873 assert( addr->encoding() < callee_saved_regs, "addr must be saved"); 2874 assert(count->encoding() < callee_saved_regs, "count must be saved"); 2875 2876 BLOCK_COMMENT("PreBarrier"); 2877 2878#ifdef AARCH64 2879 callee_saved_regs = round_to(callee_saved_regs, 2); 2880 for (int i = 0; i < callee_saved_regs; i += 2) { 2881 __ raw_push(as_Register(i), as_Register(i+1)); 2882 } 2883#else 2884 RegisterSet saved_regs = RegisterSet(R0, as_Register(callee_saved_regs-1)); 2885 __ push(saved_regs | R9ifScratched); 2886#endif // AARCH64 2887 2888 if (addr != R0) { 2889 assert_different_registers(count, R0); 2890 __ mov(R0, addr); 2891 } 2892#ifdef AARCH64 2893 __ zero_extend(R1, count, 32); // BarrierSet::static_write_ref_array_pre takes size_t 2894#else 2895 if (count != R1) { 2896 __ mov(R1, count); 2897 } 2898#endif // AARCH64 2899 2900 __ call(CAST_FROM_FN_PTR(address, BarrierSet::static_write_ref_array_pre)); 2901 2902#ifdef AARCH64 2903 for (int i = callee_saved_regs - 2; i >= 0; i -= 2) { 2904 __ raw_pop(as_Register(i), as_Register(i+1)); 2905 } 2906#else 2907 __ pop(saved_regs | R9ifScratched); 2908#endif // AARCH64 2909 } 2910 } 2911#endif // INCLUDE_ALL_GCS 2912 2913 // 2914 // Generate post-write barrier for array. 2915 // 2916 // Input: 2917 // addr - register containing starting address (can be scratched) 2918 // count - register containing element count, 32-bit int (can be scratched) 2919 // tmp - scratch register 2920 // 2921 // Note: LR can be scratched but might be equal to addr, count or tmp 2922 // Blows all volatile registers (R0-R3 on 32-bit ARM, R0-R18 on AArch64, Rtemp, LR). 2923 void gen_write_ref_array_post_barrier(Register addr, Register count, Register tmp) { 2924 assert_different_registers(addr, count, tmp); 2925 BarrierSet* bs = Universe::heap()->barrier_set(); 2926 2927 switch (bs->kind()) { 2928 case BarrierSet::G1SATBCTLogging: 2929 { 2930 BLOCK_COMMENT("G1PostBarrier"); 2931 if (addr != R0) { 2932 assert_different_registers(count, R0); 2933 __ mov(R0, addr); 2934 } 2935#ifdef AARCH64 2936 __ zero_extend(R1, count, 32); // BarrierSet::static_write_ref_array_post takes size_t 2937#else 2938 if (count != R1) { 2939 __ mov(R1, count); 2940 } 2941#if R9_IS_SCRATCHED 2942 // Safer to save R9 here since callers may have been written 2943 // assuming R9 survives. This is suboptimal but is not in 2944 // general worth optimizing for the few platforms where R9 2945 // is scratched. Note that the optimization might not be to 2946 // difficult for this particular call site. 2947 __ push(R9); 2948#endif 2949#endif // !AARCH64 2950 __ call(CAST_FROM_FN_PTR(address, BarrierSet::static_write_ref_array_post)); 2951#ifndef AARCH64 2952#if R9_IS_SCRATCHED 2953 __ pop(R9); 2954#endif 2955#endif // !AARCH64 2956 } 2957 break; 2958 case BarrierSet::CardTableForRS: 2959 case BarrierSet::CardTableExtension: 2960 { 2961 BLOCK_COMMENT("CardTablePostBarrier"); 2962 CardTableModRefBS* ct = barrier_set_cast<CardTableModRefBS>(bs); 2963 assert(sizeof(*ct->byte_map_base) == sizeof(jbyte), "adjust this code"); 2964 2965 Label L_cardtable_loop; 2966 2967 __ add_ptr_scaled_int32(count, addr, count, LogBytesPerHeapOop); 2968 __ sub(count, count, BytesPerHeapOop); // last addr 2969 2970 __ logical_shift_right(addr, addr, CardTableModRefBS::card_shift); 2971 __ logical_shift_right(count, count, CardTableModRefBS::card_shift); 2972 __ sub(count, count, addr); // nb of cards 2973 2974 // warning: Rthread has not been preserved 2975 __ mov_address(tmp, (address) ct->byte_map_base, symbolic_Relocation::card_table_reference); 2976 __ add(addr,tmp, addr); 2977 2978 Register zero = __ zero_register(tmp); 2979 2980 __ BIND(L_cardtable_loop); 2981 __ strb(zero, Address(addr, 1, post_indexed)); 2982 __ subs(count, count, 1); 2983 __ b(L_cardtable_loop, ge); 2984 } 2985 break; 2986 case BarrierSet::ModRef: 2987 break; 2988 default: 2989 ShouldNotReachHere(); 2990 } 2991 } 2992 2993 // Generates pattern of code to be placed after raw data copying in generate_oop_copy 2994 // Includes return from arraycopy stub. 2995 // 2996 // Arguments: 2997 // to: destination pointer after copying. 2998 // if 'forward' then 'to' == upper bound, else 'to' == beginning of the modified region 2999 // count: total number of copied elements, 32-bit int 3000 // 3001 // Blows all volatile (R0-R3 on 32-bit ARM, R0-R18 on AArch64, Rtemp, LR) and 'to', 'count', 'tmp' registers. 3002 void oop_arraycopy_stub_epilogue_helper(Register to, Register count, Register tmp, bool status, bool forward) { 3003 assert_different_registers(to, count, tmp); 3004 3005 if (forward) { 3006 // 'to' is upper bound of the modified region 3007 // restore initial dst: 3008 __ sub_ptr_scaled_int32(to, to, count, LogBytesPerHeapOop); 3009 } 3010 3011 // 'to' is the beginning of the region 3012 3013 gen_write_ref_array_post_barrier(to, count, tmp); 3014 3015 if (status) { 3016 __ mov(R0, 0); // OK 3017 } 3018 3019#ifdef AARCH64 3020 __ raw_pop(LR, ZR); 3021 __ ret(); 3022#else 3023 __ pop(PC); 3024#endif // AARCH64 3025 } 3026 3027 3028 // Generate stub for assign-compatible oop copy. If "aligned" is true, the 3029 // "from" and "to" addresses are assumed to be heapword aligned. 3030 // 3031 // If "disjoint" is true, arrays are assumed to be disjoint, otherwise they may overlap and 3032 // "nooverlap_target" must be specified as the address to jump if they don't. 3033 // 3034 // Arguments for generated stub: 3035 // from: R0 3036 // to: R1 3037 // count: R2 treated as signed 32-bit int 3038 // 3039 address generate_oop_copy(bool aligned, const char * name, bool status, bool disjoint, address nooverlap_target = NULL) { 3040 __ align(CodeEntryAlignment); 3041 StubCodeMark mark(this, "StubRoutines", name); 3042 address start = __ pc(); 3043 3044 Register from = R0; 3045 Register to = R1; 3046 Register count = R2; 3047 Register tmp1 = R3; 3048 Register tmp2 = R12; 3049 3050 3051 if (!aligned) { 3052 BLOCK_COMMENT("Entry:"); 3053 } 3054 3055 __ zap_high_non_significant_bits(R2); 3056 3057 if (!disjoint) { 3058 assert (nooverlap_target != NULL, "must be specified for conjoint case"); 3059 array_overlap_test(nooverlap_target, LogBytesPerHeapOop, tmp1, tmp2); 3060 } 3061 3062 inc_counter_np(SharedRuntime::_oop_array_copy_ctr, tmp1, tmp2); 3063 3064 // Conjoint case: since execution reaches this point, the arrays overlap, so performing backward copy 3065 // Disjoint case: perform forward copy 3066 bool forward = disjoint; 3067 3068 const int bytes_per_count = BytesPerHeapOop; 3069 const int log_bytes_per_count = LogBytesPerHeapOop; 3070 3071 const Register saved_count = LR; 3072 const int callee_saved_regs = 3; // R0-R2 3073 3074 // LR is used later to save barrier args 3075#ifdef AARCH64 3076 __ raw_push(LR, ZR); 3077#else 3078 __ push(LR); 3079#endif // AARCH64 3080 3081#if INCLUDE_ALL_GCS 3082 gen_write_ref_array_pre_barrier(to, count, callee_saved_regs); 3083#endif // INCLUDE_ALL_GCS 3084 3085 // save arguments for barrier generation (after the pre barrier) 3086 __ mov(saved_count, count); 3087 3088 if (!forward) { 3089 __ add_ptr_scaled_int32(to, to, count, log_bytes_per_count); 3090 __ add_ptr_scaled_int32(from, from, count, log_bytes_per_count); 3091 } 3092 3093 // for short arrays, just do single element copy 3094 Label L_small_array; 3095 const int small_copy_limit = (8*wordSize + 7)/bytes_per_count; // XXX optim: tune the limit higher ? 3096 __ cmp_32(count, small_copy_limit); 3097 __ b(L_small_array, le); 3098 3099 bool from_is_aligned = (bytes_per_count >= 8); 3100 if (aligned && forward && (HeapWordSize % 8 == 0)) { 3101 // if 'from' is heapword aligned and HeapWordSize is divisible by 8, 3102 // then from is aligned by 8 3103 from_is_aligned = true; 3104 } 3105 3106 int count_required_to_align = from_is_aligned ? 0 : align_src(from, to, count, tmp1, bytes_per_count, forward); 3107 assert (small_copy_limit >= count_required_to_align, "alignment could exhaust count"); 3108 3109 // now 'from' is aligned 3110 3111 bool to_is_aligned = false; 3112 3113 if (bytes_per_count >= wordSize) { 3114 // 'to' is aligned by bytes_per_count, so it is aligned by wordSize 3115 to_is_aligned = true; 3116 } else { 3117 if (aligned && (8 % HeapWordSize == 0) && (HeapWordSize % wordSize == 0)) { 3118 // Originally 'from' and 'to' were heapword aligned; 3119 // (from - to) has not been changed, so since now 'from' is 8-byte aligned, then it is also heapword aligned, 3120 // so 'to' is also heapword aligned and thus aligned by wordSize. 3121 to_is_aligned = true; 3122 } 3123 } 3124 3125 Label L_unaligned_dst; 3126 3127 if (!to_is_aligned) { 3128 BLOCK_COMMENT("Check dst alignment:"); 3129 __ tst(to, wordSize - 1); 3130 __ b(L_unaligned_dst, ne); // 'to' is not aligned 3131 } 3132 3133 int min_copy; 3134 if (forward) { 3135 min_copy = generate_forward_aligned_copy_loop(from, to, count, bytes_per_count); 3136 } else { 3137 min_copy = generate_backward_aligned_copy_loop(from, to, count, bytes_per_count); 3138 } 3139 assert(small_copy_limit >= count_required_to_align + min_copy, "first loop might exhaust count"); 3140 3141 oop_arraycopy_stub_epilogue_helper(to, saved_count, /* tmp */ tmp1, status, forward); 3142 3143 { 3144 copy_small_array(from, to, count, tmp1, noreg, bytes_per_count, forward, L_small_array); 3145 3146 oop_arraycopy_stub_epilogue_helper(to, saved_count, /* tmp */ tmp1, status, forward); 3147 } 3148 3149 if (!to_is_aligned) { 3150 // !to_is_aligned <=> UseCompressedOops && AArch64 3151 __ BIND(L_unaligned_dst); 3152#ifdef AARCH64 3153 assert (UseCompressedOops, "unaligned oop array copy may be requested only with UseCompressedOops"); 3154#else 3155 ShouldNotReachHere(); 3156#endif // AARCH64 3157 int min_copy_shifted = align_dst_and_generate_shifted_copy_loop(from, to, count, bytes_per_count, forward); 3158 assert (small_copy_limit >= count_required_to_align + min_copy_shifted, "first loop might exhaust count"); 3159 3160 oop_arraycopy_stub_epilogue_helper(to, saved_count, /* tmp */ tmp1, status, forward); 3161 } 3162 3163 return start; 3164 } 3165 3166 // Generate 'unsafe' array copy stub 3167 // Though just as safe as the other stubs, it takes an unscaled 3168 // size_t argument instead of an element count. 3169 // 3170 // Arguments for generated stub: 3171 // from: R0 3172 // to: R1 3173 // count: R2 byte count, treated as ssize_t, can be zero 3174 // 3175 // Examines the alignment of the operands and dispatches 3176 // to a long, int, short, or byte copy loop. 3177 // 3178 address generate_unsafe_copy(const char* name) { 3179 3180 const Register R0_from = R0; // source array address 3181 const Register R1_to = R1; // destination array address 3182 const Register R2_count = R2; // elements count 3183 3184 const Register R3_bits = R3; // test copy of low bits 3185 3186 __ align(CodeEntryAlignment); 3187 StubCodeMark mark(this, "StubRoutines", name); 3188 address start = __ pc(); 3189#ifdef AARCH64 3190 __ NOT_IMPLEMENTED(); 3191 start = NULL; 3192#else 3193 const Register tmp = Rtemp; 3194 3195 // bump this on entry, not on exit: 3196 inc_counter_np(SharedRuntime::_unsafe_array_copy_ctr, R3, tmp); 3197 3198 __ orr(R3_bits, R0_from, R1_to); 3199 __ orr(R3_bits, R2_count, R3_bits); 3200 3201 __ tst(R3_bits, BytesPerLong-1); 3202 __ mov(R2_count,AsmOperand(R2_count,asr,LogBytesPerLong), eq); 3203 __ jump(StubRoutines::_jlong_arraycopy, relocInfo::runtime_call_type, tmp, eq); 3204 3205 __ tst(R3_bits, BytesPerInt-1); 3206 __ mov(R2_count,AsmOperand(R2_count,asr,LogBytesPerInt), eq); 3207 __ jump(StubRoutines::_jint_arraycopy, relocInfo::runtime_call_type, tmp, eq); 3208 3209 __ tst(R3_bits, BytesPerShort-1); 3210 __ mov(R2_count,AsmOperand(R2_count,asr,LogBytesPerShort), eq); 3211 __ jump(StubRoutines::_jshort_arraycopy, relocInfo::runtime_call_type, tmp, eq); 3212 3213 __ jump(StubRoutines::_jbyte_arraycopy, relocInfo::runtime_call_type, tmp); 3214#endif 3215 return start; 3216 } 3217 3218 // Helper for generating a dynamic type check. 3219 // Smashes only the given temp registers. 3220 void generate_type_check(Register sub_klass, 3221 Register super_check_offset, 3222 Register super_klass, 3223 Register tmp1, 3224 Register tmp2, 3225 Register tmp3, 3226 Label& L_success) { 3227 assert_different_registers(sub_klass, super_check_offset, super_klass, tmp1, tmp2, tmp3); 3228 3229 BLOCK_COMMENT("type_check:"); 3230 3231 // If the pointers are equal, we are done (e.g., String[] elements). 3232 3233 __ cmp(super_klass, sub_klass); 3234 __ b(L_success, eq); // fast success 3235 3236 3237 Label L_loop, L_fail; 3238 3239 int sc_offset = in_bytes(Klass::secondary_super_cache_offset()); 3240 3241 // Check the supertype display: 3242 __ ldr(tmp1, Address(sub_klass, super_check_offset)); 3243 __ cmp(tmp1, super_klass); 3244 __ b(L_success, eq); 3245 3246 __ cmp(super_check_offset, sc_offset); 3247 __ b(L_fail, ne); // failure 3248 3249 BLOCK_COMMENT("type_check_slow_path:"); 3250 3251 // a couple of useful fields in sub_klass: 3252 int ss_offset = in_bytes(Klass::secondary_supers_offset()); 3253 3254 // Do a linear scan of the secondary super-klass chain. 3255 3256#ifndef PRODUCT 3257 int* pst_counter = &SharedRuntime::_partial_subtype_ctr; 3258 __ inc_counter((address) pst_counter, tmp1, tmp2); 3259#endif 3260 3261 Register scan_temp = tmp1; 3262 Register count_temp = tmp2; 3263 3264 // We will consult the secondary-super array. 3265 __ ldr(scan_temp, Address(sub_klass, ss_offset)); 3266 3267 Register search_key = super_klass; 3268 3269 // Load the array length. 3270 __ ldr_s32(count_temp, Address(scan_temp, Array<Klass*>::length_offset_in_bytes())); 3271 __ add(scan_temp, scan_temp, Array<Klass*>::base_offset_in_bytes()); 3272 3273 __ add(count_temp, count_temp, 1); 3274 3275 // Top of search loop 3276 __ bind(L_loop); 3277 // Notes: 3278 // scan_temp starts at the array elements 3279 // count_temp is 1+size 3280 3281 __ subs(count_temp, count_temp, 1); 3282 __ b(L_fail, eq); // not found 3283 3284 // Load next super to check 3285 // In the array of super classes elements are pointer sized. 3286 int element_size = wordSize; 3287 __ ldr(tmp3, Address(scan_temp, element_size, post_indexed)); 3288 3289 // Look for Rsuper_klass on Rsub_klass's secondary super-class-overflow list 3290 __ cmp(tmp3, search_key); 3291 3292 // A miss means we are NOT a subtype and need to keep looping 3293 __ b(L_loop, ne); 3294 3295 // Falling out the bottom means we found a hit; we ARE a subtype 3296 3297 // Success. Cache the super we found and proceed in triumph. 3298 __ str(super_klass, Address(sub_klass, sc_offset)); 3299 3300 // Jump to success 3301 __ b(L_success); 3302 3303 // Fall through on failure! 3304 __ bind(L_fail); 3305 } 3306 3307 // Generate stub for checked oop copy. 3308 // 3309 // Arguments for generated stub: 3310 // from: R0 3311 // to: R1 3312 // count: R2 treated as signed 32-bit int 3313 // ckoff: R3 (super_check_offset) 3314 // ckval: R4 (AArch64) / SP[0] (32-bit ARM) (super_klass) 3315 // ret: R0 zero for success; (-1^K) where K is partial transfer count (32-bit) 3316 // 3317 address generate_checkcast_copy(const char * name) { 3318 __ align(CodeEntryAlignment); 3319 StubCodeMark mark(this, "StubRoutines", name); 3320 address start = __ pc(); 3321 3322 const Register from = R0; // source array address 3323 const Register to = R1; // destination array address 3324 const Register count = R2; // elements count 3325 3326 const Register R3_ckoff = R3; // super_check_offset 3327 const Register R4_ckval = R4; // super_klass 3328 3329 const int callee_saved_regs = AARCH64_ONLY(5) NOT_AARCH64(4); // LR saved differently 3330 3331 Label load_element, store_element, do_card_marks, fail; 3332 3333 BLOCK_COMMENT("Entry:"); 3334 3335 __ zap_high_non_significant_bits(R2); 3336 3337#ifdef AARCH64 3338 __ raw_push(LR, ZR); 3339 __ raw_push(R19, R20); 3340#else 3341 int pushed = 0; 3342 __ push(LR); 3343 pushed+=1; 3344#endif // AARCH64 3345 3346#if INCLUDE_ALL_GCS 3347 gen_write_ref_array_pre_barrier(to, count, callee_saved_regs); 3348#endif // INCLUDE_ALL_GCS 3349 3350#ifndef AARCH64 3351 const RegisterSet caller_saved_regs = RegisterSet(R4,R6) | RegisterSet(R8,R9) | altFP_7_11; 3352 __ push(caller_saved_regs); 3353 assert(caller_saved_regs.size() == 6, "check the count"); 3354 pushed+=6; 3355 3356 __ ldr(R4_ckval,Address(SP, wordSize*pushed)); // read the argument that was on the stack 3357#endif // !AARCH64 3358 3359 // Save arguments for barrier generation (after the pre barrier): 3360 // - must be a caller saved register and not LR 3361 // - ARM32: avoid R10 in case RThread is needed 3362 const Register saved_count = AARCH64_ONLY(R19) NOT_AARCH64(altFP_7_11); 3363#ifdef AARCH64 3364 __ mov_w(saved_count, count); 3365 __ cbnz_w(count, load_element); // and test count 3366#else 3367 __ movs(saved_count, count); // and test count 3368 __ b(load_element,ne); 3369#endif // AARCH64 3370 3371 // nothing to copy 3372 __ mov(R0, 0); 3373 3374#ifdef AARCH64 3375 __ raw_pop(R19, R20); 3376 __ raw_pop(LR, ZR); 3377 __ ret(); 3378#else 3379 __ pop(caller_saved_regs); 3380 __ pop(PC); 3381#endif // AARCH64 3382 3383 // ======== begin loop ======== 3384 // (Loop is rotated; its entry is load_element.) 3385 __ align(OptoLoopAlignment); 3386 __ BIND(store_element); 3387 if (UseCompressedOops) { 3388 __ store_heap_oop(R5, Address(to, BytesPerHeapOop, post_indexed)); // store the oop, changes flags 3389 __ subs_32(count,count,1); 3390 } else { 3391 __ subs_32(count,count,1); 3392 __ str(R5, Address(to, BytesPerHeapOop, post_indexed)); // store the oop 3393 } 3394 __ b(do_card_marks, eq); // count exhausted 3395 3396 // ======== loop entry is here ======== 3397 __ BIND(load_element); 3398 __ load_heap_oop(R5, Address(from, BytesPerHeapOop, post_indexed)); // load the oop 3399 __ cbz(R5, store_element); // NULL 3400 3401 __ load_klass(R6, R5); 3402 3403 generate_type_check(R6, R3_ckoff, R4_ckval, /*tmps*/ R12, R8, R9, 3404 // branch to this on success: 3405 store_element); 3406 // ======== end loop ======== 3407 3408 // It was a real error; we must depend on the caller to finish the job. 3409 // Register count has number of *remaining* oops, saved_count number of *total* oops. 3410 // Emit GC store barriers for the oops we have copied 3411 // and report their number to the caller (0 or (-1^n)) 3412 __ BIND(fail); 3413 3414 // Note: fail marked by the fact that count differs from saved_count 3415 3416 __ BIND(do_card_marks); 3417 3418 Register copied = AARCH64_ONLY(R20) NOT_AARCH64(R4); // saved 3419 Label L_not_copied; 3420 3421 __ subs_32(copied, saved_count, count); // copied count (in saved reg) 3422 __ b(L_not_copied, eq); // nothing was copied, skip post barrier 3423 __ sub(to, to, AsmOperand(copied, lsl, LogBytesPerHeapOop)); // initial to value 3424 __ mov(R12, copied); // count arg scratched by post barrier 3425 3426 gen_write_ref_array_post_barrier(to, R12, R3); 3427 3428 assert_different_registers(R3,R12,LR,copied,saved_count); 3429 inc_counter_np(SharedRuntime::_checkcast_array_copy_ctr, R3, R12); 3430 3431 __ BIND(L_not_copied); 3432 __ cmp_32(copied, saved_count); // values preserved in saved registers 3433 3434#ifdef AARCH64 3435 __ csinv(R0, ZR, copied, eq); // 0 if all copied else NOT(copied) 3436 __ raw_pop(R19, R20); 3437 __ raw_pop(LR, ZR); 3438 __ ret(); 3439#else 3440 __ mov(R0, 0, eq); // 0 if all copied 3441 __ mvn(R0, copied, ne); // else NOT(copied) 3442 __ pop(caller_saved_regs); 3443 __ pop(PC); 3444#endif // AARCH64 3445 3446 return start; 3447 } 3448 3449 // Perform range checks on the proposed arraycopy. 3450 // Kills the two temps, but nothing else. 3451 void arraycopy_range_checks(Register src, // source array oop 3452 Register src_pos, // source position (32-bit int) 3453 Register dst, // destination array oop 3454 Register dst_pos, // destination position (32-bit int) 3455 Register length, // length of copy (32-bit int) 3456 Register temp1, Register temp2, 3457 Label& L_failed) { 3458 3459 BLOCK_COMMENT("arraycopy_range_checks:"); 3460 3461 // if (src_pos + length > arrayOop(src)->length() ) FAIL; 3462 3463 const Register array_length = temp1; // scratch 3464 const Register end_pos = temp2; // scratch 3465 3466 __ add_32(end_pos, length, src_pos); // src_pos + length 3467 __ ldr_s32(array_length, Address(src, arrayOopDesc::length_offset_in_bytes())); 3468 __ cmp_32(end_pos, array_length); 3469 __ b(L_failed, hi); 3470 3471 // if (dst_pos + length > arrayOop(dst)->length() ) FAIL; 3472 __ add_32(end_pos, length, dst_pos); // dst_pos + length 3473 __ ldr_s32(array_length, Address(dst, arrayOopDesc::length_offset_in_bytes())); 3474 __ cmp_32(end_pos, array_length); 3475 __ b(L_failed, hi); 3476 3477 BLOCK_COMMENT("arraycopy_range_checks done"); 3478 } 3479 3480 // 3481 // Generate generic array copy stubs 3482 // 3483 // Input: 3484 // R0 - src oop 3485 // R1 - src_pos (32-bit int) 3486 // R2 - dst oop 3487 // R3 - dst_pos (32-bit int) 3488 // R4 (AArch64) / SP[0] (32-bit ARM) - element count (32-bit int) 3489 // 3490 // Output: (32-bit int) 3491 // R0 == 0 - success 3492 // R0 < 0 - need to call System.arraycopy 3493 // 3494 address generate_generic_copy(const char *name) { 3495 Label L_failed, L_objArray; 3496 3497 // Input registers 3498 const Register src = R0; // source array oop 3499 const Register src_pos = R1; // source position 3500 const Register dst = R2; // destination array oop 3501 const Register dst_pos = R3; // destination position 3502 3503 // registers used as temp 3504 const Register R5_src_klass = R5; // source array klass 3505 const Register R6_dst_klass = R6; // destination array klass 3506 const Register R_lh = AARCH64_ONLY(R7) NOT_AARCH64(altFP_7_11); // layout handler 3507 const Register R8_temp = R8; 3508 3509 __ align(CodeEntryAlignment); 3510 StubCodeMark mark(this, "StubRoutines", name); 3511 address start = __ pc(); 3512 3513 __ zap_high_non_significant_bits(R1); 3514 __ zap_high_non_significant_bits(R3); 3515 __ zap_high_non_significant_bits(R4); 3516 3517#ifndef AARCH64 3518 int pushed = 0; 3519 const RegisterSet saved_regs = RegisterSet(R4,R6) | RegisterSet(R8,R9) | altFP_7_11; 3520 __ push(saved_regs); 3521 assert(saved_regs.size() == 6, "check the count"); 3522 pushed+=6; 3523#endif // !AARCH64 3524 3525 // bump this on entry, not on exit: 3526 inc_counter_np(SharedRuntime::_generic_array_copy_ctr, R5, R12); 3527 3528 const Register length = R4; // elements count 3529#ifndef AARCH64 3530 __ ldr(length, Address(SP,4*pushed)); 3531#endif // !AARCH64 3532 3533 3534 //----------------------------------------------------------------------- 3535 // Assembler stubs will be used for this call to arraycopy 3536 // if the following conditions are met: 3537 // 3538 // (1) src and dst must not be null. 3539 // (2) src_pos must not be negative. 3540 // (3) dst_pos must not be negative. 3541 // (4) length must not be negative. 3542 // (5) src klass and dst klass should be the same and not NULL. 3543 // (6) src and dst should be arrays. 3544 // (7) src_pos + length must not exceed length of src. 3545 // (8) dst_pos + length must not exceed length of dst. 3546 BLOCK_COMMENT("arraycopy initial argument checks"); 3547 3548 // if (src == NULL) return -1; 3549 __ cbz(src, L_failed); 3550 3551 // if (src_pos < 0) return -1; 3552 __ cmp_32(src_pos, 0); 3553 __ b(L_failed, lt); 3554 3555 // if (dst == NULL) return -1; 3556 __ cbz(dst, L_failed); 3557 3558 // if (dst_pos < 0) return -1; 3559 __ cmp_32(dst_pos, 0); 3560 __ b(L_failed, lt); 3561 3562 // if (length < 0) return -1; 3563 __ cmp_32(length, 0); 3564 __ b(L_failed, lt); 3565 3566 BLOCK_COMMENT("arraycopy argument klass checks"); 3567 // get src->klass() 3568 __ load_klass(R5_src_klass, src); 3569 3570 // Load layout helper 3571 // 3572 // |array_tag| | header_size | element_type | |log2_element_size| 3573 // 32 30 24 16 8 2 0 3574 // 3575 // array_tag: typeArray = 0x3, objArray = 0x2, non-array = 0x0 3576 // 3577 3578 int lh_offset = in_bytes(Klass::layout_helper_offset()); 3579 __ ldr_u32(R_lh, Address(R5_src_klass, lh_offset)); 3580 3581 __ load_klass(R6_dst_klass, dst); 3582 3583 // Handle objArrays completely differently... 3584 juint objArray_lh = Klass::array_layout_helper(T_OBJECT); 3585 __ mov_slow(R8_temp, objArray_lh); 3586 __ cmp_32(R_lh, R8_temp); 3587 __ b(L_objArray,eq); 3588 3589 // if (src->klass() != dst->klass()) return -1; 3590 __ cmp(R5_src_klass, R6_dst_klass); 3591 __ b(L_failed, ne); 3592 3593 // if (!src->is_Array()) return -1; 3594 __ cmp_32(R_lh, Klass::_lh_neutral_value); // < 0 3595 __ b(L_failed, ge); 3596 3597 arraycopy_range_checks(src, src_pos, dst, dst_pos, length, 3598 R8_temp, R6_dst_klass, L_failed); 3599 3600 { 3601 // TypeArrayKlass 3602 // 3603 // src_addr = (src + array_header_in_bytes()) + (src_pos << log2elemsize); 3604 // dst_addr = (dst + array_header_in_bytes()) + (dst_pos << log2elemsize); 3605 // 3606 3607 const Register R6_offset = R6_dst_klass; // array offset 3608 const Register R12_elsize = R12; // log2 element size 3609 3610 __ logical_shift_right(R6_offset, R_lh, Klass::_lh_header_size_shift); 3611 __ andr(R6_offset, R6_offset, (unsigned int)Klass::_lh_header_size_mask); // array_offset 3612 __ add(src, src, R6_offset); // src array offset 3613 __ add(dst, dst, R6_offset); // dst array offset 3614 __ andr(R12_elsize, R_lh, (unsigned int)Klass::_lh_log2_element_size_mask); // log2 element size 3615 3616 // next registers should be set before the jump to corresponding stub 3617 const Register from = R0; // source array address 3618 const Register to = R1; // destination array address 3619 const Register count = R2; // elements count 3620 3621 // 'from', 'to', 'count' registers should be set in this order 3622 // since they are the same as 'src', 'src_pos', 'dst'. 3623 3624#ifdef AARCH64 3625 3626 BLOCK_COMMENT("choose copy loop based on element size and scale indexes"); 3627 Label Lbyte, Lshort, Lint, Llong; 3628 3629 __ cbz(R12_elsize, Lbyte); 3630 3631 assert (LogBytesPerShort < LogBytesPerInt && LogBytesPerInt < LogBytesPerLong, "must be"); 3632 __ cmp(R12_elsize, LogBytesPerInt); 3633 __ b(Lint, eq); 3634 __ b(Llong, gt); 3635 3636 __ BIND(Lshort); 3637 __ add_ptr_scaled_int32(from, src, src_pos, LogBytesPerShort); 3638 __ add_ptr_scaled_int32(to, dst, dst_pos, LogBytesPerShort); 3639 __ mov(count, length); 3640 __ b(StubRoutines::_jshort_arraycopy); 3641 3642 __ BIND(Lint); 3643 __ add_ptr_scaled_int32(from, src, src_pos, LogBytesPerInt); 3644 __ add_ptr_scaled_int32(to, dst, dst_pos, LogBytesPerInt); 3645 __ mov(count, length); 3646 __ b(StubRoutines::_jint_arraycopy); 3647 3648 __ BIND(Lbyte); 3649 __ add_ptr_scaled_int32(from, src, src_pos, 0); 3650 __ add_ptr_scaled_int32(to, dst, dst_pos, 0); 3651 __ mov(count, length); 3652 __ b(StubRoutines::_jbyte_arraycopy); 3653 3654 __ BIND(Llong); 3655 __ add_ptr_scaled_int32(from, src, src_pos, LogBytesPerLong); 3656 __ add_ptr_scaled_int32(to, dst, dst_pos, LogBytesPerLong); 3657 __ mov(count, length); 3658 __ b(StubRoutines::_jlong_arraycopy); 3659 3660#else // AARCH64 3661 3662 BLOCK_COMMENT("scale indexes to element size"); 3663 __ add(from, src, AsmOperand(src_pos, lsl, R12_elsize)); // src_addr 3664 __ add(to, dst, AsmOperand(dst_pos, lsl, R12_elsize)); // dst_addr 3665 3666 __ mov(count, length); // length 3667 3668 // XXX optim: avoid later push in arraycopy variants ? 3669 3670 __ pop(saved_regs); 3671 3672 BLOCK_COMMENT("choose copy loop based on element size"); 3673 __ cmp(R12_elsize, 0); 3674 __ b(StubRoutines::_jbyte_arraycopy,eq); 3675 3676 __ cmp(R12_elsize, LogBytesPerShort); 3677 __ b(StubRoutines::_jshort_arraycopy,eq); 3678 3679 __ cmp(R12_elsize, LogBytesPerInt); 3680 __ b(StubRoutines::_jint_arraycopy,eq); 3681 3682 __ b(StubRoutines::_jlong_arraycopy); 3683 3684#endif // AARCH64 3685 } 3686 3687 // ObjArrayKlass 3688 __ BIND(L_objArray); 3689 // live at this point: R5_src_klass, R6_dst_klass, src[_pos], dst[_pos], length 3690 3691 Label L_plain_copy, L_checkcast_copy; 3692 // test array classes for subtyping 3693 __ cmp(R5_src_klass, R6_dst_klass); // usual case is exact equality 3694 __ b(L_checkcast_copy, ne); 3695 3696 BLOCK_COMMENT("Identically typed arrays"); 3697 { 3698 // Identically typed arrays can be copied without element-wise checks. 3699 arraycopy_range_checks(src, src_pos, dst, dst_pos, length, 3700 R8_temp, R_lh, L_failed); 3701 3702 // next registers should be set before the jump to corresponding stub 3703 const Register from = R0; // source array address 3704 const Register to = R1; // destination array address 3705 const Register count = R2; // elements count 3706 3707 __ add(src, src, arrayOopDesc::base_offset_in_bytes(T_OBJECT)); //src offset 3708 __ add(dst, dst, arrayOopDesc::base_offset_in_bytes(T_OBJECT)); //dst offset 3709 __ add_ptr_scaled_int32(from, src, src_pos, LogBytesPerHeapOop); // src_addr 3710 __ add_ptr_scaled_int32(to, dst, dst_pos, LogBytesPerHeapOop); // dst_addr 3711 __ BIND(L_plain_copy); 3712 __ mov(count, length); 3713 3714#ifndef AARCH64 3715 __ pop(saved_regs); // XXX optim: avoid later push in oop_arraycopy ? 3716#endif // !AARCH64 3717 __ b(StubRoutines::_oop_arraycopy); 3718 } 3719 3720 { 3721 __ BIND(L_checkcast_copy); 3722 // live at this point: R5_src_klass, R6_dst_klass 3723 3724 // Before looking at dst.length, make sure dst is also an objArray. 3725 __ ldr_u32(R8_temp, Address(R6_dst_klass, lh_offset)); 3726 __ cmp_32(R_lh, R8_temp); 3727 __ b(L_failed, ne); 3728 3729 // It is safe to examine both src.length and dst.length. 3730 3731 arraycopy_range_checks(src, src_pos, dst, dst_pos, length, 3732 R8_temp, R_lh, L_failed); 3733 3734 // next registers should be set before the jump to corresponding stub 3735 const Register from = R0; // source array address 3736 const Register to = R1; // destination array address 3737 const Register count = R2; // elements count 3738 3739 // Marshal the base address arguments now, freeing registers. 3740 __ add(src, src, arrayOopDesc::base_offset_in_bytes(T_OBJECT)); //src offset 3741 __ add(dst, dst, arrayOopDesc::base_offset_in_bytes(T_OBJECT)); //dst offset 3742 __ add_ptr_scaled_int32(from, src, src_pos, LogBytesPerHeapOop); // src_addr 3743 __ add_ptr_scaled_int32(to, dst, dst_pos, LogBytesPerHeapOop); // dst_addr 3744 3745 __ mov(count, length); // length (reloaded) 3746 3747 Register sco_temp = R3; // this register is free now 3748 assert_different_registers(from, to, count, sco_temp, 3749 R6_dst_klass, R5_src_klass); 3750 3751 // Generate the type check. 3752 int sco_offset = in_bytes(Klass::super_check_offset_offset()); 3753 __ ldr_u32(sco_temp, Address(R6_dst_klass, sco_offset)); 3754 generate_type_check(R5_src_klass, sco_temp, R6_dst_klass, 3755 R8_temp, R9, 3756 AARCH64_ONLY(R10) NOT_AARCH64(R12), 3757 L_plain_copy); 3758 3759 // Fetch destination element klass from the ObjArrayKlass header. 3760 int ek_offset = in_bytes(ObjArrayKlass::element_klass_offset()); 3761 3762 // the checkcast_copy loop needs two extra arguments: 3763 const Register Rdst_elem_klass = AARCH64_ONLY(R4) NOT_AARCH64(R3); 3764 __ ldr(Rdst_elem_klass, Address(R6_dst_klass, ek_offset)); // dest elem klass 3765#ifndef AARCH64 3766 __ pop(saved_regs); // XXX optim: avoid later push in oop_arraycopy ? 3767 __ str(Rdst_elem_klass, Address(SP,0)); // dest elem klass argument 3768#endif // !AARCH64 3769 __ ldr_u32(R3, Address(Rdst_elem_klass, sco_offset)); // sco of elem klass 3770 __ b(StubRoutines::_checkcast_arraycopy); 3771 } 3772 3773 __ BIND(L_failed); 3774 3775#ifndef AARCH64 3776 __ pop(saved_regs); 3777#endif // !AARCH64 3778 __ mvn(R0, 0); // failure, with 0 copied 3779 __ ret(); 3780 3781 return start; 3782 } 3783 3784 // Safefetch stubs. 3785 void generate_safefetch(const char* name, int size, address* entry, address* fault_pc, address* continuation_pc) { 3786 // safefetch signatures: 3787 // int SafeFetch32(int* adr, int errValue); 3788 // intptr_t SafeFetchN (intptr_t* adr, intptr_t errValue); 3789 // 3790 // arguments: 3791 // R0 = adr 3792 // R1 = errValue 3793 // 3794 // result: 3795 // R0 = *adr or errValue 3796 3797 StubCodeMark mark(this, "StubRoutines", name); 3798 3799 // Entry point, pc or function descriptor. 3800 *entry = __ pc(); 3801 3802 // Load *adr into c_rarg2, may fault. 3803 *fault_pc = __ pc(); 3804 3805 switch (size) { 3806 case 4: // int32_t 3807 __ ldr_s32(R1, Address(R0)); 3808 break; 3809 3810 case 8: // int64_t 3811#ifdef AARCH64 3812 __ ldr(R1, Address(R0)); 3813#else 3814 Unimplemented(); 3815#endif // AARCH64 3816 break; 3817 3818 default: 3819 ShouldNotReachHere(); 3820 } 3821 3822 // return errValue or *adr 3823 *continuation_pc = __ pc(); 3824 __ mov(R0, R1); 3825 __ ret(); 3826 } 3827 3828 void generate_arraycopy_stubs() { 3829 3830 // Note: the disjoint stubs must be generated first, some of 3831 // the conjoint stubs use them. 3832 3833 bool status = false; // non failing C2 stubs need not return a status in R0 3834 3835#ifdef TEST_C2_GENERIC_ARRAYCOPY /* Internal development flag */ 3836 // With this flag, the C2 stubs are tested by generating calls to 3837 // generic_arraycopy instead of Runtime1::arraycopy 3838 3839 // Runtime1::arraycopy return a status in R0 (0 if OK, else ~copied) 3840 // and the result is tested to see whether the arraycopy stub should 3841 // be called. 3842 3843 // When we test arraycopy this way, we must generate extra code in the 3844 // arraycopy methods callable from C2 generic_arraycopy to set the 3845 // status to 0 for those who always succeed (calling the slow path stub might 3846 // lead to errors since the copy has already been performed). 3847 3848 status = true; // generate a status compatible with C1 calls 3849#endif 3850 3851 // these need always status in case they are called from generic_arraycopy 3852 StubRoutines::_jbyte_disjoint_arraycopy = generate_primitive_copy(false, "jbyte_disjoint_arraycopy", true, 1, true); 3853 StubRoutines::_jshort_disjoint_arraycopy = generate_primitive_copy(false, "jshort_disjoint_arraycopy", true, 2, true); 3854 StubRoutines::_jint_disjoint_arraycopy = generate_primitive_copy(false, "jint_disjoint_arraycopy", true, 4, true); 3855 StubRoutines::_jlong_disjoint_arraycopy = generate_primitive_copy(false, "jlong_disjoint_arraycopy", true, 8, true); 3856 StubRoutines::_oop_disjoint_arraycopy = generate_oop_copy (false, "oop_disjoint_arraycopy", true, true); 3857 3858 StubRoutines::_arrayof_jbyte_disjoint_arraycopy = generate_primitive_copy(true, "arrayof_jbyte_disjoint_arraycopy", status, 1, true); 3859 StubRoutines::_arrayof_jshort_disjoint_arraycopy = generate_primitive_copy(true, "arrayof_jshort_disjoint_arraycopy",status, 2, true); 3860 StubRoutines::_arrayof_jint_disjoint_arraycopy = generate_primitive_copy(true, "arrayof_jint_disjoint_arraycopy", status, 4, true); 3861 StubRoutines::_arrayof_jlong_disjoint_arraycopy = generate_primitive_copy(true, "arrayof_jlong_disjoint_arraycopy", status, 8, true); 3862 StubRoutines::_arrayof_oop_disjoint_arraycopy = generate_oop_copy (true, "arrayof_oop_disjoint_arraycopy", status, true); 3863 3864 // these need always status in case they are called from generic_arraycopy 3865 StubRoutines::_jbyte_arraycopy = generate_primitive_copy(false, "jbyte_arraycopy", true, 1, false, StubRoutines::_jbyte_disjoint_arraycopy); 3866 StubRoutines::_jshort_arraycopy = generate_primitive_copy(false, "jshort_arraycopy", true, 2, false, StubRoutines::_jshort_disjoint_arraycopy); 3867 StubRoutines::_jint_arraycopy = generate_primitive_copy(false, "jint_arraycopy", true, 4, false, StubRoutines::_jint_disjoint_arraycopy); 3868 StubRoutines::_jlong_arraycopy = generate_primitive_copy(false, "jlong_arraycopy", true, 8, false, StubRoutines::_jlong_disjoint_arraycopy); 3869 StubRoutines::_oop_arraycopy = generate_oop_copy (false, "oop_arraycopy", true, false, StubRoutines::_oop_disjoint_arraycopy); 3870 3871 StubRoutines::_arrayof_jbyte_arraycopy = generate_primitive_copy(true, "arrayof_jbyte_arraycopy", status, 1, false, StubRoutines::_arrayof_jbyte_disjoint_arraycopy); 3872 StubRoutines::_arrayof_jshort_arraycopy = generate_primitive_copy(true, "arrayof_jshort_arraycopy", status, 2, false, StubRoutines::_arrayof_jshort_disjoint_arraycopy); 3873#ifdef _LP64 3874 // since sizeof(jint) < sizeof(HeapWord), there's a different flavor: 3875 StubRoutines::_arrayof_jint_arraycopy = generate_primitive_copy(true, "arrayof_jint_arraycopy", status, 4, false, StubRoutines::_arrayof_jint_disjoint_arraycopy); 3876#else 3877 StubRoutines::_arrayof_jint_arraycopy = StubRoutines::_jint_arraycopy; 3878#endif 3879 if (BytesPerHeapOop < HeapWordSize) { 3880 StubRoutines::_arrayof_oop_arraycopy = generate_oop_copy (true, "arrayof_oop_arraycopy", status, false, StubRoutines::_arrayof_oop_disjoint_arraycopy); 3881 } else { 3882 StubRoutines::_arrayof_oop_arraycopy = StubRoutines::_oop_arraycopy; 3883 } 3884 StubRoutines::_arrayof_jlong_arraycopy = StubRoutines::_jlong_arraycopy; 3885 3886 StubRoutines::_checkcast_arraycopy = generate_checkcast_copy("checkcast_arraycopy"); 3887 StubRoutines::_unsafe_arraycopy = generate_unsafe_copy("unsafe_arraycopy"); 3888 StubRoutines::_generic_arraycopy = generate_generic_copy("generic_arraycopy"); 3889 3890 3891 } 3892 3893#ifndef AARCH64 3894#define COMPILE_CRYPTO 3895#include "stubRoutinesCrypto_arm.cpp" 3896#else 3897 3898#ifdef COMPILER2 3899 // Arguments: 3900 // 3901 // Inputs: 3902 // c_rarg0 - source byte array address 3903 // c_rarg1 - destination byte array address 3904 // c_rarg2 - K (key) in little endian int array 3905 // 3906 address generate_aescrypt_encryptBlock() { 3907 __ align(CodeEntryAlignment); 3908 StubCodeMark mark(this, "StubRoutines", "aescrypt_encryptBlock"); 3909 3910 Label L_doLast; 3911 3912 const Register from = c_rarg0; // source array address 3913 const Register to = c_rarg1; // destination array address 3914 const Register key = c_rarg2; // key array address 3915 const Register keylen = R8; 3916 3917 address start = __ pc(); 3918 __ stp(FP, LR, Address(SP, -2 * wordSize, pre_indexed)); 3919 __ mov(FP, SP); 3920 3921 __ ldr_w(keylen, Address(key, arrayOopDesc::length_offset_in_bytes() - arrayOopDesc::base_offset_in_bytes(T_INT))); 3922 3923 __ vld1(V0, Address(from), MacroAssembler::VELEM_SIZE_8, 128); // get 16 bytes of input 3924 3925 __ vld1(V1, V2, V3, V4, Address(key, 64, post_indexed), MacroAssembler::VELEM_SIZE_8, 128); 3926 3927 int quad = 1; 3928 __ rev32(V1, V1, MacroAssembler::VELEM_SIZE_8, quad); 3929 __ rev32(V2, V2, MacroAssembler::VELEM_SIZE_8, quad); 3930 __ rev32(V3, V3, MacroAssembler::VELEM_SIZE_8, quad); 3931 __ rev32(V4, V4, MacroAssembler::VELEM_SIZE_8, quad); 3932 __ aese(V0, V1); 3933 __ aesmc(V0, V0); 3934 __ aese(V0, V2); 3935 __ aesmc(V0, V0); 3936 __ aese(V0, V3); 3937 __ aesmc(V0, V0); 3938 __ aese(V0, V4); 3939 __ aesmc(V0, V0); 3940 3941 __ vld1(V1, V2, V3, V4, Address(key, 64, post_indexed), MacroAssembler::VELEM_SIZE_8, 128); 3942 __ rev32(V1, V1, MacroAssembler::VELEM_SIZE_8, quad); 3943 __ rev32(V2, V2, MacroAssembler::VELEM_SIZE_8, quad); 3944 __ rev32(V3, V3, MacroAssembler::VELEM_SIZE_8, quad); 3945 __ rev32(V4, V4, MacroAssembler::VELEM_SIZE_8, quad); 3946 __ aese(V0, V1); 3947 __ aesmc(V0, V0); 3948 __ aese(V0, V2); 3949 __ aesmc(V0, V0); 3950 __ aese(V0, V3); 3951 __ aesmc(V0, V0); 3952 __ aese(V0, V4); 3953 __ aesmc(V0, V0); 3954 3955 __ vld1(V1, V2, Address(key, 32, post_indexed), MacroAssembler::VELEM_SIZE_8, 128); 3956 __ rev32(V1, V1, MacroAssembler::VELEM_SIZE_8, quad); 3957 __ rev32(V2, V2, MacroAssembler::VELEM_SIZE_8, quad); 3958 3959 __ cmp_w(keylen, 44); 3960 __ b(L_doLast, eq); 3961 3962 __ aese(V0, V1); 3963 __ aesmc(V0, V0); 3964 __ aese(V0, V2); 3965 __ aesmc(V0, V0); 3966 3967 __ vld1(V1, V2, Address(key, 32, post_indexed), MacroAssembler::VELEM_SIZE_8, 128); 3968 __ rev32(V1, V1, MacroAssembler::VELEM_SIZE_8, quad); 3969 __ rev32(V2, V2, MacroAssembler::VELEM_SIZE_8, quad); 3970 3971 __ cmp_w(keylen, 52); 3972 __ b(L_doLast, eq); 3973 3974 __ aese(V0, V1); 3975 __ aesmc(V0, V0); 3976 __ aese(V0, V2); 3977 __ aesmc(V0, V0); 3978 3979 __ vld1(V1, V2, Address(key, 32, post_indexed), MacroAssembler::VELEM_SIZE_8, 128); 3980 __ rev32(V1, V1, MacroAssembler::VELEM_SIZE_8, quad); 3981 __ rev32(V2, V2, MacroAssembler::VELEM_SIZE_8, quad); 3982 3983 __ BIND(L_doLast); 3984 3985 __ aese(V0, V1); 3986 __ aesmc(V0, V0); 3987 __ aese(V0, V2); 3988 3989 __ vld1(V1, Address(key), MacroAssembler::VELEM_SIZE_8, 128); 3990 __ rev32(V1, V1, MacroAssembler::VELEM_SIZE_8, quad); 3991 __ eor(V0, V0, V1, MacroAssembler::VELEM_SIZE_8, quad); 3992 3993 __ vst1(V0, Address(to), MacroAssembler::VELEM_SIZE_8, 128); 3994 3995 __ mov(R0, 0); 3996 3997 __ mov(SP, FP); 3998 __ ldp(FP, LR, Address(SP, 2 * wordSize, post_indexed)); 3999 __ ret(LR); 4000 4001 return start; 4002 } 4003 4004 // Arguments: 4005 // 4006 // Inputs: 4007 // c_rarg0 - source byte array address 4008 // c_rarg1 - destination byte array address 4009 // c_rarg2 - K (key) in little endian int array 4010 // 4011 address generate_aescrypt_decryptBlock() { 4012 assert(UseAES, "need AES instructions and misaligned SSE support"); 4013 __ align(CodeEntryAlignment); 4014 StubCodeMark mark(this, "StubRoutines", "aescrypt_decryptBlock"); 4015 Label L_doLast; 4016 4017 const Register from = c_rarg0; // source array address 4018 const Register to = c_rarg1; // destination array address 4019 const Register key = c_rarg2; // key array address 4020 const Register keylen = R8; 4021 4022 address start = __ pc(); 4023 __ stp(FP, LR, Address(SP, -2 * wordSize, pre_indexed)); 4024 __ mov(FP, SP); 4025 4026 __ ldr_w(keylen, Address(key, arrayOopDesc::length_offset_in_bytes() - arrayOopDesc::base_offset_in_bytes(T_INT))); 4027 4028 __ vld1(V0, Address(from), MacroAssembler::VELEM_SIZE_8, 128); // get 16 bytes of input 4029 4030 __ vld1(V5, Address(key, 16, post_indexed), MacroAssembler::VELEM_SIZE_8, 128); 4031 4032 int quad = 1; 4033 __ rev32(V5, V5, MacroAssembler::VELEM_SIZE_8, quad); 4034 4035 __ vld1(V1, V2, V3, V4, Address(key, 64, post_indexed), MacroAssembler::VELEM_SIZE_8, 128); 4036 __ rev32(V1, V1, MacroAssembler::VELEM_SIZE_8, quad); 4037 __ rev32(V2, V2, MacroAssembler::VELEM_SIZE_8, quad); 4038 __ rev32(V3, V3, MacroAssembler::VELEM_SIZE_8, quad); 4039 __ rev32(V4, V4, MacroAssembler::VELEM_SIZE_8, quad); 4040 __ aesd(V0, V1); 4041 __ aesimc(V0, V0); 4042 __ aesd(V0, V2); 4043 __ aesimc(V0, V0); 4044 __ aesd(V0, V3); 4045 __ aesimc(V0, V0); 4046 __ aesd(V0, V4); 4047 __ aesimc(V0, V0); 4048 4049 __ vld1(V1, V2, V3, V4, Address(key, 64, post_indexed), MacroAssembler::VELEM_SIZE_8, 128); 4050 __ rev32(V1, V1, MacroAssembler::VELEM_SIZE_8, quad); 4051 __ rev32(V2, V2, MacroAssembler::VELEM_SIZE_8, quad); 4052 __ rev32(V3, V3, MacroAssembler::VELEM_SIZE_8, quad); 4053 __ rev32(V4, V4, MacroAssembler::VELEM_SIZE_8, quad); 4054 __ aesd(V0, V1); 4055 __ aesimc(V0, V0); 4056 __ aesd(V0, V2); 4057 __ aesimc(V0, V0); 4058 __ aesd(V0, V3); 4059 __ aesimc(V0, V0); 4060 __ aesd(V0, V4); 4061 __ aesimc(V0, V0); 4062 4063 __ vld1(V1, V2, Address(key, 32, post_indexed), MacroAssembler::VELEM_SIZE_8, 128); 4064 __ rev32(V1, V1, MacroAssembler::VELEM_SIZE_8, quad); 4065 __ rev32(V2, V2, MacroAssembler::VELEM_SIZE_8, quad); 4066 4067 __ cmp_w(keylen, 44); 4068 __ b(L_doLast, eq); 4069 4070 __ aesd(V0, V1); 4071 __ aesimc(V0, V0); 4072 __ aesd(V0, V2); 4073 __ aesimc(V0, V0); 4074 4075 __ vld1(V1, V2, Address(key, 32, post_indexed), MacroAssembler::VELEM_SIZE_8, 128); 4076 __ rev32(V1, V1, MacroAssembler::VELEM_SIZE_8, quad); 4077 __ rev32(V2, V2, MacroAssembler::VELEM_SIZE_8, quad); 4078 4079 __ cmp_w(keylen, 52); 4080 __ b(L_doLast, eq); 4081 4082 __ aesd(V0, V1); 4083 __ aesimc(V0, V0); 4084 __ aesd(V0, V2); 4085 __ aesimc(V0, V0); 4086 4087 __ vld1(V1, V2, Address(key, 32, post_indexed), MacroAssembler::VELEM_SIZE_8, 128); 4088 __ rev32(V1, V1, MacroAssembler::VELEM_SIZE_8, quad); 4089 __ rev32(V2, V2, MacroAssembler::VELEM_SIZE_8, quad); 4090 4091 __ BIND(L_doLast); 4092 4093 __ aesd(V0, V1); 4094 __ aesimc(V0, V0); 4095 __ aesd(V0, V2); 4096 4097 __ eor(V0, V0, V5, MacroAssembler::VELEM_SIZE_8, quad); 4098 4099 __ vst1(V0, Address(to), MacroAssembler::VELEM_SIZE_8, 128); 4100 4101 __ mov(R0, 0); 4102 4103 __ mov(SP, FP); 4104 __ ldp(FP, LR, Address(SP, 2 * wordSize, post_indexed)); 4105 __ ret(LR); 4106 4107 4108 return start; 4109 } 4110 4111 // Arguments: 4112 // 4113 // Inputs: 4114 // c_rarg0 - source byte array address 4115 // c_rarg1 - destination byte array address 4116 // c_rarg2 - K (key) in little endian int array 4117 // c_rarg3 - r vector byte array address 4118 // c_rarg4 - input length 4119 // 4120 // Output: 4121 // x0 - input length 4122 // 4123 address generate_cipherBlockChaining_encryptAESCrypt() { 4124 assert(UseAES, "need AES instructions and misaligned SSE support"); 4125 __ align(CodeEntryAlignment); 4126 StubCodeMark mark(this, "StubRoutines", "cipherBlockChaining_encryptAESCrypt"); 4127 4128 Label L_loadkeys_44, L_loadkeys_52, L_aes_loop, L_rounds_44, L_rounds_52; 4129 4130 const Register from = c_rarg0; // source array address 4131 const Register to = c_rarg1; // destination array address 4132 const Register key = c_rarg2; // key array address 4133 const Register rvec = c_rarg3; // r byte array initialized from initvector array address 4134 // and left with the results of the last encryption block 4135 const Register len_reg = c_rarg4; // src len (must be multiple of blocksize 16) 4136 const Register keylen = R8; 4137 4138 address start = __ pc(); 4139 __ stp(FP, LR, Address(SP, -2 * wordSize, pre_indexed)); 4140 __ mov(FP, SP); 4141 4142 __ mov(R9, len_reg); 4143 __ ldr_w(keylen, Address(key, arrayOopDesc::length_offset_in_bytes() - arrayOopDesc::base_offset_in_bytes(T_INT))); 4144 4145 __ vld1(V0, Address(rvec), MacroAssembler::VELEM_SIZE_8, 128); 4146 4147 __ cmp_w(keylen, 52); 4148 __ b(L_loadkeys_44, cc); 4149 __ b(L_loadkeys_52, eq); 4150 4151 __ vld1(V17, V18, Address(key, 32, post_indexed), MacroAssembler::VELEM_SIZE_8, 128); 4152 4153 int quad = 1; 4154 __ rev32(V17, V17, MacroAssembler::VELEM_SIZE_8, quad); 4155 __ rev32(V18, V18, MacroAssembler::VELEM_SIZE_8, quad); 4156 __ BIND(L_loadkeys_52); 4157 __ vld1(V19, V20, Address(key, 32, post_indexed), MacroAssembler::VELEM_SIZE_8, 128); 4158 __ rev32(V19, V19, MacroAssembler::VELEM_SIZE_8, quad); 4159 __ rev32(V20, V20, MacroAssembler::VELEM_SIZE_8, quad); 4160 __ BIND(L_loadkeys_44); 4161 __ vld1(V21, V22, V23, V24, Address(key, 64, post_indexed), MacroAssembler::VELEM_SIZE_8, 128); 4162 __ rev32(V21, V21, MacroAssembler::VELEM_SIZE_8, quad); 4163 __ rev32(V22, V22, MacroAssembler::VELEM_SIZE_8, quad); 4164 __ rev32(V23, V23, MacroAssembler::VELEM_SIZE_8, quad); 4165 __ rev32(V24, V24, MacroAssembler::VELEM_SIZE_8, quad); 4166 __ vld1(V25, V26, V27, V28, Address(key, 64, post_indexed), MacroAssembler::VELEM_SIZE_8, 128); 4167 __ rev32(V25, V25, MacroAssembler::VELEM_SIZE_8, quad); 4168 __ rev32(V26, V26, MacroAssembler::VELEM_SIZE_8, quad); 4169 __ rev32(V27, V27, MacroAssembler::VELEM_SIZE_8, quad); 4170 __ rev32(V28, V28, MacroAssembler::VELEM_SIZE_8, quad); 4171 __ vld1(V29, V30, V31, Address(key), MacroAssembler::VELEM_SIZE_8, 128); 4172 __ rev32(V29, V29, MacroAssembler::VELEM_SIZE_8, quad); 4173 __ rev32(V30, V30, MacroAssembler::VELEM_SIZE_8, quad); 4174 __ rev32(V31, V31, MacroAssembler::VELEM_SIZE_8, quad); 4175 4176 __ BIND(L_aes_loop); 4177 __ vld1(V1, Address(from, 16, post_indexed), MacroAssembler::VELEM_SIZE_8, 128); 4178 __ eor(V0, V0, V1, MacroAssembler::VELEM_SIZE_8, quad); 4179 4180 __ b(L_rounds_44, cc); 4181 __ b(L_rounds_52, eq); 4182 4183 __ aese(V0, V17); 4184 __ aesmc(V0, V0); 4185 __ aese(V0, V18); 4186 __ aesmc(V0, V0); 4187 __ BIND(L_rounds_52); 4188 __ aese(V0, V19); 4189 __ aesmc(V0, V0); 4190 __ aese(V0, V20); 4191 __ aesmc(V0, V0); 4192 __ BIND(L_rounds_44); 4193 __ aese(V0, V21); 4194 __ aesmc(V0, V0); 4195 __ aese(V0, V22); 4196 __ aesmc(V0, V0); 4197 __ aese(V0, V23); 4198 __ aesmc(V0, V0); 4199 __ aese(V0, V24); 4200 __ aesmc(V0, V0); 4201 __ aese(V0, V25); 4202 __ aesmc(V0, V0); 4203 __ aese(V0, V26); 4204 __ aesmc(V0, V0); 4205 __ aese(V0, V27); 4206 __ aesmc(V0, V0); 4207 __ aese(V0, V28); 4208 __ aesmc(V0, V0); 4209 __ aese(V0, V29); 4210 __ aesmc(V0, V0); 4211 __ aese(V0, V30); 4212 __ eor(V0, V0, V31, MacroAssembler::VELEM_SIZE_8, quad); 4213 4214 __ vst1(V0, Address(to, 16, post_indexed), MacroAssembler::VELEM_SIZE_8, 128); 4215 __ sub(len_reg, len_reg, 16); 4216 __ cbnz(len_reg, L_aes_loop); 4217 4218 __ vst1(V0, Address(rvec), MacroAssembler::VELEM_SIZE_8, 128); 4219 4220 __ mov(R0, R9); 4221 4222 __ mov(SP, FP); 4223 __ ldp(FP, LR, Address(SP, 2 * wordSize, post_indexed)); 4224 __ ret(LR); 4225 4226 return start; 4227 } 4228 4229 // Arguments: 4230 // 4231 // Inputs: 4232 // c_rarg0 - source byte array address 4233 // c_rarg1 - destination byte array address 4234 // c_rarg2 - K (key) in little endian int array 4235 // c_rarg3 - r vector byte array address 4236 // c_rarg4 - input length 4237 // 4238 // Output: 4239 // rax - input length 4240 // 4241 address generate_cipherBlockChaining_decryptAESCrypt() { 4242 assert(UseAES, "need AES instructions and misaligned SSE support"); 4243 __ align(CodeEntryAlignment); 4244 StubCodeMark mark(this, "StubRoutines", "cipherBlockChaining_decryptAESCrypt"); 4245 4246 Label L_loadkeys_44, L_loadkeys_52, L_aes_loop, L_rounds_44, L_rounds_52; 4247 4248 const Register from = c_rarg0; // source array address 4249 const Register to = c_rarg1; // destination array address 4250 const Register key = c_rarg2; // key array address 4251 const Register rvec = c_rarg3; // r byte array initialized from initvector array address 4252 // and left with the results of the last encryption block 4253 const Register len_reg = c_rarg4; // src len (must be multiple of blocksize 16) 4254 const Register keylen = R8; 4255 4256 address start = __ pc(); 4257 __ stp(FP, LR, Address(SP, -2 * wordSize, pre_indexed)); 4258 __ mov(FP, SP); 4259 4260 __ mov(R9, len_reg); 4261 __ ldr_w(keylen, Address(key, arrayOopDesc::length_offset_in_bytes() - arrayOopDesc::base_offset_in_bytes(T_INT))); 4262 4263 __ vld1(V2, Address(rvec), MacroAssembler::VELEM_SIZE_8, 128); 4264 4265 __ vld1(V31, Address(key, 16, post_indexed), MacroAssembler::VELEM_SIZE_8, 128); 4266 4267 int quad = 1; 4268 __ rev32(V31, V31, MacroAssembler::VELEM_SIZE_8, quad); 4269 4270 __ cmp_w(keylen, 52); 4271 __ b(L_loadkeys_44, cc); 4272 __ b(L_loadkeys_52, eq); 4273 4274 __ vld1(V17, V18, Address(key, 32, post_indexed), MacroAssembler::VELEM_SIZE_8, 128); 4275 __ rev32(V17, V17, MacroAssembler::VELEM_SIZE_8, quad); 4276 __ rev32(V18, V18, MacroAssembler::VELEM_SIZE_8, quad); 4277 __ BIND(L_loadkeys_52); 4278 __ vld1(V19, V20, Address(key, 32, post_indexed), MacroAssembler::VELEM_SIZE_8, 128); 4279 __ rev32(V19, V19, MacroAssembler::VELEM_SIZE_8, quad); 4280 __ rev32(V20, V20, MacroAssembler::VELEM_SIZE_8, quad); 4281 __ BIND(L_loadkeys_44); 4282 __ vld1(V21, V22, V23, V24, Address(key, 64, post_indexed), MacroAssembler::VELEM_SIZE_8, 128); 4283 __ rev32(V21, V21, MacroAssembler::VELEM_SIZE_8, quad); 4284 __ rev32(V22, V22, MacroAssembler::VELEM_SIZE_8, quad); 4285 __ rev32(V23, V23, MacroAssembler::VELEM_SIZE_8, quad); 4286 __ rev32(V24, V24, MacroAssembler::VELEM_SIZE_8, quad); 4287 __ vld1(V25, V26, V27, V28, Address(key, 64, post_indexed), MacroAssembler::VELEM_SIZE_8, 128); 4288 __ rev32(V25, V25, MacroAssembler::VELEM_SIZE_8, quad); 4289 __ rev32(V26, V26, MacroAssembler::VELEM_SIZE_8, quad); 4290 __ rev32(V27, V27, MacroAssembler::VELEM_SIZE_8, quad); 4291 __ rev32(V28, V28, MacroAssembler::VELEM_SIZE_8, quad); 4292 __ vld1(V29, V30, Address(key), MacroAssembler::VELEM_SIZE_8, 128); 4293 __ rev32(V29, V29, MacroAssembler::VELEM_SIZE_8, quad); 4294 __ rev32(V30, V30, MacroAssembler::VELEM_SIZE_8, quad); 4295 4296 __ BIND(L_aes_loop); 4297 __ vld1(V0, Address(from, 16, post_indexed), MacroAssembler::VELEM_SIZE_8, 128); 4298 __ orr(V1, V0, V0, MacroAssembler::VELEM_SIZE_8, quad); 4299 4300 __ b(L_rounds_44, cc); 4301 __ b(L_rounds_52, eq); 4302 4303 __ aesd(V0, V17); 4304 __ aesimc(V0, V0); 4305 __ aesd(V0, V17); 4306 __ aesimc(V0, V0); 4307 __ BIND(L_rounds_52); 4308 __ aesd(V0, V19); 4309 __ aesimc(V0, V0); 4310 __ aesd(V0, V20); 4311 __ aesimc(V0, V0); 4312 __ BIND(L_rounds_44); 4313 __ aesd(V0, V21); 4314 __ aesimc(V0, V0); 4315 __ aesd(V0, V22); 4316 __ aesimc(V0, V0); 4317 __ aesd(V0, V23); 4318 __ aesimc(V0, V0); 4319 __ aesd(V0, V24); 4320 __ aesimc(V0, V0); 4321 __ aesd(V0, V25); 4322 __ aesimc(V0, V0); 4323 __ aesd(V0, V26); 4324 __ aesimc(V0, V0); 4325 __ aesd(V0, V27); 4326 __ aesimc(V0, V0); 4327 __ aesd(V0, V28); 4328 __ aesimc(V0, V0); 4329 __ aesd(V0, V29); 4330 __ aesimc(V0, V0); 4331 __ aesd(V0, V30); 4332 __ eor(V0, V0, V31, MacroAssembler::VELEM_SIZE_8, quad); 4333 __ eor(V0, V0, V2, MacroAssembler::VELEM_SIZE_8, quad); 4334 4335 __ vst1(V0, Address(to, 16, post_indexed), MacroAssembler::VELEM_SIZE_8, 128); 4336 __ orr(V2, V1, V1, MacroAssembler::VELEM_SIZE_8, quad); 4337 4338 __ sub(len_reg, len_reg, 16); 4339 __ cbnz(len_reg, L_aes_loop); 4340 4341 __ vst1(V2, Address(rvec), MacroAssembler::VELEM_SIZE_8, 128); 4342 4343 __ mov(R0, R9); 4344 4345 __ mov(SP, FP); 4346 __ ldp(FP, LR, Address(SP, 2 * wordSize, post_indexed)); 4347 __ ret(LR); 4348 4349 return start; 4350 } 4351 4352#endif // COMPILER2 4353#endif // AARCH64 4354 4355 private: 4356 4357#undef __ 4358#define __ masm-> 4359 4360 //------------------------------------------------------------------------------------------------------------------------ 4361 // Continuation point for throwing of implicit exceptions that are not handled in 4362 // the current activation. Fabricates an exception oop and initiates normal 4363 // exception dispatching in this frame. 4364 address generate_throw_exception(const char* name, address runtime_entry) { 4365 int insts_size = 128; 4366 int locs_size = 32; 4367 CodeBuffer code(name, insts_size, locs_size); 4368 OopMapSet* oop_maps; 4369 int frame_size; 4370 int frame_complete; 4371 4372 oop_maps = new OopMapSet(); 4373 MacroAssembler* masm = new MacroAssembler(&code); 4374 4375 address start = __ pc(); 4376 4377 frame_size = 2; 4378 __ mov(Rexception_pc, LR); 4379 __ raw_push(FP, LR); 4380 4381 frame_complete = __ pc() - start; 4382 4383 // Any extra arguments are already supposed to be R1 and R2 4384 __ mov(R0, Rthread); 4385 4386 int pc_offset = __ set_last_Java_frame(SP, FP, false, Rtemp); 4387 assert(((__ pc()) - start) == __ offset(), "warning: start differs from code_begin"); 4388 __ call(runtime_entry); 4389 if (pc_offset == -1) { 4390 pc_offset = __ offset(); 4391 } 4392 4393 // Generate oop map 4394 OopMap* map = new OopMap(frame_size*VMRegImpl::slots_per_word, 0); 4395 oop_maps->add_gc_map(pc_offset, map); 4396 __ reset_last_Java_frame(Rtemp); // Rtemp free since scratched by far call 4397 4398 __ raw_pop(FP, LR); 4399 __ jump(StubRoutines::forward_exception_entry(), relocInfo::runtime_call_type, Rtemp); 4400 4401 RuntimeStub* stub = RuntimeStub::new_runtime_stub(name, &code, frame_complete, 4402 frame_size, oop_maps, false); 4403 return stub->entry_point(); 4404 } 4405 4406 //--------------------------------------------------------------------------- 4407 // Initialization 4408 4409 void generate_initial() { 4410 // Generates all stubs and initializes the entry points 4411 4412 //------------------------------------------------------------------------------------------------------------------------ 4413 // entry points that exist in all platforms 4414 // Note: This is code that could be shared among different platforms - however the benefit seems to be smaller than 4415 // the disadvantage of having a much more complicated generator structure. See also comment in stubRoutines.hpp. 4416 StubRoutines::_forward_exception_entry = generate_forward_exception(); 4417 4418 StubRoutines::_call_stub_entry = 4419 generate_call_stub(StubRoutines::_call_stub_return_address); 4420 // is referenced by megamorphic call 4421 StubRoutines::_catch_exception_entry = generate_catch_exception(); 4422 4423 // stub for throwing stack overflow error used both by interpreter and compiler 4424 StubRoutines::_throw_StackOverflowError_entry = generate_throw_exception("StackOverflowError throw_exception", CAST_FROM_FN_PTR(address, SharedRuntime::throw_StackOverflowError)); 4425 4426#ifndef AARCH64 4427 // integer division used both by interpreter and compiler 4428 StubRoutines::Arm::_idiv_irem_entry = generate_idiv_irem(); 4429 4430 StubRoutines::_atomic_add_entry = generate_atomic_add(); 4431 StubRoutines::_atomic_xchg_entry = generate_atomic_xchg(); 4432 StubRoutines::_atomic_cmpxchg_entry = generate_atomic_cmpxchg(); 4433 StubRoutines::_atomic_cmpxchg_long_entry = generate_atomic_cmpxchg_long(); 4434 StubRoutines::_atomic_load_long_entry = generate_atomic_load_long(); 4435 StubRoutines::_atomic_store_long_entry = generate_atomic_store_long(); 4436#endif // !AARCH64 4437 } 4438 4439 void generate_all() { 4440 // Generates all stubs and initializes the entry points 4441 4442#ifdef COMPILER2 4443 // Generate partial_subtype_check first here since its code depends on 4444 // UseZeroBaseCompressedOops which is defined after heap initialization. 4445 StubRoutines::Arm::_partial_subtype_check = generate_partial_subtype_check(); 4446#endif 4447 // These entry points require SharedInfo::stack0 to be set up in non-core builds 4448 // and need to be relocatable, so they each fabricate a RuntimeStub internally. 4449 StubRoutines::_throw_AbstractMethodError_entry = generate_throw_exception("AbstractMethodError throw_exception", CAST_FROM_FN_PTR(address, SharedRuntime::throw_AbstractMethodError)); 4450 StubRoutines::_throw_IncompatibleClassChangeError_entry= generate_throw_exception("IncompatibleClassChangeError throw_exception", CAST_FROM_FN_PTR(address, SharedRuntime::throw_IncompatibleClassChangeError)); 4451 StubRoutines::_throw_NullPointerException_at_call_entry= generate_throw_exception("NullPointerException at call throw_exception", CAST_FROM_FN_PTR(address, SharedRuntime::throw_NullPointerException_at_call)); 4452 4453 //------------------------------------------------------------------------------------------------------------------------ 4454 // entry points that are platform specific 4455 4456 // support for verify_oop (must happen after universe_init) 4457 StubRoutines::_verify_oop_subroutine_entry = generate_verify_oop(); 4458 4459 // arraycopy stubs used by compilers 4460 generate_arraycopy_stubs(); 4461 4462 // Safefetch stubs. 4463 generate_safefetch("SafeFetch32", sizeof(int), &StubRoutines::_safefetch32_entry, 4464 &StubRoutines::_safefetch32_fault_pc, 4465 &StubRoutines::_safefetch32_continuation_pc); 4466#ifdef AARCH64 4467 generate_safefetch("SafeFetchN", wordSize, &StubRoutines::_safefetchN_entry, 4468 &StubRoutines::_safefetchN_fault_pc, 4469 &StubRoutines::_safefetchN_continuation_pc); 4470#ifdef COMPILER2 4471 if (UseAESIntrinsics) { 4472 StubRoutines::_aescrypt_encryptBlock = generate_aescrypt_encryptBlock(); 4473 StubRoutines::_aescrypt_decryptBlock = generate_aescrypt_decryptBlock(); 4474 StubRoutines::_cipherBlockChaining_encryptAESCrypt = generate_cipherBlockChaining_encryptAESCrypt(); 4475 StubRoutines::_cipherBlockChaining_decryptAESCrypt = generate_cipherBlockChaining_decryptAESCrypt(); 4476 } 4477#endif 4478#else 4479 assert (sizeof(int) == wordSize, "32-bit architecture"); 4480 StubRoutines::_safefetchN_entry = StubRoutines::_safefetch32_entry; 4481 StubRoutines::_safefetchN_fault_pc = StubRoutines::_safefetch32_fault_pc; 4482 StubRoutines::_safefetchN_continuation_pc = StubRoutines::_safefetch32_continuation_pc; 4483#endif // AARCH64 4484 4485#ifdef COMPILE_CRYPTO 4486 // generate AES intrinsics code 4487 if (UseAESIntrinsics) { 4488 aes_init(); 4489 StubRoutines::_aescrypt_encryptBlock = generate_aescrypt_encryptBlock(); 4490 StubRoutines::_aescrypt_decryptBlock = generate_aescrypt_decryptBlock(); 4491 StubRoutines::_cipherBlockChaining_encryptAESCrypt = generate_cipherBlockChaining_encryptAESCrypt(); 4492 StubRoutines::_cipherBlockChaining_decryptAESCrypt = generate_cipherBlockChaining_decryptAESCrypt(); 4493 } 4494#endif // COMPILE_CRYPTO 4495 } 4496 4497 4498 public: 4499 StubGenerator(CodeBuffer* code, bool all) : StubCodeGenerator(code) { 4500 if (all) { 4501 generate_all(); 4502 } else { 4503 generate_initial(); 4504 } 4505 } 4506}; // end class declaration 4507 4508void StubGenerator_generate(CodeBuffer* code, bool all) { 4509 StubGenerator g(code, all); 4510} 4511