stubGenerator_x86_64.cpp revision 342:37f87013dfd8
1/* 2 * Copyright 2003-2007 Sun Microsystems, Inc. All Rights Reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. 8 * 9 * This code is distributed in the hope that it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 12 * version 2 for more details (a copy is included in the LICENSE file that 13 * accompanied this code). 14 * 15 * You should have received a copy of the GNU General Public License version 16 * 2 along with this work; if not, write to the Free Software Foundation, 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 18 * 19 * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara, 20 * CA 95054 USA or visit www.sun.com if you need additional information or 21 * have any questions. 22 * 23 */ 24 25#include "incls/_precompiled.incl" 26#include "incls/_stubGenerator_x86_64.cpp.incl" 27 28// Declaration and definition of StubGenerator (no .hpp file). 29// For a more detailed description of the stub routine structure 30// see the comment in stubRoutines.hpp 31 32#define __ _masm-> 33#define TIMES_OOP (UseCompressedOops ? Address::times_4 : Address::times_8) 34 35#ifdef PRODUCT 36#define BLOCK_COMMENT(str) /* nothing */ 37#else 38#define BLOCK_COMMENT(str) __ block_comment(str) 39#endif 40 41#define BIND(label) bind(label); BLOCK_COMMENT(#label ":") 42const int MXCSR_MASK = 0xFFC0; // Mask out any pending exceptions 43 44// Stub Code definitions 45 46static address handle_unsafe_access() { 47 JavaThread* thread = JavaThread::current(); 48 address pc = thread->saved_exception_pc(); 49 // pc is the instruction which we must emulate 50 // doing a no-op is fine: return garbage from the load 51 // therefore, compute npc 52 address npc = Assembler::locate_next_instruction(pc); 53 54 // request an async exception 55 thread->set_pending_unsafe_access_error(); 56 57 // return address of next instruction to execute 58 return npc; 59} 60 61class StubGenerator: public StubCodeGenerator { 62 private: 63 64#ifdef PRODUCT 65#define inc_counter_np(counter) (0) 66#else 67 void inc_counter_np_(int& counter) { 68 __ incrementl(ExternalAddress((address)&counter)); 69 } 70#define inc_counter_np(counter) \ 71 BLOCK_COMMENT("inc_counter " #counter); \ 72 inc_counter_np_(counter); 73#endif 74 75 // Call stubs are used to call Java from C 76 // 77 // Linux Arguments: 78 // c_rarg0: call wrapper address address 79 // c_rarg1: result address 80 // c_rarg2: result type BasicType 81 // c_rarg3: method methodOop 82 // c_rarg4: (interpreter) entry point address 83 // c_rarg5: parameters intptr_t* 84 // 16(rbp): parameter size (in words) int 85 // 24(rbp): thread Thread* 86 // 87 // [ return_from_Java ] <--- rsp 88 // [ argument word n ] 89 // ... 90 // -12 [ argument word 1 ] 91 // -11 [ saved r15 ] <--- rsp_after_call 92 // -10 [ saved r14 ] 93 // -9 [ saved r13 ] 94 // -8 [ saved r12 ] 95 // -7 [ saved rbx ] 96 // -6 [ call wrapper ] 97 // -5 [ result ] 98 // -4 [ result type ] 99 // -3 [ method ] 100 // -2 [ entry point ] 101 // -1 [ parameters ] 102 // 0 [ saved rbp ] <--- rbp 103 // 1 [ return address ] 104 // 2 [ parameter size ] 105 // 3 [ thread ] 106 // 107 // Windows Arguments: 108 // c_rarg0: call wrapper address address 109 // c_rarg1: result address 110 // c_rarg2: result type BasicType 111 // c_rarg3: method methodOop 112 // 48(rbp): (interpreter) entry point address 113 // 56(rbp): parameters intptr_t* 114 // 64(rbp): parameter size (in words) int 115 // 72(rbp): thread Thread* 116 // 117 // [ return_from_Java ] <--- rsp 118 // [ argument word n ] 119 // ... 120 // -8 [ argument word 1 ] 121 // -7 [ saved r15 ] <--- rsp_after_call 122 // -6 [ saved r14 ] 123 // -5 [ saved r13 ] 124 // -4 [ saved r12 ] 125 // -3 [ saved rdi ] 126 // -2 [ saved rsi ] 127 // -1 [ saved rbx ] 128 // 0 [ saved rbp ] <--- rbp 129 // 1 [ return address ] 130 // 2 [ call wrapper ] 131 // 3 [ result ] 132 // 4 [ result type ] 133 // 5 [ method ] 134 // 6 [ entry point ] 135 // 7 [ parameters ] 136 // 8 [ parameter size ] 137 // 9 [ thread ] 138 // 139 // Windows reserves the callers stack space for arguments 1-4. 140 // We spill c_rarg0-c_rarg3 to this space. 141 142 // Call stub stack layout word offsets from rbp 143 enum call_stub_layout { 144#ifdef _WIN64 145 rsp_after_call_off = -7, 146 r15_off = rsp_after_call_off, 147 r14_off = -6, 148 r13_off = -5, 149 r12_off = -4, 150 rdi_off = -3, 151 rsi_off = -2, 152 rbx_off = -1, 153 rbp_off = 0, 154 retaddr_off = 1, 155 call_wrapper_off = 2, 156 result_off = 3, 157 result_type_off = 4, 158 method_off = 5, 159 entry_point_off = 6, 160 parameters_off = 7, 161 parameter_size_off = 8, 162 thread_off = 9 163#else 164 rsp_after_call_off = -12, 165 mxcsr_off = rsp_after_call_off, 166 r15_off = -11, 167 r14_off = -10, 168 r13_off = -9, 169 r12_off = -8, 170 rbx_off = -7, 171 call_wrapper_off = -6, 172 result_off = -5, 173 result_type_off = -4, 174 method_off = -3, 175 entry_point_off = -2, 176 parameters_off = -1, 177 rbp_off = 0, 178 retaddr_off = 1, 179 parameter_size_off = 2, 180 thread_off = 3 181#endif 182 }; 183 184 address generate_call_stub(address& return_address) { 185 assert((int)frame::entry_frame_after_call_words == -(int)rsp_after_call_off + 1 && 186 (int)frame::entry_frame_call_wrapper_offset == (int)call_wrapper_off, 187 "adjust this code"); 188 StubCodeMark mark(this, "StubRoutines", "call_stub"); 189 address start = __ pc(); 190 191 // same as in generate_catch_exception()! 192 const Address rsp_after_call(rbp, rsp_after_call_off * wordSize); 193 194 const Address call_wrapper (rbp, call_wrapper_off * wordSize); 195 const Address result (rbp, result_off * wordSize); 196 const Address result_type (rbp, result_type_off * wordSize); 197 const Address method (rbp, method_off * wordSize); 198 const Address entry_point (rbp, entry_point_off * wordSize); 199 const Address parameters (rbp, parameters_off * wordSize); 200 const Address parameter_size(rbp, parameter_size_off * wordSize); 201 202 // same as in generate_catch_exception()! 203 const Address thread (rbp, thread_off * wordSize); 204 205 const Address r15_save(rbp, r15_off * wordSize); 206 const Address r14_save(rbp, r14_off * wordSize); 207 const Address r13_save(rbp, r13_off * wordSize); 208 const Address r12_save(rbp, r12_off * wordSize); 209 const Address rbx_save(rbp, rbx_off * wordSize); 210 211 // stub code 212 __ enter(); 213 __ subq(rsp, -rsp_after_call_off * wordSize); 214 215 // save register parameters 216#ifndef _WIN64 217 __ movq(parameters, c_rarg5); // parameters 218 __ movq(entry_point, c_rarg4); // entry_point 219#endif 220 221 __ movq(method, c_rarg3); // method 222 __ movl(result_type, c_rarg2); // result type 223 __ movq(result, c_rarg1); // result 224 __ movq(call_wrapper, c_rarg0); // call wrapper 225 226 // save regs belonging to calling function 227 __ movq(rbx_save, rbx); 228 __ movq(r12_save, r12); 229 __ movq(r13_save, r13); 230 __ movq(r14_save, r14); 231 __ movq(r15_save, r15); 232 233#ifdef _WIN64 234 const Address rdi_save(rbp, rdi_off * wordSize); 235 const Address rsi_save(rbp, rsi_off * wordSize); 236 237 __ movq(rsi_save, rsi); 238 __ movq(rdi_save, rdi); 239#else 240 const Address mxcsr_save(rbp, mxcsr_off * wordSize); 241 { 242 Label skip_ldmx; 243 __ stmxcsr(mxcsr_save); 244 __ movl(rax, mxcsr_save); 245 __ andl(rax, MXCSR_MASK); // Only check control and mask bits 246 ExternalAddress mxcsr_std(StubRoutines::amd64::mxcsr_std()); 247 __ cmp32(rax, mxcsr_std); 248 __ jcc(Assembler::equal, skip_ldmx); 249 __ ldmxcsr(mxcsr_std); 250 __ bind(skip_ldmx); 251 } 252#endif 253 254 // Load up thread register 255 __ movq(r15_thread, thread); 256 __ reinit_heapbase(); 257 258#ifdef ASSERT 259 // make sure we have no pending exceptions 260 { 261 Label L; 262 __ cmpq(Address(r15_thread, Thread::pending_exception_offset()), (int)NULL_WORD); 263 __ jcc(Assembler::equal, L); 264 __ stop("StubRoutines::call_stub: entered with pending exception"); 265 __ bind(L); 266 } 267#endif 268 269 // pass parameters if any 270 BLOCK_COMMENT("pass parameters if any"); 271 Label parameters_done; 272 __ movl(c_rarg3, parameter_size); 273 __ testl(c_rarg3, c_rarg3); 274 __ jcc(Assembler::zero, parameters_done); 275 276 Label loop; 277 __ movq(c_rarg2, parameters); // parameter pointer 278 __ movl(c_rarg1, c_rarg3); // parameter counter is in c_rarg1 279 __ BIND(loop); 280 if (TaggedStackInterpreter) { 281 __ movq(rax, Address(c_rarg2, 0)); // get tag 282 __ addq(c_rarg2, wordSize); // advance to next tag 283 __ pushq(rax); // pass tag 284 } 285 __ movq(rax, Address(c_rarg2, 0)); // get parameter 286 __ addq(c_rarg2, wordSize); // advance to next parameter 287 __ decrementl(c_rarg1); // decrement counter 288 __ pushq(rax); // pass parameter 289 __ jcc(Assembler::notZero, loop); 290 291 // call Java function 292 __ BIND(parameters_done); 293 __ movq(rbx, method); // get methodOop 294 __ movq(c_rarg1, entry_point); // get entry_point 295 __ movq(r13, rsp); // set sender sp 296 BLOCK_COMMENT("call Java function"); 297 __ call(c_rarg1); 298 299 BLOCK_COMMENT("call_stub_return_address:"); 300 return_address = __ pc(); 301 302 // store result depending on type (everything that is not 303 // T_OBJECT, T_LONG, T_FLOAT or T_DOUBLE is treated as T_INT) 304 __ movq(c_rarg0, result); 305 Label is_long, is_float, is_double, exit; 306 __ movl(c_rarg1, result_type); 307 __ cmpl(c_rarg1, T_OBJECT); 308 __ jcc(Assembler::equal, is_long); 309 __ cmpl(c_rarg1, T_LONG); 310 __ jcc(Assembler::equal, is_long); 311 __ cmpl(c_rarg1, T_FLOAT); 312 __ jcc(Assembler::equal, is_float); 313 __ cmpl(c_rarg1, T_DOUBLE); 314 __ jcc(Assembler::equal, is_double); 315 316 // handle T_INT case 317 __ movl(Address(c_rarg0, 0), rax); 318 319 __ BIND(exit); 320 321 // pop parameters 322 __ leaq(rsp, rsp_after_call); 323 324#ifdef ASSERT 325 // verify that threads correspond 326 { 327 Label L, S; 328 __ cmpq(r15_thread, thread); 329 __ jcc(Assembler::notEqual, S); 330 __ get_thread(rbx); 331 __ cmpq(r15_thread, rbx); 332 __ jcc(Assembler::equal, L); 333 __ bind(S); 334 __ jcc(Assembler::equal, L); 335 __ stop("StubRoutines::call_stub: threads must correspond"); 336 __ bind(L); 337 } 338#endif 339 340 // restore regs belonging to calling function 341 __ movq(r15, r15_save); 342 __ movq(r14, r14_save); 343 __ movq(r13, r13_save); 344 __ movq(r12, r12_save); 345 __ movq(rbx, rbx_save); 346 347#ifdef _WIN64 348 __ movq(rdi, rdi_save); 349 __ movq(rsi, rsi_save); 350#else 351 __ ldmxcsr(mxcsr_save); 352#endif 353 354 // restore rsp 355 __ addq(rsp, -rsp_after_call_off * wordSize); 356 357 // return 358 __ popq(rbp); 359 __ ret(0); 360 361 // handle return types different from T_INT 362 __ BIND(is_long); 363 __ movq(Address(c_rarg0, 0), rax); 364 __ jmp(exit); 365 366 __ BIND(is_float); 367 __ movflt(Address(c_rarg0, 0), xmm0); 368 __ jmp(exit); 369 370 __ BIND(is_double); 371 __ movdbl(Address(c_rarg0, 0), xmm0); 372 __ jmp(exit); 373 374 return start; 375 } 376 377 // Return point for a Java call if there's an exception thrown in 378 // Java code. The exception is caught and transformed into a 379 // pending exception stored in JavaThread that can be tested from 380 // within the VM. 381 // 382 // Note: Usually the parameters are removed by the callee. In case 383 // of an exception crossing an activation frame boundary, that is 384 // not the case if the callee is compiled code => need to setup the 385 // rsp. 386 // 387 // rax: exception oop 388 389 address generate_catch_exception() { 390 StubCodeMark mark(this, "StubRoutines", "catch_exception"); 391 address start = __ pc(); 392 393 // same as in generate_call_stub(): 394 const Address rsp_after_call(rbp, rsp_after_call_off * wordSize); 395 const Address thread (rbp, thread_off * wordSize); 396 397#ifdef ASSERT 398 // verify that threads correspond 399 { 400 Label L, S; 401 __ cmpq(r15_thread, thread); 402 __ jcc(Assembler::notEqual, S); 403 __ get_thread(rbx); 404 __ cmpq(r15_thread, rbx); 405 __ jcc(Assembler::equal, L); 406 __ bind(S); 407 __ stop("StubRoutines::catch_exception: threads must correspond"); 408 __ bind(L); 409 } 410#endif 411 412 // set pending exception 413 __ verify_oop(rax); 414 415 __ movq(Address(r15_thread, Thread::pending_exception_offset()), rax); 416 __ lea(rscratch1, ExternalAddress((address)__FILE__)); 417 __ movq(Address(r15_thread, Thread::exception_file_offset()), rscratch1); 418 __ movl(Address(r15_thread, Thread::exception_line_offset()), (int) __LINE__); 419 420 // complete return to VM 421 assert(StubRoutines::_call_stub_return_address != NULL, 422 "_call_stub_return_address must have been generated before"); 423 __ jump(RuntimeAddress(StubRoutines::_call_stub_return_address)); 424 425 return start; 426 } 427 428 // Continuation point for runtime calls returning with a pending 429 // exception. The pending exception check happened in the runtime 430 // or native call stub. The pending exception in Thread is 431 // converted into a Java-level exception. 432 // 433 // Contract with Java-level exception handlers: 434 // rax: exception 435 // rdx: throwing pc 436 // 437 // NOTE: At entry of this stub, exception-pc must be on stack !! 438 439 address generate_forward_exception() { 440 StubCodeMark mark(this, "StubRoutines", "forward exception"); 441 address start = __ pc(); 442 443 // Upon entry, the sp points to the return address returning into 444 // Java (interpreted or compiled) code; i.e., the return address 445 // becomes the throwing pc. 446 // 447 // Arguments pushed before the runtime call are still on the stack 448 // but the exception handler will reset the stack pointer -> 449 // ignore them. A potential result in registers can be ignored as 450 // well. 451 452#ifdef ASSERT 453 // make sure this code is only executed if there is a pending exception 454 { 455 Label L; 456 __ cmpq(Address(r15_thread, Thread::pending_exception_offset()), (int) NULL); 457 __ jcc(Assembler::notEqual, L); 458 __ stop("StubRoutines::forward exception: no pending exception (1)"); 459 __ bind(L); 460 } 461#endif 462 463 // compute exception handler into rbx 464 __ movq(c_rarg0, Address(rsp, 0)); 465 BLOCK_COMMENT("call exception_handler_for_return_address"); 466 __ call_VM_leaf(CAST_FROM_FN_PTR(address, 467 SharedRuntime::exception_handler_for_return_address), 468 c_rarg0); 469 __ movq(rbx, rax); 470 471 // setup rax & rdx, remove return address & clear pending exception 472 __ popq(rdx); 473 __ movq(rax, Address(r15_thread, Thread::pending_exception_offset())); 474 __ movptr(Address(r15_thread, Thread::pending_exception_offset()), (int)NULL_WORD); 475 476#ifdef ASSERT 477 // make sure exception is set 478 { 479 Label L; 480 __ testq(rax, rax); 481 __ jcc(Assembler::notEqual, L); 482 __ stop("StubRoutines::forward exception: no pending exception (2)"); 483 __ bind(L); 484 } 485#endif 486 487 // continue at exception handler (return address removed) 488 // rax: exception 489 // rbx: exception handler 490 // rdx: throwing pc 491 __ verify_oop(rax); 492 __ jmp(rbx); 493 494 return start; 495 } 496 497 // Support for jint atomic::xchg(jint exchange_value, volatile jint* dest) 498 // 499 // Arguments : 500 // c_rarg0: exchange_value 501 // c_rarg0: dest 502 // 503 // Result: 504 // *dest <- ex, return (orig *dest) 505 address generate_atomic_xchg() { 506 StubCodeMark mark(this, "StubRoutines", "atomic_xchg"); 507 address start = __ pc(); 508 509 __ movl(rax, c_rarg0); // Copy to eax we need a return value anyhow 510 __ xchgl(rax, Address(c_rarg1, 0)); // automatic LOCK 511 __ ret(0); 512 513 return start; 514 } 515 516 // Support for intptr_t atomic::xchg_ptr(intptr_t exchange_value, volatile intptr_t* dest) 517 // 518 // Arguments : 519 // c_rarg0: exchange_value 520 // c_rarg1: dest 521 // 522 // Result: 523 // *dest <- ex, return (orig *dest) 524 address generate_atomic_xchg_ptr() { 525 StubCodeMark mark(this, "StubRoutines", "atomic_xchg_ptr"); 526 address start = __ pc(); 527 528 __ movq(rax, c_rarg0); // Copy to eax we need a return value anyhow 529 __ xchgq(rax, Address(c_rarg1, 0)); // automatic LOCK 530 __ ret(0); 531 532 return start; 533 } 534 535 // Support for jint atomic::atomic_cmpxchg(jint exchange_value, volatile jint* dest, 536 // jint compare_value) 537 // 538 // Arguments : 539 // c_rarg0: exchange_value 540 // c_rarg1: dest 541 // c_rarg2: compare_value 542 // 543 // Result: 544 // if ( compare_value == *dest ) { 545 // *dest = exchange_value 546 // return compare_value; 547 // else 548 // return *dest; 549 address generate_atomic_cmpxchg() { 550 StubCodeMark mark(this, "StubRoutines", "atomic_cmpxchg"); 551 address start = __ pc(); 552 553 __ movl(rax, c_rarg2); 554 if ( os::is_MP() ) __ lock(); 555 __ cmpxchgl(c_rarg0, Address(c_rarg1, 0)); 556 __ ret(0); 557 558 return start; 559 } 560 561 // Support for jint atomic::atomic_cmpxchg_long(jlong exchange_value, 562 // volatile jlong* dest, 563 // jlong compare_value) 564 // Arguments : 565 // c_rarg0: exchange_value 566 // c_rarg1: dest 567 // c_rarg2: compare_value 568 // 569 // Result: 570 // if ( compare_value == *dest ) { 571 // *dest = exchange_value 572 // return compare_value; 573 // else 574 // return *dest; 575 address generate_atomic_cmpxchg_long() { 576 StubCodeMark mark(this, "StubRoutines", "atomic_cmpxchg_long"); 577 address start = __ pc(); 578 579 __ movq(rax, c_rarg2); 580 if ( os::is_MP() ) __ lock(); 581 __ cmpxchgq(c_rarg0, Address(c_rarg1, 0)); 582 __ ret(0); 583 584 return start; 585 } 586 587 // Support for jint atomic::add(jint add_value, volatile jint* dest) 588 // 589 // Arguments : 590 // c_rarg0: add_value 591 // c_rarg1: dest 592 // 593 // Result: 594 // *dest += add_value 595 // return *dest; 596 address generate_atomic_add() { 597 StubCodeMark mark(this, "StubRoutines", "atomic_add"); 598 address start = __ pc(); 599 600 __ movl(rax, c_rarg0); 601 if ( os::is_MP() ) __ lock(); 602 __ xaddl(Address(c_rarg1, 0), c_rarg0); 603 __ addl(rax, c_rarg0); 604 __ ret(0); 605 606 return start; 607 } 608 609 // Support for intptr_t atomic::add_ptr(intptr_t add_value, volatile intptr_t* dest) 610 // 611 // Arguments : 612 // c_rarg0: add_value 613 // c_rarg1: dest 614 // 615 // Result: 616 // *dest += add_value 617 // return *dest; 618 address generate_atomic_add_ptr() { 619 StubCodeMark mark(this, "StubRoutines", "atomic_add_ptr"); 620 address start = __ pc(); 621 622 __ movq(rax, c_rarg0); // Copy to eax we need a return value anyhow 623 if ( os::is_MP() ) __ lock(); 624 __ xaddl(Address(c_rarg1, 0), c_rarg0); 625 __ addl(rax, c_rarg0); 626 __ ret(0); 627 628 return start; 629 } 630 631 // Support for intptr_t OrderAccess::fence() 632 // 633 // Arguments : 634 // 635 // Result: 636 address generate_orderaccess_fence() { 637 StubCodeMark mark(this, "StubRoutines", "orderaccess_fence"); 638 address start = __ pc(); 639 __ mfence(); 640 __ ret(0); 641 642 return start; 643 } 644 645 // Support for intptr_t get_previous_fp() 646 // 647 // This routine is used to find the previous frame pointer for the 648 // caller (current_frame_guess). This is used as part of debugging 649 // ps() is seemingly lost trying to find frames. 650 // This code assumes that caller current_frame_guess) has a frame. 651 address generate_get_previous_fp() { 652 StubCodeMark mark(this, "StubRoutines", "get_previous_fp"); 653 const Address old_fp(rbp, 0); 654 const Address older_fp(rax, 0); 655 address start = __ pc(); 656 657 __ enter(); 658 __ movq(rax, old_fp); // callers fp 659 __ movq(rax, older_fp); // the frame for ps() 660 __ popq(rbp); 661 __ ret(0); 662 663 return start; 664 } 665 666 //---------------------------------------------------------------------------------------------------- 667 // Support for void verify_mxcsr() 668 // 669 // This routine is used with -Xcheck:jni to verify that native 670 // JNI code does not return to Java code without restoring the 671 // MXCSR register to our expected state. 672 673 address generate_verify_mxcsr() { 674 StubCodeMark mark(this, "StubRoutines", "verify_mxcsr"); 675 address start = __ pc(); 676 677 const Address mxcsr_save(rsp, 0); 678 679 if (CheckJNICalls) { 680 Label ok_ret; 681 __ pushq(rax); 682 __ subq(rsp, wordSize); // allocate a temp location 683 __ stmxcsr(mxcsr_save); 684 __ movl(rax, mxcsr_save); 685 __ andl(rax, MXCSR_MASK); // Only check control and mask bits 686 __ cmpl(rax, *(int *)(StubRoutines::amd64::mxcsr_std())); 687 __ jcc(Assembler::equal, ok_ret); 688 689 __ warn("MXCSR changed by native JNI code, use -XX:+RestoreMXCSROnJNICall"); 690 691 __ ldmxcsr(ExternalAddress(StubRoutines::amd64::mxcsr_std())); 692 693 __ bind(ok_ret); 694 __ addq(rsp, wordSize); 695 __ popq(rax); 696 } 697 698 __ ret(0); 699 700 return start; 701 } 702 703 address generate_f2i_fixup() { 704 StubCodeMark mark(this, "StubRoutines", "f2i_fixup"); 705 Address inout(rsp, 5 * wordSize); // return address + 4 saves 706 707 address start = __ pc(); 708 709 Label L; 710 711 __ pushq(rax); 712 __ pushq(c_rarg3); 713 __ pushq(c_rarg2); 714 __ pushq(c_rarg1); 715 716 __ movl(rax, 0x7f800000); 717 __ xorl(c_rarg3, c_rarg3); 718 __ movl(c_rarg2, inout); 719 __ movl(c_rarg1, c_rarg2); 720 __ andl(c_rarg1, 0x7fffffff); 721 __ cmpl(rax, c_rarg1); // NaN? -> 0 722 __ jcc(Assembler::negative, L); 723 __ testl(c_rarg2, c_rarg2); // signed ? min_jint : max_jint 724 __ movl(c_rarg3, 0x80000000); 725 __ movl(rax, 0x7fffffff); 726 __ cmovl(Assembler::positive, c_rarg3, rax); 727 728 __ bind(L); 729 __ movq(inout, c_rarg3); 730 731 __ popq(c_rarg1); 732 __ popq(c_rarg2); 733 __ popq(c_rarg3); 734 __ popq(rax); 735 736 __ ret(0); 737 738 return start; 739 } 740 741 address generate_f2l_fixup() { 742 StubCodeMark mark(this, "StubRoutines", "f2l_fixup"); 743 Address inout(rsp, 5 * wordSize); // return address + 4 saves 744 address start = __ pc(); 745 746 Label L; 747 748 __ pushq(rax); 749 __ pushq(c_rarg3); 750 __ pushq(c_rarg2); 751 __ pushq(c_rarg1); 752 753 __ movl(rax, 0x7f800000); 754 __ xorl(c_rarg3, c_rarg3); 755 __ movl(c_rarg2, inout); 756 __ movl(c_rarg1, c_rarg2); 757 __ andl(c_rarg1, 0x7fffffff); 758 __ cmpl(rax, c_rarg1); // NaN? -> 0 759 __ jcc(Assembler::negative, L); 760 __ testl(c_rarg2, c_rarg2); // signed ? min_jlong : max_jlong 761 __ mov64(c_rarg3, 0x8000000000000000); 762 __ mov64(rax, 0x7fffffffffffffff); 763 __ cmovq(Assembler::positive, c_rarg3, rax); 764 765 __ bind(L); 766 __ movq(inout, c_rarg3); 767 768 __ popq(c_rarg1); 769 __ popq(c_rarg2); 770 __ popq(c_rarg3); 771 __ popq(rax); 772 773 __ ret(0); 774 775 return start; 776 } 777 778 address generate_d2i_fixup() { 779 StubCodeMark mark(this, "StubRoutines", "d2i_fixup"); 780 Address inout(rsp, 6 * wordSize); // return address + 5 saves 781 782 address start = __ pc(); 783 784 Label L; 785 786 __ pushq(rax); 787 __ pushq(c_rarg3); 788 __ pushq(c_rarg2); 789 __ pushq(c_rarg1); 790 __ pushq(c_rarg0); 791 792 __ movl(rax, 0x7ff00000); 793 __ movq(c_rarg2, inout); 794 __ movl(c_rarg3, c_rarg2); 795 __ movq(c_rarg1, c_rarg2); 796 __ movq(c_rarg0, c_rarg2); 797 __ negl(c_rarg3); 798 __ shrq(c_rarg1, 0x20); 799 __ orl(c_rarg3, c_rarg2); 800 __ andl(c_rarg1, 0x7fffffff); 801 __ xorl(c_rarg2, c_rarg2); 802 __ shrl(c_rarg3, 0x1f); 803 __ orl(c_rarg1, c_rarg3); 804 __ cmpl(rax, c_rarg1); 805 __ jcc(Assembler::negative, L); // NaN -> 0 806 __ testq(c_rarg0, c_rarg0); // signed ? min_jint : max_jint 807 __ movl(c_rarg2, 0x80000000); 808 __ movl(rax, 0x7fffffff); 809 __ cmovl(Assembler::positive, c_rarg2, rax); 810 811 __ bind(L); 812 __ movq(inout, c_rarg2); 813 814 __ popq(c_rarg0); 815 __ popq(c_rarg1); 816 __ popq(c_rarg2); 817 __ popq(c_rarg3); 818 __ popq(rax); 819 820 __ ret(0); 821 822 return start; 823 } 824 825 address generate_d2l_fixup() { 826 StubCodeMark mark(this, "StubRoutines", "d2l_fixup"); 827 Address inout(rsp, 6 * wordSize); // return address + 5 saves 828 829 address start = __ pc(); 830 831 Label L; 832 833 __ pushq(rax); 834 __ pushq(c_rarg3); 835 __ pushq(c_rarg2); 836 __ pushq(c_rarg1); 837 __ pushq(c_rarg0); 838 839 __ movl(rax, 0x7ff00000); 840 __ movq(c_rarg2, inout); 841 __ movl(c_rarg3, c_rarg2); 842 __ movq(c_rarg1, c_rarg2); 843 __ movq(c_rarg0, c_rarg2); 844 __ negl(c_rarg3); 845 __ shrq(c_rarg1, 0x20); 846 __ orl(c_rarg3, c_rarg2); 847 __ andl(c_rarg1, 0x7fffffff); 848 __ xorl(c_rarg2, c_rarg2); 849 __ shrl(c_rarg3, 0x1f); 850 __ orl(c_rarg1, c_rarg3); 851 __ cmpl(rax, c_rarg1); 852 __ jcc(Assembler::negative, L); // NaN -> 0 853 __ testq(c_rarg0, c_rarg0); // signed ? min_jlong : max_jlong 854 __ mov64(c_rarg2, 0x8000000000000000); 855 __ mov64(rax, 0x7fffffffffffffff); 856 __ cmovq(Assembler::positive, c_rarg2, rax); 857 858 __ bind(L); 859 __ movq(inout, c_rarg2); 860 861 __ popq(c_rarg0); 862 __ popq(c_rarg1); 863 __ popq(c_rarg2); 864 __ popq(c_rarg3); 865 __ popq(rax); 866 867 __ ret(0); 868 869 return start; 870 } 871 872 address generate_fp_mask(const char *stub_name, int64_t mask) { 873 StubCodeMark mark(this, "StubRoutines", stub_name); 874 875 __ align(16); 876 address start = __ pc(); 877 878 __ emit_data64( mask, relocInfo::none ); 879 __ emit_data64( mask, relocInfo::none ); 880 881 return start; 882 } 883 884 // The following routine generates a subroutine to throw an 885 // asynchronous UnknownError when an unsafe access gets a fault that 886 // could not be reasonably prevented by the programmer. (Example: 887 // SIGBUS/OBJERR.) 888 address generate_handler_for_unsafe_access() { 889 StubCodeMark mark(this, "StubRoutines", "handler_for_unsafe_access"); 890 address start = __ pc(); 891 892 __ pushq(0); // hole for return address-to-be 893 __ pushaq(); // push registers 894 Address next_pc(rsp, RegisterImpl::number_of_registers * BytesPerWord); 895 896 __ subq(rsp, frame::arg_reg_save_area_bytes); 897 BLOCK_COMMENT("call handle_unsafe_access"); 898 __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, handle_unsafe_access))); 899 __ addq(rsp, frame::arg_reg_save_area_bytes); 900 901 __ movq(next_pc, rax); // stuff next address 902 __ popaq(); 903 __ ret(0); // jump to next address 904 905 return start; 906 } 907 908 // Non-destructive plausibility checks for oops 909 // 910 // Arguments: 911 // all args on stack! 912 // 913 // Stack after saving c_rarg3: 914 // [tos + 0]: saved c_rarg3 915 // [tos + 1]: saved c_rarg2 916 // [tos + 2]: saved r12 (several TemplateTable methods use it) 917 // [tos + 3]: saved flags 918 // [tos + 4]: return address 919 // * [tos + 5]: error message (char*) 920 // * [tos + 6]: object to verify (oop) 921 // * [tos + 7]: saved rax - saved by caller and bashed 922 // * = popped on exit 923 address generate_verify_oop() { 924 StubCodeMark mark(this, "StubRoutines", "verify_oop"); 925 address start = __ pc(); 926 927 Label exit, error; 928 929 __ pushfq(); 930 __ incrementl(ExternalAddress((address) StubRoutines::verify_oop_count_addr())); 931 932 __ pushq(r12); 933 934 // save c_rarg2 and c_rarg3 935 __ pushq(c_rarg2); 936 __ pushq(c_rarg3); 937 938 enum { 939 // After previous pushes. 940 oop_to_verify = 6 * wordSize, 941 saved_rax = 7 * wordSize, 942 943 // Before the call to MacroAssembler::debug(), see below. 944 return_addr = 16 * wordSize, 945 error_msg = 17 * wordSize 946 }; 947 948 // get object 949 __ movq(rax, Address(rsp, oop_to_verify)); 950 951 // make sure object is 'reasonable' 952 __ testq(rax, rax); 953 __ jcc(Assembler::zero, exit); // if obj is NULL it is OK 954 // Check if the oop is in the right area of memory 955 __ movq(c_rarg2, rax); 956 __ movptr(c_rarg3, (int64_t) Universe::verify_oop_mask()); 957 __ andq(c_rarg2, c_rarg3); 958 __ movptr(c_rarg3, (int64_t) Universe::verify_oop_bits()); 959 __ cmpq(c_rarg2, c_rarg3); 960 __ jcc(Assembler::notZero, error); 961 962 // set r12 to heapbase for load_klass() 963 __ reinit_heapbase(); 964 965 // make sure klass is 'reasonable' 966 __ load_klass(rax, rax); // get klass 967 __ testq(rax, rax); 968 __ jcc(Assembler::zero, error); // if klass is NULL it is broken 969 // Check if the klass is in the right area of memory 970 __ movq(c_rarg2, rax); 971 __ movptr(c_rarg3, (int64_t) Universe::verify_klass_mask()); 972 __ andq(c_rarg2, c_rarg3); 973 __ movptr(c_rarg3, (int64_t) Universe::verify_klass_bits()); 974 __ cmpq(c_rarg2, c_rarg3); 975 __ jcc(Assembler::notZero, error); 976 977 // make sure klass' klass is 'reasonable' 978 __ load_klass(rax, rax); 979 __ testq(rax, rax); 980 __ jcc(Assembler::zero, error); // if klass' klass is NULL it is broken 981 // Check if the klass' klass is in the right area of memory 982 __ movptr(c_rarg3, (int64_t) Universe::verify_klass_mask()); 983 __ andq(rax, c_rarg3); 984 __ movptr(c_rarg3, (int64_t) Universe::verify_klass_bits()); 985 __ cmpq(rax, c_rarg3); 986 __ jcc(Assembler::notZero, error); 987 988 // return if everything seems ok 989 __ bind(exit); 990 __ movq(rax, Address(rsp, saved_rax)); // get saved rax back 991 __ popq(c_rarg3); // restore c_rarg3 992 __ popq(c_rarg2); // restore c_rarg2 993 __ popq(r12); // restore r12 994 __ popfq(); // restore flags 995 __ ret(3 * wordSize); // pop caller saved stuff 996 997 // handle errors 998 __ bind(error); 999 __ movq(rax, Address(rsp, saved_rax)); // get saved rax back 1000 __ popq(c_rarg3); // get saved c_rarg3 back 1001 __ popq(c_rarg2); // get saved c_rarg2 back 1002 __ popq(r12); // get saved r12 back 1003 __ popfq(); // get saved flags off stack -- 1004 // will be ignored 1005 1006 __ pushaq(); // push registers 1007 // (rip is already 1008 // already pushed) 1009 // debug(char* msg, int64_t pc, int64_t regs[]) 1010 // We've popped the registers we'd saved (c_rarg3, c_rarg2 and flags), and 1011 // pushed all the registers, so now the stack looks like: 1012 // [tos + 0] 16 saved registers 1013 // [tos + 16] return address 1014 // * [tos + 17] error message (char*) 1015 // * [tos + 18] object to verify (oop) 1016 // * [tos + 19] saved rax - saved by caller and bashed 1017 // * = popped on exit 1018 1019 __ movq(c_rarg0, Address(rsp, error_msg)); // pass address of error message 1020 __ movq(c_rarg1, Address(rsp, return_addr)); // pass return address 1021 __ movq(c_rarg2, rsp); // pass address of regs on stack 1022 __ movq(r12, rsp); // remember rsp 1023 __ subq(rsp, frame::arg_reg_save_area_bytes);// windows 1024 __ andq(rsp, -16); // align stack as required by ABI 1025 BLOCK_COMMENT("call MacroAssembler::debug"); 1026 __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, MacroAssembler::debug))); 1027 __ movq(rsp, r12); // restore rsp 1028 __ popaq(); // pop registers (includes r12) 1029 __ ret(3 * wordSize); // pop caller saved stuff 1030 1031 return start; 1032 } 1033 1034 static address disjoint_byte_copy_entry; 1035 static address disjoint_short_copy_entry; 1036 static address disjoint_int_copy_entry; 1037 static address disjoint_long_copy_entry; 1038 static address disjoint_oop_copy_entry; 1039 1040 static address byte_copy_entry; 1041 static address short_copy_entry; 1042 static address int_copy_entry; 1043 static address long_copy_entry; 1044 static address oop_copy_entry; 1045 1046 static address checkcast_copy_entry; 1047 1048 // 1049 // Verify that a register contains clean 32-bits positive value 1050 // (high 32-bits are 0) so it could be used in 64-bits shifts. 1051 // 1052 // Input: 1053 // Rint - 32-bits value 1054 // Rtmp - scratch 1055 // 1056 void assert_clean_int(Register Rint, Register Rtmp) { 1057#ifdef ASSERT 1058 Label L; 1059 assert_different_registers(Rtmp, Rint); 1060 __ movslq(Rtmp, Rint); 1061 __ cmpq(Rtmp, Rint); 1062 __ jcc(Assembler::equal, L); 1063 __ stop("high 32-bits of int value are not 0"); 1064 __ bind(L); 1065#endif 1066 } 1067 1068 // Generate overlap test for array copy stubs 1069 // 1070 // Input: 1071 // c_rarg0 - from 1072 // c_rarg1 - to 1073 // c_rarg2 - element count 1074 // 1075 // Output: 1076 // rax - &from[element count - 1] 1077 // 1078 void array_overlap_test(address no_overlap_target, Address::ScaleFactor sf) { 1079 assert(no_overlap_target != NULL, "must be generated"); 1080 array_overlap_test(no_overlap_target, NULL, sf); 1081 } 1082 void array_overlap_test(Label& L_no_overlap, Address::ScaleFactor sf) { 1083 array_overlap_test(NULL, &L_no_overlap, sf); 1084 } 1085 void array_overlap_test(address no_overlap_target, Label* NOLp, Address::ScaleFactor sf) { 1086 const Register from = c_rarg0; 1087 const Register to = c_rarg1; 1088 const Register count = c_rarg2; 1089 const Register end_from = rax; 1090 1091 __ cmpq(to, from); 1092 __ leaq(end_from, Address(from, count, sf, 0)); 1093 if (NOLp == NULL) { 1094 ExternalAddress no_overlap(no_overlap_target); 1095 __ jump_cc(Assembler::belowEqual, no_overlap); 1096 __ cmpq(to, end_from); 1097 __ jump_cc(Assembler::aboveEqual, no_overlap); 1098 } else { 1099 __ jcc(Assembler::belowEqual, (*NOLp)); 1100 __ cmpq(to, end_from); 1101 __ jcc(Assembler::aboveEqual, (*NOLp)); 1102 } 1103 } 1104 1105 // Shuffle first three arg regs on Windows into Linux/Solaris locations. 1106 // 1107 // Outputs: 1108 // rdi - rcx 1109 // rsi - rdx 1110 // rdx - r8 1111 // rcx - r9 1112 // 1113 // Registers r9 and r10 are used to save rdi and rsi on Windows, which latter 1114 // are non-volatile. r9 and r10 should not be used by the caller. 1115 // 1116 void setup_arg_regs(int nargs = 3) { 1117 const Register saved_rdi = r9; 1118 const Register saved_rsi = r10; 1119 assert(nargs == 3 || nargs == 4, "else fix"); 1120#ifdef _WIN64 1121 assert(c_rarg0 == rcx && c_rarg1 == rdx && c_rarg2 == r8 && c_rarg3 == r9, 1122 "unexpected argument registers"); 1123 if (nargs >= 4) 1124 __ movq(rax, r9); // r9 is also saved_rdi 1125 __ movq(saved_rdi, rdi); 1126 __ movq(saved_rsi, rsi); 1127 __ movq(rdi, rcx); // c_rarg0 1128 __ movq(rsi, rdx); // c_rarg1 1129 __ movq(rdx, r8); // c_rarg2 1130 if (nargs >= 4) 1131 __ movq(rcx, rax); // c_rarg3 (via rax) 1132#else 1133 assert(c_rarg0 == rdi && c_rarg1 == rsi && c_rarg2 == rdx && c_rarg3 == rcx, 1134 "unexpected argument registers"); 1135#endif 1136 } 1137 1138 void restore_arg_regs() { 1139 const Register saved_rdi = r9; 1140 const Register saved_rsi = r10; 1141#ifdef _WIN64 1142 __ movq(rdi, saved_rdi); 1143 __ movq(rsi, saved_rsi); 1144#endif 1145 } 1146 1147 // Generate code for an array write pre barrier 1148 // 1149 // addr - starting address 1150 // count - element count 1151 // 1152 // Destroy no registers! 1153 // 1154 void gen_write_ref_array_pre_barrier(Register addr, Register count) { 1155 BarrierSet* bs = Universe::heap()->barrier_set(); 1156 switch (bs->kind()) { 1157 case BarrierSet::G1SATBCT: 1158 case BarrierSet::G1SATBCTLogging: 1159 { 1160 __ pushaq(); // push registers 1161 if (count == c_rarg0) { 1162 if (addr == c_rarg1) { 1163 // exactly backwards!! 1164 __ xchgq(c_rarg1, c_rarg0); 1165 } else { 1166 __ movq(c_rarg1, count); 1167 __ movq(c_rarg0, addr); 1168 } 1169 1170 } else { 1171 __ movq(c_rarg0, addr); 1172 __ movq(c_rarg1, count); 1173 } 1174 __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, BarrierSet::static_write_ref_array_pre))); 1175 __ popaq(); 1176 } 1177 break; 1178 case BarrierSet::CardTableModRef: 1179 case BarrierSet::CardTableExtension: 1180 case BarrierSet::ModRef: 1181 break; 1182 default: 1183 ShouldNotReachHere(); 1184 1185 } 1186 } 1187 1188 // 1189 // Generate code for an array write post barrier 1190 // 1191 // Input: 1192 // start - register containing starting address of destination array 1193 // end - register containing ending address of destination array 1194 // scratch - scratch register 1195 // 1196 // The input registers are overwritten. 1197 // The ending address is inclusive. 1198 void gen_write_ref_array_post_barrier(Register start, Register end, Register scratch) { 1199 assert_different_registers(start, end, scratch); 1200 BarrierSet* bs = Universe::heap()->barrier_set(); 1201 switch (bs->kind()) { 1202 case BarrierSet::G1SATBCT: 1203 case BarrierSet::G1SATBCTLogging: 1204 1205 { 1206 __ pushaq(); // push registers (overkill) 1207 // must compute element count unless barrier set interface is changed (other platforms supply count) 1208 assert_different_registers(start, end, scratch); 1209 __ leaq(scratch, Address(end, wordSize)); 1210 __ subq(scratch, start); 1211 __ shrq(scratch, LogBytesPerWord); 1212 __ movq(c_rarg0, start); 1213 __ movq(c_rarg1, scratch); 1214 __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, BarrierSet::static_write_ref_array_post))); 1215 __ popaq(); 1216 } 1217 break; 1218 case BarrierSet::CardTableModRef: 1219 case BarrierSet::CardTableExtension: 1220 { 1221 CardTableModRefBS* ct = (CardTableModRefBS*)bs; 1222 assert(sizeof(*ct->byte_map_base) == sizeof(jbyte), "adjust this code"); 1223 1224 Label L_loop; 1225 1226 __ shrq(start, CardTableModRefBS::card_shift); 1227 __ shrq(end, CardTableModRefBS::card_shift); 1228 __ subq(end, start); // number of bytes to copy 1229 1230 const Register count = end; // 'end' register contains bytes count now 1231 __ lea(scratch, ExternalAddress((address)ct->byte_map_base)); 1232 __ addq(start, scratch); 1233 __ BIND(L_loop); 1234 __ movb(Address(start, count, Address::times_1), 0); 1235 __ decrementq(count); 1236 __ jcc(Assembler::greaterEqual, L_loop); 1237 } 1238 break; 1239 default: 1240 ShouldNotReachHere(); 1241 1242 } 1243 } 1244 1245 // Copy big chunks forward 1246 // 1247 // Inputs: 1248 // end_from - source arrays end address 1249 // end_to - destination array end address 1250 // qword_count - 64-bits element count, negative 1251 // to - scratch 1252 // L_copy_32_bytes - entry label 1253 // L_copy_8_bytes - exit label 1254 // 1255 void copy_32_bytes_forward(Register end_from, Register end_to, 1256 Register qword_count, Register to, 1257 Label& L_copy_32_bytes, Label& L_copy_8_bytes) { 1258 DEBUG_ONLY(__ stop("enter at entry label, not here")); 1259 Label L_loop; 1260 __ align(16); 1261 __ BIND(L_loop); 1262 __ movq(to, Address(end_from, qword_count, Address::times_8, -24)); 1263 __ movq(Address(end_to, qword_count, Address::times_8, -24), to); 1264 __ movq(to, Address(end_from, qword_count, Address::times_8, -16)); 1265 __ movq(Address(end_to, qword_count, Address::times_8, -16), to); 1266 __ movq(to, Address(end_from, qword_count, Address::times_8, - 8)); 1267 __ movq(Address(end_to, qword_count, Address::times_8, - 8), to); 1268 __ movq(to, Address(end_from, qword_count, Address::times_8, - 0)); 1269 __ movq(Address(end_to, qword_count, Address::times_8, - 0), to); 1270 __ BIND(L_copy_32_bytes); 1271 __ addq(qword_count, 4); 1272 __ jcc(Assembler::lessEqual, L_loop); 1273 __ subq(qword_count, 4); 1274 __ jcc(Assembler::less, L_copy_8_bytes); // Copy trailing qwords 1275 } 1276 1277 1278 // Copy big chunks backward 1279 // 1280 // Inputs: 1281 // from - source arrays address 1282 // dest - destination array address 1283 // qword_count - 64-bits element count 1284 // to - scratch 1285 // L_copy_32_bytes - entry label 1286 // L_copy_8_bytes - exit label 1287 // 1288 void copy_32_bytes_backward(Register from, Register dest, 1289 Register qword_count, Register to, 1290 Label& L_copy_32_bytes, Label& L_copy_8_bytes) { 1291 DEBUG_ONLY(__ stop("enter at entry label, not here")); 1292 Label L_loop; 1293 __ align(16); 1294 __ BIND(L_loop); 1295 __ movq(to, Address(from, qword_count, Address::times_8, 24)); 1296 __ movq(Address(dest, qword_count, Address::times_8, 24), to); 1297 __ movq(to, Address(from, qword_count, Address::times_8, 16)); 1298 __ movq(Address(dest, qword_count, Address::times_8, 16), to); 1299 __ movq(to, Address(from, qword_count, Address::times_8, 8)); 1300 __ movq(Address(dest, qword_count, Address::times_8, 8), to); 1301 __ movq(to, Address(from, qword_count, Address::times_8, 0)); 1302 __ movq(Address(dest, qword_count, Address::times_8, 0), to); 1303 __ BIND(L_copy_32_bytes); 1304 __ subq(qword_count, 4); 1305 __ jcc(Assembler::greaterEqual, L_loop); 1306 __ addq(qword_count, 4); 1307 __ jcc(Assembler::greater, L_copy_8_bytes); // Copy trailing qwords 1308 } 1309 1310 1311 // Arguments: 1312 // aligned - true => Input and output aligned on a HeapWord == 8-byte boundary 1313 // ignored 1314 // name - stub name string 1315 // 1316 // Inputs: 1317 // c_rarg0 - source array address 1318 // c_rarg1 - destination array address 1319 // c_rarg2 - element count, treated as ssize_t, can be zero 1320 // 1321 // If 'from' and/or 'to' are aligned on 4-, 2-, or 1-byte boundaries, 1322 // we let the hardware handle it. The one to eight bytes within words, 1323 // dwords or qwords that span cache line boundaries will still be loaded 1324 // and stored atomically. 1325 // 1326 // Side Effects: 1327 // disjoint_byte_copy_entry is set to the no-overlap entry point 1328 // used by generate_conjoint_byte_copy(). 1329 // 1330 address generate_disjoint_byte_copy(bool aligned, const char *name) { 1331 __ align(CodeEntryAlignment); 1332 StubCodeMark mark(this, "StubRoutines", name); 1333 address start = __ pc(); 1334 1335 Label L_copy_32_bytes, L_copy_8_bytes, L_copy_4_bytes, L_copy_2_bytes; 1336 Label L_copy_byte, L_exit; 1337 const Register from = rdi; // source array address 1338 const Register to = rsi; // destination array address 1339 const Register count = rdx; // elements count 1340 const Register byte_count = rcx; 1341 const Register qword_count = count; 1342 const Register end_from = from; // source array end address 1343 const Register end_to = to; // destination array end address 1344 // End pointers are inclusive, and if count is not zero they point 1345 // to the last unit copied: end_to[0] := end_from[0] 1346 1347 __ enter(); // required for proper stackwalking of RuntimeStub frame 1348 assert_clean_int(c_rarg2, rax); // Make sure 'count' is clean int. 1349 1350 disjoint_byte_copy_entry = __ pc(); 1351 BLOCK_COMMENT("Entry:"); 1352 // caller can pass a 64-bit byte count here (from Unsafe.copyMemory) 1353 1354 setup_arg_regs(); // from => rdi, to => rsi, count => rdx 1355 // r9 and r10 may be used to save non-volatile registers 1356 1357 // 'from', 'to' and 'count' are now valid 1358 __ movq(byte_count, count); 1359 __ shrq(count, 3); // count => qword_count 1360 1361 // Copy from low to high addresses. Use 'to' as scratch. 1362 __ leaq(end_from, Address(from, qword_count, Address::times_8, -8)); 1363 __ leaq(end_to, Address(to, qword_count, Address::times_8, -8)); 1364 __ negq(qword_count); // make the count negative 1365 __ jmp(L_copy_32_bytes); 1366 1367 // Copy trailing qwords 1368 __ BIND(L_copy_8_bytes); 1369 __ movq(rax, Address(end_from, qword_count, Address::times_8, 8)); 1370 __ movq(Address(end_to, qword_count, Address::times_8, 8), rax); 1371 __ incrementq(qword_count); 1372 __ jcc(Assembler::notZero, L_copy_8_bytes); 1373 1374 // Check for and copy trailing dword 1375 __ BIND(L_copy_4_bytes); 1376 __ testq(byte_count, 4); 1377 __ jccb(Assembler::zero, L_copy_2_bytes); 1378 __ movl(rax, Address(end_from, 8)); 1379 __ movl(Address(end_to, 8), rax); 1380 1381 __ addq(end_from, 4); 1382 __ addq(end_to, 4); 1383 1384 // Check for and copy trailing word 1385 __ BIND(L_copy_2_bytes); 1386 __ testq(byte_count, 2); 1387 __ jccb(Assembler::zero, L_copy_byte); 1388 __ movw(rax, Address(end_from, 8)); 1389 __ movw(Address(end_to, 8), rax); 1390 1391 __ addq(end_from, 2); 1392 __ addq(end_to, 2); 1393 1394 // Check for and copy trailing byte 1395 __ BIND(L_copy_byte); 1396 __ testq(byte_count, 1); 1397 __ jccb(Assembler::zero, L_exit); 1398 __ movb(rax, Address(end_from, 8)); 1399 __ movb(Address(end_to, 8), rax); 1400 1401 __ BIND(L_exit); 1402 inc_counter_np(SharedRuntime::_jbyte_array_copy_ctr); 1403 restore_arg_regs(); 1404 __ xorq(rax, rax); // return 0 1405 __ leave(); // required for proper stackwalking of RuntimeStub frame 1406 __ ret(0); 1407 1408 // Copy in 32-bytes chunks 1409 copy_32_bytes_forward(end_from, end_to, qword_count, rax, L_copy_32_bytes, L_copy_8_bytes); 1410 __ jmp(L_copy_4_bytes); 1411 1412 return start; 1413 } 1414 1415 // Arguments: 1416 // aligned - true => Input and output aligned on a HeapWord == 8-byte boundary 1417 // ignored 1418 // name - stub name string 1419 // 1420 // Inputs: 1421 // c_rarg0 - source array address 1422 // c_rarg1 - destination array address 1423 // c_rarg2 - element count, treated as ssize_t, can be zero 1424 // 1425 // If 'from' and/or 'to' are aligned on 4-, 2-, or 1-byte boundaries, 1426 // we let the hardware handle it. The one to eight bytes within words, 1427 // dwords or qwords that span cache line boundaries will still be loaded 1428 // and stored atomically. 1429 // 1430 address generate_conjoint_byte_copy(bool aligned, const char *name) { 1431 __ align(CodeEntryAlignment); 1432 StubCodeMark mark(this, "StubRoutines", name); 1433 address start = __ pc(); 1434 1435 Label L_copy_32_bytes, L_copy_8_bytes, L_copy_4_bytes, L_copy_2_bytes; 1436 const Register from = rdi; // source array address 1437 const Register to = rsi; // destination array address 1438 const Register count = rdx; // elements count 1439 const Register byte_count = rcx; 1440 const Register qword_count = count; 1441 1442 __ enter(); // required for proper stackwalking of RuntimeStub frame 1443 assert_clean_int(c_rarg2, rax); // Make sure 'count' is clean int. 1444 1445 byte_copy_entry = __ pc(); 1446 BLOCK_COMMENT("Entry:"); 1447 // caller can pass a 64-bit byte count here (from Unsafe.copyMemory) 1448 1449 array_overlap_test(disjoint_byte_copy_entry, Address::times_1); 1450 setup_arg_regs(); // from => rdi, to => rsi, count => rdx 1451 // r9 and r10 may be used to save non-volatile registers 1452 1453 // 'from', 'to' and 'count' are now valid 1454 __ movq(byte_count, count); 1455 __ shrq(count, 3); // count => qword_count 1456 1457 // Copy from high to low addresses. 1458 1459 // Check for and copy trailing byte 1460 __ testq(byte_count, 1); 1461 __ jcc(Assembler::zero, L_copy_2_bytes); 1462 __ movb(rax, Address(from, byte_count, Address::times_1, -1)); 1463 __ movb(Address(to, byte_count, Address::times_1, -1), rax); 1464 __ decrementq(byte_count); // Adjust for possible trailing word 1465 1466 // Check for and copy trailing word 1467 __ BIND(L_copy_2_bytes); 1468 __ testq(byte_count, 2); 1469 __ jcc(Assembler::zero, L_copy_4_bytes); 1470 __ movw(rax, Address(from, byte_count, Address::times_1, -2)); 1471 __ movw(Address(to, byte_count, Address::times_1, -2), rax); 1472 1473 // Check for and copy trailing dword 1474 __ BIND(L_copy_4_bytes); 1475 __ testq(byte_count, 4); 1476 __ jcc(Assembler::zero, L_copy_32_bytes); 1477 __ movl(rax, Address(from, qword_count, Address::times_8)); 1478 __ movl(Address(to, qword_count, Address::times_8), rax); 1479 __ jmp(L_copy_32_bytes); 1480 1481 // Copy trailing qwords 1482 __ BIND(L_copy_8_bytes); 1483 __ movq(rax, Address(from, qword_count, Address::times_8, -8)); 1484 __ movq(Address(to, qword_count, Address::times_8, -8), rax); 1485 __ decrementq(qword_count); 1486 __ jcc(Assembler::notZero, L_copy_8_bytes); 1487 1488 inc_counter_np(SharedRuntime::_jbyte_array_copy_ctr); 1489 restore_arg_regs(); 1490 __ xorq(rax, rax); // return 0 1491 __ leave(); // required for proper stackwalking of RuntimeStub frame 1492 __ ret(0); 1493 1494 // Copy in 32-bytes chunks 1495 copy_32_bytes_backward(from, to, qword_count, rax, L_copy_32_bytes, L_copy_8_bytes); 1496 1497 inc_counter_np(SharedRuntime::_jbyte_array_copy_ctr); 1498 restore_arg_regs(); 1499 __ xorq(rax, rax); // return 0 1500 __ leave(); // required for proper stackwalking of RuntimeStub frame 1501 __ ret(0); 1502 1503 return start; 1504 } 1505 1506 // Arguments: 1507 // aligned - true => Input and output aligned on a HeapWord == 8-byte boundary 1508 // ignored 1509 // name - stub name string 1510 // 1511 // Inputs: 1512 // c_rarg0 - source array address 1513 // c_rarg1 - destination array address 1514 // c_rarg2 - element count, treated as ssize_t, can be zero 1515 // 1516 // If 'from' and/or 'to' are aligned on 4- or 2-byte boundaries, we 1517 // let the hardware handle it. The two or four words within dwords 1518 // or qwords that span cache line boundaries will still be loaded 1519 // and stored atomically. 1520 // 1521 // Side Effects: 1522 // disjoint_short_copy_entry is set to the no-overlap entry point 1523 // used by generate_conjoint_short_copy(). 1524 // 1525 address generate_disjoint_short_copy(bool aligned, const char *name) { 1526 __ align(CodeEntryAlignment); 1527 StubCodeMark mark(this, "StubRoutines", name); 1528 address start = __ pc(); 1529 1530 Label L_copy_32_bytes, L_copy_8_bytes, L_copy_4_bytes,L_copy_2_bytes,L_exit; 1531 const Register from = rdi; // source array address 1532 const Register to = rsi; // destination array address 1533 const Register count = rdx; // elements count 1534 const Register word_count = rcx; 1535 const Register qword_count = count; 1536 const Register end_from = from; // source array end address 1537 const Register end_to = to; // destination array end address 1538 // End pointers are inclusive, and if count is not zero they point 1539 // to the last unit copied: end_to[0] := end_from[0] 1540 1541 __ enter(); // required for proper stackwalking of RuntimeStub frame 1542 assert_clean_int(c_rarg2, rax); // Make sure 'count' is clean int. 1543 1544 disjoint_short_copy_entry = __ pc(); 1545 BLOCK_COMMENT("Entry:"); 1546 // caller can pass a 64-bit byte count here (from Unsafe.copyMemory) 1547 1548 setup_arg_regs(); // from => rdi, to => rsi, count => rdx 1549 // r9 and r10 may be used to save non-volatile registers 1550 1551 // 'from', 'to' and 'count' are now valid 1552 __ movq(word_count, count); 1553 __ shrq(count, 2); // count => qword_count 1554 1555 // Copy from low to high addresses. Use 'to' as scratch. 1556 __ leaq(end_from, Address(from, qword_count, Address::times_8, -8)); 1557 __ leaq(end_to, Address(to, qword_count, Address::times_8, -8)); 1558 __ negq(qword_count); 1559 __ jmp(L_copy_32_bytes); 1560 1561 // Copy trailing qwords 1562 __ BIND(L_copy_8_bytes); 1563 __ movq(rax, Address(end_from, qword_count, Address::times_8, 8)); 1564 __ movq(Address(end_to, qword_count, Address::times_8, 8), rax); 1565 __ incrementq(qword_count); 1566 __ jcc(Assembler::notZero, L_copy_8_bytes); 1567 1568 // Original 'dest' is trashed, so we can't use it as a 1569 // base register for a possible trailing word copy 1570 1571 // Check for and copy trailing dword 1572 __ BIND(L_copy_4_bytes); 1573 __ testq(word_count, 2); 1574 __ jccb(Assembler::zero, L_copy_2_bytes); 1575 __ movl(rax, Address(end_from, 8)); 1576 __ movl(Address(end_to, 8), rax); 1577 1578 __ addq(end_from, 4); 1579 __ addq(end_to, 4); 1580 1581 // Check for and copy trailing word 1582 __ BIND(L_copy_2_bytes); 1583 __ testq(word_count, 1); 1584 __ jccb(Assembler::zero, L_exit); 1585 __ movw(rax, Address(end_from, 8)); 1586 __ movw(Address(end_to, 8), rax); 1587 1588 __ BIND(L_exit); 1589 inc_counter_np(SharedRuntime::_jshort_array_copy_ctr); 1590 restore_arg_regs(); 1591 __ xorq(rax, rax); // return 0 1592 __ leave(); // required for proper stackwalking of RuntimeStub frame 1593 __ ret(0); 1594 1595 // Copy in 32-bytes chunks 1596 copy_32_bytes_forward(end_from, end_to, qword_count, rax, L_copy_32_bytes, L_copy_8_bytes); 1597 __ jmp(L_copy_4_bytes); 1598 1599 return start; 1600 } 1601 1602 // Arguments: 1603 // aligned - true => Input and output aligned on a HeapWord == 8-byte boundary 1604 // ignored 1605 // name - stub name string 1606 // 1607 // Inputs: 1608 // c_rarg0 - source array address 1609 // c_rarg1 - destination array address 1610 // c_rarg2 - element count, treated as ssize_t, can be zero 1611 // 1612 // If 'from' and/or 'to' are aligned on 4- or 2-byte boundaries, we 1613 // let the hardware handle it. The two or four words within dwords 1614 // or qwords that span cache line boundaries will still be loaded 1615 // and stored atomically. 1616 // 1617 address generate_conjoint_short_copy(bool aligned, const char *name) { 1618 __ align(CodeEntryAlignment); 1619 StubCodeMark mark(this, "StubRoutines", name); 1620 address start = __ pc(); 1621 1622 Label L_copy_32_bytes, L_copy_8_bytes, L_copy_4_bytes; 1623 const Register from = rdi; // source array address 1624 const Register to = rsi; // destination array address 1625 const Register count = rdx; // elements count 1626 const Register word_count = rcx; 1627 const Register qword_count = count; 1628 1629 __ enter(); // required for proper stackwalking of RuntimeStub frame 1630 assert_clean_int(c_rarg2, rax); // Make sure 'count' is clean int. 1631 1632 short_copy_entry = __ pc(); 1633 BLOCK_COMMENT("Entry:"); 1634 // caller can pass a 64-bit byte count here (from Unsafe.copyMemory) 1635 1636 array_overlap_test(disjoint_short_copy_entry, Address::times_2); 1637 setup_arg_regs(); // from => rdi, to => rsi, count => rdx 1638 // r9 and r10 may be used to save non-volatile registers 1639 1640 // 'from', 'to' and 'count' are now valid 1641 __ movq(word_count, count); 1642 __ shrq(count, 2); // count => qword_count 1643 1644 // Copy from high to low addresses. Use 'to' as scratch. 1645 1646 // Check for and copy trailing word 1647 __ testq(word_count, 1); 1648 __ jccb(Assembler::zero, L_copy_4_bytes); 1649 __ movw(rax, Address(from, word_count, Address::times_2, -2)); 1650 __ movw(Address(to, word_count, Address::times_2, -2), rax); 1651 1652 // Check for and copy trailing dword 1653 __ BIND(L_copy_4_bytes); 1654 __ testq(word_count, 2); 1655 __ jcc(Assembler::zero, L_copy_32_bytes); 1656 __ movl(rax, Address(from, qword_count, Address::times_8)); 1657 __ movl(Address(to, qword_count, Address::times_8), rax); 1658 __ jmp(L_copy_32_bytes); 1659 1660 // Copy trailing qwords 1661 __ BIND(L_copy_8_bytes); 1662 __ movq(rax, Address(from, qword_count, Address::times_8, -8)); 1663 __ movq(Address(to, qword_count, Address::times_8, -8), rax); 1664 __ decrementq(qword_count); 1665 __ jcc(Assembler::notZero, L_copy_8_bytes); 1666 1667 inc_counter_np(SharedRuntime::_jshort_array_copy_ctr); 1668 restore_arg_regs(); 1669 __ xorq(rax, rax); // return 0 1670 __ leave(); // required for proper stackwalking of RuntimeStub frame 1671 __ ret(0); 1672 1673 // Copy in 32-bytes chunks 1674 copy_32_bytes_backward(from, to, qword_count, rax, L_copy_32_bytes, L_copy_8_bytes); 1675 1676 inc_counter_np(SharedRuntime::_jshort_array_copy_ctr); 1677 restore_arg_regs(); 1678 __ xorq(rax, rax); // return 0 1679 __ leave(); // required for proper stackwalking of RuntimeStub frame 1680 __ ret(0); 1681 1682 return start; 1683 } 1684 1685 // Arguments: 1686 // aligned - true => Input and output aligned on a HeapWord == 8-byte boundary 1687 // ignored 1688 // is_oop - true => oop array, so generate store check code 1689 // name - stub name string 1690 // 1691 // Inputs: 1692 // c_rarg0 - source array address 1693 // c_rarg1 - destination array address 1694 // c_rarg2 - element count, treated as ssize_t, can be zero 1695 // 1696 // If 'from' and/or 'to' are aligned on 4-byte boundaries, we let 1697 // the hardware handle it. The two dwords within qwords that span 1698 // cache line boundaries will still be loaded and stored atomicly. 1699 // 1700 // Side Effects: 1701 // disjoint_int_copy_entry is set to the no-overlap entry point 1702 // used by generate_conjoint_int_oop_copy(). 1703 // 1704 address generate_disjoint_int_oop_copy(bool aligned, bool is_oop, const char *name) { 1705 __ align(CodeEntryAlignment); 1706 StubCodeMark mark(this, "StubRoutines", name); 1707 address start = __ pc(); 1708 1709 Label L_copy_32_bytes, L_copy_8_bytes, L_copy_4_bytes, L_exit; 1710 const Register from = rdi; // source array address 1711 const Register to = rsi; // destination array address 1712 const Register count = rdx; // elements count 1713 const Register dword_count = rcx; 1714 const Register qword_count = count; 1715 const Register end_from = from; // source array end address 1716 const Register end_to = to; // destination array end address 1717 const Register saved_to = r11; // saved destination array address 1718 // End pointers are inclusive, and if count is not zero they point 1719 // to the last unit copied: end_to[0] := end_from[0] 1720 1721 __ enter(); // required for proper stackwalking of RuntimeStub frame 1722 assert_clean_int(c_rarg2, rax); // Make sure 'count' is clean int. 1723 1724 (is_oop ? disjoint_oop_copy_entry : disjoint_int_copy_entry) = __ pc(); 1725 1726 if (is_oop) { 1727 // no registers are destroyed by this call 1728 gen_write_ref_array_pre_barrier(/* dest */ c_rarg1, /* count */ c_rarg2); 1729 } 1730 1731 BLOCK_COMMENT("Entry:"); 1732 // caller can pass a 64-bit byte count here (from Unsafe.copyMemory) 1733 1734 setup_arg_regs(); // from => rdi, to => rsi, count => rdx 1735 // r9 and r10 may be used to save non-volatile registers 1736 1737 if (is_oop) { 1738 __ movq(saved_to, to); 1739 } 1740 1741 // 'from', 'to' and 'count' are now valid 1742 __ movq(dword_count, count); 1743 __ shrq(count, 1); // count => qword_count 1744 1745 // Copy from low to high addresses. Use 'to' as scratch. 1746 __ leaq(end_from, Address(from, qword_count, Address::times_8, -8)); 1747 __ leaq(end_to, Address(to, qword_count, Address::times_8, -8)); 1748 __ negq(qword_count); 1749 __ jmp(L_copy_32_bytes); 1750 1751 // Copy trailing qwords 1752 __ BIND(L_copy_8_bytes); 1753 __ movq(rax, Address(end_from, qword_count, Address::times_8, 8)); 1754 __ movq(Address(end_to, qword_count, Address::times_8, 8), rax); 1755 __ incrementq(qword_count); 1756 __ jcc(Assembler::notZero, L_copy_8_bytes); 1757 1758 // Check for and copy trailing dword 1759 __ BIND(L_copy_4_bytes); 1760 __ testq(dword_count, 1); // Only byte test since the value is 0 or 1 1761 __ jccb(Assembler::zero, L_exit); 1762 __ movl(rax, Address(end_from, 8)); 1763 __ movl(Address(end_to, 8), rax); 1764 1765 __ BIND(L_exit); 1766 if (is_oop) { 1767 __ leaq(end_to, Address(saved_to, dword_count, Address::times_4, -4)); 1768 gen_write_ref_array_post_barrier(saved_to, end_to, rax); 1769 } 1770 inc_counter_np(SharedRuntime::_jint_array_copy_ctr); 1771 restore_arg_regs(); 1772 __ xorq(rax, rax); // return 0 1773 __ leave(); // required for proper stackwalking of RuntimeStub frame 1774 __ ret(0); 1775 1776 // Copy 32-bytes chunks 1777 copy_32_bytes_forward(end_from, end_to, qword_count, rax, L_copy_32_bytes, L_copy_8_bytes); 1778 __ jmp(L_copy_4_bytes); 1779 1780 return start; 1781 } 1782 1783 // Arguments: 1784 // aligned - true => Input and output aligned on a HeapWord == 8-byte boundary 1785 // ignored 1786 // is_oop - true => oop array, so generate store check code 1787 // name - stub name string 1788 // 1789 // Inputs: 1790 // c_rarg0 - source array address 1791 // c_rarg1 - destination array address 1792 // c_rarg2 - element count, treated as ssize_t, can be zero 1793 // 1794 // If 'from' and/or 'to' are aligned on 4-byte boundaries, we let 1795 // the hardware handle it. The two dwords within qwords that span 1796 // cache line boundaries will still be loaded and stored atomicly. 1797 // 1798 address generate_conjoint_int_oop_copy(bool aligned, bool is_oop, const char *name) { 1799 __ align(CodeEntryAlignment); 1800 StubCodeMark mark(this, "StubRoutines", name); 1801 address start = __ pc(); 1802 1803 Label L_copy_32_bytes, L_copy_8_bytes, L_copy_2_bytes, L_exit; 1804 const Register from = rdi; // source array address 1805 const Register to = rsi; // destination array address 1806 const Register count = rdx; // elements count 1807 const Register dword_count = rcx; 1808 const Register qword_count = count; 1809 1810 __ enter(); // required for proper stackwalking of RuntimeStub frame 1811 assert_clean_int(c_rarg2, rax); // Make sure 'count' is clean int. 1812 1813 if (is_oop) { 1814 // no registers are destroyed by this call 1815 gen_write_ref_array_pre_barrier(/* dest */ c_rarg1, /* count */ c_rarg2); 1816 } 1817 1818 (is_oop ? oop_copy_entry : int_copy_entry) = __ pc(); 1819 BLOCK_COMMENT("Entry:"); 1820 // caller can pass a 64-bit byte count here (from Unsafe.copyMemory) 1821 1822 array_overlap_test(is_oop ? disjoint_oop_copy_entry : disjoint_int_copy_entry, 1823 Address::times_4); 1824 setup_arg_regs(); // from => rdi, to => rsi, count => rdx 1825 // r9 and r10 may be used to save non-volatile registers 1826 1827 assert_clean_int(count, rax); // Make sure 'count' is clean int. 1828 // 'from', 'to' and 'count' are now valid 1829 __ movq(dword_count, count); 1830 __ shrq(count, 1); // count => qword_count 1831 1832 // Copy from high to low addresses. Use 'to' as scratch. 1833 1834 // Check for and copy trailing dword 1835 __ testq(dword_count, 1); 1836 __ jcc(Assembler::zero, L_copy_32_bytes); 1837 __ movl(rax, Address(from, dword_count, Address::times_4, -4)); 1838 __ movl(Address(to, dword_count, Address::times_4, -4), rax); 1839 __ jmp(L_copy_32_bytes); 1840 1841 // Copy trailing qwords 1842 __ BIND(L_copy_8_bytes); 1843 __ movq(rax, Address(from, qword_count, Address::times_8, -8)); 1844 __ movq(Address(to, qword_count, Address::times_8, -8), rax); 1845 __ decrementq(qword_count); 1846 __ jcc(Assembler::notZero, L_copy_8_bytes); 1847 1848 inc_counter_np(SharedRuntime::_jint_array_copy_ctr); 1849 if (is_oop) { 1850 __ jmp(L_exit); 1851 } 1852 restore_arg_regs(); 1853 __ xorq(rax, rax); // return 0 1854 __ leave(); // required for proper stackwalking of RuntimeStub frame 1855 __ ret(0); 1856 1857 // Copy in 32-bytes chunks 1858 copy_32_bytes_backward(from, to, qword_count, rax, L_copy_32_bytes, L_copy_8_bytes); 1859 1860 inc_counter_np(SharedRuntime::_jint_array_copy_ctr); 1861 __ bind(L_exit); 1862 if (is_oop) { 1863 Register end_to = rdx; 1864 __ leaq(end_to, Address(to, dword_count, Address::times_4, -4)); 1865 gen_write_ref_array_post_barrier(to, end_to, rax); 1866 } 1867 restore_arg_regs(); 1868 __ xorq(rax, rax); // return 0 1869 __ leave(); // required for proper stackwalking of RuntimeStub frame 1870 __ ret(0); 1871 1872 return start; 1873 } 1874 1875 // Arguments: 1876 // aligned - true => Input and output aligned on a HeapWord boundary == 8 bytes 1877 // ignored 1878 // is_oop - true => oop array, so generate store check code 1879 // name - stub name string 1880 // 1881 // Inputs: 1882 // c_rarg0 - source array address 1883 // c_rarg1 - destination array address 1884 // c_rarg2 - element count, treated as ssize_t, can be zero 1885 // 1886 // Side Effects: 1887 // disjoint_oop_copy_entry or disjoint_long_copy_entry is set to the 1888 // no-overlap entry point used by generate_conjoint_long_oop_copy(). 1889 // 1890 address generate_disjoint_long_oop_copy(bool aligned, bool is_oop, const char *name) { 1891 __ align(CodeEntryAlignment); 1892 StubCodeMark mark(this, "StubRoutines", name); 1893 address start = __ pc(); 1894 1895 Label L_copy_32_bytes, L_copy_8_bytes, L_exit; 1896 const Register from = rdi; // source array address 1897 const Register to = rsi; // destination array address 1898 const Register qword_count = rdx; // elements count 1899 const Register end_from = from; // source array end address 1900 const Register end_to = rcx; // destination array end address 1901 const Register saved_to = to; 1902 // End pointers are inclusive, and if count is not zero they point 1903 // to the last unit copied: end_to[0] := end_from[0] 1904 1905 __ enter(); // required for proper stackwalking of RuntimeStub frame 1906 // Save no-overlap entry point for generate_conjoint_long_oop_copy() 1907 assert_clean_int(c_rarg2, rax); // Make sure 'count' is clean int. 1908 1909 if (is_oop) { 1910 disjoint_oop_copy_entry = __ pc(); 1911 // no registers are destroyed by this call 1912 gen_write_ref_array_pre_barrier(/* dest */ c_rarg1, /* count */ c_rarg2); 1913 } else { 1914 disjoint_long_copy_entry = __ pc(); 1915 } 1916 BLOCK_COMMENT("Entry:"); 1917 // caller can pass a 64-bit byte count here (from Unsafe.copyMemory) 1918 1919 setup_arg_regs(); // from => rdi, to => rsi, count => rdx 1920 // r9 and r10 may be used to save non-volatile registers 1921 1922 // 'from', 'to' and 'qword_count' are now valid 1923 1924 // Copy from low to high addresses. Use 'to' as scratch. 1925 __ leaq(end_from, Address(from, qword_count, Address::times_8, -8)); 1926 __ leaq(end_to, Address(to, qword_count, Address::times_8, -8)); 1927 __ negq(qword_count); 1928 __ jmp(L_copy_32_bytes); 1929 1930 // Copy trailing qwords 1931 __ BIND(L_copy_8_bytes); 1932 __ movq(rax, Address(end_from, qword_count, Address::times_8, 8)); 1933 __ movq(Address(end_to, qword_count, Address::times_8, 8), rax); 1934 __ incrementq(qword_count); 1935 __ jcc(Assembler::notZero, L_copy_8_bytes); 1936 1937 if (is_oop) { 1938 __ jmp(L_exit); 1939 } else { 1940 inc_counter_np(SharedRuntime::_jlong_array_copy_ctr); 1941 restore_arg_regs(); 1942 __ xorq(rax, rax); // return 0 1943 __ leave(); // required for proper stackwalking of RuntimeStub frame 1944 __ ret(0); 1945 } 1946 1947 // Copy 64-byte chunks 1948 copy_32_bytes_forward(end_from, end_to, qword_count, rax, L_copy_32_bytes, L_copy_8_bytes); 1949 1950 if (is_oop) { 1951 __ BIND(L_exit); 1952 gen_write_ref_array_post_barrier(saved_to, end_to, rax); 1953 inc_counter_np(SharedRuntime::_oop_array_copy_ctr); 1954 } else { 1955 inc_counter_np(SharedRuntime::_jlong_array_copy_ctr); 1956 } 1957 restore_arg_regs(); 1958 __ xorq(rax, rax); // return 0 1959 __ leave(); // required for proper stackwalking of RuntimeStub frame 1960 __ ret(0); 1961 1962 return start; 1963 } 1964 1965 // Arguments: 1966 // aligned - true => Input and output aligned on a HeapWord boundary == 8 bytes 1967 // ignored 1968 // is_oop - true => oop array, so generate store check code 1969 // name - stub name string 1970 // 1971 // Inputs: 1972 // c_rarg0 - source array address 1973 // c_rarg1 - destination array address 1974 // c_rarg2 - element count, treated as ssize_t, can be zero 1975 // 1976 address generate_conjoint_long_oop_copy(bool aligned, bool is_oop, const char *name) { 1977 __ align(CodeEntryAlignment); 1978 StubCodeMark mark(this, "StubRoutines", name); 1979 address start = __ pc(); 1980 1981 Label L_copy_32_bytes, L_copy_8_bytes, L_exit; 1982 const Register from = rdi; // source array address 1983 const Register to = rsi; // destination array address 1984 const Register qword_count = rdx; // elements count 1985 const Register saved_count = rcx; 1986 1987 __ enter(); // required for proper stackwalking of RuntimeStub frame 1988 assert_clean_int(c_rarg2, rax); // Make sure 'count' is clean int. 1989 1990 address disjoint_copy_entry = NULL; 1991 if (is_oop) { 1992 assert(!UseCompressedOops, "shouldn't be called for compressed oops"); 1993 disjoint_copy_entry = disjoint_oop_copy_entry; 1994 oop_copy_entry = __ pc(); 1995 array_overlap_test(disjoint_oop_copy_entry, Address::times_8); 1996 } else { 1997 disjoint_copy_entry = disjoint_long_copy_entry; 1998 long_copy_entry = __ pc(); 1999 array_overlap_test(disjoint_long_copy_entry, Address::times_8); 2000 } 2001 BLOCK_COMMENT("Entry:"); 2002 // caller can pass a 64-bit byte count here (from Unsafe.copyMemory) 2003 2004 array_overlap_test(disjoint_copy_entry, Address::times_8); 2005 setup_arg_regs(); // from => rdi, to => rsi, count => rdx 2006 // r9 and r10 may be used to save non-volatile registers 2007 2008 // 'from', 'to' and 'qword_count' are now valid 2009 2010 if (is_oop) { 2011 // Save to and count for store barrier 2012 __ movq(saved_count, qword_count); 2013 // No registers are destroyed by this call 2014 gen_write_ref_array_pre_barrier(to, saved_count); 2015 } 2016 2017 __ jmp(L_copy_32_bytes); 2018 2019 // Copy trailing qwords 2020 __ BIND(L_copy_8_bytes); 2021 __ movq(rax, Address(from, qword_count, Address::times_8, -8)); 2022 __ movq(Address(to, qword_count, Address::times_8, -8), rax); 2023 __ decrementq(qword_count); 2024 __ jcc(Assembler::notZero, L_copy_8_bytes); 2025 2026 if (is_oop) { 2027 __ jmp(L_exit); 2028 } else { 2029 inc_counter_np(SharedRuntime::_jlong_array_copy_ctr); 2030 restore_arg_regs(); 2031 __ xorq(rax, rax); // return 0 2032 __ leave(); // required for proper stackwalking of RuntimeStub frame 2033 __ ret(0); 2034 } 2035 2036 // Copy in 32-bytes chunks 2037 copy_32_bytes_backward(from, to, qword_count, rax, L_copy_32_bytes, L_copy_8_bytes); 2038 2039 if (is_oop) { 2040 __ BIND(L_exit); 2041 __ leaq(rcx, Address(to, saved_count, Address::times_8, -8)); 2042 gen_write_ref_array_post_barrier(to, rcx, rax); 2043 inc_counter_np(SharedRuntime::_oop_array_copy_ctr); 2044 } else { 2045 inc_counter_np(SharedRuntime::_jlong_array_copy_ctr); 2046 } 2047 restore_arg_regs(); 2048 __ xorq(rax, rax); // return 0 2049 __ leave(); // required for proper stackwalking of RuntimeStub frame 2050 __ ret(0); 2051 2052 return start; 2053 } 2054 2055 2056 // Helper for generating a dynamic type check. 2057 // Smashes no registers. 2058 void generate_type_check(Register sub_klass, 2059 Register super_check_offset, 2060 Register super_klass, 2061 Label& L_success) { 2062 assert_different_registers(sub_klass, super_check_offset, super_klass); 2063 2064 BLOCK_COMMENT("type_check:"); 2065 2066 Label L_miss; 2067 2068 // a couple of useful fields in sub_klass: 2069 int ss_offset = (klassOopDesc::header_size() * HeapWordSize + 2070 Klass::secondary_supers_offset_in_bytes()); 2071 int sc_offset = (klassOopDesc::header_size() * HeapWordSize + 2072 Klass::secondary_super_cache_offset_in_bytes()); 2073 Address secondary_supers_addr(sub_klass, ss_offset); 2074 Address super_cache_addr( sub_klass, sc_offset); 2075 2076 // if the pointers are equal, we are done (e.g., String[] elements) 2077 __ cmpq(super_klass, sub_klass); 2078 __ jcc(Assembler::equal, L_success); 2079 2080 // check the supertype display: 2081 Address super_check_addr(sub_klass, super_check_offset, Address::times_1, 0); 2082 __ cmpq(super_klass, super_check_addr); // test the super type 2083 __ jcc(Assembler::equal, L_success); 2084 2085 // if it was a primary super, we can just fail immediately 2086 __ cmpl(super_check_offset, sc_offset); 2087 __ jcc(Assembler::notEqual, L_miss); 2088 2089 // Now do a linear scan of the secondary super-klass chain. 2090 // The repne_scan instruction uses fixed registers, which we must spill. 2091 // (We need a couple more temps in any case.) 2092 // This code is rarely used, so simplicity is a virtue here. 2093 inc_counter_np(SharedRuntime::_partial_subtype_ctr); 2094 { 2095 __ pushq(rax); 2096 __ pushq(rcx); 2097 __ pushq(rdi); 2098 assert_different_registers(sub_klass, super_klass, rax, rcx, rdi); 2099 2100 __ movq(rdi, secondary_supers_addr); 2101 // Load the array length. 2102 __ movl(rcx, Address(rdi, arrayOopDesc::length_offset_in_bytes())); 2103 // Skip to start of data. 2104 __ addq(rdi, arrayOopDesc::base_offset_in_bytes(T_OBJECT)); 2105 // Scan rcx words at [rdi] for occurance of rax 2106 // Set NZ/Z based on last compare 2107 __ movq(rax, super_klass); 2108 if (UseCompressedOops) { 2109 // Compare against compressed form. Don't need to uncompress because 2110 // looks like orig rax is restored in popq below. 2111 __ encode_heap_oop(rax); 2112 __ repne_scanl(); 2113 } else { 2114 __ repne_scanq(); 2115 } 2116 2117 // Unspill the temp. registers: 2118 __ popq(rdi); 2119 __ popq(rcx); 2120 __ popq(rax); 2121 2122 __ jcc(Assembler::notEqual, L_miss); 2123 } 2124 2125 // Success. Cache the super we found and proceed in triumph. 2126 __ movq(super_cache_addr, super_klass); // note: rax is dead 2127 __ jmp(L_success); 2128 2129 // Fall through on failure! 2130 __ BIND(L_miss); 2131 } 2132 2133 // 2134 // Generate checkcasting array copy stub 2135 // 2136 // Input: 2137 // c_rarg0 - source array address 2138 // c_rarg1 - destination array address 2139 // c_rarg2 - element count, treated as ssize_t, can be zero 2140 // c_rarg3 - size_t ckoff (super_check_offset) 2141 // not Win64 2142 // c_rarg4 - oop ckval (super_klass) 2143 // Win64 2144 // rsp+40 - oop ckval (super_klass) 2145 // 2146 // Output: 2147 // rax == 0 - success 2148 // rax == -1^K - failure, where K is partial transfer count 2149 // 2150 address generate_checkcast_copy(const char *name) { 2151 2152 Label L_load_element, L_store_element, L_do_card_marks, L_done; 2153 2154 // Input registers (after setup_arg_regs) 2155 const Register from = rdi; // source array address 2156 const Register to = rsi; // destination array address 2157 const Register length = rdx; // elements count 2158 const Register ckoff = rcx; // super_check_offset 2159 const Register ckval = r8; // super_klass 2160 2161 // Registers used as temps (r13, r14 are save-on-entry) 2162 const Register end_from = from; // source array end address 2163 const Register end_to = r13; // destination array end address 2164 const Register count = rdx; // -(count_remaining) 2165 const Register r14_length = r14; // saved copy of length 2166 // End pointers are inclusive, and if length is not zero they point 2167 // to the last unit copied: end_to[0] := end_from[0] 2168 2169 const Register rax_oop = rax; // actual oop copied 2170 const Register r11_klass = r11; // oop._klass 2171 2172 //--------------------------------------------------------------- 2173 // Assembler stub will be used for this call to arraycopy 2174 // if the two arrays are subtypes of Object[] but the 2175 // destination array type is not equal to or a supertype 2176 // of the source type. Each element must be separately 2177 // checked. 2178 2179 __ align(CodeEntryAlignment); 2180 StubCodeMark mark(this, "StubRoutines", name); 2181 address start = __ pc(); 2182 2183 __ enter(); // required for proper stackwalking of RuntimeStub frame 2184 2185 checkcast_copy_entry = __ pc(); 2186 BLOCK_COMMENT("Entry:"); 2187 2188#ifdef ASSERT 2189 // caller guarantees that the arrays really are different 2190 // otherwise, we would have to make conjoint checks 2191 { Label L; 2192 array_overlap_test(L, TIMES_OOP); 2193 __ stop("checkcast_copy within a single array"); 2194 __ bind(L); 2195 } 2196#endif //ASSERT 2197 2198 // allocate spill slots for r13, r14 2199 enum { 2200 saved_r13_offset, 2201 saved_r14_offset, 2202 saved_rbp_offset, 2203 saved_rip_offset, 2204 saved_rarg0_offset 2205 }; 2206 __ subq(rsp, saved_rbp_offset * wordSize); 2207 __ movq(Address(rsp, saved_r13_offset * wordSize), r13); 2208 __ movq(Address(rsp, saved_r14_offset * wordSize), r14); 2209 setup_arg_regs(4); // from => rdi, to => rsi, length => rdx 2210 // ckoff => rcx, ckval => r8 2211 // r9 and r10 may be used to save non-volatile registers 2212#ifdef _WIN64 2213 // last argument (#4) is on stack on Win64 2214 const int ckval_offset = saved_rarg0_offset + 4; 2215 __ movq(ckval, Address(rsp, ckval_offset * wordSize)); 2216#endif 2217 2218 // check that int operands are properly extended to size_t 2219 assert_clean_int(length, rax); 2220 assert_clean_int(ckoff, rax); 2221 2222#ifdef ASSERT 2223 BLOCK_COMMENT("assert consistent ckoff/ckval"); 2224 // The ckoff and ckval must be mutually consistent, 2225 // even though caller generates both. 2226 { Label L; 2227 int sco_offset = (klassOopDesc::header_size() * HeapWordSize + 2228 Klass::super_check_offset_offset_in_bytes()); 2229 __ cmpl(ckoff, Address(ckval, sco_offset)); 2230 __ jcc(Assembler::equal, L); 2231 __ stop("super_check_offset inconsistent"); 2232 __ bind(L); 2233 } 2234#endif //ASSERT 2235 2236 // Loop-invariant addresses. They are exclusive end pointers. 2237 Address end_from_addr(from, length, TIMES_OOP, 0); 2238 Address end_to_addr(to, length, TIMES_OOP, 0); 2239 // Loop-variant addresses. They assume post-incremented count < 0. 2240 Address from_element_addr(end_from, count, TIMES_OOP, 0); 2241 Address to_element_addr(end_to, count, TIMES_OOP, 0); 2242 2243 gen_write_ref_array_pre_barrier(to, count); 2244 2245 // Copy from low to high addresses, indexed from the end of each array. 2246 __ leaq(end_from, end_from_addr); 2247 __ leaq(end_to, end_to_addr); 2248 __ movq(r14_length, length); // save a copy of the length 2249 assert(length == count, ""); // else fix next line: 2250 __ negq(count); // negate and test the length 2251 __ jcc(Assembler::notZero, L_load_element); 2252 2253 // Empty array: Nothing to do. 2254 __ xorq(rax, rax); // return 0 on (trivial) success 2255 __ jmp(L_done); 2256 2257 // ======== begin loop ======== 2258 // (Loop is rotated; its entry is L_load_element.) 2259 // Loop control: 2260 // for (count = -count; count != 0; count++) 2261 // Base pointers src, dst are biased by 8*(count-1),to last element. 2262 __ align(16); 2263 2264 __ BIND(L_store_element); 2265 __ store_heap_oop(to_element_addr, rax_oop); // store the oop 2266 __ incrementq(count); // increment the count toward zero 2267 __ jcc(Assembler::zero, L_do_card_marks); 2268 2269 // ======== loop entry is here ======== 2270 __ BIND(L_load_element); 2271 __ load_heap_oop(rax_oop, from_element_addr); // load the oop 2272 __ testq(rax_oop, rax_oop); 2273 __ jcc(Assembler::zero, L_store_element); 2274 2275 __ load_klass(r11_klass, rax_oop);// query the object klass 2276 generate_type_check(r11_klass, ckoff, ckval, L_store_element); 2277 // ======== end loop ======== 2278 2279 // It was a real error; we must depend on the caller to finish the job. 2280 // Register rdx = -1 * number of *remaining* oops, r14 = *total* oops. 2281 // Emit GC store barriers for the oops we have copied (r14 + rdx), 2282 // and report their number to the caller. 2283 assert_different_registers(rax, r14_length, count, to, end_to, rcx); 2284 __ leaq(end_to, to_element_addr); 2285 gen_write_ref_array_post_barrier(to, end_to, rcx); 2286 __ movq(rax, r14_length); // original oops 2287 __ addq(rax, count); // K = (original - remaining) oops 2288 __ notq(rax); // report (-1^K) to caller 2289 __ jmp(L_done); 2290 2291 // Come here on success only. 2292 __ BIND(L_do_card_marks); 2293 __ addq(end_to, -wordSize); // make an inclusive end pointer 2294 gen_write_ref_array_post_barrier(to, end_to, rcx); 2295 __ xorq(rax, rax); // return 0 on success 2296 2297 // Common exit point (success or failure). 2298 __ BIND(L_done); 2299 __ movq(r13, Address(rsp, saved_r13_offset * wordSize)); 2300 __ movq(r14, Address(rsp, saved_r14_offset * wordSize)); 2301 inc_counter_np(SharedRuntime::_checkcast_array_copy_ctr); 2302 restore_arg_regs(); 2303 __ leave(); // required for proper stackwalking of RuntimeStub frame 2304 __ ret(0); 2305 2306 return start; 2307 } 2308 2309 // 2310 // Generate 'unsafe' array copy stub 2311 // Though just as safe as the other stubs, it takes an unscaled 2312 // size_t argument instead of an element count. 2313 // 2314 // Input: 2315 // c_rarg0 - source array address 2316 // c_rarg1 - destination array address 2317 // c_rarg2 - byte count, treated as ssize_t, can be zero 2318 // 2319 // Examines the alignment of the operands and dispatches 2320 // to a long, int, short, or byte copy loop. 2321 // 2322 address generate_unsafe_copy(const char *name) { 2323 2324 Label L_long_aligned, L_int_aligned, L_short_aligned; 2325 2326 // Input registers (before setup_arg_regs) 2327 const Register from = c_rarg0; // source array address 2328 const Register to = c_rarg1; // destination array address 2329 const Register size = c_rarg2; // byte count (size_t) 2330 2331 // Register used as a temp 2332 const Register bits = rax; // test copy of low bits 2333 2334 __ align(CodeEntryAlignment); 2335 StubCodeMark mark(this, "StubRoutines", name); 2336 address start = __ pc(); 2337 2338 __ enter(); // required for proper stackwalking of RuntimeStub frame 2339 2340 // bump this on entry, not on exit: 2341 inc_counter_np(SharedRuntime::_unsafe_array_copy_ctr); 2342 2343 __ movq(bits, from); 2344 __ orq(bits, to); 2345 __ orq(bits, size); 2346 2347 __ testb(bits, BytesPerLong-1); 2348 __ jccb(Assembler::zero, L_long_aligned); 2349 2350 __ testb(bits, BytesPerInt-1); 2351 __ jccb(Assembler::zero, L_int_aligned); 2352 2353 __ testb(bits, BytesPerShort-1); 2354 __ jump_cc(Assembler::notZero, RuntimeAddress(byte_copy_entry)); 2355 2356 __ BIND(L_short_aligned); 2357 __ shrq(size, LogBytesPerShort); // size => short_count 2358 __ jump(RuntimeAddress(short_copy_entry)); 2359 2360 __ BIND(L_int_aligned); 2361 __ shrq(size, LogBytesPerInt); // size => int_count 2362 __ jump(RuntimeAddress(int_copy_entry)); 2363 2364 __ BIND(L_long_aligned); 2365 __ shrq(size, LogBytesPerLong); // size => qword_count 2366 __ jump(RuntimeAddress(long_copy_entry)); 2367 2368 return start; 2369 } 2370 2371 // Perform range checks on the proposed arraycopy. 2372 // Kills temp, but nothing else. 2373 // Also, clean the sign bits of src_pos and dst_pos. 2374 void arraycopy_range_checks(Register src, // source array oop (c_rarg0) 2375 Register src_pos, // source position (c_rarg1) 2376 Register dst, // destination array oo (c_rarg2) 2377 Register dst_pos, // destination position (c_rarg3) 2378 Register length, 2379 Register temp, 2380 Label& L_failed) { 2381 BLOCK_COMMENT("arraycopy_range_checks:"); 2382 2383 // if (src_pos + length > arrayOop(src)->length()) FAIL; 2384 __ movl(temp, length); 2385 __ addl(temp, src_pos); // src_pos + length 2386 __ cmpl(temp, Address(src, arrayOopDesc::length_offset_in_bytes())); 2387 __ jcc(Assembler::above, L_failed); 2388 2389 // if (dst_pos + length > arrayOop(dst)->length()) FAIL; 2390 __ movl(temp, length); 2391 __ addl(temp, dst_pos); // dst_pos + length 2392 __ cmpl(temp, Address(dst, arrayOopDesc::length_offset_in_bytes())); 2393 __ jcc(Assembler::above, L_failed); 2394 2395 // Have to clean up high 32-bits of 'src_pos' and 'dst_pos'. 2396 // Move with sign extension can be used since they are positive. 2397 __ movslq(src_pos, src_pos); 2398 __ movslq(dst_pos, dst_pos); 2399 2400 BLOCK_COMMENT("arraycopy_range_checks done"); 2401 } 2402 2403 // 2404 // Generate generic array copy stubs 2405 // 2406 // Input: 2407 // c_rarg0 - src oop 2408 // c_rarg1 - src_pos (32-bits) 2409 // c_rarg2 - dst oop 2410 // c_rarg3 - dst_pos (32-bits) 2411 // not Win64 2412 // c_rarg4 - element count (32-bits) 2413 // Win64 2414 // rsp+40 - element count (32-bits) 2415 // 2416 // Output: 2417 // rax == 0 - success 2418 // rax == -1^K - failure, where K is partial transfer count 2419 // 2420 address generate_generic_copy(const char *name) { 2421 2422 Label L_failed, L_failed_0, L_objArray; 2423 Label L_copy_bytes, L_copy_shorts, L_copy_ints, L_copy_longs; 2424 2425 // Input registers 2426 const Register src = c_rarg0; // source array oop 2427 const Register src_pos = c_rarg1; // source position 2428 const Register dst = c_rarg2; // destination array oop 2429 const Register dst_pos = c_rarg3; // destination position 2430 // elements count is on stack on Win64 2431#ifdef _WIN64 2432#define C_RARG4 Address(rsp, 6 * wordSize) 2433#else 2434#define C_RARG4 c_rarg4 2435#endif 2436 2437 { int modulus = CodeEntryAlignment; 2438 int target = modulus - 5; // 5 = sizeof jmp(L_failed) 2439 int advance = target - (__ offset() % modulus); 2440 if (advance < 0) advance += modulus; 2441 if (advance > 0) __ nop(advance); 2442 } 2443 StubCodeMark mark(this, "StubRoutines", name); 2444 2445 // Short-hop target to L_failed. Makes for denser prologue code. 2446 __ BIND(L_failed_0); 2447 __ jmp(L_failed); 2448 assert(__ offset() % CodeEntryAlignment == 0, "no further alignment needed"); 2449 2450 __ align(CodeEntryAlignment); 2451 address start = __ pc(); 2452 2453 __ enter(); // required for proper stackwalking of RuntimeStub frame 2454 2455 // bump this on entry, not on exit: 2456 inc_counter_np(SharedRuntime::_generic_array_copy_ctr); 2457 2458 //----------------------------------------------------------------------- 2459 // Assembler stub will be used for this call to arraycopy 2460 // if the following conditions are met: 2461 // 2462 // (1) src and dst must not be null. 2463 // (2) src_pos must not be negative. 2464 // (3) dst_pos must not be negative. 2465 // (4) length must not be negative. 2466 // (5) src klass and dst klass should be the same and not NULL. 2467 // (6) src and dst should be arrays. 2468 // (7) src_pos + length must not exceed length of src. 2469 // (8) dst_pos + length must not exceed length of dst. 2470 // 2471 2472 // if (src == NULL) return -1; 2473 __ testq(src, src); // src oop 2474 size_t j1off = __ offset(); 2475 __ jccb(Assembler::zero, L_failed_0); 2476 2477 // if (src_pos < 0) return -1; 2478 __ testl(src_pos, src_pos); // src_pos (32-bits) 2479 __ jccb(Assembler::negative, L_failed_0); 2480 2481 // if (dst == NULL) return -1; 2482 __ testq(dst, dst); // dst oop 2483 __ jccb(Assembler::zero, L_failed_0); 2484 2485 // if (dst_pos < 0) return -1; 2486 __ testl(dst_pos, dst_pos); // dst_pos (32-bits) 2487 size_t j4off = __ offset(); 2488 __ jccb(Assembler::negative, L_failed_0); 2489 2490 // The first four tests are very dense code, 2491 // but not quite dense enough to put four 2492 // jumps in a 16-byte instruction fetch buffer. 2493 // That's good, because some branch predicters 2494 // do not like jumps so close together. 2495 // Make sure of this. 2496 guarantee(((j1off ^ j4off) & ~15) != 0, "I$ line of 1st & 4th jumps"); 2497 2498 // registers used as temp 2499 const Register r11_length = r11; // elements count to copy 2500 const Register r10_src_klass = r10; // array klass 2501 const Register r9_dst_klass = r9; // dest array klass 2502 2503 // if (length < 0) return -1; 2504 __ movl(r11_length, C_RARG4); // length (elements count, 32-bits value) 2505 __ testl(r11_length, r11_length); 2506 __ jccb(Assembler::negative, L_failed_0); 2507 2508 __ load_klass(r10_src_klass, src); 2509#ifdef ASSERT 2510 // assert(src->klass() != NULL); 2511 BLOCK_COMMENT("assert klasses not null"); 2512 { Label L1, L2; 2513 __ testq(r10_src_klass, r10_src_klass); 2514 __ jcc(Assembler::notZero, L2); // it is broken if klass is NULL 2515 __ bind(L1); 2516 __ stop("broken null klass"); 2517 __ bind(L2); 2518 __ load_klass(r9_dst_klass, dst); 2519 __ cmpq(r9_dst_klass, 0); 2520 __ jcc(Assembler::equal, L1); // this would be broken also 2521 BLOCK_COMMENT("assert done"); 2522 } 2523#endif 2524 2525 // Load layout helper (32-bits) 2526 // 2527 // |array_tag| | header_size | element_type | |log2_element_size| 2528 // 32 30 24 16 8 2 0 2529 // 2530 // array_tag: typeArray = 0x3, objArray = 0x2, non-array = 0x0 2531 // 2532 2533 int lh_offset = klassOopDesc::header_size() * HeapWordSize + 2534 Klass::layout_helper_offset_in_bytes(); 2535 2536 const Register rax_lh = rax; // layout helper 2537 2538 __ movl(rax_lh, Address(r10_src_klass, lh_offset)); 2539 2540 // Handle objArrays completely differently... 2541 jint objArray_lh = Klass::array_layout_helper(T_OBJECT); 2542 __ cmpl(rax_lh, objArray_lh); 2543 __ jcc(Assembler::equal, L_objArray); 2544 2545 // if (src->klass() != dst->klass()) return -1; 2546 __ load_klass(r9_dst_klass, dst); 2547 __ cmpq(r10_src_klass, r9_dst_klass); 2548 __ jcc(Assembler::notEqual, L_failed); 2549 2550 // if (!src->is_Array()) return -1; 2551 __ cmpl(rax_lh, Klass::_lh_neutral_value); 2552 __ jcc(Assembler::greaterEqual, L_failed); 2553 2554 // At this point, it is known to be a typeArray (array_tag 0x3). 2555#ifdef ASSERT 2556 { Label L; 2557 __ cmpl(rax_lh, (Klass::_lh_array_tag_type_value << Klass::_lh_array_tag_shift)); 2558 __ jcc(Assembler::greaterEqual, L); 2559 __ stop("must be a primitive array"); 2560 __ bind(L); 2561 } 2562#endif 2563 2564 arraycopy_range_checks(src, src_pos, dst, dst_pos, r11_length, 2565 r10, L_failed); 2566 2567 // typeArrayKlass 2568 // 2569 // src_addr = (src + array_header_in_bytes()) + (src_pos << log2elemsize); 2570 // dst_addr = (dst + array_header_in_bytes()) + (dst_pos << log2elemsize); 2571 // 2572 2573 const Register r10_offset = r10; // array offset 2574 const Register rax_elsize = rax_lh; // element size 2575 2576 __ movl(r10_offset, rax_lh); 2577 __ shrl(r10_offset, Klass::_lh_header_size_shift); 2578 __ andq(r10_offset, Klass::_lh_header_size_mask); // array_offset 2579 __ addq(src, r10_offset); // src array offset 2580 __ addq(dst, r10_offset); // dst array offset 2581 BLOCK_COMMENT("choose copy loop based on element size"); 2582 __ andl(rax_lh, Klass::_lh_log2_element_size_mask); // rax_lh -> rax_elsize 2583 2584 // next registers should be set before the jump to corresponding stub 2585 const Register from = c_rarg0; // source array address 2586 const Register to = c_rarg1; // destination array address 2587 const Register count = c_rarg2; // elements count 2588 2589 // 'from', 'to', 'count' registers should be set in such order 2590 // since they are the same as 'src', 'src_pos', 'dst'. 2591 2592 __ BIND(L_copy_bytes); 2593 __ cmpl(rax_elsize, 0); 2594 __ jccb(Assembler::notEqual, L_copy_shorts); 2595 __ leaq(from, Address(src, src_pos, Address::times_1, 0));// src_addr 2596 __ leaq(to, Address(dst, dst_pos, Address::times_1, 0));// dst_addr 2597 __ movslq(count, r11_length); // length 2598 __ jump(RuntimeAddress(byte_copy_entry)); 2599 2600 __ BIND(L_copy_shorts); 2601 __ cmpl(rax_elsize, LogBytesPerShort); 2602 __ jccb(Assembler::notEqual, L_copy_ints); 2603 __ leaq(from, Address(src, src_pos, Address::times_2, 0));// src_addr 2604 __ leaq(to, Address(dst, dst_pos, Address::times_2, 0));// dst_addr 2605 __ movslq(count, r11_length); // length 2606 __ jump(RuntimeAddress(short_copy_entry)); 2607 2608 __ BIND(L_copy_ints); 2609 __ cmpl(rax_elsize, LogBytesPerInt); 2610 __ jccb(Assembler::notEqual, L_copy_longs); 2611 __ leaq(from, Address(src, src_pos, Address::times_4, 0));// src_addr 2612 __ leaq(to, Address(dst, dst_pos, Address::times_4, 0));// dst_addr 2613 __ movslq(count, r11_length); // length 2614 __ jump(RuntimeAddress(int_copy_entry)); 2615 2616 __ BIND(L_copy_longs); 2617#ifdef ASSERT 2618 { Label L; 2619 __ cmpl(rax_elsize, LogBytesPerLong); 2620 __ jcc(Assembler::equal, L); 2621 __ stop("must be long copy, but elsize is wrong"); 2622 __ bind(L); 2623 } 2624#endif 2625 __ leaq(from, Address(src, src_pos, Address::times_8, 0));// src_addr 2626 __ leaq(to, Address(dst, dst_pos, Address::times_8, 0));// dst_addr 2627 __ movslq(count, r11_length); // length 2628 __ jump(RuntimeAddress(long_copy_entry)); 2629 2630 // objArrayKlass 2631 __ BIND(L_objArray); 2632 // live at this point: r10_src_klass, src[_pos], dst[_pos] 2633 2634 Label L_plain_copy, L_checkcast_copy; 2635 // test array classes for subtyping 2636 __ load_klass(r9_dst_klass, dst); 2637 __ cmpq(r10_src_klass, r9_dst_klass); // usual case is exact equality 2638 __ jcc(Assembler::notEqual, L_checkcast_copy); 2639 2640 // Identically typed arrays can be copied without element-wise checks. 2641 arraycopy_range_checks(src, src_pos, dst, dst_pos, r11_length, 2642 r10, L_failed); 2643 2644 __ leaq(from, Address(src, src_pos, TIMES_OOP, 2645 arrayOopDesc::base_offset_in_bytes(T_OBJECT))); // src_addr 2646 __ leaq(to, Address(dst, dst_pos, TIMES_OOP, 2647 arrayOopDesc::base_offset_in_bytes(T_OBJECT))); // dst_addr 2648 __ movslq(count, r11_length); // length 2649 __ BIND(L_plain_copy); 2650 __ jump(RuntimeAddress(oop_copy_entry)); 2651 2652 __ BIND(L_checkcast_copy); 2653 // live at this point: r10_src_klass, !r11_length 2654 { 2655 // assert(r11_length == C_RARG4); // will reload from here 2656 Register r11_dst_klass = r11; 2657 __ load_klass(r11_dst_klass, dst); 2658 2659 // Before looking at dst.length, make sure dst is also an objArray. 2660 __ cmpl(Address(r11_dst_klass, lh_offset), objArray_lh); 2661 __ jcc(Assembler::notEqual, L_failed); 2662 2663 // It is safe to examine both src.length and dst.length. 2664#ifndef _WIN64 2665 arraycopy_range_checks(src, src_pos, dst, dst_pos, C_RARG4, 2666 rax, L_failed); 2667#else 2668 __ movl(r11_length, C_RARG4); // reload 2669 arraycopy_range_checks(src, src_pos, dst, dst_pos, r11_length, 2670 rax, L_failed); 2671 __ load_klass(r11_dst_klass, dst); // reload 2672#endif 2673 2674 // Marshal the base address arguments now, freeing registers. 2675 __ leaq(from, Address(src, src_pos, TIMES_OOP, 2676 arrayOopDesc::base_offset_in_bytes(T_OBJECT))); 2677 __ leaq(to, Address(dst, dst_pos, TIMES_OOP, 2678 arrayOopDesc::base_offset_in_bytes(T_OBJECT))); 2679 __ movl(count, C_RARG4); // length (reloaded) 2680 Register sco_temp = c_rarg3; // this register is free now 2681 assert_different_registers(from, to, count, sco_temp, 2682 r11_dst_klass, r10_src_klass); 2683 assert_clean_int(count, sco_temp); 2684 2685 // Generate the type check. 2686 int sco_offset = (klassOopDesc::header_size() * HeapWordSize + 2687 Klass::super_check_offset_offset_in_bytes()); 2688 __ movl(sco_temp, Address(r11_dst_klass, sco_offset)); 2689 assert_clean_int(sco_temp, rax); 2690 generate_type_check(r10_src_klass, sco_temp, r11_dst_klass, L_plain_copy); 2691 2692 // Fetch destination element klass from the objArrayKlass header. 2693 int ek_offset = (klassOopDesc::header_size() * HeapWordSize + 2694 objArrayKlass::element_klass_offset_in_bytes()); 2695 __ movq(r11_dst_klass, Address(r11_dst_klass, ek_offset)); 2696 __ movl(sco_temp, Address(r11_dst_klass, sco_offset)); 2697 assert_clean_int(sco_temp, rax); 2698 2699 // the checkcast_copy loop needs two extra arguments: 2700 assert(c_rarg3 == sco_temp, "#3 already in place"); 2701 __ movq(C_RARG4, r11_dst_klass); // dst.klass.element_klass 2702 __ jump(RuntimeAddress(checkcast_copy_entry)); 2703 } 2704 2705 __ BIND(L_failed); 2706 __ xorq(rax, rax); 2707 __ notq(rax); // return -1 2708 __ leave(); // required for proper stackwalking of RuntimeStub frame 2709 __ ret(0); 2710 2711 return start; 2712 } 2713 2714#undef length_arg 2715 2716 void generate_arraycopy_stubs() { 2717 // Call the conjoint generation methods immediately after 2718 // the disjoint ones so that short branches from the former 2719 // to the latter can be generated. 2720 StubRoutines::_jbyte_disjoint_arraycopy = generate_disjoint_byte_copy(false, "jbyte_disjoint_arraycopy"); 2721 StubRoutines::_jbyte_arraycopy = generate_conjoint_byte_copy(false, "jbyte_arraycopy"); 2722 2723 StubRoutines::_jshort_disjoint_arraycopy = generate_disjoint_short_copy(false, "jshort_disjoint_arraycopy"); 2724 StubRoutines::_jshort_arraycopy = generate_conjoint_short_copy(false, "jshort_arraycopy"); 2725 2726 StubRoutines::_jint_disjoint_arraycopy = generate_disjoint_int_oop_copy(false, false, "jint_disjoint_arraycopy"); 2727 StubRoutines::_jint_arraycopy = generate_conjoint_int_oop_copy(false, false, "jint_arraycopy"); 2728 2729 StubRoutines::_jlong_disjoint_arraycopy = generate_disjoint_long_oop_copy(false, false, "jlong_disjoint_arraycopy"); 2730 StubRoutines::_jlong_arraycopy = generate_conjoint_long_oop_copy(false, false, "jlong_arraycopy"); 2731 2732 2733 if (UseCompressedOops) { 2734 StubRoutines::_oop_disjoint_arraycopy = generate_disjoint_int_oop_copy(false, true, "oop_disjoint_arraycopy"); 2735 StubRoutines::_oop_arraycopy = generate_conjoint_int_oop_copy(false, true, "oop_arraycopy"); 2736 } else { 2737 StubRoutines::_oop_disjoint_arraycopy = generate_disjoint_long_oop_copy(false, true, "oop_disjoint_arraycopy"); 2738 StubRoutines::_oop_arraycopy = generate_conjoint_long_oop_copy(false, true, "oop_arraycopy"); 2739 } 2740 2741 StubRoutines::_checkcast_arraycopy = generate_checkcast_copy("checkcast_arraycopy"); 2742 StubRoutines::_unsafe_arraycopy = generate_unsafe_copy("unsafe_arraycopy"); 2743 StubRoutines::_generic_arraycopy = generate_generic_copy("generic_arraycopy"); 2744 2745 // We don't generate specialized code for HeapWord-aligned source 2746 // arrays, so just use the code we've already generated 2747 StubRoutines::_arrayof_jbyte_disjoint_arraycopy = StubRoutines::_jbyte_disjoint_arraycopy; 2748 StubRoutines::_arrayof_jbyte_arraycopy = StubRoutines::_jbyte_arraycopy; 2749 2750 StubRoutines::_arrayof_jshort_disjoint_arraycopy = StubRoutines::_jshort_disjoint_arraycopy; 2751 StubRoutines::_arrayof_jshort_arraycopy = StubRoutines::_jshort_arraycopy; 2752 2753 StubRoutines::_arrayof_jint_disjoint_arraycopy = StubRoutines::_jint_disjoint_arraycopy; 2754 StubRoutines::_arrayof_jint_arraycopy = StubRoutines::_jint_arraycopy; 2755 2756 StubRoutines::_arrayof_jlong_disjoint_arraycopy = StubRoutines::_jlong_disjoint_arraycopy; 2757 StubRoutines::_arrayof_jlong_arraycopy = StubRoutines::_jlong_arraycopy; 2758 2759 StubRoutines::_arrayof_oop_disjoint_arraycopy = StubRoutines::_oop_disjoint_arraycopy; 2760 StubRoutines::_arrayof_oop_arraycopy = StubRoutines::_oop_arraycopy; 2761 } 2762 2763#undef __ 2764#define __ masm-> 2765 2766 // Continuation point for throwing of implicit exceptions that are 2767 // not handled in the current activation. Fabricates an exception 2768 // oop and initiates normal exception dispatching in this 2769 // frame. Since we need to preserve callee-saved values (currently 2770 // only for C2, but done for C1 as well) we need a callee-saved oop 2771 // map and therefore have to make these stubs into RuntimeStubs 2772 // rather than BufferBlobs. If the compiler needs all registers to 2773 // be preserved between the fault point and the exception handler 2774 // then it must assume responsibility for that in 2775 // AbstractCompiler::continuation_for_implicit_null_exception or 2776 // continuation_for_implicit_division_by_zero_exception. All other 2777 // implicit exceptions (e.g., NullPointerException or 2778 // AbstractMethodError on entry) are either at call sites or 2779 // otherwise assume that stack unwinding will be initiated, so 2780 // caller saved registers were assumed volatile in the compiler. 2781 address generate_throw_exception(const char* name, 2782 address runtime_entry, 2783 bool restore_saved_exception_pc) { 2784 // Information about frame layout at time of blocking runtime call. 2785 // Note that we only have to preserve callee-saved registers since 2786 // the compilers are responsible for supplying a continuation point 2787 // if they expect all registers to be preserved. 2788 enum layout { 2789 rbp_off = frame::arg_reg_save_area_bytes/BytesPerInt, 2790 rbp_off2, 2791 return_off, 2792 return_off2, 2793 framesize // inclusive of return address 2794 }; 2795 2796 int insts_size = 512; 2797 int locs_size = 64; 2798 2799 CodeBuffer code(name, insts_size, locs_size); 2800 OopMapSet* oop_maps = new OopMapSet(); 2801 MacroAssembler* masm = new MacroAssembler(&code); 2802 2803 address start = __ pc(); 2804 2805 // This is an inlined and slightly modified version of call_VM 2806 // which has the ability to fetch the return PC out of 2807 // thread-local storage and also sets up last_Java_sp slightly 2808 // differently than the real call_VM 2809 if (restore_saved_exception_pc) { 2810 __ movq(rax, 2811 Address(r15_thread, 2812 in_bytes(JavaThread::saved_exception_pc_offset()))); 2813 __ pushq(rax); 2814 } 2815 2816 __ enter(); // required for proper stackwalking of RuntimeStub frame 2817 2818 assert(is_even(framesize/2), "sp not 16-byte aligned"); 2819 2820 // return address and rbp are already in place 2821 __ subq(rsp, (framesize-4) << LogBytesPerInt); // prolog 2822 2823 int frame_complete = __ pc() - start; 2824 2825 // Set up last_Java_sp and last_Java_fp 2826 __ set_last_Java_frame(rsp, rbp, NULL); 2827 2828 // Call runtime 2829 __ movq(c_rarg0, r15_thread); 2830 BLOCK_COMMENT("call runtime_entry"); 2831 __ call(RuntimeAddress(runtime_entry)); 2832 2833 // Generate oop map 2834 OopMap* map = new OopMap(framesize, 0); 2835 2836 oop_maps->add_gc_map(__ pc() - start, map); 2837 2838 __ reset_last_Java_frame(true, false); 2839 2840 __ leave(); // required for proper stackwalking of RuntimeStub frame 2841 2842 // check for pending exceptions 2843#ifdef ASSERT 2844 Label L; 2845 __ cmpq(Address(r15_thread, Thread::pending_exception_offset()), 2846 (int) NULL); 2847 __ jcc(Assembler::notEqual, L); 2848 __ should_not_reach_here(); 2849 __ bind(L); 2850#endif // ASSERT 2851 __ jump(RuntimeAddress(StubRoutines::forward_exception_entry())); 2852 2853 2854 // codeBlob framesize is in words (not VMRegImpl::slot_size) 2855 RuntimeStub* stub = 2856 RuntimeStub::new_runtime_stub(name, 2857 &code, 2858 frame_complete, 2859 (framesize >> (LogBytesPerWord - LogBytesPerInt)), 2860 oop_maps, false); 2861 return stub->entry_point(); 2862 } 2863 2864 // Initialization 2865 void generate_initial() { 2866 // Generates all stubs and initializes the entry points 2867 2868 // This platform-specific stub is needed by generate_call_stub() 2869 StubRoutines::amd64::_mxcsr_std = generate_fp_mask("mxcsr_std", 0x0000000000001F80); 2870 2871 // entry points that exist in all platforms Note: This is code 2872 // that could be shared among different platforms - however the 2873 // benefit seems to be smaller than the disadvantage of having a 2874 // much more complicated generator structure. See also comment in 2875 // stubRoutines.hpp. 2876 2877 StubRoutines::_forward_exception_entry = generate_forward_exception(); 2878 2879 StubRoutines::_call_stub_entry = 2880 generate_call_stub(StubRoutines::_call_stub_return_address); 2881 2882 // is referenced by megamorphic call 2883 StubRoutines::_catch_exception_entry = generate_catch_exception(); 2884 2885 // atomic calls 2886 StubRoutines::_atomic_xchg_entry = generate_atomic_xchg(); 2887 StubRoutines::_atomic_xchg_ptr_entry = generate_atomic_xchg_ptr(); 2888 StubRoutines::_atomic_cmpxchg_entry = generate_atomic_cmpxchg(); 2889 StubRoutines::_atomic_cmpxchg_long_entry = generate_atomic_cmpxchg_long(); 2890 StubRoutines::_atomic_add_entry = generate_atomic_add(); 2891 StubRoutines::_atomic_add_ptr_entry = generate_atomic_add_ptr(); 2892 StubRoutines::_fence_entry = generate_orderaccess_fence(); 2893 2894 StubRoutines::_handler_for_unsafe_access_entry = 2895 generate_handler_for_unsafe_access(); 2896 2897 // platform dependent 2898 StubRoutines::amd64::_get_previous_fp_entry = generate_get_previous_fp(); 2899 2900 StubRoutines::amd64::_verify_mxcsr_entry = generate_verify_mxcsr(); 2901 } 2902 2903 void generate_all() { 2904 // Generates all stubs and initializes the entry points 2905 2906 // These entry points require SharedInfo::stack0 to be set up in 2907 // non-core builds and need to be relocatable, so they each 2908 // fabricate a RuntimeStub internally. 2909 StubRoutines::_throw_AbstractMethodError_entry = 2910 generate_throw_exception("AbstractMethodError throw_exception", 2911 CAST_FROM_FN_PTR(address, 2912 SharedRuntime:: 2913 throw_AbstractMethodError), 2914 false); 2915 2916 StubRoutines::_throw_IncompatibleClassChangeError_entry = 2917 generate_throw_exception("IncompatibleClassChangeError throw_exception", 2918 CAST_FROM_FN_PTR(address, 2919 SharedRuntime:: 2920 throw_IncompatibleClassChangeError), 2921 false); 2922 2923 StubRoutines::_throw_ArithmeticException_entry = 2924 generate_throw_exception("ArithmeticException throw_exception", 2925 CAST_FROM_FN_PTR(address, 2926 SharedRuntime:: 2927 throw_ArithmeticException), 2928 true); 2929 2930 StubRoutines::_throw_NullPointerException_entry = 2931 generate_throw_exception("NullPointerException throw_exception", 2932 CAST_FROM_FN_PTR(address, 2933 SharedRuntime:: 2934 throw_NullPointerException), 2935 true); 2936 2937 StubRoutines::_throw_NullPointerException_at_call_entry = 2938 generate_throw_exception("NullPointerException at call throw_exception", 2939 CAST_FROM_FN_PTR(address, 2940 SharedRuntime:: 2941 throw_NullPointerException_at_call), 2942 false); 2943 2944 StubRoutines::_throw_StackOverflowError_entry = 2945 generate_throw_exception("StackOverflowError throw_exception", 2946 CAST_FROM_FN_PTR(address, 2947 SharedRuntime:: 2948 throw_StackOverflowError), 2949 false); 2950 2951 // entry points that are platform specific 2952 StubRoutines::amd64::_f2i_fixup = generate_f2i_fixup(); 2953 StubRoutines::amd64::_f2l_fixup = generate_f2l_fixup(); 2954 StubRoutines::amd64::_d2i_fixup = generate_d2i_fixup(); 2955 StubRoutines::amd64::_d2l_fixup = generate_d2l_fixup(); 2956 2957 StubRoutines::amd64::_float_sign_mask = generate_fp_mask("float_sign_mask", 0x7FFFFFFF7FFFFFFF); 2958 StubRoutines::amd64::_float_sign_flip = generate_fp_mask("float_sign_flip", 0x8000000080000000); 2959 StubRoutines::amd64::_double_sign_mask = generate_fp_mask("double_sign_mask", 0x7FFFFFFFFFFFFFFF); 2960 StubRoutines::amd64::_double_sign_flip = generate_fp_mask("double_sign_flip", 0x8000000000000000); 2961 2962 // support for verify_oop (must happen after universe_init) 2963 StubRoutines::_verify_oop_subroutine_entry = generate_verify_oop(); 2964 2965 // arraycopy stubs used by compilers 2966 generate_arraycopy_stubs(); 2967 } 2968 2969 public: 2970 StubGenerator(CodeBuffer* code, bool all) : StubCodeGenerator(code) { 2971 if (all) { 2972 generate_all(); 2973 } else { 2974 generate_initial(); 2975 } 2976 } 2977}; // end class declaration 2978 2979address StubGenerator::disjoint_byte_copy_entry = NULL; 2980address StubGenerator::disjoint_short_copy_entry = NULL; 2981address StubGenerator::disjoint_int_copy_entry = NULL; 2982address StubGenerator::disjoint_long_copy_entry = NULL; 2983address StubGenerator::disjoint_oop_copy_entry = NULL; 2984 2985address StubGenerator::byte_copy_entry = NULL; 2986address StubGenerator::short_copy_entry = NULL; 2987address StubGenerator::int_copy_entry = NULL; 2988address StubGenerator::long_copy_entry = NULL; 2989address StubGenerator::oop_copy_entry = NULL; 2990 2991address StubGenerator::checkcast_copy_entry = NULL; 2992 2993void StubGenerator_generate(CodeBuffer* code, bool all) { 2994 StubGenerator g(code, all); 2995} 2996