macro.cpp revision 1472:c18cbe5936b8
1/* 2 * Copyright (c) 2005, 2009, Oracle and/or its affiliates. All rights reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. 8 * 9 * This code is distributed in the hope that it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 12 * version 2 for more details (a copy is included in the LICENSE file that 13 * accompanied this code). 14 * 15 * You should have received a copy of the GNU General Public License version 16 * 2 along with this work; if not, write to the Free Software Foundation, 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 18 * 19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 20 * or visit www.oracle.com if you need additional information or have any 21 * questions. 22 * 23 */ 24 25#include "incls/_precompiled.incl" 26#include "incls/_macro.cpp.incl" 27 28 29// 30// Replace any references to "oldref" in inputs to "use" with "newref". 31// Returns the number of replacements made. 32// 33int PhaseMacroExpand::replace_input(Node *use, Node *oldref, Node *newref) { 34 int nreplacements = 0; 35 uint req = use->req(); 36 for (uint j = 0; j < use->len(); j++) { 37 Node *uin = use->in(j); 38 if (uin == oldref) { 39 if (j < req) 40 use->set_req(j, newref); 41 else 42 use->set_prec(j, newref); 43 nreplacements++; 44 } else if (j >= req && uin == NULL) { 45 break; 46 } 47 } 48 return nreplacements; 49} 50 51void PhaseMacroExpand::copy_call_debug_info(CallNode *oldcall, CallNode * newcall) { 52 // Copy debug information and adjust JVMState information 53 uint old_dbg_start = oldcall->tf()->domain()->cnt(); 54 uint new_dbg_start = newcall->tf()->domain()->cnt(); 55 int jvms_adj = new_dbg_start - old_dbg_start; 56 assert (new_dbg_start == newcall->req(), "argument count mismatch"); 57 58 Dict* sosn_map = new Dict(cmpkey,hashkey); 59 for (uint i = old_dbg_start; i < oldcall->req(); i++) { 60 Node* old_in = oldcall->in(i); 61 // Clone old SafePointScalarObjectNodes, adjusting their field contents. 62 if (old_in != NULL && old_in->is_SafePointScalarObject()) { 63 SafePointScalarObjectNode* old_sosn = old_in->as_SafePointScalarObject(); 64 uint old_unique = C->unique(); 65 Node* new_in = old_sosn->clone(jvms_adj, sosn_map); 66 if (old_unique != C->unique()) { 67 new_in->set_req(0, newcall->in(0)); // reset control edge 68 new_in = transform_later(new_in); // Register new node. 69 } 70 old_in = new_in; 71 } 72 newcall->add_req(old_in); 73 } 74 75 newcall->set_jvms(oldcall->jvms()); 76 for (JVMState *jvms = newcall->jvms(); jvms != NULL; jvms = jvms->caller()) { 77 jvms->set_map(newcall); 78 jvms->set_locoff(jvms->locoff()+jvms_adj); 79 jvms->set_stkoff(jvms->stkoff()+jvms_adj); 80 jvms->set_monoff(jvms->monoff()+jvms_adj); 81 jvms->set_scloff(jvms->scloff()+jvms_adj); 82 jvms->set_endoff(jvms->endoff()+jvms_adj); 83 } 84} 85 86Node* PhaseMacroExpand::opt_bits_test(Node* ctrl, Node* region, int edge, Node* word, int mask, int bits, bool return_fast_path) { 87 Node* cmp; 88 if (mask != 0) { 89 Node* and_node = transform_later(new (C, 3) AndXNode(word, MakeConX(mask))); 90 cmp = transform_later(new (C, 3) CmpXNode(and_node, MakeConX(bits))); 91 } else { 92 cmp = word; 93 } 94 Node* bol = transform_later(new (C, 2) BoolNode(cmp, BoolTest::ne)); 95 IfNode* iff = new (C, 2) IfNode( ctrl, bol, PROB_MIN, COUNT_UNKNOWN ); 96 transform_later(iff); 97 98 // Fast path taken. 99 Node *fast_taken = transform_later( new (C, 1) IfFalseNode(iff) ); 100 101 // Fast path not-taken, i.e. slow path 102 Node *slow_taken = transform_later( new (C, 1) IfTrueNode(iff) ); 103 104 if (return_fast_path) { 105 region->init_req(edge, slow_taken); // Capture slow-control 106 return fast_taken; 107 } else { 108 region->init_req(edge, fast_taken); // Capture fast-control 109 return slow_taken; 110 } 111} 112 113//--------------------copy_predefined_input_for_runtime_call-------------------- 114void PhaseMacroExpand::copy_predefined_input_for_runtime_call(Node * ctrl, CallNode* oldcall, CallNode* call) { 115 // Set fixed predefined input arguments 116 call->init_req( TypeFunc::Control, ctrl ); 117 call->init_req( TypeFunc::I_O , oldcall->in( TypeFunc::I_O) ); 118 call->init_req( TypeFunc::Memory , oldcall->in( TypeFunc::Memory ) ); // ????? 119 call->init_req( TypeFunc::ReturnAdr, oldcall->in( TypeFunc::ReturnAdr ) ); 120 call->init_req( TypeFunc::FramePtr, oldcall->in( TypeFunc::FramePtr ) ); 121} 122 123//------------------------------make_slow_call--------------------------------- 124CallNode* PhaseMacroExpand::make_slow_call(CallNode *oldcall, const TypeFunc* slow_call_type, address slow_call, const char* leaf_name, Node* slow_path, Node* parm0, Node* parm1) { 125 126 // Slow-path call 127 int size = slow_call_type->domain()->cnt(); 128 CallNode *call = leaf_name 129 ? (CallNode*)new (C, size) CallLeafNode ( slow_call_type, slow_call, leaf_name, TypeRawPtr::BOTTOM ) 130 : (CallNode*)new (C, size) CallStaticJavaNode( slow_call_type, slow_call, OptoRuntime::stub_name(slow_call), oldcall->jvms()->bci(), TypeRawPtr::BOTTOM ); 131 132 // Slow path call has no side-effects, uses few values 133 copy_predefined_input_for_runtime_call(slow_path, oldcall, call ); 134 if (parm0 != NULL) call->init_req(TypeFunc::Parms+0, parm0); 135 if (parm1 != NULL) call->init_req(TypeFunc::Parms+1, parm1); 136 copy_call_debug_info(oldcall, call); 137 call->set_cnt(PROB_UNLIKELY_MAG(4)); // Same effect as RC_UNCOMMON. 138 _igvn.hash_delete(oldcall); 139 _igvn.subsume_node(oldcall, call); 140 transform_later(call); 141 142 return call; 143} 144 145void PhaseMacroExpand::extract_call_projections(CallNode *call) { 146 _fallthroughproj = NULL; 147 _fallthroughcatchproj = NULL; 148 _ioproj_fallthrough = NULL; 149 _ioproj_catchall = NULL; 150 _catchallcatchproj = NULL; 151 _memproj_fallthrough = NULL; 152 _memproj_catchall = NULL; 153 _resproj = NULL; 154 for (DUIterator_Fast imax, i = call->fast_outs(imax); i < imax; i++) { 155 ProjNode *pn = call->fast_out(i)->as_Proj(); 156 switch (pn->_con) { 157 case TypeFunc::Control: 158 { 159 // For Control (fallthrough) and I_O (catch_all_index) we have CatchProj -> Catch -> Proj 160 _fallthroughproj = pn; 161 DUIterator_Fast jmax, j = pn->fast_outs(jmax); 162 const Node *cn = pn->fast_out(j); 163 if (cn->is_Catch()) { 164 ProjNode *cpn = NULL; 165 for (DUIterator_Fast kmax, k = cn->fast_outs(kmax); k < kmax; k++) { 166 cpn = cn->fast_out(k)->as_Proj(); 167 assert(cpn->is_CatchProj(), "must be a CatchProjNode"); 168 if (cpn->_con == CatchProjNode::fall_through_index) 169 _fallthroughcatchproj = cpn; 170 else { 171 assert(cpn->_con == CatchProjNode::catch_all_index, "must be correct index."); 172 _catchallcatchproj = cpn; 173 } 174 } 175 } 176 break; 177 } 178 case TypeFunc::I_O: 179 if (pn->_is_io_use) 180 _ioproj_catchall = pn; 181 else 182 _ioproj_fallthrough = pn; 183 break; 184 case TypeFunc::Memory: 185 if (pn->_is_io_use) 186 _memproj_catchall = pn; 187 else 188 _memproj_fallthrough = pn; 189 break; 190 case TypeFunc::Parms: 191 _resproj = pn; 192 break; 193 default: 194 assert(false, "unexpected projection from allocation node."); 195 } 196 } 197 198} 199 200// Eliminate a card mark sequence. p2x is a ConvP2XNode 201void PhaseMacroExpand::eliminate_card_mark(Node* p2x) { 202 assert(p2x->Opcode() == Op_CastP2X, "ConvP2XNode required"); 203 if (!UseG1GC) { 204 // vanilla/CMS post barrier 205 Node *shift = p2x->unique_out(); 206 Node *addp = shift->unique_out(); 207 for (DUIterator_Last jmin, j = addp->last_outs(jmin); j >= jmin; --j) { 208 Node *st = addp->last_out(j); 209 assert(st->is_Store(), "store required"); 210 _igvn.replace_node(st, st->in(MemNode::Memory)); 211 } 212 } else { 213 // G1 pre/post barriers 214 assert(p2x->outcnt() == 2, "expects 2 users: Xor and URShift nodes"); 215 // It could be only one user, URShift node, in Object.clone() instrinsic 216 // but the new allocation is passed to arraycopy stub and it could not 217 // be scalar replaced. So we don't check the case. 218 219 // Remove G1 post barrier. 220 221 // Search for CastP2X->Xor->URShift->Cmp path which 222 // checks if the store done to a different from the value's region. 223 // And replace Cmp with #0 (false) to collapse G1 post barrier. 224 Node* xorx = NULL; 225 for (DUIterator_Fast imax, i = p2x->fast_outs(imax); i < imax; i++) { 226 Node* u = p2x->fast_out(i); 227 if (u->Opcode() == Op_XorX) { 228 xorx = u; 229 break; 230 } 231 } 232 assert(xorx != NULL, "missing G1 post barrier"); 233 Node* shift = xorx->unique_out(); 234 Node* cmpx = shift->unique_out(); 235 assert(cmpx->is_Cmp() && cmpx->unique_out()->is_Bool() && 236 cmpx->unique_out()->as_Bool()->_test._test == BoolTest::ne, 237 "missing region check in G1 post barrier"); 238 _igvn.replace_node(cmpx, makecon(TypeInt::CC_EQ)); 239 240 // Remove G1 pre barrier. 241 242 // Search "if (marking != 0)" check and set it to "false". 243 Node* this_region = p2x->in(0); 244 assert(this_region != NULL, ""); 245 // There is no G1 pre barrier if previous stored value is NULL 246 // (for example, after initialization). 247 if (this_region->is_Region() && this_region->req() == 3) { 248 int ind = 1; 249 if (!this_region->in(ind)->is_IfFalse()) { 250 ind = 2; 251 } 252 if (this_region->in(ind)->is_IfFalse()) { 253 Node* bol = this_region->in(ind)->in(0)->in(1); 254 assert(bol->is_Bool(), ""); 255 cmpx = bol->in(1); 256 if (bol->as_Bool()->_test._test == BoolTest::ne && 257 cmpx->is_Cmp() && cmpx->in(2) == intcon(0) && 258 cmpx->in(1)->is_Load()) { 259 Node* adr = cmpx->in(1)->as_Load()->in(MemNode::Address); 260 const int marking_offset = in_bytes(JavaThread::satb_mark_queue_offset() + 261 PtrQueue::byte_offset_of_active()); 262 if (adr->is_AddP() && adr->in(AddPNode::Base) == top() && 263 adr->in(AddPNode::Address)->Opcode() == Op_ThreadLocal && 264 adr->in(AddPNode::Offset) == MakeConX(marking_offset)) { 265 _igvn.replace_node(cmpx, makecon(TypeInt::CC_EQ)); 266 } 267 } 268 } 269 } 270 // Now CastP2X can be removed since it is used only on dead path 271 // which currently still alive until igvn optimize it. 272 assert(p2x->unique_out()->Opcode() == Op_URShiftX, ""); 273 _igvn.replace_node(p2x, top()); 274 } 275} 276 277// Search for a memory operation for the specified memory slice. 278static Node *scan_mem_chain(Node *mem, int alias_idx, int offset, Node *start_mem, Node *alloc, PhaseGVN *phase) { 279 Node *orig_mem = mem; 280 Node *alloc_mem = alloc->in(TypeFunc::Memory); 281 const TypeOopPtr *tinst = phase->C->get_adr_type(alias_idx)->isa_oopptr(); 282 while (true) { 283 if (mem == alloc_mem || mem == start_mem ) { 284 return mem; // hit one of our sentinels 285 } else if (mem->is_MergeMem()) { 286 mem = mem->as_MergeMem()->memory_at(alias_idx); 287 } else if (mem->is_Proj() && mem->as_Proj()->_con == TypeFunc::Memory) { 288 Node *in = mem->in(0); 289 // we can safely skip over safepoints, calls, locks and membars because we 290 // already know that the object is safe to eliminate. 291 if (in->is_Initialize() && in->as_Initialize()->allocation() == alloc) { 292 return in; 293 } else if (in->is_Call()) { 294 CallNode *call = in->as_Call(); 295 if (!call->may_modify(tinst, phase)) { 296 mem = call->in(TypeFunc::Memory); 297 } 298 mem = in->in(TypeFunc::Memory); 299 } else if (in->is_MemBar()) { 300 mem = in->in(TypeFunc::Memory); 301 } else { 302 assert(false, "unexpected projection"); 303 } 304 } else if (mem->is_Store()) { 305 const TypePtr* atype = mem->as_Store()->adr_type(); 306 int adr_idx = Compile::current()->get_alias_index(atype); 307 if (adr_idx == alias_idx) { 308 assert(atype->isa_oopptr(), "address type must be oopptr"); 309 int adr_offset = atype->offset(); 310 uint adr_iid = atype->is_oopptr()->instance_id(); 311 // Array elements references have the same alias_idx 312 // but different offset and different instance_id. 313 if (adr_offset == offset && adr_iid == alloc->_idx) 314 return mem; 315 } else { 316 assert(adr_idx == Compile::AliasIdxRaw, "address must match or be raw"); 317 } 318 mem = mem->in(MemNode::Memory); 319 } else if (mem->is_ClearArray()) { 320 if (!ClearArrayNode::step_through(&mem, alloc->_idx, phase)) { 321 // Can not bypass initialization of the instance 322 // we are looking. 323 debug_only(intptr_t offset;) 324 assert(alloc == AllocateNode::Ideal_allocation(mem->in(3), phase, offset), "sanity"); 325 InitializeNode* init = alloc->as_Allocate()->initialization(); 326 // We are looking for stored value, return Initialize node 327 // or memory edge from Allocate node. 328 if (init != NULL) 329 return init; 330 else 331 return alloc->in(TypeFunc::Memory); // It will produce zero value (see callers). 332 } 333 // Otherwise skip it (the call updated 'mem' value). 334 } else if (mem->Opcode() == Op_SCMemProj) { 335 assert(mem->in(0)->is_LoadStore(), "sanity"); 336 const TypePtr* atype = mem->in(0)->in(MemNode::Address)->bottom_type()->is_ptr(); 337 int adr_idx = Compile::current()->get_alias_index(atype); 338 if (adr_idx == alias_idx) { 339 assert(false, "Object is not scalar replaceable if a LoadStore node access its field"); 340 return NULL; 341 } 342 mem = mem->in(0)->in(MemNode::Memory); 343 } else { 344 return mem; 345 } 346 assert(mem != orig_mem, "dead memory loop"); 347 } 348} 349 350// 351// Given a Memory Phi, compute a value Phi containing the values from stores 352// on the input paths. 353// Note: this function is recursive, its depth is limied by the "level" argument 354// Returns the computed Phi, or NULL if it cannot compute it. 355Node *PhaseMacroExpand::value_from_mem_phi(Node *mem, BasicType ft, const Type *phi_type, const TypeOopPtr *adr_t, Node *alloc, Node_Stack *value_phis, int level) { 356 assert(mem->is_Phi(), "sanity"); 357 int alias_idx = C->get_alias_index(adr_t); 358 int offset = adr_t->offset(); 359 int instance_id = adr_t->instance_id(); 360 361 // Check if an appropriate value phi already exists. 362 Node* region = mem->in(0); 363 for (DUIterator_Fast kmax, k = region->fast_outs(kmax); k < kmax; k++) { 364 Node* phi = region->fast_out(k); 365 if (phi->is_Phi() && phi != mem && 366 phi->as_Phi()->is_same_inst_field(phi_type, instance_id, alias_idx, offset)) { 367 return phi; 368 } 369 } 370 // Check if an appropriate new value phi already exists. 371 Node* new_phi = NULL; 372 uint size = value_phis->size(); 373 for (uint i=0; i < size; i++) { 374 if ( mem->_idx == value_phis->index_at(i) ) { 375 return value_phis->node_at(i); 376 } 377 } 378 379 if (level <= 0) { 380 return NULL; // Give up: phi tree too deep 381 } 382 Node *start_mem = C->start()->proj_out(TypeFunc::Memory); 383 Node *alloc_mem = alloc->in(TypeFunc::Memory); 384 385 uint length = mem->req(); 386 GrowableArray <Node *> values(length, length, NULL); 387 388 // create a new Phi for the value 389 PhiNode *phi = new (C, length) PhiNode(mem->in(0), phi_type, NULL, instance_id, alias_idx, offset); 390 transform_later(phi); 391 value_phis->push(phi, mem->_idx); 392 393 for (uint j = 1; j < length; j++) { 394 Node *in = mem->in(j); 395 if (in == NULL || in->is_top()) { 396 values.at_put(j, in); 397 } else { 398 Node *val = scan_mem_chain(in, alias_idx, offset, start_mem, alloc, &_igvn); 399 if (val == start_mem || val == alloc_mem) { 400 // hit a sentinel, return appropriate 0 value 401 values.at_put(j, _igvn.zerocon(ft)); 402 continue; 403 } 404 if (val->is_Initialize()) { 405 val = val->as_Initialize()->find_captured_store(offset, type2aelembytes(ft), &_igvn); 406 } 407 if (val == NULL) { 408 return NULL; // can't find a value on this path 409 } 410 if (val == mem) { 411 values.at_put(j, mem); 412 } else if (val->is_Store()) { 413 values.at_put(j, val->in(MemNode::ValueIn)); 414 } else if(val->is_Proj() && val->in(0) == alloc) { 415 values.at_put(j, _igvn.zerocon(ft)); 416 } else if (val->is_Phi()) { 417 val = value_from_mem_phi(val, ft, phi_type, adr_t, alloc, value_phis, level-1); 418 if (val == NULL) { 419 return NULL; 420 } 421 values.at_put(j, val); 422 } else if (val->Opcode() == Op_SCMemProj) { 423 assert(val->in(0)->is_LoadStore(), "sanity"); 424 assert(false, "Object is not scalar replaceable if a LoadStore node access its field"); 425 return NULL; 426 } else { 427#ifdef ASSERT 428 val->dump(); 429 assert(false, "unknown node on this path"); 430#endif 431 return NULL; // unknown node on this path 432 } 433 } 434 } 435 // Set Phi's inputs 436 for (uint j = 1; j < length; j++) { 437 if (values.at(j) == mem) { 438 phi->init_req(j, phi); 439 } else { 440 phi->init_req(j, values.at(j)); 441 } 442 } 443 return phi; 444} 445 446// Search the last value stored into the object's field. 447Node *PhaseMacroExpand::value_from_mem(Node *sfpt_mem, BasicType ft, const Type *ftype, const TypeOopPtr *adr_t, Node *alloc) { 448 assert(adr_t->is_known_instance_field(), "instance required"); 449 int instance_id = adr_t->instance_id(); 450 assert((uint)instance_id == alloc->_idx, "wrong allocation"); 451 452 int alias_idx = C->get_alias_index(adr_t); 453 int offset = adr_t->offset(); 454 Node *start_mem = C->start()->proj_out(TypeFunc::Memory); 455 Node *alloc_ctrl = alloc->in(TypeFunc::Control); 456 Node *alloc_mem = alloc->in(TypeFunc::Memory); 457 Arena *a = Thread::current()->resource_area(); 458 VectorSet visited(a); 459 460 461 bool done = sfpt_mem == alloc_mem; 462 Node *mem = sfpt_mem; 463 while (!done) { 464 if (visited.test_set(mem->_idx)) { 465 return NULL; // found a loop, give up 466 } 467 mem = scan_mem_chain(mem, alias_idx, offset, start_mem, alloc, &_igvn); 468 if (mem == start_mem || mem == alloc_mem) { 469 done = true; // hit a sentinel, return appropriate 0 value 470 } else if (mem->is_Initialize()) { 471 mem = mem->as_Initialize()->find_captured_store(offset, type2aelembytes(ft), &_igvn); 472 if (mem == NULL) { 473 done = true; // Something go wrong. 474 } else if (mem->is_Store()) { 475 const TypePtr* atype = mem->as_Store()->adr_type(); 476 assert(C->get_alias_index(atype) == Compile::AliasIdxRaw, "store is correct memory slice"); 477 done = true; 478 } 479 } else if (mem->is_Store()) { 480 const TypeOopPtr* atype = mem->as_Store()->adr_type()->isa_oopptr(); 481 assert(atype != NULL, "address type must be oopptr"); 482 assert(C->get_alias_index(atype) == alias_idx && 483 atype->is_known_instance_field() && atype->offset() == offset && 484 atype->instance_id() == instance_id, "store is correct memory slice"); 485 done = true; 486 } else if (mem->is_Phi()) { 487 // try to find a phi's unique input 488 Node *unique_input = NULL; 489 Node *top = C->top(); 490 for (uint i = 1; i < mem->req(); i++) { 491 Node *n = scan_mem_chain(mem->in(i), alias_idx, offset, start_mem, alloc, &_igvn); 492 if (n == NULL || n == top || n == mem) { 493 continue; 494 } else if (unique_input == NULL) { 495 unique_input = n; 496 } else if (unique_input != n) { 497 unique_input = top; 498 break; 499 } 500 } 501 if (unique_input != NULL && unique_input != top) { 502 mem = unique_input; 503 } else { 504 done = true; 505 } 506 } else { 507 assert(false, "unexpected node"); 508 } 509 } 510 if (mem != NULL) { 511 if (mem == start_mem || mem == alloc_mem) { 512 // hit a sentinel, return appropriate 0 value 513 return _igvn.zerocon(ft); 514 } else if (mem->is_Store()) { 515 return mem->in(MemNode::ValueIn); 516 } else if (mem->is_Phi()) { 517 // attempt to produce a Phi reflecting the values on the input paths of the Phi 518 Node_Stack value_phis(a, 8); 519 Node * phi = value_from_mem_phi(mem, ft, ftype, adr_t, alloc, &value_phis, ValueSearchLimit); 520 if (phi != NULL) { 521 return phi; 522 } else { 523 // Kill all new Phis 524 while(value_phis.is_nonempty()) { 525 Node* n = value_phis.node(); 526 _igvn.hash_delete(n); 527 _igvn.subsume_node(n, C->top()); 528 value_phis.pop(); 529 } 530 } 531 } 532 } 533 // Something go wrong. 534 return NULL; 535} 536 537// Check the possibility of scalar replacement. 538bool PhaseMacroExpand::can_eliminate_allocation(AllocateNode *alloc, GrowableArray <SafePointNode *>& safepoints) { 539 // Scan the uses of the allocation to check for anything that would 540 // prevent us from eliminating it. 541 NOT_PRODUCT( const char* fail_eliminate = NULL; ) 542 DEBUG_ONLY( Node* disq_node = NULL; ) 543 bool can_eliminate = true; 544 545 Node* res = alloc->result_cast(); 546 const TypeOopPtr* res_type = NULL; 547 if (res == NULL) { 548 // All users were eliminated. 549 } else if (!res->is_CheckCastPP()) { 550 alloc->_is_scalar_replaceable = false; // don't try again 551 NOT_PRODUCT(fail_eliminate = "Allocation does not have unique CheckCastPP";) 552 can_eliminate = false; 553 } else { 554 res_type = _igvn.type(res)->isa_oopptr(); 555 if (res_type == NULL) { 556 NOT_PRODUCT(fail_eliminate = "Neither instance or array allocation";) 557 can_eliminate = false; 558 } else if (res_type->isa_aryptr()) { 559 int length = alloc->in(AllocateNode::ALength)->find_int_con(-1); 560 if (length < 0) { 561 NOT_PRODUCT(fail_eliminate = "Array's size is not constant";) 562 can_eliminate = false; 563 } 564 } 565 } 566 567 if (can_eliminate && res != NULL) { 568 for (DUIterator_Fast jmax, j = res->fast_outs(jmax); 569 j < jmax && can_eliminate; j++) { 570 Node* use = res->fast_out(j); 571 572 if (use->is_AddP()) { 573 const TypePtr* addp_type = _igvn.type(use)->is_ptr(); 574 int offset = addp_type->offset(); 575 576 if (offset == Type::OffsetTop || offset == Type::OffsetBot) { 577 NOT_PRODUCT(fail_eliminate = "Undefined field referrence";) 578 can_eliminate = false; 579 break; 580 } 581 for (DUIterator_Fast kmax, k = use->fast_outs(kmax); 582 k < kmax && can_eliminate; k++) { 583 Node* n = use->fast_out(k); 584 if (!n->is_Store() && n->Opcode() != Op_CastP2X) { 585 DEBUG_ONLY(disq_node = n;) 586 if (n->is_Load() || n->is_LoadStore()) { 587 NOT_PRODUCT(fail_eliminate = "Field load";) 588 } else { 589 NOT_PRODUCT(fail_eliminate = "Not store field referrence";) 590 } 591 can_eliminate = false; 592 } 593 } 594 } else if (use->is_SafePoint()) { 595 SafePointNode* sfpt = use->as_SafePoint(); 596 if (sfpt->is_Call() && sfpt->as_Call()->has_non_debug_use(res)) { 597 // Object is passed as argument. 598 DEBUG_ONLY(disq_node = use;) 599 NOT_PRODUCT(fail_eliminate = "Object is passed as argument";) 600 can_eliminate = false; 601 } 602 Node* sfptMem = sfpt->memory(); 603 if (sfptMem == NULL || sfptMem->is_top()) { 604 DEBUG_ONLY(disq_node = use;) 605 NOT_PRODUCT(fail_eliminate = "NULL or TOP memory";) 606 can_eliminate = false; 607 } else { 608 safepoints.append_if_missing(sfpt); 609 } 610 } else if (use->Opcode() != Op_CastP2X) { // CastP2X is used by card mark 611 if (use->is_Phi()) { 612 if (use->outcnt() == 1 && use->unique_out()->Opcode() == Op_Return) { 613 NOT_PRODUCT(fail_eliminate = "Object is return value";) 614 } else { 615 NOT_PRODUCT(fail_eliminate = "Object is referenced by Phi";) 616 } 617 DEBUG_ONLY(disq_node = use;) 618 } else { 619 if (use->Opcode() == Op_Return) { 620 NOT_PRODUCT(fail_eliminate = "Object is return value";) 621 }else { 622 NOT_PRODUCT(fail_eliminate = "Object is referenced by node";) 623 } 624 DEBUG_ONLY(disq_node = use;) 625 } 626 can_eliminate = false; 627 } 628 } 629 } 630 631#ifndef PRODUCT 632 if (PrintEliminateAllocations) { 633 if (can_eliminate) { 634 tty->print("Scalar "); 635 if (res == NULL) 636 alloc->dump(); 637 else 638 res->dump(); 639 } else { 640 tty->print("NotScalar (%s)", fail_eliminate); 641 if (res == NULL) 642 alloc->dump(); 643 else 644 res->dump(); 645#ifdef ASSERT 646 if (disq_node != NULL) { 647 tty->print(" >>>> "); 648 disq_node->dump(); 649 } 650#endif /*ASSERT*/ 651 } 652 } 653#endif 654 return can_eliminate; 655} 656 657// Do scalar replacement. 658bool PhaseMacroExpand::scalar_replacement(AllocateNode *alloc, GrowableArray <SafePointNode *>& safepoints) { 659 GrowableArray <SafePointNode *> safepoints_done; 660 661 ciKlass* klass = NULL; 662 ciInstanceKlass* iklass = NULL; 663 int nfields = 0; 664 int array_base; 665 int element_size; 666 BasicType basic_elem_type; 667 ciType* elem_type; 668 669 Node* res = alloc->result_cast(); 670 const TypeOopPtr* res_type = NULL; 671 if (res != NULL) { // Could be NULL when there are no users 672 res_type = _igvn.type(res)->isa_oopptr(); 673 } 674 675 if (res != NULL) { 676 klass = res_type->klass(); 677 if (res_type->isa_instptr()) { 678 // find the fields of the class which will be needed for safepoint debug information 679 assert(klass->is_instance_klass(), "must be an instance klass."); 680 iklass = klass->as_instance_klass(); 681 nfields = iklass->nof_nonstatic_fields(); 682 } else { 683 // find the array's elements which will be needed for safepoint debug information 684 nfields = alloc->in(AllocateNode::ALength)->find_int_con(-1); 685 assert(klass->is_array_klass() && nfields >= 0, "must be an array klass."); 686 elem_type = klass->as_array_klass()->element_type(); 687 basic_elem_type = elem_type->basic_type(); 688 array_base = arrayOopDesc::base_offset_in_bytes(basic_elem_type); 689 element_size = type2aelembytes(basic_elem_type); 690 } 691 } 692 // 693 // Process the safepoint uses 694 // 695 while (safepoints.length() > 0) { 696 SafePointNode* sfpt = safepoints.pop(); 697 Node* mem = sfpt->memory(); 698 uint first_ind = sfpt->req(); 699 SafePointScalarObjectNode* sobj = new (C, 1) SafePointScalarObjectNode(res_type, 700#ifdef ASSERT 701 alloc, 702#endif 703 first_ind, nfields); 704 sobj->init_req(0, sfpt->in(TypeFunc::Control)); 705 transform_later(sobj); 706 707 // Scan object's fields adding an input to the safepoint for each field. 708 for (int j = 0; j < nfields; j++) { 709 intptr_t offset; 710 ciField* field = NULL; 711 if (iklass != NULL) { 712 field = iklass->nonstatic_field_at(j); 713 offset = field->offset(); 714 elem_type = field->type(); 715 basic_elem_type = field->layout_type(); 716 } else { 717 offset = array_base + j * (intptr_t)element_size; 718 } 719 720 const Type *field_type; 721 // The next code is taken from Parse::do_get_xxx(). 722 if (basic_elem_type == T_OBJECT || basic_elem_type == T_ARRAY) { 723 if (!elem_type->is_loaded()) { 724 field_type = TypeInstPtr::BOTTOM; 725 } else if (field != NULL && field->is_constant()) { 726 // This can happen if the constant oop is non-perm. 727 ciObject* con = field->constant_value().as_object(); 728 // Do not "join" in the previous type; it doesn't add value, 729 // and may yield a vacuous result if the field is of interface type. 730 field_type = TypeOopPtr::make_from_constant(con)->isa_oopptr(); 731 assert(field_type != NULL, "field singleton type must be consistent"); 732 } else { 733 field_type = TypeOopPtr::make_from_klass(elem_type->as_klass()); 734 } 735 if (UseCompressedOops) { 736 field_type = field_type->make_narrowoop(); 737 basic_elem_type = T_NARROWOOP; 738 } 739 } else { 740 field_type = Type::get_const_basic_type(basic_elem_type); 741 } 742 743 const TypeOopPtr *field_addr_type = res_type->add_offset(offset)->isa_oopptr(); 744 745 Node *field_val = value_from_mem(mem, basic_elem_type, field_type, field_addr_type, alloc); 746 if (field_val == NULL) { 747 // we weren't able to find a value for this field, 748 // give up on eliminating this allocation 749 alloc->_is_scalar_replaceable = false; // don't try again 750 // remove any extra entries we added to the safepoint 751 uint last = sfpt->req() - 1; 752 for (int k = 0; k < j; k++) { 753 sfpt->del_req(last--); 754 } 755 // rollback processed safepoints 756 while (safepoints_done.length() > 0) { 757 SafePointNode* sfpt_done = safepoints_done.pop(); 758 // remove any extra entries we added to the safepoint 759 last = sfpt_done->req() - 1; 760 for (int k = 0; k < nfields; k++) { 761 sfpt_done->del_req(last--); 762 } 763 JVMState *jvms = sfpt_done->jvms(); 764 jvms->set_endoff(sfpt_done->req()); 765 // Now make a pass over the debug information replacing any references 766 // to SafePointScalarObjectNode with the allocated object. 767 int start = jvms->debug_start(); 768 int end = jvms->debug_end(); 769 for (int i = start; i < end; i++) { 770 if (sfpt_done->in(i)->is_SafePointScalarObject()) { 771 SafePointScalarObjectNode* scobj = sfpt_done->in(i)->as_SafePointScalarObject(); 772 if (scobj->first_index() == sfpt_done->req() && 773 scobj->n_fields() == (uint)nfields) { 774 assert(scobj->alloc() == alloc, "sanity"); 775 sfpt_done->set_req(i, res); 776 } 777 } 778 } 779 } 780#ifndef PRODUCT 781 if (PrintEliminateAllocations) { 782 if (field != NULL) { 783 tty->print("=== At SafePoint node %d can't find value of Field: ", 784 sfpt->_idx); 785 field->print(); 786 int field_idx = C->get_alias_index(field_addr_type); 787 tty->print(" (alias_idx=%d)", field_idx); 788 } else { // Array's element 789 tty->print("=== At SafePoint node %d can't find value of array element [%d]", 790 sfpt->_idx, j); 791 } 792 tty->print(", which prevents elimination of: "); 793 if (res == NULL) 794 alloc->dump(); 795 else 796 res->dump(); 797 } 798#endif 799 return false; 800 } 801 if (UseCompressedOops && field_type->isa_narrowoop()) { 802 // Enable "DecodeN(EncodeP(Allocate)) --> Allocate" transformation 803 // to be able scalar replace the allocation. 804 if (field_val->is_EncodeP()) { 805 field_val = field_val->in(1); 806 } else { 807 field_val = transform_later(new (C, 2) DecodeNNode(field_val, field_val->bottom_type()->make_ptr())); 808 } 809 } 810 sfpt->add_req(field_val); 811 } 812 JVMState *jvms = sfpt->jvms(); 813 jvms->set_endoff(sfpt->req()); 814 // Now make a pass over the debug information replacing any references 815 // to the allocated object with "sobj" 816 int start = jvms->debug_start(); 817 int end = jvms->debug_end(); 818 for (int i = start; i < end; i++) { 819 if (sfpt->in(i) == res) { 820 sfpt->set_req(i, sobj); 821 } 822 } 823 safepoints_done.append_if_missing(sfpt); // keep it for rollback 824 } 825 return true; 826} 827 828// Process users of eliminated allocation. 829void PhaseMacroExpand::process_users_of_allocation(AllocateNode *alloc) { 830 Node* res = alloc->result_cast(); 831 if (res != NULL) { 832 for (DUIterator_Last jmin, j = res->last_outs(jmin); j >= jmin; ) { 833 Node *use = res->last_out(j); 834 uint oc1 = res->outcnt(); 835 836 if (use->is_AddP()) { 837 for (DUIterator_Last kmin, k = use->last_outs(kmin); k >= kmin; ) { 838 Node *n = use->last_out(k); 839 uint oc2 = use->outcnt(); 840 if (n->is_Store()) { 841#ifdef ASSERT 842 // Verify that there is no dependent MemBarVolatile nodes, 843 // they should be removed during IGVN, see MemBarNode::Ideal(). 844 for (DUIterator_Fast pmax, p = n->fast_outs(pmax); 845 p < pmax; p++) { 846 Node* mb = n->fast_out(p); 847 assert(mb->is_Initialize() || !mb->is_MemBar() || 848 mb->req() <= MemBarNode::Precedent || 849 mb->in(MemBarNode::Precedent) != n, 850 "MemBarVolatile should be eliminated for non-escaping object"); 851 } 852#endif 853 _igvn.replace_node(n, n->in(MemNode::Memory)); 854 } else { 855 eliminate_card_mark(n); 856 } 857 k -= (oc2 - use->outcnt()); 858 } 859 } else { 860 eliminate_card_mark(use); 861 } 862 j -= (oc1 - res->outcnt()); 863 } 864 assert(res->outcnt() == 0, "all uses of allocated objects must be deleted"); 865 _igvn.remove_dead_node(res); 866 } 867 868 // 869 // Process other users of allocation's projections 870 // 871 if (_resproj != NULL && _resproj->outcnt() != 0) { 872 for (DUIterator_Last jmin, j = _resproj->last_outs(jmin); j >= jmin; ) { 873 Node *use = _resproj->last_out(j); 874 uint oc1 = _resproj->outcnt(); 875 if (use->is_Initialize()) { 876 // Eliminate Initialize node. 877 InitializeNode *init = use->as_Initialize(); 878 assert(init->outcnt() <= 2, "only a control and memory projection expected"); 879 Node *ctrl_proj = init->proj_out(TypeFunc::Control); 880 if (ctrl_proj != NULL) { 881 assert(init->in(TypeFunc::Control) == _fallthroughcatchproj, "allocation control projection"); 882 _igvn.replace_node(ctrl_proj, _fallthroughcatchproj); 883 } 884 Node *mem_proj = init->proj_out(TypeFunc::Memory); 885 if (mem_proj != NULL) { 886 Node *mem = init->in(TypeFunc::Memory); 887#ifdef ASSERT 888 if (mem->is_MergeMem()) { 889 assert(mem->in(TypeFunc::Memory) == _memproj_fallthrough, "allocation memory projection"); 890 } else { 891 assert(mem == _memproj_fallthrough, "allocation memory projection"); 892 } 893#endif 894 _igvn.replace_node(mem_proj, mem); 895 } 896 } else if (use->is_AddP()) { 897 // raw memory addresses used only by the initialization 898 _igvn.replace_node(use, C->top()); 899 } else { 900 assert(false, "only Initialize or AddP expected"); 901 } 902 j -= (oc1 - _resproj->outcnt()); 903 } 904 } 905 if (_fallthroughcatchproj != NULL) { 906 _igvn.replace_node(_fallthroughcatchproj, alloc->in(TypeFunc::Control)); 907 } 908 if (_memproj_fallthrough != NULL) { 909 _igvn.replace_node(_memproj_fallthrough, alloc->in(TypeFunc::Memory)); 910 } 911 if (_memproj_catchall != NULL) { 912 _igvn.replace_node(_memproj_catchall, C->top()); 913 } 914 if (_ioproj_fallthrough != NULL) { 915 _igvn.replace_node(_ioproj_fallthrough, alloc->in(TypeFunc::I_O)); 916 } 917 if (_ioproj_catchall != NULL) { 918 _igvn.replace_node(_ioproj_catchall, C->top()); 919 } 920 if (_catchallcatchproj != NULL) { 921 _igvn.replace_node(_catchallcatchproj, C->top()); 922 } 923} 924 925bool PhaseMacroExpand::eliminate_allocate_node(AllocateNode *alloc) { 926 927 if (!EliminateAllocations || !alloc->_is_scalar_replaceable) { 928 return false; 929 } 930 931 extract_call_projections(alloc); 932 933 GrowableArray <SafePointNode *> safepoints; 934 if (!can_eliminate_allocation(alloc, safepoints)) { 935 return false; 936 } 937 938 if (!scalar_replacement(alloc, safepoints)) { 939 return false; 940 } 941 942 CompileLog* log = C->log(); 943 if (log != NULL) { 944 Node* klass = alloc->in(AllocateNode::KlassNode); 945 const TypeKlassPtr* tklass = _igvn.type(klass)->is_klassptr(); 946 log->head("eliminate_allocation type='%d'", 947 log->identify(tklass->klass())); 948 JVMState* p = alloc->jvms(); 949 while (p != NULL) { 950 log->elem("jvms bci='%d' method='%d'", p->bci(), log->identify(p->method())); 951 p = p->caller(); 952 } 953 log->tail("eliminate_allocation"); 954 } 955 956 process_users_of_allocation(alloc); 957 958#ifndef PRODUCT 959 if (PrintEliminateAllocations) { 960 if (alloc->is_AllocateArray()) 961 tty->print_cr("++++ Eliminated: %d AllocateArray", alloc->_idx); 962 else 963 tty->print_cr("++++ Eliminated: %d Allocate", alloc->_idx); 964 } 965#endif 966 967 return true; 968} 969 970 971//---------------------------set_eden_pointers------------------------- 972void PhaseMacroExpand::set_eden_pointers(Node* &eden_top_adr, Node* &eden_end_adr) { 973 if (UseTLAB) { // Private allocation: load from TLS 974 Node* thread = transform_later(new (C, 1) ThreadLocalNode()); 975 int tlab_top_offset = in_bytes(JavaThread::tlab_top_offset()); 976 int tlab_end_offset = in_bytes(JavaThread::tlab_end_offset()); 977 eden_top_adr = basic_plus_adr(top()/*not oop*/, thread, tlab_top_offset); 978 eden_end_adr = basic_plus_adr(top()/*not oop*/, thread, tlab_end_offset); 979 } else { // Shared allocation: load from globals 980 CollectedHeap* ch = Universe::heap(); 981 address top_adr = (address)ch->top_addr(); 982 address end_adr = (address)ch->end_addr(); 983 eden_top_adr = makecon(TypeRawPtr::make(top_adr)); 984 eden_end_adr = basic_plus_adr(eden_top_adr, end_adr - top_adr); 985 } 986} 987 988 989Node* PhaseMacroExpand::make_load(Node* ctl, Node* mem, Node* base, int offset, const Type* value_type, BasicType bt) { 990 Node* adr = basic_plus_adr(base, offset); 991 const TypePtr* adr_type = adr->bottom_type()->is_ptr(); 992 Node* value = LoadNode::make(_igvn, ctl, mem, adr, adr_type, value_type, bt); 993 transform_later(value); 994 return value; 995} 996 997 998Node* PhaseMacroExpand::make_store(Node* ctl, Node* mem, Node* base, int offset, Node* value, BasicType bt) { 999 Node* adr = basic_plus_adr(base, offset); 1000 mem = StoreNode::make(_igvn, ctl, mem, adr, NULL, value, bt); 1001 transform_later(mem); 1002 return mem; 1003} 1004 1005//============================================================================= 1006// 1007// A L L O C A T I O N 1008// 1009// Allocation attempts to be fast in the case of frequent small objects. 1010// It breaks down like this: 1011// 1012// 1) Size in doublewords is computed. This is a constant for objects and 1013// variable for most arrays. Doubleword units are used to avoid size 1014// overflow of huge doubleword arrays. We need doublewords in the end for 1015// rounding. 1016// 1017// 2) Size is checked for being 'too large'. Too-large allocations will go 1018// the slow path into the VM. The slow path can throw any required 1019// exceptions, and does all the special checks for very large arrays. The 1020// size test can constant-fold away for objects. For objects with 1021// finalizers it constant-folds the otherway: you always go slow with 1022// finalizers. 1023// 1024// 3) If NOT using TLABs, this is the contended loop-back point. 1025// Load-Locked the heap top. If using TLABs normal-load the heap top. 1026// 1027// 4) Check that heap top + size*8 < max. If we fail go the slow ` route. 1028// NOTE: "top+size*8" cannot wrap the 4Gig line! Here's why: for largish 1029// "size*8" we always enter the VM, where "largish" is a constant picked small 1030// enough that there's always space between the eden max and 4Gig (old space is 1031// there so it's quite large) and large enough that the cost of entering the VM 1032// is dwarfed by the cost to initialize the space. 1033// 1034// 5) If NOT using TLABs, Store-Conditional the adjusted heap top back 1035// down. If contended, repeat at step 3. If using TLABs normal-store 1036// adjusted heap top back down; there is no contention. 1037// 1038// 6) If !ZeroTLAB then Bulk-clear the object/array. Fill in klass & mark 1039// fields. 1040// 1041// 7) Merge with the slow-path; cast the raw memory pointer to the correct 1042// oop flavor. 1043// 1044//============================================================================= 1045// FastAllocateSizeLimit value is in DOUBLEWORDS. 1046// Allocations bigger than this always go the slow route. 1047// This value must be small enough that allocation attempts that need to 1048// trigger exceptions go the slow route. Also, it must be small enough so 1049// that heap_top + size_in_bytes does not wrap around the 4Gig limit. 1050//=============================================================================j// 1051// %%% Here is an old comment from parseHelper.cpp; is it outdated? 1052// The allocator will coalesce int->oop copies away. See comment in 1053// coalesce.cpp about how this works. It depends critically on the exact 1054// code shape produced here, so if you are changing this code shape 1055// make sure the GC info for the heap-top is correct in and around the 1056// slow-path call. 1057// 1058 1059void PhaseMacroExpand::expand_allocate_common( 1060 AllocateNode* alloc, // allocation node to be expanded 1061 Node* length, // array length for an array allocation 1062 const TypeFunc* slow_call_type, // Type of slow call 1063 address slow_call_address // Address of slow call 1064 ) 1065{ 1066 1067 Node* ctrl = alloc->in(TypeFunc::Control); 1068 Node* mem = alloc->in(TypeFunc::Memory); 1069 Node* i_o = alloc->in(TypeFunc::I_O); 1070 Node* size_in_bytes = alloc->in(AllocateNode::AllocSize); 1071 Node* klass_node = alloc->in(AllocateNode::KlassNode); 1072 Node* initial_slow_test = alloc->in(AllocateNode::InitialTest); 1073 1074 assert(ctrl != NULL, "must have control"); 1075 // We need a Region and corresponding Phi's to merge the slow-path and fast-path results. 1076 // they will not be used if "always_slow" is set 1077 enum { slow_result_path = 1, fast_result_path = 2 }; 1078 Node *result_region; 1079 Node *result_phi_rawmem; 1080 Node *result_phi_rawoop; 1081 Node *result_phi_i_o; 1082 1083 // The initial slow comparison is a size check, the comparison 1084 // we want to do is a BoolTest::gt 1085 bool always_slow = false; 1086 int tv = _igvn.find_int_con(initial_slow_test, -1); 1087 if (tv >= 0) { 1088 always_slow = (tv == 1); 1089 initial_slow_test = NULL; 1090 } else { 1091 initial_slow_test = BoolNode::make_predicate(initial_slow_test, &_igvn); 1092 } 1093 1094 if (C->env()->dtrace_alloc_probes() || 1095 !UseTLAB && (!Universe::heap()->supports_inline_contig_alloc() || 1096 (UseConcMarkSweepGC && CMSIncrementalMode))) { 1097 // Force slow-path allocation 1098 always_slow = true; 1099 initial_slow_test = NULL; 1100 } 1101 1102 1103 enum { too_big_or_final_path = 1, need_gc_path = 2 }; 1104 Node *slow_region = NULL; 1105 Node *toobig_false = ctrl; 1106 1107 assert (initial_slow_test == NULL || !always_slow, "arguments must be consistent"); 1108 // generate the initial test if necessary 1109 if (initial_slow_test != NULL ) { 1110 slow_region = new (C, 3) RegionNode(3); 1111 1112 // Now make the initial failure test. Usually a too-big test but 1113 // might be a TRUE for finalizers or a fancy class check for 1114 // newInstance0. 1115 IfNode *toobig_iff = new (C, 2) IfNode(ctrl, initial_slow_test, PROB_MIN, COUNT_UNKNOWN); 1116 transform_later(toobig_iff); 1117 // Plug the failing-too-big test into the slow-path region 1118 Node *toobig_true = new (C, 1) IfTrueNode( toobig_iff ); 1119 transform_later(toobig_true); 1120 slow_region ->init_req( too_big_or_final_path, toobig_true ); 1121 toobig_false = new (C, 1) IfFalseNode( toobig_iff ); 1122 transform_later(toobig_false); 1123 } else { // No initial test, just fall into next case 1124 toobig_false = ctrl; 1125 debug_only(slow_region = NodeSentinel); 1126 } 1127 1128 Node *slow_mem = mem; // save the current memory state for slow path 1129 // generate the fast allocation code unless we know that the initial test will always go slow 1130 if (!always_slow) { 1131 // Fast path modifies only raw memory. 1132 if (mem->is_MergeMem()) { 1133 mem = mem->as_MergeMem()->memory_at(Compile::AliasIdxRaw); 1134 } 1135 1136 Node* eden_top_adr; 1137 Node* eden_end_adr; 1138 1139 set_eden_pointers(eden_top_adr, eden_end_adr); 1140 1141 // Load Eden::end. Loop invariant and hoisted. 1142 // 1143 // Note: We set the control input on "eden_end" and "old_eden_top" when using 1144 // a TLAB to work around a bug where these values were being moved across 1145 // a safepoint. These are not oops, so they cannot be include in the oop 1146 // map, but the can be changed by a GC. The proper way to fix this would 1147 // be to set the raw memory state when generating a SafepointNode. However 1148 // this will require extensive changes to the loop optimization in order to 1149 // prevent a degradation of the optimization. 1150 // See comment in memnode.hpp, around line 227 in class LoadPNode. 1151 Node *eden_end = make_load(ctrl, mem, eden_end_adr, 0, TypeRawPtr::BOTTOM, T_ADDRESS); 1152 1153 // allocate the Region and Phi nodes for the result 1154 result_region = new (C, 3) RegionNode(3); 1155 result_phi_rawmem = new (C, 3) PhiNode( result_region, Type::MEMORY, TypeRawPtr::BOTTOM ); 1156 result_phi_rawoop = new (C, 3) PhiNode( result_region, TypeRawPtr::BOTTOM ); 1157 result_phi_i_o = new (C, 3) PhiNode( result_region, Type::ABIO ); // I/O is used for Prefetch 1158 1159 // We need a Region for the loop-back contended case. 1160 enum { fall_in_path = 1, contended_loopback_path = 2 }; 1161 Node *contended_region; 1162 Node *contended_phi_rawmem; 1163 if( UseTLAB ) { 1164 contended_region = toobig_false; 1165 contended_phi_rawmem = mem; 1166 } else { 1167 contended_region = new (C, 3) RegionNode(3); 1168 contended_phi_rawmem = new (C, 3) PhiNode( contended_region, Type::MEMORY, TypeRawPtr::BOTTOM); 1169 // Now handle the passing-too-big test. We fall into the contended 1170 // loop-back merge point. 1171 contended_region ->init_req( fall_in_path, toobig_false ); 1172 contended_phi_rawmem->init_req( fall_in_path, mem ); 1173 transform_later(contended_region); 1174 transform_later(contended_phi_rawmem); 1175 } 1176 1177 // Load(-locked) the heap top. 1178 // See note above concerning the control input when using a TLAB 1179 Node *old_eden_top = UseTLAB 1180 ? new (C, 3) LoadPNode ( ctrl, contended_phi_rawmem, eden_top_adr, TypeRawPtr::BOTTOM, TypeRawPtr::BOTTOM ) 1181 : new (C, 3) LoadPLockedNode( contended_region, contended_phi_rawmem, eden_top_adr ); 1182 1183 transform_later(old_eden_top); 1184 // Add to heap top to get a new heap top 1185 Node *new_eden_top = new (C, 4) AddPNode( top(), old_eden_top, size_in_bytes ); 1186 transform_later(new_eden_top); 1187 // Check for needing a GC; compare against heap end 1188 Node *needgc_cmp = new (C, 3) CmpPNode( new_eden_top, eden_end ); 1189 transform_later(needgc_cmp); 1190 Node *needgc_bol = new (C, 2) BoolNode( needgc_cmp, BoolTest::ge ); 1191 transform_later(needgc_bol); 1192 IfNode *needgc_iff = new (C, 2) IfNode(contended_region, needgc_bol, PROB_UNLIKELY_MAG(4), COUNT_UNKNOWN ); 1193 transform_later(needgc_iff); 1194 1195 // Plug the failing-heap-space-need-gc test into the slow-path region 1196 Node *needgc_true = new (C, 1) IfTrueNode( needgc_iff ); 1197 transform_later(needgc_true); 1198 if( initial_slow_test ) { 1199 slow_region ->init_req( need_gc_path, needgc_true ); 1200 // This completes all paths into the slow merge point 1201 transform_later(slow_region); 1202 } else { // No initial slow path needed! 1203 // Just fall from the need-GC path straight into the VM call. 1204 slow_region = needgc_true; 1205 } 1206 // No need for a GC. Setup for the Store-Conditional 1207 Node *needgc_false = new (C, 1) IfFalseNode( needgc_iff ); 1208 transform_later(needgc_false); 1209 1210 // Grab regular I/O before optional prefetch may change it. 1211 // Slow-path does no I/O so just set it to the original I/O. 1212 result_phi_i_o->init_req( slow_result_path, i_o ); 1213 1214 i_o = prefetch_allocation(i_o, needgc_false, contended_phi_rawmem, 1215 old_eden_top, new_eden_top, length); 1216 1217 // Store (-conditional) the modified eden top back down. 1218 // StorePConditional produces flags for a test PLUS a modified raw 1219 // memory state. 1220 Node *store_eden_top; 1221 Node *fast_oop_ctrl; 1222 if( UseTLAB ) { 1223 store_eden_top = new (C, 4) StorePNode( needgc_false, contended_phi_rawmem, eden_top_adr, TypeRawPtr::BOTTOM, new_eden_top ); 1224 transform_later(store_eden_top); 1225 fast_oop_ctrl = needgc_false; // No contention, so this is the fast path 1226 } else { 1227 store_eden_top = new (C, 5) StorePConditionalNode( needgc_false, contended_phi_rawmem, eden_top_adr, new_eden_top, old_eden_top ); 1228 transform_later(store_eden_top); 1229 Node *contention_check = new (C, 2) BoolNode( store_eden_top, BoolTest::ne ); 1230 transform_later(contention_check); 1231 store_eden_top = new (C, 1) SCMemProjNode(store_eden_top); 1232 transform_later(store_eden_top); 1233 1234 // If not using TLABs, check to see if there was contention. 1235 IfNode *contention_iff = new (C, 2) IfNode ( needgc_false, contention_check, PROB_MIN, COUNT_UNKNOWN ); 1236 transform_later(contention_iff); 1237 Node *contention_true = new (C, 1) IfTrueNode( contention_iff ); 1238 transform_later(contention_true); 1239 // If contention, loopback and try again. 1240 contended_region->init_req( contended_loopback_path, contention_true ); 1241 contended_phi_rawmem->init_req( contended_loopback_path, store_eden_top ); 1242 1243 // Fast-path succeeded with no contention! 1244 Node *contention_false = new (C, 1) IfFalseNode( contention_iff ); 1245 transform_later(contention_false); 1246 fast_oop_ctrl = contention_false; 1247 } 1248 1249 // Rename successful fast-path variables to make meaning more obvious 1250 Node* fast_oop = old_eden_top; 1251 Node* fast_oop_rawmem = store_eden_top; 1252 fast_oop_rawmem = initialize_object(alloc, 1253 fast_oop_ctrl, fast_oop_rawmem, fast_oop, 1254 klass_node, length, size_in_bytes); 1255 1256 if (C->env()->dtrace_extended_probes()) { 1257 // Slow-path call 1258 int size = TypeFunc::Parms + 2; 1259 CallLeafNode *call = new (C, size) CallLeafNode(OptoRuntime::dtrace_object_alloc_Type(), 1260 CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_object_alloc_base), 1261 "dtrace_object_alloc", 1262 TypeRawPtr::BOTTOM); 1263 1264 // Get base of thread-local storage area 1265 Node* thread = new (C, 1) ThreadLocalNode(); 1266 transform_later(thread); 1267 1268 call->init_req(TypeFunc::Parms+0, thread); 1269 call->init_req(TypeFunc::Parms+1, fast_oop); 1270 call->init_req( TypeFunc::Control, fast_oop_ctrl ); 1271 call->init_req( TypeFunc::I_O , top() ) ; // does no i/o 1272 call->init_req( TypeFunc::Memory , fast_oop_rawmem ); 1273 call->init_req( TypeFunc::ReturnAdr, alloc->in(TypeFunc::ReturnAdr) ); 1274 call->init_req( TypeFunc::FramePtr, alloc->in(TypeFunc::FramePtr) ); 1275 transform_later(call); 1276 fast_oop_ctrl = new (C, 1) ProjNode(call,TypeFunc::Control); 1277 transform_later(fast_oop_ctrl); 1278 fast_oop_rawmem = new (C, 1) ProjNode(call,TypeFunc::Memory); 1279 transform_later(fast_oop_rawmem); 1280 } 1281 1282 // Plug in the successful fast-path into the result merge point 1283 result_region ->init_req( fast_result_path, fast_oop_ctrl ); 1284 result_phi_rawoop->init_req( fast_result_path, fast_oop ); 1285 result_phi_i_o ->init_req( fast_result_path, i_o ); 1286 result_phi_rawmem->init_req( fast_result_path, fast_oop_rawmem ); 1287 } else { 1288 slow_region = ctrl; 1289 } 1290 1291 // Generate slow-path call 1292 CallNode *call = new (C, slow_call_type->domain()->cnt()) 1293 CallStaticJavaNode(slow_call_type, slow_call_address, 1294 OptoRuntime::stub_name(slow_call_address), 1295 alloc->jvms()->bci(), 1296 TypePtr::BOTTOM); 1297 call->init_req( TypeFunc::Control, slow_region ); 1298 call->init_req( TypeFunc::I_O , top() ) ; // does no i/o 1299 call->init_req( TypeFunc::Memory , slow_mem ); // may gc ptrs 1300 call->init_req( TypeFunc::ReturnAdr, alloc->in(TypeFunc::ReturnAdr) ); 1301 call->init_req( TypeFunc::FramePtr, alloc->in(TypeFunc::FramePtr) ); 1302 1303 call->init_req(TypeFunc::Parms+0, klass_node); 1304 if (length != NULL) { 1305 call->init_req(TypeFunc::Parms+1, length); 1306 } 1307 1308 // Copy debug information and adjust JVMState information, then replace 1309 // allocate node with the call 1310 copy_call_debug_info((CallNode *) alloc, call); 1311 if (!always_slow) { 1312 call->set_cnt(PROB_UNLIKELY_MAG(4)); // Same effect as RC_UNCOMMON. 1313 } 1314 _igvn.hash_delete(alloc); 1315 _igvn.subsume_node(alloc, call); 1316 transform_later(call); 1317 1318 // Identify the output projections from the allocate node and 1319 // adjust any references to them. 1320 // The control and io projections look like: 1321 // 1322 // v---Proj(ctrl) <-----+ v---CatchProj(ctrl) 1323 // Allocate Catch 1324 // ^---Proj(io) <-------+ ^---CatchProj(io) 1325 // 1326 // We are interested in the CatchProj nodes. 1327 // 1328 extract_call_projections(call); 1329 1330 // An allocate node has separate memory projections for the uses on the control and i_o paths 1331 // Replace uses of the control memory projection with result_phi_rawmem (unless we are only generating a slow call) 1332 if (!always_slow && _memproj_fallthrough != NULL) { 1333 for (DUIterator_Fast imax, i = _memproj_fallthrough->fast_outs(imax); i < imax; i++) { 1334 Node *use = _memproj_fallthrough->fast_out(i); 1335 _igvn.hash_delete(use); 1336 imax -= replace_input(use, _memproj_fallthrough, result_phi_rawmem); 1337 _igvn._worklist.push(use); 1338 // back up iterator 1339 --i; 1340 } 1341 } 1342 // Now change uses of _memproj_catchall to use _memproj_fallthrough and delete _memproj_catchall so 1343 // we end up with a call that has only 1 memory projection 1344 if (_memproj_catchall != NULL ) { 1345 if (_memproj_fallthrough == NULL) { 1346 _memproj_fallthrough = new (C, 1) ProjNode(call, TypeFunc::Memory); 1347 transform_later(_memproj_fallthrough); 1348 } 1349 for (DUIterator_Fast imax, i = _memproj_catchall->fast_outs(imax); i < imax; i++) { 1350 Node *use = _memproj_catchall->fast_out(i); 1351 _igvn.hash_delete(use); 1352 imax -= replace_input(use, _memproj_catchall, _memproj_fallthrough); 1353 _igvn._worklist.push(use); 1354 // back up iterator 1355 --i; 1356 } 1357 } 1358 1359 // An allocate node has separate i_o projections for the uses on the control and i_o paths 1360 // Replace uses of the control i_o projection with result_phi_i_o (unless we are only generating a slow call) 1361 if (_ioproj_fallthrough == NULL) { 1362 _ioproj_fallthrough = new (C, 1) ProjNode(call, TypeFunc::I_O); 1363 transform_later(_ioproj_fallthrough); 1364 } else if (!always_slow) { 1365 for (DUIterator_Fast imax, i = _ioproj_fallthrough->fast_outs(imax); i < imax; i++) { 1366 Node *use = _ioproj_fallthrough->fast_out(i); 1367 1368 _igvn.hash_delete(use); 1369 imax -= replace_input(use, _ioproj_fallthrough, result_phi_i_o); 1370 _igvn._worklist.push(use); 1371 // back up iterator 1372 --i; 1373 } 1374 } 1375 // Now change uses of _ioproj_catchall to use _ioproj_fallthrough and delete _ioproj_catchall so 1376 // we end up with a call that has only 1 control projection 1377 if (_ioproj_catchall != NULL ) { 1378 for (DUIterator_Fast imax, i = _ioproj_catchall->fast_outs(imax); i < imax; i++) { 1379 Node *use = _ioproj_catchall->fast_out(i); 1380 _igvn.hash_delete(use); 1381 imax -= replace_input(use, _ioproj_catchall, _ioproj_fallthrough); 1382 _igvn._worklist.push(use); 1383 // back up iterator 1384 --i; 1385 } 1386 } 1387 1388 // if we generated only a slow call, we are done 1389 if (always_slow) 1390 return; 1391 1392 1393 if (_fallthroughcatchproj != NULL) { 1394 ctrl = _fallthroughcatchproj->clone(); 1395 transform_later(ctrl); 1396 _igvn.replace_node(_fallthroughcatchproj, result_region); 1397 } else { 1398 ctrl = top(); 1399 } 1400 Node *slow_result; 1401 if (_resproj == NULL) { 1402 // no uses of the allocation result 1403 slow_result = top(); 1404 } else { 1405 slow_result = _resproj->clone(); 1406 transform_later(slow_result); 1407 _igvn.replace_node(_resproj, result_phi_rawoop); 1408 } 1409 1410 // Plug slow-path into result merge point 1411 result_region ->init_req( slow_result_path, ctrl ); 1412 result_phi_rawoop->init_req( slow_result_path, slow_result); 1413 result_phi_rawmem->init_req( slow_result_path, _memproj_fallthrough ); 1414 transform_later(result_region); 1415 transform_later(result_phi_rawoop); 1416 transform_later(result_phi_rawmem); 1417 transform_later(result_phi_i_o); 1418 // This completes all paths into the result merge point 1419} 1420 1421 1422// Helper for PhaseMacroExpand::expand_allocate_common. 1423// Initializes the newly-allocated storage. 1424Node* 1425PhaseMacroExpand::initialize_object(AllocateNode* alloc, 1426 Node* control, Node* rawmem, Node* object, 1427 Node* klass_node, Node* length, 1428 Node* size_in_bytes) { 1429 InitializeNode* init = alloc->initialization(); 1430 // Store the klass & mark bits 1431 Node* mark_node = NULL; 1432 // For now only enable fast locking for non-array types 1433 if (UseBiasedLocking && (length == NULL)) { 1434 mark_node = make_load(NULL, rawmem, klass_node, Klass::prototype_header_offset_in_bytes() + sizeof(oopDesc), TypeRawPtr::BOTTOM, T_ADDRESS); 1435 } else { 1436 mark_node = makecon(TypeRawPtr::make((address)markOopDesc::prototype())); 1437 } 1438 rawmem = make_store(control, rawmem, object, oopDesc::mark_offset_in_bytes(), mark_node, T_ADDRESS); 1439 1440 rawmem = make_store(control, rawmem, object, oopDesc::klass_offset_in_bytes(), klass_node, T_OBJECT); 1441 int header_size = alloc->minimum_header_size(); // conservatively small 1442 1443 // Array length 1444 if (length != NULL) { // Arrays need length field 1445 rawmem = make_store(control, rawmem, object, arrayOopDesc::length_offset_in_bytes(), length, T_INT); 1446 // conservatively small header size: 1447 header_size = arrayOopDesc::base_offset_in_bytes(T_BYTE); 1448 ciKlass* k = _igvn.type(klass_node)->is_klassptr()->klass(); 1449 if (k->is_array_klass()) // we know the exact header size in most cases: 1450 header_size = Klass::layout_helper_header_size(k->layout_helper()); 1451 } 1452 1453 // Clear the object body, if necessary. 1454 if (init == NULL) { 1455 // The init has somehow disappeared; be cautious and clear everything. 1456 // 1457 // This can happen if a node is allocated but an uncommon trap occurs 1458 // immediately. In this case, the Initialize gets associated with the 1459 // trap, and may be placed in a different (outer) loop, if the Allocate 1460 // is in a loop. If (this is rare) the inner loop gets unrolled, then 1461 // there can be two Allocates to one Initialize. The answer in all these 1462 // edge cases is safety first. It is always safe to clear immediately 1463 // within an Allocate, and then (maybe or maybe not) clear some more later. 1464 if (!ZeroTLAB) 1465 rawmem = ClearArrayNode::clear_memory(control, rawmem, object, 1466 header_size, size_in_bytes, 1467 &_igvn); 1468 } else { 1469 if (!init->is_complete()) { 1470 // Try to win by zeroing only what the init does not store. 1471 // We can also try to do some peephole optimizations, 1472 // such as combining some adjacent subword stores. 1473 rawmem = init->complete_stores(control, rawmem, object, 1474 header_size, size_in_bytes, &_igvn); 1475 } 1476 // We have no more use for this link, since the AllocateNode goes away: 1477 init->set_req(InitializeNode::RawAddress, top()); 1478 // (If we keep the link, it just confuses the register allocator, 1479 // who thinks he sees a real use of the address by the membar.) 1480 } 1481 1482 return rawmem; 1483} 1484 1485// Generate prefetch instructions for next allocations. 1486Node* PhaseMacroExpand::prefetch_allocation(Node* i_o, Node*& needgc_false, 1487 Node*& contended_phi_rawmem, 1488 Node* old_eden_top, Node* new_eden_top, 1489 Node* length) { 1490 enum { fall_in_path = 1, pf_path = 2 }; 1491 if( UseTLAB && AllocatePrefetchStyle == 2 ) { 1492 // Generate prefetch allocation with watermark check. 1493 // As an allocation hits the watermark, we will prefetch starting 1494 // at a "distance" away from watermark. 1495 1496 Node *pf_region = new (C, 3) RegionNode(3); 1497 Node *pf_phi_rawmem = new (C, 3) PhiNode( pf_region, Type::MEMORY, 1498 TypeRawPtr::BOTTOM ); 1499 // I/O is used for Prefetch 1500 Node *pf_phi_abio = new (C, 3) PhiNode( pf_region, Type::ABIO ); 1501 1502 Node *thread = new (C, 1) ThreadLocalNode(); 1503 transform_later(thread); 1504 1505 Node *eden_pf_adr = new (C, 4) AddPNode( top()/*not oop*/, thread, 1506 _igvn.MakeConX(in_bytes(JavaThread::tlab_pf_top_offset())) ); 1507 transform_later(eden_pf_adr); 1508 1509 Node *old_pf_wm = new (C, 3) LoadPNode( needgc_false, 1510 contended_phi_rawmem, eden_pf_adr, 1511 TypeRawPtr::BOTTOM, TypeRawPtr::BOTTOM ); 1512 transform_later(old_pf_wm); 1513 1514 // check against new_eden_top 1515 Node *need_pf_cmp = new (C, 3) CmpPNode( new_eden_top, old_pf_wm ); 1516 transform_later(need_pf_cmp); 1517 Node *need_pf_bol = new (C, 2) BoolNode( need_pf_cmp, BoolTest::ge ); 1518 transform_later(need_pf_bol); 1519 IfNode *need_pf_iff = new (C, 2) IfNode( needgc_false, need_pf_bol, 1520 PROB_UNLIKELY_MAG(4), COUNT_UNKNOWN ); 1521 transform_later(need_pf_iff); 1522 1523 // true node, add prefetchdistance 1524 Node *need_pf_true = new (C, 1) IfTrueNode( need_pf_iff ); 1525 transform_later(need_pf_true); 1526 1527 Node *need_pf_false = new (C, 1) IfFalseNode( need_pf_iff ); 1528 transform_later(need_pf_false); 1529 1530 Node *new_pf_wmt = new (C, 4) AddPNode( top(), old_pf_wm, 1531 _igvn.MakeConX(AllocatePrefetchDistance) ); 1532 transform_later(new_pf_wmt ); 1533 new_pf_wmt->set_req(0, need_pf_true); 1534 1535 Node *store_new_wmt = new (C, 4) StorePNode( need_pf_true, 1536 contended_phi_rawmem, eden_pf_adr, 1537 TypeRawPtr::BOTTOM, new_pf_wmt ); 1538 transform_later(store_new_wmt); 1539 1540 // adding prefetches 1541 pf_phi_abio->init_req( fall_in_path, i_o ); 1542 1543 Node *prefetch_adr; 1544 Node *prefetch; 1545 uint lines = AllocatePrefetchDistance / AllocatePrefetchStepSize; 1546 uint step_size = AllocatePrefetchStepSize; 1547 uint distance = 0; 1548 1549 for ( uint i = 0; i < lines; i++ ) { 1550 prefetch_adr = new (C, 4) AddPNode( old_pf_wm, new_pf_wmt, 1551 _igvn.MakeConX(distance) ); 1552 transform_later(prefetch_adr); 1553 prefetch = new (C, 3) PrefetchWriteNode( i_o, prefetch_adr ); 1554 transform_later(prefetch); 1555 distance += step_size; 1556 i_o = prefetch; 1557 } 1558 pf_phi_abio->set_req( pf_path, i_o ); 1559 1560 pf_region->init_req( fall_in_path, need_pf_false ); 1561 pf_region->init_req( pf_path, need_pf_true ); 1562 1563 pf_phi_rawmem->init_req( fall_in_path, contended_phi_rawmem ); 1564 pf_phi_rawmem->init_req( pf_path, store_new_wmt ); 1565 1566 transform_later(pf_region); 1567 transform_later(pf_phi_rawmem); 1568 transform_later(pf_phi_abio); 1569 1570 needgc_false = pf_region; 1571 contended_phi_rawmem = pf_phi_rawmem; 1572 i_o = pf_phi_abio; 1573 } else if( UseTLAB && AllocatePrefetchStyle == 3 ) { 1574 // Insert a prefetch for each allocation only on the fast-path 1575 Node *pf_region = new (C, 3) RegionNode(3); 1576 Node *pf_phi_rawmem = new (C, 3) PhiNode( pf_region, Type::MEMORY, 1577 TypeRawPtr::BOTTOM ); 1578 1579 // Generate several prefetch instructions only for arrays. 1580 uint lines = (length != NULL) ? AllocatePrefetchLines : 1; 1581 uint step_size = AllocatePrefetchStepSize; 1582 uint distance = AllocatePrefetchDistance; 1583 1584 // Next cache address. 1585 Node *cache_adr = new (C, 4) AddPNode(old_eden_top, old_eden_top, 1586 _igvn.MakeConX(distance)); 1587 transform_later(cache_adr); 1588 cache_adr = new (C, 2) CastP2XNode(needgc_false, cache_adr); 1589 transform_later(cache_adr); 1590 Node* mask = _igvn.MakeConX(~(intptr_t)(step_size-1)); 1591 cache_adr = new (C, 3) AndXNode(cache_adr, mask); 1592 transform_later(cache_adr); 1593 cache_adr = new (C, 2) CastX2PNode(cache_adr); 1594 transform_later(cache_adr); 1595 1596 // Prefetch 1597 Node *prefetch = new (C, 3) PrefetchWriteNode( contended_phi_rawmem, cache_adr ); 1598 prefetch->set_req(0, needgc_false); 1599 transform_later(prefetch); 1600 contended_phi_rawmem = prefetch; 1601 Node *prefetch_adr; 1602 distance = step_size; 1603 for ( uint i = 1; i < lines; i++ ) { 1604 prefetch_adr = new (C, 4) AddPNode( cache_adr, cache_adr, 1605 _igvn.MakeConX(distance) ); 1606 transform_later(prefetch_adr); 1607 prefetch = new (C, 3) PrefetchWriteNode( contended_phi_rawmem, prefetch_adr ); 1608 transform_later(prefetch); 1609 distance += step_size; 1610 contended_phi_rawmem = prefetch; 1611 } 1612 } else if( AllocatePrefetchStyle > 0 ) { 1613 // Insert a prefetch for each allocation only on the fast-path 1614 Node *prefetch_adr; 1615 Node *prefetch; 1616 // Generate several prefetch instructions only for arrays. 1617 uint lines = (length != NULL) ? AllocatePrefetchLines : 1; 1618 uint step_size = AllocatePrefetchStepSize; 1619 uint distance = AllocatePrefetchDistance; 1620 for ( uint i = 0; i < lines; i++ ) { 1621 prefetch_adr = new (C, 4) AddPNode( old_eden_top, new_eden_top, 1622 _igvn.MakeConX(distance) ); 1623 transform_later(prefetch_adr); 1624 prefetch = new (C, 3) PrefetchWriteNode( i_o, prefetch_adr ); 1625 // Do not let it float too high, since if eden_top == eden_end, 1626 // both might be null. 1627 if( i == 0 ) { // Set control for first prefetch, next follows it 1628 prefetch->init_req(0, needgc_false); 1629 } 1630 transform_later(prefetch); 1631 distance += step_size; 1632 i_o = prefetch; 1633 } 1634 } 1635 return i_o; 1636} 1637 1638 1639void PhaseMacroExpand::expand_allocate(AllocateNode *alloc) { 1640 expand_allocate_common(alloc, NULL, 1641 OptoRuntime::new_instance_Type(), 1642 OptoRuntime::new_instance_Java()); 1643} 1644 1645void PhaseMacroExpand::expand_allocate_array(AllocateArrayNode *alloc) { 1646 Node* length = alloc->in(AllocateNode::ALength); 1647 expand_allocate_common(alloc, length, 1648 OptoRuntime::new_array_Type(), 1649 OptoRuntime::new_array_Java()); 1650} 1651 1652 1653// we have determined that this lock/unlock can be eliminated, we simply 1654// eliminate the node without expanding it. 1655// 1656// Note: The membar's associated with the lock/unlock are currently not 1657// eliminated. This should be investigated as a future enhancement. 1658// 1659bool PhaseMacroExpand::eliminate_locking_node(AbstractLockNode *alock) { 1660 1661 if (!alock->is_eliminated()) { 1662 return false; 1663 } 1664 if (alock->is_Lock() && !alock->is_coarsened()) { 1665 // Create new "eliminated" BoxLock node and use it 1666 // in monitor debug info for the same object. 1667 BoxLockNode* oldbox = alock->box_node()->as_BoxLock(); 1668 Node* obj = alock->obj_node(); 1669 if (!oldbox->is_eliminated()) { 1670 BoxLockNode* newbox = oldbox->clone()->as_BoxLock(); 1671 newbox->set_eliminated(); 1672 transform_later(newbox); 1673 // Replace old box node with new box for all users 1674 // of the same object. 1675 for (uint i = 0; i < oldbox->outcnt();) { 1676 1677 bool next_edge = true; 1678 Node* u = oldbox->raw_out(i); 1679 if (u == alock) { 1680 i++; 1681 continue; // It will be removed below 1682 } 1683 if (u->is_Lock() && 1684 u->as_Lock()->obj_node() == obj && 1685 // oldbox could be referenced in debug info also 1686 u->as_Lock()->box_node() == oldbox) { 1687 assert(u->as_Lock()->is_eliminated(), "sanity"); 1688 _igvn.hash_delete(u); 1689 u->set_req(TypeFunc::Parms + 1, newbox); 1690 next_edge = false; 1691#ifdef ASSERT 1692 } else if (u->is_Unlock() && u->as_Unlock()->obj_node() == obj) { 1693 assert(u->as_Unlock()->is_eliminated(), "sanity"); 1694#endif 1695 } 1696 // Replace old box in monitor debug info. 1697 if (u->is_SafePoint() && u->as_SafePoint()->jvms()) { 1698 SafePointNode* sfn = u->as_SafePoint(); 1699 JVMState* youngest_jvms = sfn->jvms(); 1700 int max_depth = youngest_jvms->depth(); 1701 for (int depth = 1; depth <= max_depth; depth++) { 1702 JVMState* jvms = youngest_jvms->of_depth(depth); 1703 int num_mon = jvms->nof_monitors(); 1704 // Loop over monitors 1705 for (int idx = 0; idx < num_mon; idx++) { 1706 Node* obj_node = sfn->monitor_obj(jvms, idx); 1707 Node* box_node = sfn->monitor_box(jvms, idx); 1708 if (box_node == oldbox && obj_node == obj) { 1709 int j = jvms->monitor_box_offset(idx); 1710 _igvn.hash_delete(u); 1711 u->set_req(j, newbox); 1712 next_edge = false; 1713 } 1714 } // for (int idx = 0; 1715 } // for (int depth = 1; 1716 } // if (u->is_SafePoint() 1717 if (next_edge) i++; 1718 } // for (uint i = 0; i < oldbox->outcnt();) 1719 } // if (!oldbox->is_eliminated()) 1720 } // if (alock->is_Lock() && !lock->is_coarsened()) 1721 1722 CompileLog* log = C->log(); 1723 if (log != NULL) { 1724 log->head("eliminate_lock lock='%d'", 1725 alock->is_Lock()); 1726 JVMState* p = alock->jvms(); 1727 while (p != NULL) { 1728 log->elem("jvms bci='%d' method='%d'", p->bci(), log->identify(p->method())); 1729 p = p->caller(); 1730 } 1731 log->tail("eliminate_lock"); 1732 } 1733 1734 #ifndef PRODUCT 1735 if (PrintEliminateLocks) { 1736 if (alock->is_Lock()) { 1737 tty->print_cr("++++ Eliminating: %d Lock", alock->_idx); 1738 } else { 1739 tty->print_cr("++++ Eliminating: %d Unlock", alock->_idx); 1740 } 1741 } 1742 #endif 1743 1744 Node* mem = alock->in(TypeFunc::Memory); 1745 Node* ctrl = alock->in(TypeFunc::Control); 1746 1747 extract_call_projections(alock); 1748 // There are 2 projections from the lock. The lock node will 1749 // be deleted when its last use is subsumed below. 1750 assert(alock->outcnt() == 2 && 1751 _fallthroughproj != NULL && 1752 _memproj_fallthrough != NULL, 1753 "Unexpected projections from Lock/Unlock"); 1754 1755 Node* fallthroughproj = _fallthroughproj; 1756 Node* memproj_fallthrough = _memproj_fallthrough; 1757 1758 // The memory projection from a lock/unlock is RawMem 1759 // The input to a Lock is merged memory, so extract its RawMem input 1760 // (unless the MergeMem has been optimized away.) 1761 if (alock->is_Lock()) { 1762 // Seach for MemBarAcquire node and delete it also. 1763 MemBarNode* membar = fallthroughproj->unique_ctrl_out()->as_MemBar(); 1764 assert(membar != NULL && membar->Opcode() == Op_MemBarAcquire, ""); 1765 Node* ctrlproj = membar->proj_out(TypeFunc::Control); 1766 Node* memproj = membar->proj_out(TypeFunc::Memory); 1767 _igvn.replace_node(ctrlproj, fallthroughproj); 1768 _igvn.replace_node(memproj, memproj_fallthrough); 1769 1770 // Delete FastLock node also if this Lock node is unique user 1771 // (a loop peeling may clone a Lock node). 1772 Node* flock = alock->as_Lock()->fastlock_node(); 1773 if (flock->outcnt() == 1) { 1774 assert(flock->unique_out() == alock, "sanity"); 1775 _igvn.replace_node(flock, top()); 1776 } 1777 } 1778 1779 // Seach for MemBarRelease node and delete it also. 1780 if (alock->is_Unlock() && ctrl != NULL && ctrl->is_Proj() && 1781 ctrl->in(0)->is_MemBar()) { 1782 MemBarNode* membar = ctrl->in(0)->as_MemBar(); 1783 assert(membar->Opcode() == Op_MemBarRelease && 1784 mem->is_Proj() && membar == mem->in(0), ""); 1785 _igvn.replace_node(fallthroughproj, ctrl); 1786 _igvn.replace_node(memproj_fallthrough, mem); 1787 fallthroughproj = ctrl; 1788 memproj_fallthrough = mem; 1789 ctrl = membar->in(TypeFunc::Control); 1790 mem = membar->in(TypeFunc::Memory); 1791 } 1792 1793 _igvn.replace_node(fallthroughproj, ctrl); 1794 _igvn.replace_node(memproj_fallthrough, mem); 1795 return true; 1796} 1797 1798 1799//------------------------------expand_lock_node---------------------- 1800void PhaseMacroExpand::expand_lock_node(LockNode *lock) { 1801 1802 Node* ctrl = lock->in(TypeFunc::Control); 1803 Node* mem = lock->in(TypeFunc::Memory); 1804 Node* obj = lock->obj_node(); 1805 Node* box = lock->box_node(); 1806 Node* flock = lock->fastlock_node(); 1807 1808 // Make the merge point 1809 Node *region; 1810 Node *mem_phi; 1811 Node *slow_path; 1812 1813 if (UseOptoBiasInlining) { 1814 /* 1815 * See the full description in MacroAssembler::biased_locking_enter(). 1816 * 1817 * if( (mark_word & biased_lock_mask) == biased_lock_pattern ) { 1818 * // The object is biased. 1819 * proto_node = klass->prototype_header; 1820 * o_node = thread | proto_node; 1821 * x_node = o_node ^ mark_word; 1822 * if( (x_node & ~age_mask) == 0 ) { // Biased to the current thread ? 1823 * // Done. 1824 * } else { 1825 * if( (x_node & biased_lock_mask) != 0 ) { 1826 * // The klass's prototype header is no longer biased. 1827 * cas(&mark_word, mark_word, proto_node) 1828 * goto cas_lock; 1829 * } else { 1830 * // The klass's prototype header is still biased. 1831 * if( (x_node & epoch_mask) != 0 ) { // Expired epoch? 1832 * old = mark_word; 1833 * new = o_node; 1834 * } else { 1835 * // Different thread or anonymous biased. 1836 * old = mark_word & (epoch_mask | age_mask | biased_lock_mask); 1837 * new = thread | old; 1838 * } 1839 * // Try to rebias. 1840 * if( cas(&mark_word, old, new) == 0 ) { 1841 * // Done. 1842 * } else { 1843 * goto slow_path; // Failed. 1844 * } 1845 * } 1846 * } 1847 * } else { 1848 * // The object is not biased. 1849 * cas_lock: 1850 * if( FastLock(obj) == 0 ) { 1851 * // Done. 1852 * } else { 1853 * slow_path: 1854 * OptoRuntime::complete_monitor_locking_Java(obj); 1855 * } 1856 * } 1857 */ 1858 1859 region = new (C, 5) RegionNode(5); 1860 // create a Phi for the memory state 1861 mem_phi = new (C, 5) PhiNode( region, Type::MEMORY, TypeRawPtr::BOTTOM); 1862 1863 Node* fast_lock_region = new (C, 3) RegionNode(3); 1864 Node* fast_lock_mem_phi = new (C, 3) PhiNode( fast_lock_region, Type::MEMORY, TypeRawPtr::BOTTOM); 1865 1866 // First, check mark word for the biased lock pattern. 1867 Node* mark_node = make_load(ctrl, mem, obj, oopDesc::mark_offset_in_bytes(), TypeX_X, TypeX_X->basic_type()); 1868 1869 // Get fast path - mark word has the biased lock pattern. 1870 ctrl = opt_bits_test(ctrl, fast_lock_region, 1, mark_node, 1871 markOopDesc::biased_lock_mask_in_place, 1872 markOopDesc::biased_lock_pattern, true); 1873 // fast_lock_region->in(1) is set to slow path. 1874 fast_lock_mem_phi->init_req(1, mem); 1875 1876 // Now check that the lock is biased to the current thread and has 1877 // the same epoch and bias as Klass::_prototype_header. 1878 1879 // Special-case a fresh allocation to avoid building nodes: 1880 Node* klass_node = AllocateNode::Ideal_klass(obj, &_igvn); 1881 if (klass_node == NULL) { 1882 Node* k_adr = basic_plus_adr(obj, oopDesc::klass_offset_in_bytes()); 1883 klass_node = transform_later( LoadKlassNode::make(_igvn, mem, k_adr, _igvn.type(k_adr)->is_ptr()) ); 1884#ifdef _LP64 1885 if (UseCompressedOops && klass_node->is_DecodeN()) { 1886 assert(klass_node->in(1)->Opcode() == Op_LoadNKlass, "sanity"); 1887 klass_node->in(1)->init_req(0, ctrl); 1888 } else 1889#endif 1890 klass_node->init_req(0, ctrl); 1891 } 1892 Node *proto_node = make_load(ctrl, mem, klass_node, Klass::prototype_header_offset_in_bytes() + sizeof(oopDesc), TypeX_X, TypeX_X->basic_type()); 1893 1894 Node* thread = transform_later(new (C, 1) ThreadLocalNode()); 1895 Node* cast_thread = transform_later(new (C, 2) CastP2XNode(ctrl, thread)); 1896 Node* o_node = transform_later(new (C, 3) OrXNode(cast_thread, proto_node)); 1897 Node* x_node = transform_later(new (C, 3) XorXNode(o_node, mark_node)); 1898 1899 // Get slow path - mark word does NOT match the value. 1900 Node* not_biased_ctrl = opt_bits_test(ctrl, region, 3, x_node, 1901 (~markOopDesc::age_mask_in_place), 0); 1902 // region->in(3) is set to fast path - the object is biased to the current thread. 1903 mem_phi->init_req(3, mem); 1904 1905 1906 // Mark word does NOT match the value (thread | Klass::_prototype_header). 1907 1908 1909 // First, check biased pattern. 1910 // Get fast path - _prototype_header has the same biased lock pattern. 1911 ctrl = opt_bits_test(not_biased_ctrl, fast_lock_region, 2, x_node, 1912 markOopDesc::biased_lock_mask_in_place, 0, true); 1913 1914 not_biased_ctrl = fast_lock_region->in(2); // Slow path 1915 // fast_lock_region->in(2) - the prototype header is no longer biased 1916 // and we have to revoke the bias on this object. 1917 // We are going to try to reset the mark of this object to the prototype 1918 // value and fall through to the CAS-based locking scheme. 1919 Node* adr = basic_plus_adr(obj, oopDesc::mark_offset_in_bytes()); 1920 Node* cas = new (C, 5) StoreXConditionalNode(not_biased_ctrl, mem, adr, 1921 proto_node, mark_node); 1922 transform_later(cas); 1923 Node* proj = transform_later( new (C, 1) SCMemProjNode(cas)); 1924 fast_lock_mem_phi->init_req(2, proj); 1925 1926 1927 // Second, check epoch bits. 1928 Node* rebiased_region = new (C, 3) RegionNode(3); 1929 Node* old_phi = new (C, 3) PhiNode( rebiased_region, TypeX_X); 1930 Node* new_phi = new (C, 3) PhiNode( rebiased_region, TypeX_X); 1931 1932 // Get slow path - mark word does NOT match epoch bits. 1933 Node* epoch_ctrl = opt_bits_test(ctrl, rebiased_region, 1, x_node, 1934 markOopDesc::epoch_mask_in_place, 0); 1935 // The epoch of the current bias is not valid, attempt to rebias the object 1936 // toward the current thread. 1937 rebiased_region->init_req(2, epoch_ctrl); 1938 old_phi->init_req(2, mark_node); 1939 new_phi->init_req(2, o_node); 1940 1941 // rebiased_region->in(1) is set to fast path. 1942 // The epoch of the current bias is still valid but we know 1943 // nothing about the owner; it might be set or it might be clear. 1944 Node* cmask = MakeConX(markOopDesc::biased_lock_mask_in_place | 1945 markOopDesc::age_mask_in_place | 1946 markOopDesc::epoch_mask_in_place); 1947 Node* old = transform_later(new (C, 3) AndXNode(mark_node, cmask)); 1948 cast_thread = transform_later(new (C, 2) CastP2XNode(ctrl, thread)); 1949 Node* new_mark = transform_later(new (C, 3) OrXNode(cast_thread, old)); 1950 old_phi->init_req(1, old); 1951 new_phi->init_req(1, new_mark); 1952 1953 transform_later(rebiased_region); 1954 transform_later(old_phi); 1955 transform_later(new_phi); 1956 1957 // Try to acquire the bias of the object using an atomic operation. 1958 // If this fails we will go in to the runtime to revoke the object's bias. 1959 cas = new (C, 5) StoreXConditionalNode(rebiased_region, mem, adr, 1960 new_phi, old_phi); 1961 transform_later(cas); 1962 proj = transform_later( new (C, 1) SCMemProjNode(cas)); 1963 1964 // Get slow path - Failed to CAS. 1965 not_biased_ctrl = opt_bits_test(rebiased_region, region, 4, cas, 0, 0); 1966 mem_phi->init_req(4, proj); 1967 // region->in(4) is set to fast path - the object is rebiased to the current thread. 1968 1969 // Failed to CAS. 1970 slow_path = new (C, 3) RegionNode(3); 1971 Node *slow_mem = new (C, 3) PhiNode( slow_path, Type::MEMORY, TypeRawPtr::BOTTOM); 1972 1973 slow_path->init_req(1, not_biased_ctrl); // Capture slow-control 1974 slow_mem->init_req(1, proj); 1975 1976 // Call CAS-based locking scheme (FastLock node). 1977 1978 transform_later(fast_lock_region); 1979 transform_later(fast_lock_mem_phi); 1980 1981 // Get slow path - FastLock failed to lock the object. 1982 ctrl = opt_bits_test(fast_lock_region, region, 2, flock, 0, 0); 1983 mem_phi->init_req(2, fast_lock_mem_phi); 1984 // region->in(2) is set to fast path - the object is locked to the current thread. 1985 1986 slow_path->init_req(2, ctrl); // Capture slow-control 1987 slow_mem->init_req(2, fast_lock_mem_phi); 1988 1989 transform_later(slow_path); 1990 transform_later(slow_mem); 1991 // Reset lock's memory edge. 1992 lock->set_req(TypeFunc::Memory, slow_mem); 1993 1994 } else { 1995 region = new (C, 3) RegionNode(3); 1996 // create a Phi for the memory state 1997 mem_phi = new (C, 3) PhiNode( region, Type::MEMORY, TypeRawPtr::BOTTOM); 1998 1999 // Optimize test; set region slot 2 2000 slow_path = opt_bits_test(ctrl, region, 2, flock, 0, 0); 2001 mem_phi->init_req(2, mem); 2002 } 2003 2004 // Make slow path call 2005 CallNode *call = make_slow_call( (CallNode *) lock, OptoRuntime::complete_monitor_enter_Type(), OptoRuntime::complete_monitor_locking_Java(), NULL, slow_path, obj, box ); 2006 2007 extract_call_projections(call); 2008 2009 // Slow path can only throw asynchronous exceptions, which are always 2010 // de-opted. So the compiler thinks the slow-call can never throw an 2011 // exception. If it DOES throw an exception we would need the debug 2012 // info removed first (since if it throws there is no monitor). 2013 assert ( _ioproj_fallthrough == NULL && _ioproj_catchall == NULL && 2014 _memproj_catchall == NULL && _catchallcatchproj == NULL, "Unexpected projection from Lock"); 2015 2016 // Capture slow path 2017 // disconnect fall-through projection from call and create a new one 2018 // hook up users of fall-through projection to region 2019 Node *slow_ctrl = _fallthroughproj->clone(); 2020 transform_later(slow_ctrl); 2021 _igvn.hash_delete(_fallthroughproj); 2022 _fallthroughproj->disconnect_inputs(NULL); 2023 region->init_req(1, slow_ctrl); 2024 // region inputs are now complete 2025 transform_later(region); 2026 _igvn.replace_node(_fallthroughproj, region); 2027 2028 Node *memproj = transform_later( new(C, 1) ProjNode(call, TypeFunc::Memory) ); 2029 mem_phi->init_req(1, memproj ); 2030 transform_later(mem_phi); 2031 _igvn.replace_node(_memproj_fallthrough, mem_phi); 2032} 2033 2034//------------------------------expand_unlock_node---------------------- 2035void PhaseMacroExpand::expand_unlock_node(UnlockNode *unlock) { 2036 2037 Node* ctrl = unlock->in(TypeFunc::Control); 2038 Node* mem = unlock->in(TypeFunc::Memory); 2039 Node* obj = unlock->obj_node(); 2040 Node* box = unlock->box_node(); 2041 2042 // No need for a null check on unlock 2043 2044 // Make the merge point 2045 Node *region; 2046 Node *mem_phi; 2047 2048 if (UseOptoBiasInlining) { 2049 // Check for biased locking unlock case, which is a no-op. 2050 // See the full description in MacroAssembler::biased_locking_exit(). 2051 region = new (C, 4) RegionNode(4); 2052 // create a Phi for the memory state 2053 mem_phi = new (C, 4) PhiNode( region, Type::MEMORY, TypeRawPtr::BOTTOM); 2054 mem_phi->init_req(3, mem); 2055 2056 Node* mark_node = make_load(ctrl, mem, obj, oopDesc::mark_offset_in_bytes(), TypeX_X, TypeX_X->basic_type()); 2057 ctrl = opt_bits_test(ctrl, region, 3, mark_node, 2058 markOopDesc::biased_lock_mask_in_place, 2059 markOopDesc::biased_lock_pattern); 2060 } else { 2061 region = new (C, 3) RegionNode(3); 2062 // create a Phi for the memory state 2063 mem_phi = new (C, 3) PhiNode( region, Type::MEMORY, TypeRawPtr::BOTTOM); 2064 } 2065 2066 FastUnlockNode *funlock = new (C, 3) FastUnlockNode( ctrl, obj, box ); 2067 funlock = transform_later( funlock )->as_FastUnlock(); 2068 // Optimize test; set region slot 2 2069 Node *slow_path = opt_bits_test(ctrl, region, 2, funlock, 0, 0); 2070 2071 CallNode *call = make_slow_call( (CallNode *) unlock, OptoRuntime::complete_monitor_exit_Type(), CAST_FROM_FN_PTR(address, SharedRuntime::complete_monitor_unlocking_C), "complete_monitor_unlocking_C", slow_path, obj, box ); 2072 2073 extract_call_projections(call); 2074 2075 assert ( _ioproj_fallthrough == NULL && _ioproj_catchall == NULL && 2076 _memproj_catchall == NULL && _catchallcatchproj == NULL, "Unexpected projection from Lock"); 2077 2078 // No exceptions for unlocking 2079 // Capture slow path 2080 // disconnect fall-through projection from call and create a new one 2081 // hook up users of fall-through projection to region 2082 Node *slow_ctrl = _fallthroughproj->clone(); 2083 transform_later(slow_ctrl); 2084 _igvn.hash_delete(_fallthroughproj); 2085 _fallthroughproj->disconnect_inputs(NULL); 2086 region->init_req(1, slow_ctrl); 2087 // region inputs are now complete 2088 transform_later(region); 2089 _igvn.replace_node(_fallthroughproj, region); 2090 2091 Node *memproj = transform_later( new(C, 1) ProjNode(call, TypeFunc::Memory) ); 2092 mem_phi->init_req(1, memproj ); 2093 mem_phi->init_req(2, mem); 2094 transform_later(mem_phi); 2095 _igvn.replace_node(_memproj_fallthrough, mem_phi); 2096} 2097 2098//------------------------------expand_macro_nodes---------------------- 2099// Returns true if a failure occurred. 2100bool PhaseMacroExpand::expand_macro_nodes() { 2101 if (C->macro_count() == 0) 2102 return false; 2103 // First, attempt to eliminate locks 2104 bool progress = true; 2105 while (progress) { 2106 progress = false; 2107 for (int i = C->macro_count(); i > 0; i--) { 2108 Node * n = C->macro_node(i-1); 2109 bool success = false; 2110 debug_only(int old_macro_count = C->macro_count();); 2111 if (n->is_AbstractLock()) { 2112 success = eliminate_locking_node(n->as_AbstractLock()); 2113 } else if (n->Opcode() == Op_Opaque1 || n->Opcode() == Op_Opaque2) { 2114 _igvn.replace_node(n, n->in(1)); 2115 success = true; 2116 } 2117 assert(success == (C->macro_count() < old_macro_count), "elimination reduces macro count"); 2118 progress = progress || success; 2119 } 2120 } 2121 // Next, attempt to eliminate allocations 2122 progress = true; 2123 while (progress) { 2124 progress = false; 2125 for (int i = C->macro_count(); i > 0; i--) { 2126 Node * n = C->macro_node(i-1); 2127 bool success = false; 2128 debug_only(int old_macro_count = C->macro_count();); 2129 switch (n->class_id()) { 2130 case Node::Class_Allocate: 2131 case Node::Class_AllocateArray: 2132 success = eliminate_allocate_node(n->as_Allocate()); 2133 break; 2134 case Node::Class_Lock: 2135 case Node::Class_Unlock: 2136 assert(!n->as_AbstractLock()->is_eliminated(), "sanity"); 2137 break; 2138 default: 2139 assert(false, "unknown node type in macro list"); 2140 } 2141 assert(success == (C->macro_count() < old_macro_count), "elimination reduces macro count"); 2142 progress = progress || success; 2143 } 2144 } 2145 // Make sure expansion will not cause node limit to be exceeded. 2146 // Worst case is a macro node gets expanded into about 50 nodes. 2147 // Allow 50% more for optimization. 2148 if (C->check_node_count(C->macro_count() * 75, "out of nodes before macro expansion" ) ) 2149 return true; 2150 2151 // expand "macro" nodes 2152 // nodes are removed from the macro list as they are processed 2153 while (C->macro_count() > 0) { 2154 int macro_count = C->macro_count(); 2155 Node * n = C->macro_node(macro_count-1); 2156 assert(n->is_macro(), "only macro nodes expected here"); 2157 if (_igvn.type(n) == Type::TOP || n->in(0)->is_top() ) { 2158 // node is unreachable, so don't try to expand it 2159 C->remove_macro_node(n); 2160 continue; 2161 } 2162 switch (n->class_id()) { 2163 case Node::Class_Allocate: 2164 expand_allocate(n->as_Allocate()); 2165 break; 2166 case Node::Class_AllocateArray: 2167 expand_allocate_array(n->as_AllocateArray()); 2168 break; 2169 case Node::Class_Lock: 2170 expand_lock_node(n->as_Lock()); 2171 break; 2172 case Node::Class_Unlock: 2173 expand_unlock_node(n->as_Unlock()); 2174 break; 2175 default: 2176 assert(false, "unknown node type in macro list"); 2177 } 2178 assert(C->macro_count() < macro_count, "must have deleted a node from macro list"); 2179 if (C->failing()) return true; 2180 } 2181 2182 _igvn.set_delay_transform(false); 2183 _igvn.optimize(); 2184 return false; 2185} 2186