library_call.cpp revision 4802:f2110083203d 
1/*
2 * Copyright (c) 1999, 2013, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25#include "precompiled.hpp"
26#include "classfile/systemDictionary.hpp"
27#include "classfile/vmSymbols.hpp"
28#include "compiler/compileBroker.hpp"
29#include "compiler/compileLog.hpp"
30#include "oops/objArrayKlass.hpp"
31#include "opto/addnode.hpp"
32#include "opto/callGenerator.hpp"
33#include "opto/cfgnode.hpp"
34#include "opto/idealKit.hpp"
35#include "opto/mulnode.hpp"
36#include "opto/parse.hpp"
37#include "opto/runtime.hpp"
38#include "opto/subnode.hpp"
39#include "prims/nativeLookup.hpp"
40#include "runtime/sharedRuntime.hpp"
41#include "trace/traceMacros.hpp"
42
43class LibraryIntrinsic : public InlineCallGenerator {
44  // Extend the set of intrinsics known to the runtime:
45 public:
46 private:
47  bool             _is_virtual;
48  bool             _is_predicted;
49  vmIntrinsics::ID _intrinsic_id;
50
51 public:
52  LibraryIntrinsic(ciMethod* m, bool is_virtual, bool is_predicted, vmIntrinsics::ID id)
53    : InlineCallGenerator(m),
54      _is_virtual(is_virtual),
55      _is_predicted(is_predicted),
56      _intrinsic_id(id)
57  {
58  }
59  virtual bool is_intrinsic() const { return true; }
60  virtual bool is_virtual()   const { return _is_virtual; }
61  virtual bool is_predicted()   const { return _is_predicted; }
62  virtual JVMState* generate(JVMState* jvms);
63  virtual Node* generate_predicate(JVMState* jvms);
64  vmIntrinsics::ID intrinsic_id() const { return _intrinsic_id; }
65};
66
67
68// Local helper class for LibraryIntrinsic:
69class LibraryCallKit : public GraphKit {
70 private:
71  LibraryIntrinsic* _intrinsic;     // the library intrinsic being called
72  Node*             _result;        // the result node, if any
73  int               _reexecute_sp;  // the stack pointer when bytecode needs to be reexecuted
74
75  const TypeOopPtr* sharpen_unsafe_type(Compile::AliasType* alias_type, const TypePtr *adr_type, bool is_native_ptr = false);
76
77 public:
78  LibraryCallKit(JVMState* jvms, LibraryIntrinsic* intrinsic)
79    : GraphKit(jvms),
80      _intrinsic(intrinsic),
81      _result(NULL)
82  {
83    // Check if this is a root compile.  In that case we don't have a caller.
84    if (!jvms->has_method()) {
85      _reexecute_sp = sp();
86    } else {
87      // Find out how many arguments the interpreter needs when deoptimizing
88      // and save the stack pointer value so it can used by uncommon_trap.
89      // We find the argument count by looking at the declared signature.
90      bool ignored_will_link;
91      ciSignature* declared_signature = NULL;
92      ciMethod* ignored_callee = caller()->get_method_at_bci(bci(), ignored_will_link, &declared_signature);
93      const int nargs = declared_signature->arg_size_for_bc(caller()->java_code_at_bci(bci()));
94      _reexecute_sp = sp() + nargs;  // "push" arguments back on stack
95    }
96  }
97
98  virtual LibraryCallKit* is_LibraryCallKit() const { return (LibraryCallKit*)this; }
99
100  ciMethod*         caller()    const    { return jvms()->method(); }
101  int               bci()       const    { return jvms()->bci(); }
102  LibraryIntrinsic* intrinsic() const    { return _intrinsic; }
103  vmIntrinsics::ID  intrinsic_id() const { return _intrinsic->intrinsic_id(); }
104  ciMethod*         callee()    const    { return _intrinsic->method(); }
105
106  bool try_to_inline();
107  Node* try_to_predicate();
108
109  void push_result() {
110    // Push the result onto the stack.
111    if (!stopped() && result() != NULL) {
112      BasicType bt = result()->bottom_type()->basic_type();
113      push_node(bt, result());
114    }
115  }
116
117 private:
118  void fatal_unexpected_iid(vmIntrinsics::ID iid) {
119    fatal(err_msg_res("unexpected intrinsic %d: %s", iid, vmIntrinsics::name_at(iid)));
120  }
121
122  void  set_result(Node* n) { assert(_result == NULL, "only set once"); _result = n; }
123  void  set_result(RegionNode* region, PhiNode* value);
124  Node*     result() { return _result; }
125
126  virtual int reexecute_sp() { return _reexecute_sp; }
127
128  // Helper functions to inline natives
129  Node* generate_guard(Node* test, RegionNode* region, float true_prob);
130  Node* generate_slow_guard(Node* test, RegionNode* region);
131  Node* generate_fair_guard(Node* test, RegionNode* region);
132  Node* generate_negative_guard(Node* index, RegionNode* region,
133                                // resulting CastII of index:
134                                Node* *pos_index = NULL);
135  Node* generate_nonpositive_guard(Node* index, bool never_negative,
136                                   // resulting CastII of index:
137                                   Node* *pos_index = NULL);
138  Node* generate_limit_guard(Node* offset, Node* subseq_length,
139                             Node* array_length,
140                             RegionNode* region);
141  Node* generate_current_thread(Node* &tls_output);
142  address basictype2arraycopy(BasicType t, Node *src_offset, Node *dest_offset,
143                              bool disjoint_bases, const char* &name, bool dest_uninitialized);
144  Node* load_mirror_from_klass(Node* klass);
145  Node* load_klass_from_mirror_common(Node* mirror, bool never_see_null,
146                                      RegionNode* region, int null_path,
147                                      int offset);
148  Node* load_klass_from_mirror(Node* mirror, bool never_see_null,
149                               RegionNode* region, int null_path) {
150    int offset = java_lang_Class::klass_offset_in_bytes();
151    return load_klass_from_mirror_common(mirror, never_see_null,
152                                         region, null_path,
153                                         offset);
154  }
155  Node* load_array_klass_from_mirror(Node* mirror, bool never_see_null,
156                                     RegionNode* region, int null_path) {
157    int offset = java_lang_Class::array_klass_offset_in_bytes();
158    return load_klass_from_mirror_common(mirror, never_see_null,
159                                         region, null_path,
160                                         offset);
161  }
162  Node* generate_access_flags_guard(Node* kls,
163                                    int modifier_mask, int modifier_bits,
164                                    RegionNode* region);
165  Node* generate_interface_guard(Node* kls, RegionNode* region);
166  Node* generate_array_guard(Node* kls, RegionNode* region) {
167    return generate_array_guard_common(kls, region, false, false);
168  }
169  Node* generate_non_array_guard(Node* kls, RegionNode* region) {
170    return generate_array_guard_common(kls, region, false, true);
171  }
172  Node* generate_objArray_guard(Node* kls, RegionNode* region) {
173    return generate_array_guard_common(kls, region, true, false);
174  }
175  Node* generate_non_objArray_guard(Node* kls, RegionNode* region) {
176    return generate_array_guard_common(kls, region, true, true);
177  }
178  Node* generate_array_guard_common(Node* kls, RegionNode* region,
179                                    bool obj_array, bool not_array);
180  Node* generate_virtual_guard(Node* obj_klass, RegionNode* slow_region);
181  CallJavaNode* generate_method_call(vmIntrinsics::ID method_id,
182                                     bool is_virtual = false, bool is_static = false);
183  CallJavaNode* generate_method_call_static(vmIntrinsics::ID method_id) {
184    return generate_method_call(method_id, false, true);
185  }
186  CallJavaNode* generate_method_call_virtual(vmIntrinsics::ID method_id) {
187    return generate_method_call(method_id, true, false);
188  }
189  Node * load_field_from_object(Node * fromObj, const char * fieldName, const char * fieldTypeString, bool is_exact, bool is_static);
190
191  Node* make_string_method_node(int opcode, Node* str1_start, Node* cnt1, Node* str2_start, Node* cnt2);
192  Node* make_string_method_node(int opcode, Node* str1, Node* str2);
193  bool inline_string_compareTo();
194  bool inline_string_indexOf();
195  Node* string_indexOf(Node* string_object, ciTypeArray* target_array, jint offset, jint cache_i, jint md2_i);
196  bool inline_string_equals();
197  Node* round_double_node(Node* n);
198  bool runtime_math(const TypeFunc* call_type, address funcAddr, const char* funcName);
199  bool inline_math_native(vmIntrinsics::ID id);
200  bool inline_trig(vmIntrinsics::ID id);
201  bool inline_math(vmIntrinsics::ID id);
202  bool inline_exp();
203  bool inline_pow();
204  void finish_pow_exp(Node* result, Node* x, Node* y, const TypeFunc* call_type, address funcAddr, const char* funcName);
205  bool inline_min_max(vmIntrinsics::ID id);
206  Node* generate_min_max(vmIntrinsics::ID id, Node* x, Node* y);
207  // This returns Type::AnyPtr, RawPtr, or OopPtr.
208  int classify_unsafe_addr(Node* &base, Node* &offset);
209  Node* make_unsafe_address(Node* base, Node* offset);
210  // Helper for inline_unsafe_access.
211  // Generates the guards that check whether the result of
212  // Unsafe.getObject should be recorded in an SATB log buffer.
213  void insert_pre_barrier(Node* base_oop, Node* offset, Node* pre_val, bool need_mem_bar);
214  bool inline_unsafe_access(bool is_native_ptr, bool is_store, BasicType type, bool is_volatile);
215  bool inline_unsafe_prefetch(bool is_native_ptr, bool is_store, bool is_static);
216  bool inline_unsafe_allocate();
217  bool inline_unsafe_copyMemory();
218  bool inline_native_currentThread();
219#ifdef TRACE_HAVE_INTRINSICS
220  bool inline_native_classID();
221  bool inline_native_threadID();
222#endif
223  bool inline_native_time_funcs(address method, const char* funcName);
224  bool inline_native_isInterrupted();
225  bool inline_native_Class_query(vmIntrinsics::ID id);
226  bool inline_native_subtype_check();
227
228  bool inline_native_newArray();
229  bool inline_native_getLength();
230  bool inline_array_copyOf(bool is_copyOfRange);
231  bool inline_array_equals();
232  void copy_to_clone(Node* obj, Node* alloc_obj, Node* obj_size, bool is_array, bool card_mark);
233  bool inline_native_clone(bool is_virtual);
234  bool inline_native_Reflection_getCallerClass();
235  // Helper function for inlining native object hash method
236  bool inline_native_hashcode(bool is_virtual, bool is_static);
237  bool inline_native_getClass();
238
239  // Helper functions for inlining arraycopy
240  bool inline_arraycopy();
241  void generate_arraycopy(const TypePtr* adr_type,
242                          BasicType basic_elem_type,
243                          Node* src,  Node* src_offset,
244                          Node* dest, Node* dest_offset,
245                          Node* copy_length,
246                          bool disjoint_bases = false,
247                          bool length_never_negative = false,
248                          RegionNode* slow_region = NULL);
249  AllocateArrayNode* tightly_coupled_allocation(Node* ptr,
250                                                RegionNode* slow_region);
251  void generate_clear_array(const TypePtr* adr_type,
252                            Node* dest,
253                            BasicType basic_elem_type,
254                            Node* slice_off,
255                            Node* slice_len,
256                            Node* slice_end);
257  bool generate_block_arraycopy(const TypePtr* adr_type,
258                                BasicType basic_elem_type,
259                                AllocateNode* alloc,
260                                Node* src,  Node* src_offset,
261                                Node* dest, Node* dest_offset,
262                                Node* dest_size, bool dest_uninitialized);
263  void generate_slow_arraycopy(const TypePtr* adr_type,
264                               Node* src,  Node* src_offset,
265                               Node* dest, Node* dest_offset,
266                               Node* copy_length, bool dest_uninitialized);
267  Node* generate_checkcast_arraycopy(const TypePtr* adr_type,
268                                     Node* dest_elem_klass,
269                                     Node* src,  Node* src_offset,
270                                     Node* dest, Node* dest_offset,
271                                     Node* copy_length, bool dest_uninitialized);
272  Node* generate_generic_arraycopy(const TypePtr* adr_type,
273                                   Node* src,  Node* src_offset,
274                                   Node* dest, Node* dest_offset,
275                                   Node* copy_length, bool dest_uninitialized);
276  void generate_unchecked_arraycopy(const TypePtr* adr_type,
277                                    BasicType basic_elem_type,
278                                    bool disjoint_bases,
279                                    Node* src,  Node* src_offset,
280                                    Node* dest, Node* dest_offset,
281                                    Node* copy_length, bool dest_uninitialized);
282  typedef enum { LS_xadd, LS_xchg, LS_cmpxchg } LoadStoreKind;
283  bool inline_unsafe_load_store(BasicType type,  LoadStoreKind kind);
284  bool inline_unsafe_ordered_store(BasicType type);
285  bool inline_unsafe_fence(vmIntrinsics::ID id);
286  bool inline_fp_conversions(vmIntrinsics::ID id);
287  bool inline_number_methods(vmIntrinsics::ID id);
288  bool inline_reference_get();
289  bool inline_aescrypt_Block(vmIntrinsics::ID id);
290  bool inline_cipherBlockChaining_AESCrypt(vmIntrinsics::ID id);
291  Node* inline_cipherBlockChaining_AESCrypt_predicate(bool decrypting);
292  Node* get_key_start_from_aescrypt_object(Node* aescrypt_object);
293  bool inline_encodeISOArray();
294};
295
296
297//---------------------------make_vm_intrinsic----------------------------
298CallGenerator* Compile::make_vm_intrinsic(ciMethod* m, bool is_virtual) {
299  vmIntrinsics::ID id = m->intrinsic_id();
300  assert(id != vmIntrinsics::_none, "must be a VM intrinsic");
301
302  if (DisableIntrinsic[0] != '\0'
303      && strstr(DisableIntrinsic, vmIntrinsics::name_at(id)) != NULL) {
304    // disabled by a user request on the command line:
305    // example: -XX:DisableIntrinsic=_hashCode,_getClass
306    return NULL;
307  }
308
309  if (!m->is_loaded()) {
310    // do not attempt to inline unloaded methods
311    return NULL;
312  }
313
314  // Only a few intrinsics implement a virtual dispatch.
315  // They are expensive calls which are also frequently overridden.
316  if (is_virtual) {
317    switch (id) {
318    case vmIntrinsics::_hashCode:
319    case vmIntrinsics::_clone:
320      // OK, Object.hashCode and Object.clone intrinsics come in both flavors
321      break;
322    default:
323      return NULL;
324    }
325  }
326
327  // -XX:-InlineNatives disables nearly all intrinsics:
328  if (!InlineNatives) {
329    switch (id) {
330    case vmIntrinsics::_indexOf:
331    case vmIntrinsics::_compareTo:
332    case vmIntrinsics::_equals:
333    case vmIntrinsics::_equalsC:
334    case vmIntrinsics::_getAndAddInt:
335    case vmIntrinsics::_getAndAddLong:
336    case vmIntrinsics::_getAndSetInt:
337    case vmIntrinsics::_getAndSetLong:
338    case vmIntrinsics::_getAndSetObject:
339    case vmIntrinsics::_loadFence:
340    case vmIntrinsics::_storeFence:
341    case vmIntrinsics::_fullFence:
342      break;  // InlineNatives does not control String.compareTo
343    case vmIntrinsics::_Reference_get:
344      break;  // InlineNatives does not control Reference.get
345    default:
346      return NULL;
347    }
348  }
349
350  bool is_predicted = false;
351
352  switch (id) {
353  case vmIntrinsics::_compareTo:
354    if (!SpecialStringCompareTo)  return NULL;
355    if (!Matcher::match_rule_supported(Op_StrComp))  return NULL;
356    break;
357  case vmIntrinsics::_indexOf:
358    if (!SpecialStringIndexOf)  return NULL;
359    break;
360  case vmIntrinsics::_equals:
361    if (!SpecialStringEquals)  return NULL;
362    if (!Matcher::match_rule_supported(Op_StrEquals))  return NULL;
363    break;
364  case vmIntrinsics::_equalsC:
365    if (!SpecialArraysEquals)  return NULL;
366    if (!Matcher::match_rule_supported(Op_AryEq))  return NULL;
367    break;
368  case vmIntrinsics::_arraycopy:
369    if (!InlineArrayCopy)  return NULL;
370    break;
371  case vmIntrinsics::_copyMemory:
372    if (StubRoutines::unsafe_arraycopy() == NULL)  return NULL;
373    if (!InlineArrayCopy)  return NULL;
374    break;
375  case vmIntrinsics::_hashCode:
376    if (!InlineObjectHash)  return NULL;
377    break;
378  case vmIntrinsics::_clone:
379  case vmIntrinsics::_copyOf:
380  case vmIntrinsics::_copyOfRange:
381    if (!InlineObjectCopy)  return NULL;
382    // These also use the arraycopy intrinsic mechanism:
383    if (!InlineArrayCopy)  return NULL;
384    break;
385  case vmIntrinsics::_encodeISOArray:
386    if (!SpecialEncodeISOArray)  return NULL;
387    if (!Matcher::match_rule_supported(Op_EncodeISOArray))  return NULL;
388    break;
389  case vmIntrinsics::_checkIndex:
390    // We do not intrinsify this.  The optimizer does fine with it.
391    return NULL;
392
393  case vmIntrinsics::_getCallerClass:
394    if (!UseNewReflection)  return NULL;
395    if (!InlineReflectionGetCallerClass)  return NULL;
396    if (SystemDictionary::reflect_CallerSensitive_klass() == NULL)  return NULL;
397    break;
398
399  case vmIntrinsics::_bitCount_i:
400    if (!Matcher::match_rule_supported(Op_PopCountI)) return NULL;
401    break;
402
403  case vmIntrinsics::_bitCount_l:
404    if (!Matcher::match_rule_supported(Op_PopCountL)) return NULL;
405    break;
406
407  case vmIntrinsics::_numberOfLeadingZeros_i:
408    if (!Matcher::match_rule_supported(Op_CountLeadingZerosI)) return NULL;
409    break;
410
411  case vmIntrinsics::_numberOfLeadingZeros_l:
412    if (!Matcher::match_rule_supported(Op_CountLeadingZerosL)) return NULL;
413    break;
414
415  case vmIntrinsics::_numberOfTrailingZeros_i:
416    if (!Matcher::match_rule_supported(Op_CountTrailingZerosI)) return NULL;
417    break;
418
419  case vmIntrinsics::_numberOfTrailingZeros_l:
420    if (!Matcher::match_rule_supported(Op_CountTrailingZerosL)) return NULL;
421    break;
422
423  case vmIntrinsics::_reverseBytes_c:
424    if (!Matcher::match_rule_supported(Op_ReverseBytesUS)) return NULL;
425    break;
426  case vmIntrinsics::_reverseBytes_s:
427    if (!Matcher::match_rule_supported(Op_ReverseBytesS))  return NULL;
428    break;
429  case vmIntrinsics::_reverseBytes_i:
430    if (!Matcher::match_rule_supported(Op_ReverseBytesI))  return NULL;
431    break;
432  case vmIntrinsics::_reverseBytes_l:
433    if (!Matcher::match_rule_supported(Op_ReverseBytesL))  return NULL;
434    break;
435
436  case vmIntrinsics::_Reference_get:
437    // Use the intrinsic version of Reference.get() so that the value in
438    // the referent field can be registered by the G1 pre-barrier code.
439    // Also add memory barrier to prevent commoning reads from this field
440    // across safepoint since GC can change it value.
441    break;
442
443  case vmIntrinsics::_compareAndSwapObject:
444#ifdef _LP64
445    if (!UseCompressedOops && !Matcher::match_rule_supported(Op_CompareAndSwapP)) return NULL;
446#endif
447    break;
448
449  case vmIntrinsics::_compareAndSwapLong:
450    if (!Matcher::match_rule_supported(Op_CompareAndSwapL)) return NULL;
451    break;
452
453  case vmIntrinsics::_getAndAddInt:
454    if (!Matcher::match_rule_supported(Op_GetAndAddI)) return NULL;
455    break;
456
457  case vmIntrinsics::_getAndAddLong:
458    if (!Matcher::match_rule_supported(Op_GetAndAddL)) return NULL;
459    break;
460
461  case vmIntrinsics::_getAndSetInt:
462    if (!Matcher::match_rule_supported(Op_GetAndSetI)) return NULL;
463    break;
464
465  case vmIntrinsics::_getAndSetLong:
466    if (!Matcher::match_rule_supported(Op_GetAndSetL)) return NULL;
467    break;
468
469  case vmIntrinsics::_getAndSetObject:
470#ifdef _LP64
471    if (!UseCompressedOops && !Matcher::match_rule_supported(Op_GetAndSetP)) return NULL;
472    if (UseCompressedOops && !Matcher::match_rule_supported(Op_GetAndSetN)) return NULL;
473    break;
474#else
475    if (!Matcher::match_rule_supported(Op_GetAndSetP)) return NULL;
476    break;
477#endif
478
479  case vmIntrinsics::_aescrypt_encryptBlock:
480  case vmIntrinsics::_aescrypt_decryptBlock:
481    if (!UseAESIntrinsics) return NULL;
482    break;
483
484  case vmIntrinsics::_cipherBlockChaining_encryptAESCrypt:
485  case vmIntrinsics::_cipherBlockChaining_decryptAESCrypt:
486    if (!UseAESIntrinsics) return NULL;
487    // these two require the predicated logic
488    is_predicted = true;
489    break;
490
491 default:
492    assert(id <= vmIntrinsics::LAST_COMPILER_INLINE, "caller responsibility");
493    assert(id != vmIntrinsics::_Object_init && id != vmIntrinsics::_invoke, "enum out of order?");
494    break;
495  }
496
497  // -XX:-InlineClassNatives disables natives from the Class class.
498  // The flag applies to all reflective calls, notably Array.newArray
499  // (visible to Java programmers as Array.newInstance).
500  if (m->holder()->name() == ciSymbol::java_lang_Class() ||
501      m->holder()->name() == ciSymbol::java_lang_reflect_Array()) {
502    if (!InlineClassNatives)  return NULL;
503  }
504
505  // -XX:-InlineThreadNatives disables natives from the Thread class.
506  if (m->holder()->name() == ciSymbol::java_lang_Thread()) {
507    if (!InlineThreadNatives)  return NULL;
508  }
509
510  // -XX:-InlineMathNatives disables natives from the Math,Float and Double classes.
511  if (m->holder()->name() == ciSymbol::java_lang_Math() ||
512      m->holder()->name() == ciSymbol::java_lang_Float() ||
513      m->holder()->name() == ciSymbol::java_lang_Double()) {
514    if (!InlineMathNatives)  return NULL;
515  }
516
517  // -XX:-InlineUnsafeOps disables natives from the Unsafe class.
518  if (m->holder()->name() == ciSymbol::sun_misc_Unsafe()) {
519    if (!InlineUnsafeOps)  return NULL;
520  }
521
522  return new LibraryIntrinsic(m, is_virtual, is_predicted, (vmIntrinsics::ID) id);
523}
524
525//----------------------register_library_intrinsics-----------------------
526// Initialize this file's data structures, for each Compile instance.
527void Compile::register_library_intrinsics() {
528  // Nothing to do here.
529}
530
531JVMState* LibraryIntrinsic::generate(JVMState* jvms) {
532  LibraryCallKit kit(jvms, this);
533  Compile* C = kit.C;
534  int nodes = C->unique();
535#ifndef PRODUCT
536  if ((PrintIntrinsics || PrintInlining NOT_PRODUCT( || PrintOptoInlining) ) && Verbose) {
537    char buf[1000];
538    const char* str = vmIntrinsics::short_name_as_C_string(intrinsic_id(), buf, sizeof(buf));
539    tty->print_cr("Intrinsic %s", str);
540  }
541#endif
542  ciMethod* callee = kit.callee();
543  const int bci    = kit.bci();
544
545  // Try to inline the intrinsic.
546  if (kit.try_to_inline()) {
547    if (PrintIntrinsics || PrintInlining NOT_PRODUCT( || PrintOptoInlining) ) {
548      C->print_inlining(callee, jvms->depth() - 1, bci, is_virtual() ? "(intrinsic, virtual)" : "(intrinsic)");
549    }
550    C->gather_intrinsic_statistics(intrinsic_id(), is_virtual(), Compile::_intrinsic_worked);
551    if (C->log()) {
552      C->log()->elem("intrinsic id='%s'%s nodes='%d'",
553                     vmIntrinsics::name_at(intrinsic_id()),
554                     (is_virtual() ? " virtual='1'" : ""),
555                     C->unique() - nodes);
556    }
557    // Push the result from the inlined method onto the stack.
558    kit.push_result();
559    return kit.transfer_exceptions_into_jvms();
560  }
561
562  // The intrinsic bailed out
563  if (PrintIntrinsics || PrintInlining NOT_PRODUCT( || PrintOptoInlining) ) {
564    if (jvms->has_method()) {
565      // Not a root compile.
566      const char* msg = is_virtual() ? "failed to inline (intrinsic, virtual)" : "failed to inline (intrinsic)";
567      C->print_inlining(callee, jvms->depth() - 1, bci, msg);
568    } else {
569      // Root compile
570      tty->print("Did not generate intrinsic %s%s at bci:%d in",
571               vmIntrinsics::name_at(intrinsic_id()),
572               (is_virtual() ? " (virtual)" : ""), bci);
573    }
574  }
575  C->gather_intrinsic_statistics(intrinsic_id(), is_virtual(), Compile::_intrinsic_failed);
576  return NULL;
577}
578
579Node* LibraryIntrinsic::generate_predicate(JVMState* jvms) {
580  LibraryCallKit kit(jvms, this);
581  Compile* C = kit.C;
582  int nodes = C->unique();
583#ifndef PRODUCT
584  assert(is_predicted(), "sanity");
585  if ((PrintIntrinsics || PrintInlining NOT_PRODUCT( || PrintOptoInlining) ) && Verbose) {
586    char buf[1000];
587    const char* str = vmIntrinsics::short_name_as_C_string(intrinsic_id(), buf, sizeof(buf));
588    tty->print_cr("Predicate for intrinsic %s", str);
589  }
590#endif
591  ciMethod* callee = kit.callee();
592  const int bci    = kit.bci();
593
594  Node* slow_ctl = kit.try_to_predicate();
595  if (!kit.failing()) {
596    if (PrintIntrinsics || PrintInlining NOT_PRODUCT( || PrintOptoInlining) ) {
597      C->print_inlining(callee, jvms->depth() - 1, bci, is_virtual() ? "(intrinsic, virtual)" : "(intrinsic)");
598    }
599    C->gather_intrinsic_statistics(intrinsic_id(), is_virtual(), Compile::_intrinsic_worked);
600    if (C->log()) {
601      C->log()->elem("predicate_intrinsic id='%s'%s nodes='%d'",
602                     vmIntrinsics::name_at(intrinsic_id()),
603                     (is_virtual() ? " virtual='1'" : ""),
604                     C->unique() - nodes);
605    }
606    return slow_ctl; // Could be NULL if the check folds.
607  }
608
609  // The intrinsic bailed out
610  if (PrintIntrinsics || PrintInlining NOT_PRODUCT( || PrintOptoInlining) ) {
611    if (jvms->has_method()) {
612      // Not a root compile.
613      const char* msg = "failed to generate predicate for intrinsic";
614      C->print_inlining(kit.callee(), jvms->depth() - 1, bci, msg);
615    } else {
616      // Root compile
617      C->print_inlining_stream()->print("Did not generate predicate for intrinsic %s%s at bci:%d in",
618                                        vmIntrinsics::name_at(intrinsic_id()),
619                                        (is_virtual() ? " (virtual)" : ""), bci);
620    }
621  }
622  C->gather_intrinsic_statistics(intrinsic_id(), is_virtual(), Compile::_intrinsic_failed);
623  return NULL;
624}
625
626bool LibraryCallKit::try_to_inline() {
627  // Handle symbolic names for otherwise undistinguished boolean switches:
628  const bool is_store       = true;
629  const bool is_native_ptr  = true;
630  const bool is_static      = true;
631  const bool is_volatile    = true;
632
633  if (!jvms()->has_method()) {
634    // Root JVMState has a null method.
635    assert(map()->memory()->Opcode() == Op_Parm, "");
636    // Insert the memory aliasing node
637    set_all_memory(reset_memory());
638  }
639  assert(merged_memory(), "");
640
641
642  switch (intrinsic_id()) {
643  case vmIntrinsics::_hashCode:                 return inline_native_hashcode(intrinsic()->is_virtual(), !is_static);
644  case vmIntrinsics::_identityHashCode:         return inline_native_hashcode(/*!virtual*/ false,         is_static);
645  case vmIntrinsics::_getClass:                 return inline_native_getClass();
646
647  case vmIntrinsics::_dsin:
648  case vmIntrinsics::_dcos:
649  case vmIntrinsics::_dtan:
650  case vmIntrinsics::_dabs:
651  case vmIntrinsics::_datan2:
652  case vmIntrinsics::_dsqrt:
653  case vmIntrinsics::_dexp:
654  case vmIntrinsics::_dlog:
655  case vmIntrinsics::_dlog10:
656  case vmIntrinsics::_dpow:                     return inline_math_native(intrinsic_id());
657
658  case vmIntrinsics::_min:
659  case vmIntrinsics::_max:                      return inline_min_max(intrinsic_id());
660
661  case vmIntrinsics::_arraycopy:                return inline_arraycopy();
662
663  case vmIntrinsics::_compareTo:                return inline_string_compareTo();
664  case vmIntrinsics::_indexOf:                  return inline_string_indexOf();
665  case vmIntrinsics::_equals:                   return inline_string_equals();
666
667  case vmIntrinsics::_getObject:                return inline_unsafe_access(!is_native_ptr, !is_store, T_OBJECT,  !is_volatile);
668  case vmIntrinsics::_getBoolean:               return inline_unsafe_access(!is_native_ptr, !is_store, T_BOOLEAN, !is_volatile);
669  case vmIntrinsics::_getByte:                  return inline_unsafe_access(!is_native_ptr, !is_store, T_BYTE,    !is_volatile);
670  case vmIntrinsics::_getShort:                 return inline_unsafe_access(!is_native_ptr, !is_store, T_SHORT,   !is_volatile);
671  case vmIntrinsics::_getChar:                  return inline_unsafe_access(!is_native_ptr, !is_store, T_CHAR,    !is_volatile);
672  case vmIntrinsics::_getInt:                   return inline_unsafe_access(!is_native_ptr, !is_store, T_INT,     !is_volatile);
673  case vmIntrinsics::_getLong:                  return inline_unsafe_access(!is_native_ptr, !is_store, T_LONG,    !is_volatile);
674  case vmIntrinsics::_getFloat:                 return inline_unsafe_access(!is_native_ptr, !is_store, T_FLOAT,   !is_volatile);
675  case vmIntrinsics::_getDouble:                return inline_unsafe_access(!is_native_ptr, !is_store, T_DOUBLE,  !is_volatile);
676
677  case vmIntrinsics::_putObject:                return inline_unsafe_access(!is_native_ptr,  is_store, T_OBJECT,  !is_volatile);
678  case vmIntrinsics::_putBoolean:               return inline_unsafe_access(!is_native_ptr,  is_store, T_BOOLEAN, !is_volatile);
679  case vmIntrinsics::_putByte:                  return inline_unsafe_access(!is_native_ptr,  is_store, T_BYTE,    !is_volatile);
680  case vmIntrinsics::_putShort:                 return inline_unsafe_access(!is_native_ptr,  is_store, T_SHORT,   !is_volatile);
681  case vmIntrinsics::_putChar:                  return inline_unsafe_access(!is_native_ptr,  is_store, T_CHAR,    !is_volatile);
682  case vmIntrinsics::_putInt:                   return inline_unsafe_access(!is_native_ptr,  is_store, T_INT,     !is_volatile);
683  case vmIntrinsics::_putLong:                  return inline_unsafe_access(!is_native_ptr,  is_store, T_LONG,    !is_volatile);
684  case vmIntrinsics::_putFloat:                 return inline_unsafe_access(!is_native_ptr,  is_store, T_FLOAT,   !is_volatile);
685  case vmIntrinsics::_putDouble:                return inline_unsafe_access(!is_native_ptr,  is_store, T_DOUBLE,  !is_volatile);
686
687  case vmIntrinsics::_getByte_raw:              return inline_unsafe_access( is_native_ptr, !is_store, T_BYTE,    !is_volatile);
688  case vmIntrinsics::_getShort_raw:             return inline_unsafe_access( is_native_ptr, !is_store, T_SHORT,   !is_volatile);
689  case vmIntrinsics::_getChar_raw:              return inline_unsafe_access( is_native_ptr, !is_store, T_CHAR,    !is_volatile);
690  case vmIntrinsics::_getInt_raw:               return inline_unsafe_access( is_native_ptr, !is_store, T_INT,     !is_volatile);
691  case vmIntrinsics::_getLong_raw:              return inline_unsafe_access( is_native_ptr, !is_store, T_LONG,    !is_volatile);
692  case vmIntrinsics::_getFloat_raw:             return inline_unsafe_access( is_native_ptr, !is_store, T_FLOAT,   !is_volatile);
693  case vmIntrinsics::_getDouble_raw:            return inline_unsafe_access( is_native_ptr, !is_store, T_DOUBLE,  !is_volatile);
694  case vmIntrinsics::_getAddress_raw:           return inline_unsafe_access( is_native_ptr, !is_store, T_ADDRESS, !is_volatile);
695
696  case vmIntrinsics::_putByte_raw:              return inline_unsafe_access( is_native_ptr,  is_store, T_BYTE,    !is_volatile);
697  case vmIntrinsics::_putShort_raw:             return inline_unsafe_access( is_native_ptr,  is_store, T_SHORT,   !is_volatile);
698  case vmIntrinsics::_putChar_raw:              return inline_unsafe_access( is_native_ptr,  is_store, T_CHAR,    !is_volatile);
699  case vmIntrinsics::_putInt_raw:               return inline_unsafe_access( is_native_ptr,  is_store, T_INT,     !is_volatile);
700  case vmIntrinsics::_putLong_raw:              return inline_unsafe_access( is_native_ptr,  is_store, T_LONG,    !is_volatile);
701  case vmIntrinsics::_putFloat_raw:             return inline_unsafe_access( is_native_ptr,  is_store, T_FLOAT,   !is_volatile);
702  case vmIntrinsics::_putDouble_raw:            return inline_unsafe_access( is_native_ptr,  is_store, T_DOUBLE,  !is_volatile);
703  case vmIntrinsics::_putAddress_raw:           return inline_unsafe_access( is_native_ptr,  is_store, T_ADDRESS, !is_volatile);
704
705  case vmIntrinsics::_getObjectVolatile:        return inline_unsafe_access(!is_native_ptr, !is_store, T_OBJECT,   is_volatile);
706  case vmIntrinsics::_getBooleanVolatile:       return inline_unsafe_access(!is_native_ptr, !is_store, T_BOOLEAN,  is_volatile);
707  case vmIntrinsics::_getByteVolatile:          return inline_unsafe_access(!is_native_ptr, !is_store, T_BYTE,     is_volatile);
708  case vmIntrinsics::_getShortVolatile:         return inline_unsafe_access(!is_native_ptr, !is_store, T_SHORT,    is_volatile);
709  case vmIntrinsics::_getCharVolatile:          return inline_unsafe_access(!is_native_ptr, !is_store, T_CHAR,     is_volatile);
710  case vmIntrinsics::_getIntVolatile:           return inline_unsafe_access(!is_native_ptr, !is_store, T_INT,      is_volatile);
711  case vmIntrinsics::_getLongVolatile:          return inline_unsafe_access(!is_native_ptr, !is_store, T_LONG,     is_volatile);
712  case vmIntrinsics::_getFloatVolatile:         return inline_unsafe_access(!is_native_ptr, !is_store, T_FLOAT,    is_volatile);
713  case vmIntrinsics::_getDoubleVolatile:        return inline_unsafe_access(!is_native_ptr, !is_store, T_DOUBLE,   is_volatile);
714
715  case vmIntrinsics::_putObjectVolatile:        return inline_unsafe_access(!is_native_ptr,  is_store, T_OBJECT,   is_volatile);
716  case vmIntrinsics::_putBooleanVolatile:       return inline_unsafe_access(!is_native_ptr,  is_store, T_BOOLEAN,  is_volatile);
717  case vmIntrinsics::_putByteVolatile:          return inline_unsafe_access(!is_native_ptr,  is_store, T_BYTE,     is_volatile);
718  case vmIntrinsics::_putShortVolatile:         return inline_unsafe_access(!is_native_ptr,  is_store, T_SHORT,    is_volatile);
719  case vmIntrinsics::_putCharVolatile:          return inline_unsafe_access(!is_native_ptr,  is_store, T_CHAR,     is_volatile);
720  case vmIntrinsics::_putIntVolatile:           return inline_unsafe_access(!is_native_ptr,  is_store, T_INT,      is_volatile);
721  case vmIntrinsics::_putLongVolatile:          return inline_unsafe_access(!is_native_ptr,  is_store, T_LONG,     is_volatile);
722  case vmIntrinsics::_putFloatVolatile:         return inline_unsafe_access(!is_native_ptr,  is_store, T_FLOAT,    is_volatile);
723  case vmIntrinsics::_putDoubleVolatile:        return inline_unsafe_access(!is_native_ptr,  is_store, T_DOUBLE,   is_volatile);
724
725  case vmIntrinsics::_prefetchRead:             return inline_unsafe_prefetch(!is_native_ptr, !is_store, !is_static);
726  case vmIntrinsics::_prefetchWrite:            return inline_unsafe_prefetch(!is_native_ptr,  is_store, !is_static);
727  case vmIntrinsics::_prefetchReadStatic:       return inline_unsafe_prefetch(!is_native_ptr, !is_store,  is_static);
728  case vmIntrinsics::_prefetchWriteStatic:      return inline_unsafe_prefetch(!is_native_ptr,  is_store,  is_static);
729
730  case vmIntrinsics::_compareAndSwapObject:     return inline_unsafe_load_store(T_OBJECT, LS_cmpxchg);
731  case vmIntrinsics::_compareAndSwapInt:        return inline_unsafe_load_store(T_INT,    LS_cmpxchg);
732  case vmIntrinsics::_compareAndSwapLong:       return inline_unsafe_load_store(T_LONG,   LS_cmpxchg);
733
734  case vmIntrinsics::_putOrderedObject:         return inline_unsafe_ordered_store(T_OBJECT);
735  case vmIntrinsics::_putOrderedInt:            return inline_unsafe_ordered_store(T_INT);
736  case vmIntrinsics::_putOrderedLong:           return inline_unsafe_ordered_store(T_LONG);
737
738  case vmIntrinsics::_getAndAddInt:             return inline_unsafe_load_store(T_INT,    LS_xadd);
739  case vmIntrinsics::_getAndAddLong:            return inline_unsafe_load_store(T_LONG,   LS_xadd);
740  case vmIntrinsics::_getAndSetInt:             return inline_unsafe_load_store(T_INT,    LS_xchg);
741  case vmIntrinsics::_getAndSetLong:            return inline_unsafe_load_store(T_LONG,   LS_xchg);
742  case vmIntrinsics::_getAndSetObject:          return inline_unsafe_load_store(T_OBJECT, LS_xchg);
743
744  case vmIntrinsics::_loadFence:
745  case vmIntrinsics::_storeFence:
746  case vmIntrinsics::_fullFence:                return inline_unsafe_fence(intrinsic_id());
747
748  case vmIntrinsics::_currentThread:            return inline_native_currentThread();
749  case vmIntrinsics::_isInterrupted:            return inline_native_isInterrupted();
750
751#ifdef TRACE_HAVE_INTRINSICS
752  case vmIntrinsics::_classID:                  return inline_native_classID();
753  case vmIntrinsics::_threadID:                 return inline_native_threadID();
754  case vmIntrinsics::_counterTime:              return inline_native_time_funcs(CAST_FROM_FN_PTR(address, TRACE_TIME_METHOD), "counterTime");
755#endif
756  case vmIntrinsics::_currentTimeMillis:        return inline_native_time_funcs(CAST_FROM_FN_PTR(address, os::javaTimeMillis), "currentTimeMillis");
757  case vmIntrinsics::_nanoTime:                 return inline_native_time_funcs(CAST_FROM_FN_PTR(address, os::javaTimeNanos), "nanoTime");
758  case vmIntrinsics::_allocateInstance:         return inline_unsafe_allocate();
759  case vmIntrinsics::_copyMemory:               return inline_unsafe_copyMemory();
760  case vmIntrinsics::_newArray:                 return inline_native_newArray();
761  case vmIntrinsics::_getLength:                return inline_native_getLength();
762  case vmIntrinsics::_copyOf:                   return inline_array_copyOf(false);
763  case vmIntrinsics::_copyOfRange:              return inline_array_copyOf(true);
764  case vmIntrinsics::_equalsC:                  return inline_array_equals();
765  case vmIntrinsics::_clone:                    return inline_native_clone(intrinsic()->is_virtual());
766
767  case vmIntrinsics::_isAssignableFrom:         return inline_native_subtype_check();
768
769  case vmIntrinsics::_isInstance:
770  case vmIntrinsics::_getModifiers:
771  case vmIntrinsics::_isInterface:
772  case vmIntrinsics::_isArray:
773  case vmIntrinsics::_isPrimitive:
774  case vmIntrinsics::_getSuperclass:
775  case vmIntrinsics::_getComponentType:
776  case vmIntrinsics::_getClassAccessFlags:      return inline_native_Class_query(intrinsic_id());
777
778  case vmIntrinsics::_floatToRawIntBits:
779  case vmIntrinsics::_floatToIntBits:
780  case vmIntrinsics::_intBitsToFloat:
781  case vmIntrinsics::_doubleToRawLongBits:
782  case vmIntrinsics::_doubleToLongBits:
783  case vmIntrinsics::_longBitsToDouble:         return inline_fp_conversions(intrinsic_id());
784
785  case vmIntrinsics::_numberOfLeadingZeros_i:
786  case vmIntrinsics::_numberOfLeadingZeros_l:
787  case vmIntrinsics::_numberOfTrailingZeros_i:
788  case vmIntrinsics::_numberOfTrailingZeros_l:
789  case vmIntrinsics::_bitCount_i:
790  case vmIntrinsics::_bitCount_l:
791  case vmIntrinsics::_reverseBytes_i:
792  case vmIntrinsics::_reverseBytes_l:
793  case vmIntrinsics::_reverseBytes_s:
794  case vmIntrinsics::_reverseBytes_c:           return inline_number_methods(intrinsic_id());
795
796  case vmIntrinsics::_getCallerClass:           return inline_native_Reflection_getCallerClass();
797
798  case vmIntrinsics::_Reference_get:            return inline_reference_get();
799
800  case vmIntrinsics::_aescrypt_encryptBlock:
801  case vmIntrinsics::_aescrypt_decryptBlock:    return inline_aescrypt_Block(intrinsic_id());
802
803  case vmIntrinsics::_cipherBlockChaining_encryptAESCrypt:
804  case vmIntrinsics::_cipherBlockChaining_decryptAESCrypt:
805    return inline_cipherBlockChaining_AESCrypt(intrinsic_id());
806
807  case vmIntrinsics::_encodeISOArray:
808    return inline_encodeISOArray();
809
810  default:
811    // If you get here, it may be that someone has added a new intrinsic
812    // to the list in vmSymbols.hpp without implementing it here.
813#ifndef PRODUCT
814    if ((PrintMiscellaneous && (Verbose || WizardMode)) || PrintOpto) {
815      tty->print_cr("*** Warning: Unimplemented intrinsic %s(%d)",
816                    vmIntrinsics::name_at(intrinsic_id()), intrinsic_id());
817    }
818#endif
819    return false;
820  }
821}
822
823Node* LibraryCallKit::try_to_predicate() {
824  if (!jvms()->has_method()) {
825    // Root JVMState has a null method.
826    assert(map()->memory()->Opcode() == Op_Parm, "");
827    // Insert the memory aliasing node
828    set_all_memory(reset_memory());
829  }
830  assert(merged_memory(), "");
831
832  switch (intrinsic_id()) {
833  case vmIntrinsics::_cipherBlockChaining_encryptAESCrypt:
834    return inline_cipherBlockChaining_AESCrypt_predicate(false);
835  case vmIntrinsics::_cipherBlockChaining_decryptAESCrypt:
836    return inline_cipherBlockChaining_AESCrypt_predicate(true);
837
838  default:
839    // If you get here, it may be that someone has added a new intrinsic
840    // to the list in vmSymbols.hpp without implementing it here.
841#ifndef PRODUCT
842    if ((PrintMiscellaneous && (Verbose || WizardMode)) || PrintOpto) {
843      tty->print_cr("*** Warning: Unimplemented predicate for intrinsic %s(%d)",
844                    vmIntrinsics::name_at(intrinsic_id()), intrinsic_id());
845    }
846#endif
847    Node* slow_ctl = control();
848    set_control(top()); // No fast path instrinsic
849    return slow_ctl;
850  }
851}
852
853//------------------------------set_result-------------------------------
854// Helper function for finishing intrinsics.
855void LibraryCallKit::set_result(RegionNode* region, PhiNode* value) {
856  record_for_igvn(region);
857  set_control(_gvn.transform(region));
858  set_result( _gvn.transform(value));
859  assert(value->type()->basic_type() == result()->bottom_type()->basic_type(), "sanity");
860}
861
862//------------------------------generate_guard---------------------------
863// Helper function for generating guarded fast-slow graph structures.
864// The given 'test', if true, guards a slow path.  If the test fails
865// then a fast path can be taken.  (We generally hope it fails.)
866// In all cases, GraphKit::control() is updated to the fast path.
867// The returned value represents the control for the slow path.
868// The return value is never 'top'; it is either a valid control
869// or NULL if it is obvious that the slow path can never be taken.
870// Also, if region and the slow control are not NULL, the slow edge
871// is appended to the region.
872Node* LibraryCallKit::generate_guard(Node* test, RegionNode* region, float true_prob) {
873  if (stopped()) {
874    // Already short circuited.
875    return NULL;
876  }
877
878  // Build an if node and its projections.
879  // If test is true we take the slow path, which we assume is uncommon.
880  if (_gvn.type(test) == TypeInt::ZERO) {
881    // The slow branch is never taken.  No need to build this guard.
882    return NULL;
883  }
884
885  IfNode* iff = create_and_map_if(control(), test, true_prob, COUNT_UNKNOWN);
886
887  Node* if_slow = _gvn.transform( new (C) IfTrueNode(iff) );
888  if (if_slow == top()) {
889    // The slow branch is never taken.  No need to build this guard.
890    return NULL;
891  }
892
893  if (region != NULL)
894    region->add_req(if_slow);
895
896  Node* if_fast = _gvn.transform( new (C) IfFalseNode(iff) );
897  set_control(if_fast);
898
899  return if_slow;
900}
901
902inline Node* LibraryCallKit::generate_slow_guard(Node* test, RegionNode* region) {
903  return generate_guard(test, region, PROB_UNLIKELY_MAG(3));
904}
905inline Node* LibraryCallKit::generate_fair_guard(Node* test, RegionNode* region) {
906  return generate_guard(test, region, PROB_FAIR);
907}
908
909inline Node* LibraryCallKit::generate_negative_guard(Node* index, RegionNode* region,
910                                                     Node* *pos_index) {
911  if (stopped())
912    return NULL;                // already stopped
913  if (_gvn.type(index)->higher_equal(TypeInt::POS)) // [0,maxint]
914    return NULL;                // index is already adequately typed
915  Node* cmp_lt = _gvn.transform( new (C) CmpINode(index, intcon(0)) );
916  Node* bol_lt = _gvn.transform( new (C) BoolNode(cmp_lt, BoolTest::lt) );
917  Node* is_neg = generate_guard(bol_lt, region, PROB_MIN);
918  if (is_neg != NULL && pos_index != NULL) {
919    // Emulate effect of Parse::adjust_map_after_if.
920    Node* ccast = new (C) CastIINode(index, TypeInt::POS);
921    ccast->set_req(0, control());
922    (*pos_index) = _gvn.transform(ccast);
923  }
924  return is_neg;
925}
926
927inline Node* LibraryCallKit::generate_nonpositive_guard(Node* index, bool never_negative,
928                                                        Node* *pos_index) {
929  if (stopped())
930    return NULL;                // already stopped
931  if (_gvn.type(index)->higher_equal(TypeInt::POS1)) // [1,maxint]
932    return NULL;                // index is already adequately typed
933  Node* cmp_le = _gvn.transform( new (C) CmpINode(index, intcon(0)) );
934  BoolTest::mask le_or_eq = (never_negative ? BoolTest::eq : BoolTest::le);
935  Node* bol_le = _gvn.transform( new (C) BoolNode(cmp_le, le_or_eq) );
936  Node* is_notp = generate_guard(bol_le, NULL, PROB_MIN);
937  if (is_notp != NULL && pos_index != NULL) {
938    // Emulate effect of Parse::adjust_map_after_if.
939    Node* ccast = new (C) CastIINode(index, TypeInt::POS1);
940    ccast->set_req(0, control());
941    (*pos_index) = _gvn.transform(ccast);
942  }
943  return is_notp;
944}
945
946// Make sure that 'position' is a valid limit index, in [0..length].
947// There are two equivalent plans for checking this:
948//   A. (offset + copyLength)  unsigned<=  arrayLength
949//   B. offset  <=  (arrayLength - copyLength)
950// We require that all of the values above, except for the sum and
951// difference, are already known to be non-negative.
952// Plan A is robust in the face of overflow, if offset and copyLength
953// are both hugely positive.
954//
955// Plan B is less direct and intuitive, but it does not overflow at
956// all, since the difference of two non-negatives is always
957// representable.  Whenever Java methods must perform the equivalent
958// check they generally use Plan B instead of Plan A.
959// For the moment we use Plan A.
960inline Node* LibraryCallKit::generate_limit_guard(Node* offset,
961                                                  Node* subseq_length,
962                                                  Node* array_length,
963                                                  RegionNode* region) {
964  if (stopped())
965    return NULL;                // already stopped
966  bool zero_offset = _gvn.type(offset) == TypeInt::ZERO;
967  if (zero_offset && subseq_length->eqv_uncast(array_length))
968    return NULL;                // common case of whole-array copy
969  Node* last = subseq_length;
970  if (!zero_offset)             // last += offset
971    last = _gvn.transform( new (C) AddINode(last, offset));
972  Node* cmp_lt = _gvn.transform( new (C) CmpUNode(array_length, last) );
973  Node* bol_lt = _gvn.transform( new (C) BoolNode(cmp_lt, BoolTest::lt) );
974  Node* is_over = generate_guard(bol_lt, region, PROB_MIN);
975  return is_over;
976}
977
978
979//--------------------------generate_current_thread--------------------
980Node* LibraryCallKit::generate_current_thread(Node* &tls_output) {
981  ciKlass*    thread_klass = env()->Thread_klass();
982  const Type* thread_type  = TypeOopPtr::make_from_klass(thread_klass)->cast_to_ptr_type(TypePtr::NotNull);
983  Node* thread = _gvn.transform(new (C) ThreadLocalNode());
984  Node* p = basic_plus_adr(top()/*!oop*/, thread, in_bytes(JavaThread::threadObj_offset()));
985  Node* threadObj = make_load(NULL, p, thread_type, T_OBJECT);
986  tls_output = thread;
987  return threadObj;
988}
989
990
991//------------------------------make_string_method_node------------------------
992// Helper method for String intrinsic functions. This version is called
993// with str1 and str2 pointing to String object nodes.
994//
995Node* LibraryCallKit::make_string_method_node(int opcode, Node* str1, Node* str2) {
996  Node* no_ctrl = NULL;
997
998  // Get start addr of string
999  Node* str1_value   = load_String_value(no_ctrl, str1);
1000  Node* str1_offset  = load_String_offset(no_ctrl, str1);
1001  Node* str1_start   = array_element_address(str1_value, str1_offset, T_CHAR);
1002
1003  // Get length of string 1
1004  Node* str1_len  = load_String_length(no_ctrl, str1);
1005
1006  Node* str2_value   = load_String_value(no_ctrl, str2);
1007  Node* str2_offset  = load_String_offset(no_ctrl, str2);
1008  Node* str2_start   = array_element_address(str2_value, str2_offset, T_CHAR);
1009
1010  Node* str2_len = NULL;
1011  Node* result = NULL;
1012
1013  switch (opcode) {
1014  case Op_StrIndexOf:
1015    // Get length of string 2
1016    str2_len = load_String_length(no_ctrl, str2);
1017
1018    result = new (C) StrIndexOfNode(control(), memory(TypeAryPtr::CHARS),
1019                                 str1_start, str1_len, str2_start, str2_len);
1020    break;
1021  case Op_StrComp:
1022    // Get length of string 2
1023    str2_len = load_String_length(no_ctrl, str2);
1024
1025    result = new (C) StrCompNode(control(), memory(TypeAryPtr::CHARS),
1026                                 str1_start, str1_len, str2_start, str2_len);
1027    break;
1028  case Op_StrEquals:
1029    result = new (C) StrEqualsNode(control(), memory(TypeAryPtr::CHARS),
1030                               str1_start, str2_start, str1_len);
1031    break;
1032  default:
1033    ShouldNotReachHere();
1034    return NULL;
1035  }
1036
1037  // All these intrinsics have checks.
1038  C->set_has_split_ifs(true); // Has chance for split-if optimization
1039
1040  return _gvn.transform(result);
1041}
1042
1043// Helper method for String intrinsic functions. This version is called
1044// with str1 and str2 pointing to char[] nodes, with cnt1 and cnt2 pointing
1045// to Int nodes containing the lenghts of str1 and str2.
1046//
1047Node* LibraryCallKit::make_string_method_node(int opcode, Node* str1_start, Node* cnt1, Node* str2_start, Node* cnt2) {
1048  Node* result = NULL;
1049  switch (opcode) {
1050  case Op_StrIndexOf:
1051    result = new (C) StrIndexOfNode(control(), memory(TypeAryPtr::CHARS),
1052                                 str1_start, cnt1, str2_start, cnt2);
1053    break;
1054  case Op_StrComp:
1055    result = new (C) StrCompNode(control(), memory(TypeAryPtr::CHARS),
1056                                 str1_start, cnt1, str2_start, cnt2);
1057    break;
1058  case Op_StrEquals:
1059    result = new (C) StrEqualsNode(control(), memory(TypeAryPtr::CHARS),
1060                                 str1_start, str2_start, cnt1);
1061    break;
1062  default:
1063    ShouldNotReachHere();
1064    return NULL;
1065  }
1066
1067  // All these intrinsics have checks.
1068  C->set_has_split_ifs(true); // Has chance for split-if optimization
1069
1070  return _gvn.transform(result);
1071}
1072
1073//------------------------------inline_string_compareTo------------------------
1074// public int java.lang.String.compareTo(String anotherString);
1075bool LibraryCallKit::inline_string_compareTo() {
1076  Node* receiver = null_check(argument(0));
1077  Node* arg      = null_check(argument(1));
1078  if (stopped()) {
1079    return true;
1080  }
1081  set_result(make_string_method_node(Op_StrComp, receiver, arg));
1082  return true;
1083}
1084
1085//------------------------------inline_string_equals------------------------
1086bool LibraryCallKit::inline_string_equals() {
1087  Node* receiver = null_check_receiver();
1088  // NOTE: Do not null check argument for String.equals() because spec
1089  // allows to specify NULL as argument.
1090  Node* argument = this->argument(1);
1091  if (stopped()) {
1092    return true;
1093  }
1094
1095  // paths (plus control) merge
1096  RegionNode* region = new (C) RegionNode(5);
1097  Node* phi = new (C) PhiNode(region, TypeInt::BOOL);
1098
1099  // does source == target string?
1100  Node* cmp = _gvn.transform(new (C) CmpPNode(receiver, argument));
1101  Node* bol = _gvn.transform(new (C) BoolNode(cmp, BoolTest::eq));
1102
1103  Node* if_eq = generate_slow_guard(bol, NULL);
1104  if (if_eq != NULL) {
1105    // receiver == argument
1106    phi->init_req(2, intcon(1));
1107    region->init_req(2, if_eq);
1108  }
1109
1110  // get String klass for instanceOf
1111  ciInstanceKlass* klass = env()->String_klass();
1112
1113  if (!stopped()) {
1114    Node* inst = gen_instanceof(argument, makecon(TypeKlassPtr::make(klass)));
1115    Node* cmp  = _gvn.transform(new (C) CmpINode(inst, intcon(1)));
1116    Node* bol  = _gvn.transform(new (C) BoolNode(cmp, BoolTest::ne));
1117
1118    Node* inst_false = generate_guard(bol, NULL, PROB_MIN);
1119    //instanceOf == true, fallthrough
1120
1121    if (inst_false != NULL) {
1122      phi->init_req(3, intcon(0));
1123      region->init_req(3, inst_false);
1124    }
1125  }
1126
1127  if (!stopped()) {
1128    const TypeOopPtr* string_type = TypeOopPtr::make_from_klass(klass);
1129
1130    // Properly cast the argument to String
1131    argument = _gvn.transform(new (C) CheckCastPPNode(control(), argument, string_type));
1132    // This path is taken only when argument's type is String:NotNull.
1133    argument = cast_not_null(argument, false);
1134
1135    Node* no_ctrl = NULL;
1136
1137    // Get start addr of receiver
1138    Node* receiver_val    = load_String_value(no_ctrl, receiver);
1139    Node* receiver_offset = load_String_offset(no_ctrl, receiver);
1140    Node* receiver_start = array_element_address(receiver_val, receiver_offset, T_CHAR);
1141
1142    // Get length of receiver
1143    Node* receiver_cnt  = load_String_length(no_ctrl, receiver);
1144
1145    // Get start addr of argument
1146    Node* argument_val    = load_String_value(no_ctrl, argument);
1147    Node* argument_offset = load_String_offset(no_ctrl, argument);
1148    Node* argument_start = array_element_address(argument_val, argument_offset, T_CHAR);
1149
1150    // Get length of argument
1151    Node* argument_cnt  = load_String_length(no_ctrl, argument);
1152
1153    // Check for receiver count != argument count
1154    Node* cmp = _gvn.transform( new(C) CmpINode(receiver_cnt, argument_cnt) );
1155    Node* bol = _gvn.transform( new(C) BoolNode(cmp, BoolTest::ne) );
1156    Node* if_ne = generate_slow_guard(bol, NULL);
1157    if (if_ne != NULL) {
1158      phi->init_req(4, intcon(0));
1159      region->init_req(4, if_ne);
1160    }
1161
1162    // Check for count == 0 is done by assembler code for StrEquals.
1163
1164    if (!stopped()) {
1165      Node* equals = make_string_method_node(Op_StrEquals, receiver_start, receiver_cnt, argument_start, argument_cnt);
1166      phi->init_req(1, equals);
1167      region->init_req(1, control());
1168    }
1169  }
1170
1171  // post merge
1172  set_control(_gvn.transform(region));
1173  record_for_igvn(region);
1174
1175  set_result(_gvn.transform(phi));
1176  return true;
1177}
1178
1179//------------------------------inline_array_equals----------------------------
1180bool LibraryCallKit::inline_array_equals() {
1181  Node* arg1 = argument(0);
1182  Node* arg2 = argument(1);
1183  set_result(_gvn.transform(new (C) AryEqNode(control(), memory(TypeAryPtr::CHARS), arg1, arg2)));
1184  return true;
1185}
1186
1187// Java version of String.indexOf(constant string)
1188// class StringDecl {
1189//   StringDecl(char[] ca) {
1190//     offset = 0;
1191//     count = ca.length;
1192//     value = ca;
1193//   }
1194//   int offset;
1195//   int count;
1196//   char[] value;
1197// }
1198//
1199// static int string_indexOf_J(StringDecl string_object, char[] target_object,
1200//                             int targetOffset, int cache_i, int md2) {
1201//   int cache = cache_i;
1202//   int sourceOffset = string_object.offset;
1203//   int sourceCount = string_object.count;
1204//   int targetCount = target_object.length;
1205//
1206//   int targetCountLess1 = targetCount - 1;
1207//   int sourceEnd = sourceOffset + sourceCount - targetCountLess1;
1208//
1209//   char[] source = string_object.value;
1210//   char[] target = target_object;
1211//   int lastChar = target[targetCountLess1];
1212//
1213//  outer_loop:
1214//   for (int i = sourceOffset; i < sourceEnd; ) {
1215//     int src = source[i + targetCountLess1];
1216//     if (src == lastChar) {
1217//       // With random strings and a 4-character alphabet,
1218//       // reverse matching at this point sets up 0.8% fewer
1219//       // frames, but (paradoxically) makes 0.3% more probes.
1220//       // Since those probes are nearer the lastChar probe,
1221//       // there is may be a net D$ win with reverse matching.
1222//       // But, reversing loop inhibits unroll of inner loop
1223//       // for unknown reason.  So, does running outer loop from
1224//       // (sourceOffset - targetCountLess1) to (sourceOffset + sourceCount)
1225//       for (int j = 0; j < targetCountLess1; j++) {
1226//         if (target[targetOffset + j] != source[i+j]) {
1227//           if ((cache & (1 << source[i+j])) == 0) {
1228//             if (md2 < j+1) {
1229//               i += j+1;
1230//               continue outer_loop;
1231//             }
1232//           }
1233//           i += md2;
1234//           continue outer_loop;
1235//         }
1236//       }
1237//       return i - sourceOffset;
1238//     }
1239//     if ((cache & (1 << src)) == 0) {
1240//       i += targetCountLess1;
1241//     } // using "i += targetCount;" and an "else i++;" causes a jump to jump.
1242//     i++;
1243//   }
1244//   return -1;
1245// }
1246
1247//------------------------------string_indexOf------------------------
1248Node* LibraryCallKit::string_indexOf(Node* string_object, ciTypeArray* target_array, jint targetOffset_i,
1249                                     jint cache_i, jint md2_i) {
1250
1251  Node* no_ctrl  = NULL;
1252  float likely   = PROB_LIKELY(0.9);
1253  float unlikely = PROB_UNLIKELY(0.9);
1254
1255  const int nargs = 0; // no arguments to push back for uncommon trap in predicate
1256
1257  Node* source        = load_String_value(no_ctrl, string_object);
1258  Node* sourceOffset  = load_String_offset(no_ctrl, string_object);
1259  Node* sourceCount   = load_String_length(no_ctrl, string_object);
1260
1261  Node* target = _gvn.transform( makecon(TypeOopPtr::make_from_constant(target_array, true)) );
1262  jint target_length = target_array->length();
1263  const TypeAry* target_array_type = TypeAry::make(TypeInt::CHAR, TypeInt::make(0, target_length, Type::WidenMin));
1264  const TypeAryPtr* target_type = TypeAryPtr::make(TypePtr::BotPTR, target_array_type, target_array->klass(), true, Type::OffsetBot);
1265
1266  IdealKit kit(this, false, true);
1267#define __ kit.
1268  Node* zero             = __ ConI(0);
1269  Node* one              = __ ConI(1);
1270  Node* cache            = __ ConI(cache_i);
1271  Node* md2              = __ ConI(md2_i);
1272  Node* lastChar         = __ ConI(target_array->char_at(target_length - 1));
1273  Node* targetCount      = __ ConI(target_length);
1274  Node* targetCountLess1 = __ ConI(target_length - 1);
1275  Node* targetOffset     = __ ConI(targetOffset_i);
1276  Node* sourceEnd        = __ SubI(__ AddI(sourceOffset, sourceCount), targetCountLess1);
1277
1278  IdealVariable rtn(kit), i(kit), j(kit); __ declarations_done();
1279  Node* outer_loop = __ make_label(2 /* goto */);
1280  Node* return_    = __ make_label(1);
1281
1282  __ set(rtn,__ ConI(-1));
1283  __ loop(this, nargs, i, sourceOffset, BoolTest::lt, sourceEnd); {
1284       Node* i2  = __ AddI(__ value(i), targetCountLess1);
1285       // pin to prohibit loading of "next iteration" value which may SEGV (rare)
1286       Node* src = load_array_element(__ ctrl(), source, i2, TypeAryPtr::CHARS);
1287       __ if_then(src, BoolTest::eq, lastChar, unlikely); {
1288         __ loop(this, nargs, j, zero, BoolTest::lt, targetCountLess1); {
1289              Node* tpj = __ AddI(targetOffset, __ value(j));
1290              Node* targ = load_array_element(no_ctrl, target, tpj, target_type);
1291              Node* ipj  = __ AddI(__ value(i), __ value(j));
1292              Node* src2 = load_array_element(no_ctrl, source, ipj, TypeAryPtr::CHARS);
1293              __ if_then(targ, BoolTest::ne, src2); {
1294                __ if_then(__ AndI(cache, __ LShiftI(one, src2)), BoolTest::eq, zero); {
1295                  __ if_then(md2, BoolTest::lt, __ AddI(__ value(j), one)); {
1296                    __ increment(i, __ AddI(__ value(j), one));
1297                    __ goto_(outer_loop);
1298                  } __ end_if(); __ dead(j);
1299                }__ end_if(); __ dead(j);
1300                __ increment(i, md2);
1301                __ goto_(outer_loop);
1302              }__ end_if();
1303              __ increment(j, one);
1304         }__ end_loop(); __ dead(j);
1305         __ set(rtn, __ SubI(__ value(i), sourceOffset)); __ dead(i);
1306         __ goto_(return_);
1307       }__ end_if();
1308       __ if_then(__ AndI(cache, __ LShiftI(one, src)), BoolTest::eq, zero, likely); {
1309         __ increment(i, targetCountLess1);
1310       }__ end_if();
1311       __ increment(i, one);
1312       __ bind(outer_loop);
1313  }__ end_loop(); __ dead(i);
1314  __ bind(return_);
1315
1316  // Final sync IdealKit and GraphKit.
1317  final_sync(kit);
1318  Node* result = __ value(rtn);
1319#undef __
1320  C->set_has_loops(true);
1321  return result;
1322}
1323
1324//------------------------------inline_string_indexOf------------------------
1325bool LibraryCallKit::inline_string_indexOf() {
1326  Node* receiver = argument(0);
1327  Node* arg      = argument(1);
1328
1329  Node* result;
1330  // Disable the use of pcmpestri until it can be guaranteed that
1331  // the load doesn't cross into the uncommited space.
1332  if (Matcher::has_match_rule(Op_StrIndexOf) &&
1333      UseSSE42Intrinsics) {
1334    // Generate SSE4.2 version of indexOf
1335    // We currently only have match rules that use SSE4.2
1336
1337    receiver = null_check(receiver);
1338    arg      = null_check(arg);
1339    if (stopped()) {
1340      return true;
1341    }
1342
1343    ciInstanceKlass* str_klass = env()->String_klass();
1344    const TypeOopPtr* string_type = TypeOopPtr::make_from_klass(str_klass);
1345
1346    // Make the merge point
1347    RegionNode* result_rgn = new (C) RegionNode(4);
1348    Node*       result_phi = new (C) PhiNode(result_rgn, TypeInt::INT);
1349    Node* no_ctrl  = NULL;
1350
1351    // Get start addr of source string
1352    Node* source = load_String_value(no_ctrl, receiver);
1353    Node* source_offset = load_String_offset(no_ctrl, receiver);
1354    Node* source_start = array_element_address(source, source_offset, T_CHAR);
1355
1356    // Get length of source string
1357    Node* source_cnt  = load_String_length(no_ctrl, receiver);
1358
1359    // Get start addr of substring
1360    Node* substr = load_String_value(no_ctrl, arg);
1361    Node* substr_offset = load_String_offset(no_ctrl, arg);
1362    Node* substr_start = array_element_address(substr, substr_offset, T_CHAR);
1363
1364    // Get length of source string
1365    Node* substr_cnt  = load_String_length(no_ctrl, arg);
1366
1367    // Check for substr count > string count
1368    Node* cmp = _gvn.transform( new(C) CmpINode(substr_cnt, source_cnt) );
1369    Node* bol = _gvn.transform( new(C) BoolNode(cmp, BoolTest::gt) );
1370    Node* if_gt = generate_slow_guard(bol, NULL);
1371    if (if_gt != NULL) {
1372      result_phi->init_req(2, intcon(-1));
1373      result_rgn->init_req(2, if_gt);
1374    }
1375
1376    if (!stopped()) {
1377      // Check for substr count == 0
1378      cmp = _gvn.transform( new(C) CmpINode(substr_cnt, intcon(0)) );
1379      bol = _gvn.transform( new(C) BoolNode(cmp, BoolTest::eq) );
1380      Node* if_zero = generate_slow_guard(bol, NULL);
1381      if (if_zero != NULL) {
1382        result_phi->init_req(3, intcon(0));
1383        result_rgn->init_req(3, if_zero);
1384      }
1385    }
1386
1387    if (!stopped()) {
1388      result = make_string_method_node(Op_StrIndexOf, source_start, source_cnt, substr_start, substr_cnt);
1389      result_phi->init_req(1, result);
1390      result_rgn->init_req(1, control());
1391    }
1392    set_control(_gvn.transform(result_rgn));
1393    record_for_igvn(result_rgn);
1394    result = _gvn.transform(result_phi);
1395
1396  } else { // Use LibraryCallKit::string_indexOf
1397    // don't intrinsify if argument isn't a constant string.
1398    if (!arg->is_Con()) {
1399     return false;
1400    }
1401    const TypeOopPtr* str_type = _gvn.type(arg)->isa_oopptr();
1402    if (str_type == NULL) {
1403      return false;
1404    }
1405    ciInstanceKlass* klass = env()->String_klass();
1406    ciObject* str_const = str_type->const_oop();
1407    if (str_const == NULL || str_const->klass() != klass) {
1408      return false;
1409    }
1410    ciInstance* str = str_const->as_instance();
1411    assert(str != NULL, "must be instance");
1412
1413    ciObject* v = str->field_value_by_offset(java_lang_String::value_offset_in_bytes()).as_object();
1414    ciTypeArray* pat = v->as_type_array(); // pattern (argument) character array
1415
1416    int o;
1417    int c;
1418    if (java_lang_String::has_offset_field()) {
1419      o = str->field_value_by_offset(java_lang_String::offset_offset_in_bytes()).as_int();
1420      c = str->field_value_by_offset(java_lang_String::count_offset_in_bytes()).as_int();
1421    } else {
1422      o = 0;
1423      c = pat->length();
1424    }
1425
1426    // constant strings have no offset and count == length which
1427    // simplifies the resulting code somewhat so lets optimize for that.
1428    if (o != 0 || c != pat->length()) {
1429     return false;
1430    }
1431
1432    receiver = null_check(receiver, T_OBJECT);
1433    // NOTE: No null check on the argument is needed since it's a constant String oop.
1434    if (stopped()) {
1435      return true;
1436    }
1437
1438    // The null string as a pattern always returns 0 (match at beginning of string)
1439    if (c == 0) {
1440      set_result(intcon(0));
1441      return true;
1442    }
1443
1444    // Generate default indexOf
1445    jchar lastChar = pat->char_at(o + (c - 1));
1446    int cache = 0;
1447    int i;
1448    for (i = 0; i < c - 1; i++) {
1449      assert(i < pat->length(), "out of range");
1450      cache |= (1 << (pat->char_at(o + i) & (sizeof(cache) * BitsPerByte - 1)));
1451    }
1452
1453    int md2 = c;
1454    for (i = 0; i < c - 1; i++) {
1455      assert(i < pat->length(), "out of range");
1456      if (pat->char_at(o + i) == lastChar) {
1457        md2 = (c - 1) - i;
1458      }
1459    }
1460
1461    result = string_indexOf(receiver, pat, o, cache, md2);
1462  }
1463  set_result(result);
1464  return true;
1465}
1466
1467//--------------------------round_double_node--------------------------------
1468// Round a double node if necessary.
1469Node* LibraryCallKit::round_double_node(Node* n) {
1470  if (Matcher::strict_fp_requires_explicit_rounding && UseSSE <= 1)
1471    n = _gvn.transform(new (C) RoundDoubleNode(0, n));
1472  return n;
1473}
1474
1475//------------------------------inline_math-----------------------------------
1476// public static double Math.abs(double)
1477// public static double Math.sqrt(double)
1478// public static double Math.log(double)
1479// public static double Math.log10(double)
1480bool LibraryCallKit::inline_math(vmIntrinsics::ID id) {
1481  Node* arg = round_double_node(argument(0));
1482  Node* n;
1483  switch (id) {
1484  case vmIntrinsics::_dabs:   n = new (C) AbsDNode(                arg);  break;
1485  case vmIntrinsics::_dsqrt:  n = new (C) SqrtDNode(C, control(),  arg);  break;
1486  case vmIntrinsics::_dlog:   n = new (C) LogDNode(C, control(),   arg);  break;
1487  case vmIntrinsics::_dlog10: n = new (C) Log10DNode(C, control(), arg);  break;
1488  default:  fatal_unexpected_iid(id);  break;
1489  }
1490  set_result(_gvn.transform(n));
1491  return true;
1492}
1493
1494//------------------------------inline_trig----------------------------------
1495// Inline sin/cos/tan instructions, if possible.  If rounding is required, do
1496// argument reduction which will turn into a fast/slow diamond.
1497bool LibraryCallKit::inline_trig(vmIntrinsics::ID id) {
1498  Node* arg = round_double_node(argument(0));
1499  Node* n = NULL;
1500
1501  switch (id) {
1502  case vmIntrinsics::_dsin:  n = new (C) SinDNode(C, control(), arg);  break;
1503  case vmIntrinsics::_dcos:  n = new (C) CosDNode(C, control(), arg);  break;
1504  case vmIntrinsics::_dtan:  n = new (C) TanDNode(C, control(), arg);  break;
1505  default:  fatal_unexpected_iid(id);  break;
1506  }
1507  n = _gvn.transform(n);
1508
1509  // Rounding required?  Check for argument reduction!
1510  if (Matcher::strict_fp_requires_explicit_rounding) {
1511    static const double     pi_4 =  0.7853981633974483;
1512    static const double neg_pi_4 = -0.7853981633974483;
1513    // pi/2 in 80-bit extended precision
1514    // static const unsigned char pi_2_bits_x[] = {0x35,0xc2,0x68,0x21,0xa2,0xda,0x0f,0xc9,0xff,0x3f,0x00,0x00,0x00,0x00,0x00,0x00};
1515    // -pi/2 in 80-bit extended precision
1516    // static const unsigned char neg_pi_2_bits_x[] = {0x35,0xc2,0x68,0x21,0xa2,0xda,0x0f,0xc9,0xff,0xbf,0x00,0x00,0x00,0x00,0x00,0x00};
1517    // Cutoff value for using this argument reduction technique
1518    //static const double    pi_2_minus_epsilon =  1.564660403643354;
1519    //static const double neg_pi_2_plus_epsilon = -1.564660403643354;
1520
1521    // Pseudocode for sin:
1522    // if (x <= Math.PI / 4.0) {
1523    //   if (x >= -Math.PI / 4.0) return  fsin(x);
1524    //   if (x >= -Math.PI / 2.0) return -fcos(x + Math.PI / 2.0);
1525    // } else {
1526    //   if (x <=  Math.PI / 2.0) return  fcos(x - Math.PI / 2.0);
1527    // }
1528    // return StrictMath.sin(x);
1529
1530    // Pseudocode for cos:
1531    // if (x <= Math.PI / 4.0) {
1532    //   if (x >= -Math.PI / 4.0) return  fcos(x);
1533    //   if (x >= -Math.PI / 2.0) return  fsin(x + Math.PI / 2.0);
1534    // } else {
1535    //   if (x <=  Math.PI / 2.0) return -fsin(x - Math.PI / 2.0);
1536    // }
1537    // return StrictMath.cos(x);
1538
1539    // Actually, sticking in an 80-bit Intel value into C2 will be tough; it
1540    // requires a special machine instruction to load it.  Instead we'll try
1541    // the 'easy' case.  If we really need the extra range +/- PI/2 we'll
1542    // probably do the math inside the SIN encoding.
1543
1544    // Make the merge point
1545    RegionNode* r = new (C) RegionNode(3);
1546    Node* phi = new (C) PhiNode(r, Type::DOUBLE);
1547
1548    // Flatten arg so we need only 1 test
1549    Node *abs = _gvn.transform(new (C) AbsDNode(arg));
1550    // Node for PI/4 constant
1551    Node *pi4 = makecon(TypeD::make(pi_4));
1552    // Check PI/4 : abs(arg)
1553    Node *cmp = _gvn.transform(new (C) CmpDNode(pi4,abs));
1554    // Check: If PI/4 < abs(arg) then go slow
1555    Node *bol = _gvn.transform( new (C) BoolNode( cmp, BoolTest::lt ) );
1556    // Branch either way
1557    IfNode *iff = create_and_xform_if(control(),bol, PROB_STATIC_FREQUENT, COUNT_UNKNOWN);
1558    set_control(opt_iff(r,iff));
1559
1560    // Set fast path result
1561    phi->init_req(2, n);
1562
1563    // Slow path - non-blocking leaf call
1564    Node* call = NULL;
1565    switch (id) {
1566    case vmIntrinsics::_dsin:
1567      call = make_runtime_call(RC_LEAF, OptoRuntime::Math_D_D_Type(),
1568                               CAST_FROM_FN_PTR(address, SharedRuntime::dsin),
1569                               "Sin", NULL, arg, top());
1570      break;
1571    case vmIntrinsics::_dcos:
1572      call = make_runtime_call(RC_LEAF, OptoRuntime::Math_D_D_Type(),
1573                               CAST_FROM_FN_PTR(address, SharedRuntime::dcos),
1574                               "Cos", NULL, arg, top());
1575      break;
1576    case vmIntrinsics::_dtan:
1577      call = make_runtime_call(RC_LEAF, OptoRuntime::Math_D_D_Type(),
1578                               CAST_FROM_FN_PTR(address, SharedRuntime::dtan),
1579                               "Tan", NULL, arg, top());
1580      break;
1581    }
1582    assert(control()->in(0) == call, "");
1583    Node* slow_result = _gvn.transform(new (C) ProjNode(call, TypeFunc::Parms));
1584    r->init_req(1, control());
1585    phi->init_req(1, slow_result);
1586
1587    // Post-merge
1588    set_control(_gvn.transform(r));
1589    record_for_igvn(r);
1590    n = _gvn.transform(phi);
1591
1592    C->set_has_split_ifs(true); // Has chance for split-if optimization
1593  }
1594  set_result(n);
1595  return true;
1596}
1597
1598void LibraryCallKit::finish_pow_exp(Node* result, Node* x, Node* y, const TypeFunc* call_type, address funcAddr, const char* funcName) {
1599  //-------------------
1600  //result=(result.isNaN())? funcAddr():result;
1601  // Check: If isNaN() by checking result!=result? then either trap
1602  // or go to runtime
1603  Node* cmpisnan = _gvn.transform(new (C) CmpDNode(result, result));
1604  // Build the boolean node
1605  Node* bolisnum = _gvn.transform(new (C) BoolNode(cmpisnan, BoolTest::eq));
1606
1607  if (!too_many_traps(Deoptimization::Reason_intrinsic)) {
1608    { BuildCutout unless(this, bolisnum, PROB_STATIC_FREQUENT);
1609      // The pow or exp intrinsic returned a NaN, which requires a call
1610      // to the runtime.  Recompile with the runtime call.
1611      uncommon_trap(Deoptimization::Reason_intrinsic,
1612                    Deoptimization::Action_make_not_entrant);
1613    }
1614    set_result(result);
1615  } else {
1616    // If this inlining ever returned NaN in the past, we compile a call
1617    // to the runtime to properly handle corner cases
1618
1619    IfNode* iff = create_and_xform_if(control(), bolisnum, PROB_STATIC_FREQUENT, COUNT_UNKNOWN);
1620    Node* if_slow = _gvn.transform( new (C) IfFalseNode(iff) );
1621    Node* if_fast = _gvn.transform( new (C) IfTrueNode(iff) );
1622
1623    if (!if_slow->is_top()) {
1624      RegionNode* result_region = new (C) RegionNode(3);
1625      PhiNode*    result_val = new (C) PhiNode(result_region, Type::DOUBLE);
1626
1627      result_region->init_req(1, if_fast);
1628      result_val->init_req(1, result);
1629
1630      set_control(if_slow);
1631
1632      const TypePtr* no_memory_effects = NULL;
1633      Node* rt = make_runtime_call(RC_LEAF, call_type, funcAddr, funcName,
1634                                   no_memory_effects,
1635                                   x, top(), y, y ? top() : NULL);
1636      Node* value = _gvn.transform(new (C) ProjNode(rt, TypeFunc::Parms+0));
1637#ifdef ASSERT
1638      Node* value_top = _gvn.transform(new (C) ProjNode(rt, TypeFunc::Parms+1));
1639      assert(value_top == top(), "second value must be top");
1640#endif
1641
1642      result_region->init_req(2, control());
1643      result_val->init_req(2, value);
1644      set_result(result_region, result_val);
1645    } else {
1646      set_result(result);
1647    }
1648  }
1649}
1650
1651//------------------------------inline_exp-------------------------------------
1652// Inline exp instructions, if possible.  The Intel hardware only misses
1653// really odd corner cases (+/- Infinity).  Just uncommon-trap them.
1654bool LibraryCallKit::inline_exp() {
1655  Node* arg = round_double_node(argument(0));
1656  Node* n   = _gvn.transform(new (C) ExpDNode(C, control(), arg));
1657
1658  finish_pow_exp(n, arg, NULL, OptoRuntime::Math_D_D_Type(), CAST_FROM_FN_PTR(address, SharedRuntime::dexp), "EXP");
1659
1660  C->set_has_split_ifs(true); // Has chance for split-if optimization
1661  return true;
1662}
1663
1664//------------------------------inline_pow-------------------------------------
1665// Inline power instructions, if possible.
1666bool LibraryCallKit::inline_pow() {
1667  // Pseudocode for pow
1668  // if (x <= 0.0) {
1669  //   long longy = (long)y;
1670  //   if ((double)longy == y) { // if y is long
1671  //     if (y + 1 == y) longy = 0; // huge number: even
1672  //     result = ((1&longy) == 0)?-DPow(abs(x), y):DPow(abs(x), y);
1673  //   } else {
1674  //     result = NaN;
1675  //   }
1676  // } else {
1677  //   result = DPow(x,y);
1678  // }
1679  // if (result != result)?  {
1680  //   result = uncommon_trap() or runtime_call();
1681  // }
1682  // return result;
1683
1684  Node* x = round_double_node(argument(0));
1685  Node* y = round_double_node(argument(2));
1686
1687  Node* result = NULL;
1688
1689  if (!too_many_traps(Deoptimization::Reason_intrinsic)) {
1690    // Short form: skip the fancy tests and just check for NaN result.
1691    result = _gvn.transform(new (C) PowDNode(C, control(), x, y));
1692  } else {
1693    // If this inlining ever returned NaN in the past, include all
1694    // checks + call to the runtime.
1695
1696    // Set the merge point for If node with condition of (x <= 0.0)
1697    // There are four possible paths to region node and phi node
1698    RegionNode *r = new (C) RegionNode(4);
1699    Node *phi = new (C) PhiNode(r, Type::DOUBLE);
1700
1701    // Build the first if node: if (x <= 0.0)
1702    // Node for 0 constant
1703    Node *zeronode = makecon(TypeD::ZERO);
1704    // Check x:0
1705    Node *cmp = _gvn.transform(new (C) CmpDNode(x, zeronode));
1706    // Check: If (x<=0) then go complex path
1707    Node *bol1 = _gvn.transform( new (C) BoolNode( cmp, BoolTest::le ) );
1708    // Branch either way
1709    IfNode *if1 = create_and_xform_if(control(),bol1, PROB_STATIC_INFREQUENT, COUNT_UNKNOWN);
1710    // Fast path taken; set region slot 3
1711    Node *fast_taken = _gvn.transform( new (C) IfFalseNode(if1) );
1712    r->init_req(3,fast_taken); // Capture fast-control
1713
1714    // Fast path not-taken, i.e. slow path
1715    Node *complex_path = _gvn.transform( new (C) IfTrueNode(if1) );
1716
1717    // Set fast path result
1718    Node *fast_result = _gvn.transform( new (C) PowDNode(C, control(), x, y) );
1719    phi->init_req(3, fast_result);
1720
1721    // Complex path
1722    // Build the second if node (if y is long)
1723    // Node for (long)y
1724    Node *longy = _gvn.transform( new (C) ConvD2LNode(y));
1725    // Node for (double)((long) y)
1726    Node *doublelongy= _gvn.transform( new (C) ConvL2DNode(longy));
1727    // Check (double)((long) y) : y
1728    Node *cmplongy= _gvn.transform(new (C) CmpDNode(doublelongy, y));
1729    // Check if (y isn't long) then go to slow path
1730
1731    Node *bol2 = _gvn.transform( new (C) BoolNode( cmplongy, BoolTest::ne ) );
1732    // Branch either way
1733    IfNode *if2 = create_and_xform_if(complex_path,bol2, PROB_STATIC_INFREQUENT, COUNT_UNKNOWN);
1734    Node* ylong_path = _gvn.transform( new (C) IfFalseNode(if2));
1735
1736    Node *slow_path = _gvn.transform( new (C) IfTrueNode(if2) );
1737
1738    // Calculate DPow(abs(x), y)*(1 & (long)y)
1739    // Node for constant 1
1740    Node *conone = longcon(1);
1741    // 1& (long)y
1742    Node *signnode= _gvn.transform( new (C) AndLNode(conone, longy) );
1743
1744    // A huge number is always even. Detect a huge number by checking
1745    // if y + 1 == y and set integer to be tested for parity to 0.
1746    // Required for corner case:
1747    // (long)9.223372036854776E18 = max_jlong
1748    // (double)(long)9.223372036854776E18 = 9.223372036854776E18
1749    // max_jlong is odd but 9.223372036854776E18 is even
1750    Node* yplus1 = _gvn.transform( new (C) AddDNode(y, makecon(TypeD::make(1))));
1751    Node *cmpyplus1= _gvn.transform(new (C) CmpDNode(yplus1, y));
1752    Node *bolyplus1 = _gvn.transform( new (C) BoolNode( cmpyplus1, BoolTest::eq ) );
1753    Node* correctedsign = NULL;
1754    if (ConditionalMoveLimit != 0) {
1755      correctedsign = _gvn.transform( CMoveNode::make(C, NULL, bolyplus1, signnode, longcon(0), TypeLong::LONG));
1756    } else {
1757      IfNode *ifyplus1 = create_and_xform_if(ylong_path,bolyplus1, PROB_FAIR, COUNT_UNKNOWN);
1758      RegionNode *r = new (C) RegionNode(3);
1759      Node *phi = new (C) PhiNode(r, TypeLong::LONG);
1760      r->init_req(1, _gvn.transform( new (C) IfFalseNode(ifyplus1)));
1761      r->init_req(2, _gvn.transform( new (C) IfTrueNode(ifyplus1)));
1762      phi->init_req(1, signnode);
1763      phi->init_req(2, longcon(0));
1764      correctedsign = _gvn.transform(phi);
1765      ylong_path = _gvn.transform(r);
1766      record_for_igvn(r);
1767    }
1768
1769    // zero node
1770    Node *conzero = longcon(0);
1771    // Check (1&(long)y)==0?
1772    Node *cmpeq1 = _gvn.transform(new (C) CmpLNode(correctedsign, conzero));
1773    // Check if (1&(long)y)!=0?, if so the result is negative
1774    Node *bol3 = _gvn.transform( new (C) BoolNode( cmpeq1, BoolTest::ne ) );
1775    // abs(x)
1776    Node *absx=_gvn.transform( new (C) AbsDNode(x));
1777    // abs(x)^y
1778    Node *absxpowy = _gvn.transform( new (C) PowDNode(C, control(), absx, y) );
1779    // -abs(x)^y
1780    Node *negabsxpowy = _gvn.transform(new (C) NegDNode (absxpowy));
1781    // (1&(long)y)==1?-DPow(abs(x), y):DPow(abs(x), y)
1782    Node *signresult = NULL;
1783    if (ConditionalMoveLimit != 0) {
1784      signresult = _gvn.transform( CMoveNode::make(C, NULL, bol3, absxpowy, negabsxpowy, Type::DOUBLE));
1785    } else {
1786      IfNode *ifyeven = create_and_xform_if(ylong_path,bol3, PROB_FAIR, COUNT_UNKNOWN);
1787      RegionNode *r = new (C) RegionNode(3);
1788      Node *phi = new (C) PhiNode(r, Type::DOUBLE);
1789      r->init_req(1, _gvn.transform( new (C) IfFalseNode(ifyeven)));
1790      r->init_req(2, _gvn.transform( new (C) IfTrueNode(ifyeven)));
1791      phi->init_req(1, absxpowy);
1792      phi->init_req(2, negabsxpowy);
1793      signresult = _gvn.transform(phi);
1794      ylong_path = _gvn.transform(r);
1795      record_for_igvn(r);
1796    }
1797    // Set complex path fast result
1798    r->init_req(2, ylong_path);
1799    phi->init_req(2, signresult);
1800
1801    static const jlong nan_bits = CONST64(0x7ff8000000000000);
1802    Node *slow_result = makecon(TypeD::make(*(double*)&nan_bits)); // return NaN
1803    r->init_req(1,slow_path);
1804    phi->init_req(1,slow_result);
1805
1806    // Post merge
1807    set_control(_gvn.transform(r));
1808    record_for_igvn(r);
1809    result = _gvn.transform(phi);
1810  }
1811
1812  finish_pow_exp(result, x, y, OptoRuntime::Math_DD_D_Type(), CAST_FROM_FN_PTR(address, SharedRuntime::dpow), "POW");
1813
1814  C->set_has_split_ifs(true); // Has chance for split-if optimization
1815  return true;
1816}
1817
1818//------------------------------runtime_math-----------------------------
1819bool LibraryCallKit::runtime_math(const TypeFunc* call_type, address funcAddr, const char* funcName) {
1820  assert(call_type == OptoRuntime::Math_DD_D_Type() || call_type == OptoRuntime::Math_D_D_Type(),
1821         "must be (DD)D or (D)D type");
1822
1823  // Inputs
1824  Node* a = round_double_node(argument(0));
1825  Node* b = (call_type == OptoRuntime::Math_DD_D_Type()) ? round_double_node(argument(2)) : NULL;
1826
1827  const TypePtr* no_memory_effects = NULL;
1828  Node* trig = make_runtime_call(RC_LEAF, call_type, funcAddr, funcName,
1829                                 no_memory_effects,
1830                                 a, top(), b, b ? top() : NULL);
1831  Node* value = _gvn.transform(new (C) ProjNode(trig, TypeFunc::Parms+0));
1832#ifdef ASSERT
1833  Node* value_top = _gvn.transform(new (C) ProjNode(trig, TypeFunc::Parms+1));
1834  assert(value_top == top(), "second value must be top");
1835#endif
1836
1837  set_result(value);
1838  return true;
1839}
1840
1841//------------------------------inline_math_native-----------------------------
1842bool LibraryCallKit::inline_math_native(vmIntrinsics::ID id) {
1843#define FN_PTR(f) CAST_FROM_FN_PTR(address, f)
1844  switch (id) {
1845    // These intrinsics are not properly supported on all hardware
1846  case vmIntrinsics::_dcos:   return Matcher::has_match_rule(Op_CosD)   ? inline_trig(id) :
1847    runtime_math(OptoRuntime::Math_D_D_Type(), FN_PTR(SharedRuntime::dcos),   "COS");
1848  case vmIntrinsics::_dsin:   return Matcher::has_match_rule(Op_SinD)   ? inline_trig(id) :
1849    runtime_math(OptoRuntime::Math_D_D_Type(), FN_PTR(SharedRuntime::dsin),   "SIN");
1850  case vmIntrinsics::_dtan:   return Matcher::has_match_rule(Op_TanD)   ? inline_trig(id) :
1851    runtime_math(OptoRuntime::Math_D_D_Type(), FN_PTR(SharedRuntime::dtan),   "TAN");
1852
1853  case vmIntrinsics::_dlog:   return Matcher::has_match_rule(Op_LogD)   ? inline_math(id) :
1854    runtime_math(OptoRuntime::Math_D_D_Type(), FN_PTR(SharedRuntime::dlog),   "LOG");
1855  case vmIntrinsics::_dlog10: return Matcher::has_match_rule(Op_Log10D) ? inline_math(id) :
1856    runtime_math(OptoRuntime::Math_D_D_Type(), FN_PTR(SharedRuntime::dlog10), "LOG10");
1857
1858    // These intrinsics are supported on all hardware
1859  case vmIntrinsics::_dsqrt:  return Matcher::has_match_rule(Op_SqrtD)  ? inline_math(id) : false;
1860  case vmIntrinsics::_dabs:   return Matcher::has_match_rule(Op_AbsD)   ? inline_math(id) : false;
1861
1862  case vmIntrinsics::_dexp:   return Matcher::has_match_rule(Op_ExpD)   ? inline_exp()    :
1863    runtime_math(OptoRuntime::Math_D_D_Type(),  FN_PTR(SharedRuntime::dexp),  "EXP");
1864  case vmIntrinsics::_dpow:   return Matcher::has_match_rule(Op_PowD)   ? inline_pow()    :
1865    runtime_math(OptoRuntime::Math_DD_D_Type(), FN_PTR(SharedRuntime::dpow),  "POW");
1866#undef FN_PTR
1867
1868   // These intrinsics are not yet correctly implemented
1869  case vmIntrinsics::_datan2:
1870    return false;
1871
1872  default:
1873    fatal_unexpected_iid(id);
1874    return false;
1875  }
1876}
1877
1878static bool is_simple_name(Node* n) {
1879  return (n->req() == 1         // constant
1880          || (n->is_Type() && n->as_Type()->type()->singleton())
1881          || n->is_Proj()       // parameter or return value
1882          || n->is_Phi()        // local of some sort
1883          );
1884}
1885
1886//----------------------------inline_min_max-----------------------------------
1887bool LibraryCallKit::inline_min_max(vmIntrinsics::ID id) {
1888  set_result(generate_min_max(id, argument(0), argument(1)));
1889  return true;
1890}
1891
1892Node*
1893LibraryCallKit::generate_min_max(vmIntrinsics::ID id, Node* x0, Node* y0) {
1894  // These are the candidate return value:
1895  Node* xvalue = x0;
1896  Node* yvalue = y0;
1897
1898  if (xvalue == yvalue) {
1899    return xvalue;
1900  }
1901
1902  bool want_max = (id == vmIntrinsics::_max);
1903
1904  const TypeInt* txvalue = _gvn.type(xvalue)->isa_int();
1905  const TypeInt* tyvalue = _gvn.type(yvalue)->isa_int();
1906  if (txvalue == NULL || tyvalue == NULL)  return top();
1907  // This is not really necessary, but it is consistent with a
1908  // hypothetical MaxINode::Value method:
1909  int widen = MAX2(txvalue->_widen, tyvalue->_widen);
1910
1911  // %%% This folding logic should (ideally) be in a different place.
1912  // Some should be inside IfNode, and there to be a more reliable
1913  // transformation of ?: style patterns into cmoves.  We also want
1914  // more powerful optimizations around cmove and min/max.
1915
1916  // Try to find a dominating comparison of these guys.
1917  // It can simplify the index computation for Arrays.copyOf
1918  // and similar uses of System.arraycopy.
1919  // First, compute the normalized version of CmpI(x, y).
1920  int   cmp_op = Op_CmpI;
1921  Node* xkey = xvalue;
1922  Node* ykey = yvalue;
1923  Node* ideal_cmpxy = _gvn.transform( new(C) CmpINode(xkey, ykey) );
1924  if (ideal_cmpxy->is_Cmp()) {
1925    // E.g., if we have CmpI(length - offset, count),
1926    // it might idealize to CmpI(length, count + offset)
1927    cmp_op = ideal_cmpxy->Opcode();
1928    xkey = ideal_cmpxy->in(1);
1929    ykey = ideal_cmpxy->in(2);
1930  }
1931
1932  // Start by locating any relevant comparisons.
1933  Node* start_from = (xkey->outcnt() < ykey->outcnt()) ? xkey : ykey;
1934  Node* cmpxy = NULL;
1935  Node* cmpyx = NULL;
1936  for (DUIterator_Fast kmax, k = start_from->fast_outs(kmax); k < kmax; k++) {
1937    Node* cmp = start_from->fast_out(k);
1938    if (cmp->outcnt() > 0 &&            // must have prior uses
1939        cmp->in(0) == NULL &&           // must be context-independent
1940        cmp->Opcode() == cmp_op) {      // right kind of compare
1941      if (cmp->in(1) == xkey && cmp->in(2) == ykey)  cmpxy = cmp;
1942      if (cmp->in(1) == ykey && cmp->in(2) == xkey)  cmpyx = cmp;
1943    }
1944  }
1945
1946  const int NCMPS = 2;
1947  Node* cmps[NCMPS] = { cmpxy, cmpyx };
1948  int cmpn;
1949  for (cmpn = 0; cmpn < NCMPS; cmpn++) {
1950    if (cmps[cmpn] != NULL)  break;     // find a result
1951  }
1952  if (cmpn < NCMPS) {
1953    // Look for a dominating test that tells us the min and max.
1954    int depth = 0;                // Limit search depth for speed
1955    Node* dom = control();
1956    for (; dom != NULL; dom = IfNode::up_one_dom(dom, true)) {
1957      if (++depth >= 100)  break;
1958      Node* ifproj = dom;
1959      if (!ifproj->is_Proj())  continue;
1960      Node* iff = ifproj->in(0);
1961      if (!iff->is_If())  continue;
1962      Node* bol = iff->in(1);
1963      if (!bol->is_Bool())  continue;
1964      Node* cmp = bol->in(1);
1965      if (cmp == NULL)  continue;
1966      for (cmpn = 0; cmpn < NCMPS; cmpn++)
1967        if (cmps[cmpn] == cmp)  break;
1968      if (cmpn == NCMPS)  continue;
1969      BoolTest::mask btest = bol->as_Bool()->_test._test;
1970      if (ifproj->is_IfFalse())  btest = BoolTest(btest).negate();
1971      if (cmp->in(1) == ykey)    btest = BoolTest(btest).commute();
1972      // At this point, we know that 'x btest y' is true.
1973      switch (btest) {
1974      case BoolTest::eq:
1975        // They are proven equal, so we can collapse the min/max.
1976        // Either value is the answer.  Choose the simpler.
1977        if (is_simple_name(yvalue) && !is_simple_name(xvalue))
1978          return yvalue;
1979        return xvalue;
1980      case BoolTest::lt:          // x < y
1981      case BoolTest::le:          // x <= y
1982        return (want_max ? yvalue : xvalue);
1983      case BoolTest::gt:          // x > y
1984      case BoolTest::ge:          // x >= y
1985        return (want_max ? xvalue : yvalue);
1986      }
1987    }
1988  }
1989
1990  // We failed to find a dominating test.
1991  // Let's pick a test that might GVN with prior tests.
1992  Node*          best_bol   = NULL;
1993  BoolTest::mask best_btest = BoolTest::illegal;
1994  for (cmpn = 0; cmpn < NCMPS; cmpn++) {
1995    Node* cmp = cmps[cmpn];
1996    if (cmp == NULL)  continue;
1997    for (DUIterator_Fast jmax, j = cmp->fast_outs(jmax); j < jmax; j++) {
1998      Node* bol = cmp->fast_out(j);
1999      if (!bol->is_Bool())  continue;
2000      BoolTest::mask btest = bol->as_Bool()->_test._test;
2001      if (btest == BoolTest::eq || btest == BoolTest::ne)  continue;
2002      if (cmp->in(1) == ykey)   btest = BoolTest(btest).commute();
2003      if (bol->outcnt() > (best_bol == NULL ? 0 : best_bol->outcnt())) {
2004        best_bol   = bol->as_Bool();
2005        best_btest = btest;
2006      }
2007    }
2008  }
2009
2010  Node* answer_if_true  = NULL;
2011  Node* answer_if_false = NULL;
2012  switch (best_btest) {
2013  default:
2014    if (cmpxy == NULL)
2015      cmpxy = ideal_cmpxy;
2016    best_bol = _gvn.transform( new(C) BoolNode(cmpxy, BoolTest::lt) );
2017    // and fall through:
2018  case BoolTest::lt:          // x < y
2019  case BoolTest::le:          // x <= y
2020    answer_if_true  = (want_max ? yvalue : xvalue);
2021    answer_if_false = (want_max ? xvalue : yvalue);
2022    break;
2023  case BoolTest::gt:          // x > y
2024  case BoolTest::ge:          // x >= y
2025    answer_if_true  = (want_max ? xvalue : yvalue);
2026    answer_if_false = (want_max ? yvalue : xvalue);
2027    break;
2028  }
2029
2030  jint hi, lo;
2031  if (want_max) {
2032    // We can sharpen the minimum.
2033    hi = MAX2(txvalue->_hi, tyvalue->_hi);
2034    lo = MAX2(txvalue->_lo, tyvalue->_lo);
2035  } else {
2036    // We can sharpen the maximum.
2037    hi = MIN2(txvalue->_hi, tyvalue->_hi);
2038    lo = MIN2(txvalue->_lo, tyvalue->_lo);
2039  }
2040
2041  // Use a flow-free graph structure, to avoid creating excess control edges
2042  // which could hinder other optimizations.
2043  // Since Math.min/max is often used with arraycopy, we want
2044  // tightly_coupled_allocation to be able to see beyond min/max expressions.
2045  Node* cmov = CMoveNode::make(C, NULL, best_bol,
2046                               answer_if_false, answer_if_true,
2047                               TypeInt::make(lo, hi, widen));
2048
2049  return _gvn.transform(cmov);
2050
2051  /*
2052  // This is not as desirable as it may seem, since Min and Max
2053  // nodes do not have a full set of optimizations.
2054  // And they would interfere, anyway, with 'if' optimizations
2055  // and with CMoveI canonical forms.
2056  switch (id) {
2057  case vmIntrinsics::_min:
2058    result_val = _gvn.transform(new (C, 3) MinINode(x,y)); break;
2059  case vmIntrinsics::_max:
2060    result_val = _gvn.transform(new (C, 3) MaxINode(x,y)); break;
2061  default:
2062    ShouldNotReachHere();
2063  }
2064  */
2065}
2066
2067inline int
2068LibraryCallKit::classify_unsafe_addr(Node* &base, Node* &offset) {
2069  const TypePtr* base_type = TypePtr::NULL_PTR;
2070  if (base != NULL)  base_type = _gvn.type(base)->isa_ptr();
2071  if (base_type == NULL) {
2072    // Unknown type.
2073    return Type::AnyPtr;
2074  } else if (base_type == TypePtr::NULL_PTR) {
2075    // Since this is a NULL+long form, we have to switch to a rawptr.
2076    base   = _gvn.transform( new (C) CastX2PNode(offset) );
2077    offset = MakeConX(0);
2078    return Type::RawPtr;
2079  } else if (base_type->base() == Type::RawPtr) {
2080    return Type::RawPtr;
2081  } else if (base_type->isa_oopptr()) {
2082    // Base is never null => always a heap address.
2083    if (base_type->ptr() == TypePtr::NotNull) {
2084      return Type::OopPtr;
2085    }
2086    // Offset is small => always a heap address.
2087    const TypeX* offset_type = _gvn.type(offset)->isa_intptr_t();
2088    if (offset_type != NULL &&
2089        base_type->offset() == 0 &&     // (should always be?)
2090        offset_type->_lo >= 0 &&
2091        !MacroAssembler::needs_explicit_null_check(offset_type->_hi)) {
2092      return Type::OopPtr;
2093    }
2094    // Otherwise, it might either be oop+off or NULL+addr.
2095    return Type::AnyPtr;
2096  } else {
2097    // No information:
2098    return Type::AnyPtr;
2099  }
2100}
2101
2102inline Node* LibraryCallKit::make_unsafe_address(Node* base, Node* offset) {
2103  int kind = classify_unsafe_addr(base, offset);
2104  if (kind == Type::RawPtr) {
2105    return basic_plus_adr(top(), base, offset);
2106  } else {
2107    return basic_plus_adr(base, offset);
2108  }
2109}
2110
2111//--------------------------inline_number_methods-----------------------------
2112// inline int     Integer.numberOfLeadingZeros(int)
2113// inline int        Long.numberOfLeadingZeros(long)
2114//
2115// inline int     Integer.numberOfTrailingZeros(int)
2116// inline int        Long.numberOfTrailingZeros(long)
2117//
2118// inline int     Integer.bitCount(int)
2119// inline int        Long.bitCount(long)
2120//
2121// inline char  Character.reverseBytes(char)
2122// inline short     Short.reverseBytes(short)
2123// inline int     Integer.reverseBytes(int)
2124// inline long       Long.reverseBytes(long)
2125bool LibraryCallKit::inline_number_methods(vmIntrinsics::ID id) {
2126  Node* arg = argument(0);
2127  Node* n;
2128  switch (id) {
2129  case vmIntrinsics::_numberOfLeadingZeros_i:   n = new (C) CountLeadingZerosINode( arg);  break;
2130  case vmIntrinsics::_numberOfLeadingZeros_l:   n = new (C) CountLeadingZerosLNode( arg);  break;
2131  case vmIntrinsics::_numberOfTrailingZeros_i:  n = new (C) CountTrailingZerosINode(arg);  break;
2132  case vmIntrinsics::_numberOfTrailingZeros_l:  n = new (C) CountTrailingZerosLNode(arg);  break;
2133  case vmIntrinsics::_bitCount_i:               n = new (C) PopCountINode(          arg);  break;
2134  case vmIntrinsics::_bitCount_l:               n = new (C) PopCountLNode(          arg);  break;
2135  case vmIntrinsics::_reverseBytes_c:           n = new (C) ReverseBytesUSNode(0,   arg);  break;
2136  case vmIntrinsics::_reverseBytes_s:           n = new (C) ReverseBytesSNode( 0,   arg);  break;
2137  case vmIntrinsics::_reverseBytes_i:           n = new (C) ReverseBytesINode( 0,   arg);  break;
2138  case vmIntrinsics::_reverseBytes_l:           n = new (C) ReverseBytesLNode( 0,   arg);  break;
2139  default:  fatal_unexpected_iid(id);  break;
2140  }
2141  set_result(_gvn.transform(n));
2142  return true;
2143}
2144
2145//----------------------------inline_unsafe_access----------------------------
2146
2147const static BasicType T_ADDRESS_HOLDER = T_LONG;
2148
2149// Helper that guards and inserts a pre-barrier.
2150void LibraryCallKit::insert_pre_barrier(Node* base_oop, Node* offset,
2151                                        Node* pre_val, bool need_mem_bar) {
2152  // We could be accessing the referent field of a reference object. If so, when G1
2153  // is enabled, we need to log the value in the referent field in an SATB buffer.
2154  // This routine performs some compile time filters and generates suitable
2155  // runtime filters that guard the pre-barrier code.
2156  // Also add memory barrier for non volatile load from the referent field
2157  // to prevent commoning of loads across safepoint.
2158  if (!UseG1GC && !need_mem_bar)
2159    return;
2160
2161  // Some compile time checks.
2162
2163  // If offset is a constant, is it java_lang_ref_Reference::_reference_offset?
2164  const TypeX* otype = offset->find_intptr_t_type();
2165  if (otype != NULL && otype->is_con() &&
2166      otype->get_con() != java_lang_ref_Reference::referent_offset) {
2167    // Constant offset but not the reference_offset so just return
2168    return;
2169  }
2170
2171  // We only need to generate the runtime guards for instances.
2172  const TypeOopPtr* btype = base_oop->bottom_type()->isa_oopptr();
2173  if (btype != NULL) {
2174    if (btype->isa_aryptr()) {
2175      // Array type so nothing to do
2176      return;
2177    }
2178
2179    const TypeInstPtr* itype = btype->isa_instptr();
2180    if (itype != NULL) {
2181      // Can the klass of base_oop be statically determined to be
2182      // _not_ a sub-class of Reference and _not_ Object?
2183      ciKlass* klass = itype->klass();
2184      if ( klass->is_loaded() &&
2185          !klass->is_subtype_of(env()->Reference_klass()) &&
2186          !env()->Object_klass()->is_subtype_of(klass)) {
2187        return;
2188      }
2189    }
2190  }
2191
2192  // The compile time filters did not reject base_oop/offset so
2193  // we need to generate the following runtime filters
2194  //
2195  // if (offset == java_lang_ref_Reference::_reference_offset) {
2196  //   if (instance_of(base, java.lang.ref.Reference)) {
2197  //     pre_barrier(_, pre_val, ...);
2198  //   }
2199  // }
2200
2201  float likely   = PROB_LIKELY(  0.999);
2202  float unlikely = PROB_UNLIKELY(0.999);
2203
2204  IdealKit ideal(this);
2205#define __ ideal.
2206
2207  Node* referent_off = __ ConX(java_lang_ref_Reference::referent_offset);
2208
2209  __ if_then(offset, BoolTest::eq, referent_off, unlikely); {
2210      // Update graphKit memory and control from IdealKit.
2211      sync_kit(ideal);
2212
2213      Node* ref_klass_con = makecon(TypeKlassPtr::make(env()->Reference_klass()));
2214      Node* is_instof = gen_instanceof(base_oop, ref_klass_con);
2215
2216      // Update IdealKit memory and control from graphKit.
2217      __ sync_kit(this);
2218
2219      Node* one = __ ConI(1);
2220      // is_instof == 0 if base_oop == NULL
2221      __ if_then(is_instof, BoolTest::eq, one, unlikely); {
2222
2223        // Update graphKit from IdeakKit.
2224        sync_kit(ideal);
2225
2226        // Use the pre-barrier to record the value in the referent field
2227        pre_barrier(false /* do_load */,
2228                    __ ctrl(),
2229                    NULL /* obj */, NULL /* adr */, max_juint /* alias_idx */, NULL /* val */, NULL /* val_type */,
2230                    pre_val /* pre_val */,
2231                    T_OBJECT);
2232        if (need_mem_bar) {
2233          // Add memory barrier to prevent commoning reads from this field
2234          // across safepoint since GC can change its value.
2235          insert_mem_bar(Op_MemBarCPUOrder);
2236        }
2237        // Update IdealKit from graphKit.
2238        __ sync_kit(this);
2239
2240      } __ end_if(); // _ref_type != ref_none
2241  } __ end_if(); // offset == referent_offset
2242
2243  // Final sync IdealKit and GraphKit.
2244  final_sync(ideal);
2245#undef __
2246}
2247
2248
2249// Interpret Unsafe.fieldOffset cookies correctly:
2250extern jlong Unsafe_field_offset_to_byte_offset(jlong field_offset);
2251
2252const TypeOopPtr* LibraryCallKit::sharpen_unsafe_type(Compile::AliasType* alias_type, const TypePtr *adr_type, bool is_native_ptr) {
2253  // Attempt to infer a sharper value type from the offset and base type.
2254  ciKlass* sharpened_klass = NULL;
2255
2256  // See if it is an instance field, with an object type.
2257  if (alias_type->field() != NULL) {
2258    assert(!is_native_ptr, "native pointer op cannot use a java address");
2259    if (alias_type->field()->type()->is_klass()) {
2260      sharpened_klass = alias_type->field()->type()->as_klass();
2261    }
2262  }
2263
2264  // See if it is a narrow oop array.
2265  if (adr_type->isa_aryptr()) {
2266    if (adr_type->offset() >= objArrayOopDesc::base_offset_in_bytes()) {
2267      const TypeOopPtr *elem_type = adr_type->is_aryptr()->elem()->isa_oopptr();
2268      if (elem_type != NULL) {
2269        sharpened_klass = elem_type->klass();
2270      }
2271    }
2272  }
2273
2274  // The sharpened class might be unloaded if there is no class loader
2275  // contraint in place.
2276  if (sharpened_klass != NULL && sharpened_klass->is_loaded()) {
2277    const TypeOopPtr* tjp = TypeOopPtr::make_from_klass(sharpened_klass);
2278
2279#ifndef PRODUCT
2280    if (PrintIntrinsics || PrintInlining || PrintOptoInlining) {
2281      tty->print("  from base type: ");  adr_type->dump();
2282      tty->print("  sharpened value: ");  tjp->dump();
2283    }
2284#endif
2285    // Sharpen the value type.
2286    return tjp;
2287  }
2288  return NULL;
2289}
2290
2291bool LibraryCallKit::inline_unsafe_access(bool is_native_ptr, bool is_store, BasicType type, bool is_volatile) {
2292  if (callee()->is_static())  return false;  // caller must have the capability!
2293
2294#ifndef PRODUCT
2295  {
2296    ResourceMark rm;
2297    // Check the signatures.
2298    ciSignature* sig = callee()->signature();
2299#ifdef ASSERT
2300    if (!is_store) {
2301      // Object getObject(Object base, int/long offset), etc.
2302      BasicType rtype = sig->return_type()->basic_type();
2303      if (rtype == T_ADDRESS_HOLDER && callee()->name() == ciSymbol::getAddress_name())
2304          rtype = T_ADDRESS;  // it is really a C void*
2305      assert(rtype == type, "getter must return the expected value");
2306      if (!is_native_ptr) {
2307        assert(sig->count() == 2, "oop getter has 2 arguments");
2308        assert(sig->type_at(0)->basic_type() == T_OBJECT, "getter base is object");
2309        assert(sig->type_at(1)->basic_type() == T_LONG, "getter offset is correct");
2310      } else {
2311        assert(sig->count() == 1, "native getter has 1 argument");
2312        assert(sig->type_at(0)->basic_type() == T_LONG, "getter base is long");
2313      }
2314    } else {
2315      // void putObject(Object base, int/long offset, Object x), etc.
2316      assert(sig->return_type()->basic_type() == T_VOID, "putter must not return a value");
2317      if (!is_native_ptr) {
2318        assert(sig->count() == 3, "oop putter has 3 arguments");
2319        assert(sig->type_at(0)->basic_type() == T_OBJECT, "putter base is object");
2320        assert(sig->type_at(1)->basic_type() == T_LONG, "putter offset is correct");
2321      } else {
2322        assert(sig->count() == 2, "native putter has 2 arguments");
2323        assert(sig->type_at(0)->basic_type() == T_LONG, "putter base is long");
2324      }
2325      BasicType vtype = sig->type_at(sig->count()-1)->basic_type();
2326      if (vtype == T_ADDRESS_HOLDER && callee()->name() == ciSymbol::putAddress_name())
2327        vtype = T_ADDRESS;  // it is really a C void*
2328      assert(vtype == type, "putter must accept the expected value");
2329    }
2330#endif // ASSERT
2331 }
2332#endif //PRODUCT
2333
2334  C->set_has_unsafe_access(true);  // Mark eventual nmethod as "unsafe".
2335
2336  Node* receiver = argument(0);  // type: oop
2337
2338  // Build address expression.  See the code in inline_unsafe_prefetch.
2339  Node* adr;
2340  Node* heap_base_oop = top();
2341  Node* offset = top();
2342  Node* val;
2343
2344  if (!is_native_ptr) {
2345    // The base is either a Java object or a value produced by Unsafe.staticFieldBase
2346    Node* base = argument(1);  // type: oop
2347    // The offset is a value produced by Unsafe.staticFieldOffset or Unsafe.objectFieldOffset
2348    offset = argument(2);  // type: long
2349    // We currently rely on the cookies produced by Unsafe.xxxFieldOffset
2350    // to be plain byte offsets, which are also the same as those accepted
2351    // by oopDesc::field_base.
2352    assert(Unsafe_field_offset_to_byte_offset(11) == 11,
2353           "fieldOffset must be byte-scaled");
2354    // 32-bit machines ignore the high half!
2355    offset = ConvL2X(offset);
2356    adr = make_unsafe_address(base, offset);
2357    heap_base_oop = base;
2358    val = is_store ? argument(4) : NULL;
2359  } else {
2360    Node* ptr = argument(1);  // type: long
2361    ptr = ConvL2X(ptr);  // adjust Java long to machine word
2362    adr = make_unsafe_address(NULL, ptr);
2363    val = is_store ? argument(3) : NULL;
2364  }
2365
2366  const TypePtr *adr_type = _gvn.type(adr)->isa_ptr();
2367
2368  // First guess at the value type.
2369  const Type *value_type = Type::get_const_basic_type(type);
2370
2371  // Try to categorize the address.  If it comes up as TypeJavaPtr::BOTTOM,
2372  // there was not enough information to nail it down.
2373  Compile::AliasType* alias_type = C->alias_type(adr_type);
2374  assert(alias_type->index() != Compile::AliasIdxBot, "no bare pointers here");
2375
2376  // We will need memory barriers unless we can determine a unique
2377  // alias category for this reference.  (Note:  If for some reason
2378  // the barriers get omitted and the unsafe reference begins to "pollute"
2379  // the alias analysis of the rest of the graph, either Compile::can_alias
2380  // or Compile::must_alias will throw a diagnostic assert.)
2381  bool need_mem_bar = (alias_type->adr_type() == TypeOopPtr::BOTTOM);
2382
2383  // If we are reading the value of the referent field of a Reference
2384  // object (either by using Unsafe directly or through reflection)
2385  // then, if G1 is enabled, we need to record the referent in an
2386  // SATB log buffer using the pre-barrier mechanism.
2387  // Also we need to add memory barrier to prevent commoning reads
2388  // from this field across safepoint since GC can change its value.
2389  bool need_read_barrier = !is_native_ptr && !is_store &&
2390                           offset != top() && heap_base_oop != top();
2391
2392  if (!is_store && type == T_OBJECT) {
2393    const TypeOopPtr* tjp = sharpen_unsafe_type(alias_type, adr_type, is_native_ptr);
2394    if (tjp != NULL) {
2395      value_type = tjp;
2396    }
2397  }
2398
2399  receiver = null_check(receiver);
2400  if (stopped()) {
2401    return true;
2402  }
2403  // Heap pointers get a null-check from the interpreter,
2404  // as a courtesy.  However, this is not guaranteed by Unsafe,
2405  // and it is not possible to fully distinguish unintended nulls
2406  // from intended ones in this API.
2407
2408  if (is_volatile) {
2409    // We need to emit leading and trailing CPU membars (see below) in
2410    // addition to memory membars when is_volatile. This is a little
2411    // too strong, but avoids the need to insert per-alias-type
2412    // volatile membars (for stores; compare Parse::do_put_xxx), which
2413    // we cannot do effectively here because we probably only have a
2414    // rough approximation of type.
2415    need_mem_bar = true;
2416    // For Stores, place a memory ordering barrier now.
2417    if (is_store)
2418      insert_mem_bar(Op_MemBarRelease);
2419  }
2420
2421  // Memory barrier to prevent normal and 'unsafe' accesses from
2422  // bypassing each other.  Happens after null checks, so the
2423  // exception paths do not take memory state from the memory barrier,
2424  // so there's no problems making a strong assert about mixing users
2425  // of safe & unsafe memory.  Otherwise fails in a CTW of rt.jar
2426  // around 5701, class sun/reflect/UnsafeBooleanFieldAccessorImpl.
2427  if (need_mem_bar) insert_mem_bar(Op_MemBarCPUOrder);
2428
2429  if (!is_store) {
2430    Node* p = make_load(control(), adr, value_type, type, adr_type, is_volatile);
2431    // load value
2432    switch (type) {
2433    case T_BOOLEAN:
2434    case T_CHAR:
2435    case T_BYTE:
2436    case T_SHORT:
2437    case T_INT:
2438    case T_LONG:
2439    case T_FLOAT:
2440    case T_DOUBLE:
2441      break;
2442    case T_OBJECT:
2443      if (need_read_barrier) {
2444        insert_pre_barrier(heap_base_oop, offset, p, !(is_volatile || need_mem_bar));
2445      }
2446      break;
2447    case T_ADDRESS:
2448      // Cast to an int type.
2449      p = _gvn.transform(new (C) CastP2XNode(NULL, p));
2450      p = ConvX2L(p);
2451      break;
2452    default:
2453      fatal(err_msg_res("unexpected type %d: %s", type, type2name(type)));
2454      break;
2455    }
2456    // The load node has the control of the preceding MemBarCPUOrder.  All
2457    // following nodes will have the control of the MemBarCPUOrder inserted at
2458    // the end of this method.  So, pushing the load onto the stack at a later
2459    // point is fine.
2460    set_result(p);
2461  } else {
2462    // place effect of store into memory
2463    switch (type) {
2464    case T_DOUBLE:
2465      val = dstore_rounding(val);
2466      break;
2467    case T_ADDRESS:
2468      // Repackage the long as a pointer.
2469      val = ConvL2X(val);
2470      val = _gvn.transform( new (C) CastX2PNode(val) );
2471      break;
2472    }
2473
2474    if (type != T_OBJECT ) {
2475      (void) store_to_memory(control(), adr, val, type, adr_type, is_volatile);
2476    } else {
2477      // Possibly an oop being stored to Java heap or native memory
2478      if (!TypePtr::NULL_PTR->higher_equal(_gvn.type(heap_base_oop))) {
2479        // oop to Java heap.
2480        (void) store_oop_to_unknown(control(), heap_base_oop, adr, adr_type, val, type);
2481      } else {
2482        // We can't tell at compile time if we are storing in the Java heap or outside
2483        // of it. So we need to emit code to conditionally do the proper type of
2484        // store.
2485
2486        IdealKit ideal(this);
2487#define __ ideal.
2488        // QQQ who knows what probability is here??
2489        __ if_then(heap_base_oop, BoolTest::ne, null(), PROB_UNLIKELY(0.999)); {
2490          // Sync IdealKit and graphKit.
2491          sync_kit(ideal);
2492          Node* st = store_oop_to_unknown(control(), heap_base_oop, adr, adr_type, val, type);
2493          // Update IdealKit memory.
2494          __ sync_kit(this);
2495        } __ else_(); {
2496          __ store(__ ctrl(), adr, val, type, alias_type->index(), is_volatile);
2497        } __ end_if();
2498        // Final sync IdealKit and GraphKit.
2499        final_sync(ideal);
2500#undef __
2501      }
2502    }
2503  }
2504
2505  if (is_volatile) {
2506    if (!is_store)
2507      insert_mem_bar(Op_MemBarAcquire);
2508    else
2509      insert_mem_bar(Op_MemBarVolatile);
2510  }
2511
2512  if (need_mem_bar) insert_mem_bar(Op_MemBarCPUOrder);
2513
2514  return true;
2515}
2516
2517//----------------------------inline_unsafe_prefetch----------------------------
2518
2519bool LibraryCallKit::inline_unsafe_prefetch(bool is_native_ptr, bool is_store, bool is_static) {
2520#ifndef PRODUCT
2521  {
2522    ResourceMark rm;
2523    // Check the signatures.
2524    ciSignature* sig = callee()->signature();
2525#ifdef ASSERT
2526    // Object getObject(Object base, int/long offset), etc.
2527    BasicType rtype = sig->return_type()->basic_type();
2528    if (!is_native_ptr) {
2529      assert(sig->count() == 2, "oop prefetch has 2 arguments");
2530      assert(sig->type_at(0)->basic_type() == T_OBJECT, "prefetch base is object");
2531      assert(sig->type_at(1)->basic_type() == T_LONG, "prefetcha offset is correct");
2532    } else {
2533      assert(sig->count() == 1, "native prefetch has 1 argument");
2534      assert(sig->type_at(0)->basic_type() == T_LONG, "prefetch base is long");
2535    }
2536#endif // ASSERT
2537  }
2538#endif // !PRODUCT
2539
2540  C->set_has_unsafe_access(true);  // Mark eventual nmethod as "unsafe".
2541
2542  const int idx = is_static ? 0 : 1;
2543  if (!is_static) {
2544    null_check_receiver();
2545    if (stopped()) {
2546      return true;
2547    }
2548  }
2549
2550  // Build address expression.  See the code in inline_unsafe_access.
2551  Node *adr;
2552  if (!is_native_ptr) {
2553    // The base is either a Java object or a value produced by Unsafe.staticFieldBase
2554    Node* base   = argument(idx + 0);  // type: oop
2555    // The offset is a value produced by Unsafe.staticFieldOffset or Unsafe.objectFieldOffset
2556    Node* offset = argument(idx + 1);  // type: long
2557    // We currently rely on the cookies produced by Unsafe.xxxFieldOffset
2558    // to be plain byte offsets, which are also the same as those accepted
2559    // by oopDesc::field_base.
2560    assert(Unsafe_field_offset_to_byte_offset(11) == 11,
2561           "fieldOffset must be byte-scaled");
2562    // 32-bit machines ignore the high half!
2563    offset = ConvL2X(offset);
2564    adr = make_unsafe_address(base, offset);
2565  } else {
2566    Node* ptr = argument(idx + 0);  // type: long
2567    ptr = ConvL2X(ptr);  // adjust Java long to machine word
2568    adr = make_unsafe_address(NULL, ptr);
2569  }
2570
2571  // Generate the read or write prefetch
2572  Node *prefetch;
2573  if (is_store) {
2574    prefetch = new (C) PrefetchWriteNode(i_o(), adr);
2575  } else {
2576    prefetch = new (C) PrefetchReadNode(i_o(), adr);
2577  }
2578  prefetch->init_req(0, control());
2579  set_i_o(_gvn.transform(prefetch));
2580
2581  return true;
2582}
2583
2584//----------------------------inline_unsafe_load_store----------------------------
2585// This method serves a couple of different customers (depending on LoadStoreKind):
2586//
2587// LS_cmpxchg:
2588//   public final native boolean compareAndSwapObject(Object o, long offset, Object expected, Object x);
2589//   public final native boolean compareAndSwapInt(   Object o, long offset, int    expected, int    x);
2590//   public final native boolean compareAndSwapLong(  Object o, long offset, long   expected, long   x);
2591//
2592// LS_xadd:
2593//   public int  getAndAddInt( Object o, long offset, int  delta)
2594//   public long getAndAddLong(Object o, long offset, long delta)
2595//
2596// LS_xchg:
2597//   int    getAndSet(Object o, long offset, int    newValue)
2598//   long   getAndSet(Object o, long offset, long   newValue)
2599//   Object getAndSet(Object o, long offset, Object newValue)
2600//
2601bool LibraryCallKit::inline_unsafe_load_store(BasicType type, LoadStoreKind kind) {
2602  // This basic scheme here is the same as inline_unsafe_access, but
2603  // differs in enough details that combining them would make the code
2604  // overly confusing.  (This is a true fact! I originally combined
2605  // them, but even I was confused by it!) As much code/comments as
2606  // possible are retained from inline_unsafe_access though to make
2607  // the correspondences clearer. - dl
2608
2609  if (callee()->is_static())  return false;  // caller must have the capability!
2610
2611#ifndef PRODUCT
2612  BasicType rtype;
2613  {
2614    ResourceMark rm;
2615    // Check the signatures.
2616    ciSignature* sig = callee()->signature();
2617    rtype = sig->return_type()->basic_type();
2618    if (kind == LS_xadd || kind == LS_xchg) {
2619      // Check the signatures.
2620#ifdef ASSERT
2621      assert(rtype == type, "get and set must return the expected type");
2622      assert(sig->count() == 3, "get and set has 3 arguments");
2623      assert(sig->type_at(0)->basic_type() == T_OBJECT, "get and set base is object");
2624      assert(sig->type_at(1)->basic_type() == T_LONG, "get and set offset is long");
2625      assert(sig->type_at(2)->basic_type() == type, "get and set must take expected type as new value/delta");
2626#endif // ASSERT
2627    } else if (kind == LS_cmpxchg) {
2628      // Check the signatures.
2629#ifdef ASSERT
2630      assert(rtype == T_BOOLEAN, "CAS must return boolean");
2631      assert(sig->count() == 4, "CAS has 4 arguments");
2632      assert(sig->type_at(0)->basic_type() == T_OBJECT, "CAS base is object");
2633      assert(sig->type_at(1)->basic_type() == T_LONG, "CAS offset is long");
2634#endif // ASSERT
2635    } else {
2636      ShouldNotReachHere();
2637    }
2638  }
2639#endif //PRODUCT
2640
2641  C->set_has_unsafe_access(true);  // Mark eventual nmethod as "unsafe".
2642
2643  // Get arguments:
2644  Node* receiver = NULL;
2645  Node* base     = NULL;
2646  Node* offset   = NULL;
2647  Node* oldval   = NULL;
2648  Node* newval   = NULL;
2649  if (kind == LS_cmpxchg) {
2650    const bool two_slot_type = type2size[type] == 2;
2651    receiver = argument(0);  // type: oop
2652    base     = argument(1);  // type: oop
2653    offset   = argument(2);  // type: long
2654    oldval   = argument(4);  // type: oop, int, or long
2655    newval   = argument(two_slot_type ? 6 : 5);  // type: oop, int, or long
2656  } else if (kind == LS_xadd || kind == LS_xchg){
2657    receiver = argument(0);  // type: oop
2658    base     = argument(1);  // type: oop
2659    offset   = argument(2);  // type: long
2660    oldval   = NULL;
2661    newval   = argument(4);  // type: oop, int, or long
2662  }
2663
2664  // Null check receiver.
2665  receiver = null_check(receiver);
2666  if (stopped()) {
2667    return true;
2668  }
2669
2670  // Build field offset expression.
2671  // We currently rely on the cookies produced by Unsafe.xxxFieldOffset
2672  // to be plain byte offsets, which are also the same as those accepted
2673  // by oopDesc::field_base.
2674  assert(Unsafe_field_offset_to_byte_offset(11) == 11, "fieldOffset must be byte-scaled");
2675  // 32-bit machines ignore the high half of long offsets
2676  offset = ConvL2X(offset);
2677  Node* adr = make_unsafe_address(base, offset);
2678  const TypePtr *adr_type = _gvn.type(adr)->isa_ptr();
2679
2680  // For CAS, unlike inline_unsafe_access, there seems no point in
2681  // trying to refine types. Just use the coarse types here.
2682  const Type *value_type = Type::get_const_basic_type(type);
2683  Compile::AliasType* alias_type = C->alias_type(adr_type);
2684  assert(alias_type->index() != Compile::AliasIdxBot, "no bare pointers here");
2685
2686  if (kind == LS_xchg && type == T_OBJECT) {
2687    const TypeOopPtr* tjp = sharpen_unsafe_type(alias_type, adr_type);
2688    if (tjp != NULL) {
2689      value_type = tjp;
2690    }
2691  }
2692
2693  int alias_idx = C->get_alias_index(adr_type);
2694
2695  // Memory-model-wise, a LoadStore acts like a little synchronized
2696  // block, so needs barriers on each side.  These don't translate
2697  // into actual barriers on most machines, but we still need rest of
2698  // compiler to respect ordering.
2699
2700  insert_mem_bar(Op_MemBarRelease);
2701  insert_mem_bar(Op_MemBarCPUOrder);
2702
2703  // 4984716: MemBars must be inserted before this
2704  //          memory node in order to avoid a false
2705  //          dependency which will confuse the scheduler.
2706  Node *mem = memory(alias_idx);
2707
2708  // For now, we handle only those cases that actually exist: ints,
2709  // longs, and Object. Adding others should be straightforward.
2710  Node* load_store;
2711  switch(type) {
2712  case T_INT:
2713    if (kind == LS_xadd) {
2714      load_store = _gvn.transform(new (C) GetAndAddINode(control(), mem, adr, newval, adr_type));
2715    } else if (kind == LS_xchg) {
2716      load_store = _gvn.transform(new (C) GetAndSetINode(control(), mem, adr, newval, adr_type));
2717    } else if (kind == LS_cmpxchg) {
2718      load_store = _gvn.transform(new (C) CompareAndSwapINode(control(), mem, adr, newval, oldval));
2719    } else {
2720      ShouldNotReachHere();
2721    }
2722    break;
2723  case T_LONG:
2724    if (kind == LS_xadd) {
2725      load_store = _gvn.transform(new (C) GetAndAddLNode(control(), mem, adr, newval, adr_type));
2726    } else if (kind == LS_xchg) {
2727      load_store = _gvn.transform(new (C) GetAndSetLNode(control(), mem, adr, newval, adr_type));
2728    } else if (kind == LS_cmpxchg) {
2729      load_store = _gvn.transform(new (C) CompareAndSwapLNode(control(), mem, adr, newval, oldval));
2730    } else {
2731      ShouldNotReachHere();
2732    }
2733    break;
2734  case T_OBJECT:
2735    // Transformation of a value which could be NULL pointer (CastPP #NULL)
2736    // could be delayed during Parse (for example, in adjust_map_after_if()).
2737    // Execute transformation here to avoid barrier generation in such case.
2738    if (_gvn.type(newval) == TypePtr::NULL_PTR)
2739      newval = _gvn.makecon(TypePtr::NULL_PTR);
2740
2741    // Reference stores need a store barrier.
2742    pre_barrier(true /* do_load*/,
2743                control(), base, adr, alias_idx, newval, value_type->make_oopptr(),
2744                NULL /* pre_val*/,
2745                T_OBJECT);
2746#ifdef _LP64
2747    if (adr->bottom_type()->is_ptr_to_narrowoop()) {
2748      Node *newval_enc = _gvn.transform(new (C) EncodePNode(newval, newval->bottom_type()->make_narrowoop()));
2749      if (kind == LS_xchg) {
2750        load_store = _gvn.transform(new (C) GetAndSetNNode(control(), mem, adr,
2751                                                              newval_enc, adr_type, value_type->make_narrowoop()));
2752      } else {
2753        assert(kind == LS_cmpxchg, "wrong LoadStore operation");
2754        Node *oldval_enc = _gvn.transform(new (C) EncodePNode(oldval, oldval->bottom_type()->make_narrowoop()));
2755        load_store = _gvn.transform(new (C) CompareAndSwapNNode(control(), mem, adr,
2756                                                                   newval_enc, oldval_enc));
2757      }
2758    } else
2759#endif
2760    {
2761      if (kind == LS_xchg) {
2762        load_store = _gvn.transform(new (C) GetAndSetPNode(control(), mem, adr, newval, adr_type, value_type->is_oopptr()));
2763      } else {
2764        assert(kind == LS_cmpxchg, "wrong LoadStore operation");
2765        load_store = _gvn.transform(new (C) CompareAndSwapPNode(control(), mem, adr, newval, oldval));
2766      }
2767    }
2768    post_barrier(control(), load_store, base, adr, alias_idx, newval, T_OBJECT, true);
2769    break;
2770  default:
2771    fatal(err_msg_res("unexpected type %d: %s", type, type2name(type)));
2772    break;
2773  }
2774
2775  // SCMemProjNodes represent the memory state of a LoadStore. Their
2776  // main role is to prevent LoadStore nodes from being optimized away
2777  // when their results aren't used.
2778  Node* proj = _gvn.transform( new (C) SCMemProjNode(load_store));
2779  set_memory(proj, alias_idx);
2780
2781  // Add the trailing membar surrounding the access
2782  insert_mem_bar(Op_MemBarCPUOrder);
2783  insert_mem_bar(Op_MemBarAcquire);
2784
2785#ifdef _LP64
2786  if (type == T_OBJECT && adr->bottom_type()->is_ptr_to_narrowoop() && kind == LS_xchg) {
2787    load_store = _gvn.transform(new (C) DecodeNNode(load_store, load_store->get_ptr_type()));
2788  }
2789#endif
2790
2791  assert(type2size[load_store->bottom_type()->basic_type()] == type2size[rtype], "result type should match");
2792  set_result(load_store);
2793  return true;
2794}
2795
2796//----------------------------inline_unsafe_ordered_store----------------------
2797// public native void sun.misc.Unsafe.putOrderedObject(Object o, long offset, Object x);
2798// public native void sun.misc.Unsafe.putOrderedInt(Object o, long offset, int x);
2799// public native void sun.misc.Unsafe.putOrderedLong(Object o, long offset, long x);
2800bool LibraryCallKit::inline_unsafe_ordered_store(BasicType type) {
2801  // This is another variant of inline_unsafe_access, differing in
2802  // that it always issues store-store ("release") barrier and ensures
2803  // store-atomicity (which only matters for "long").
2804
2805  if (callee()->is_static())  return false;  // caller must have the capability!
2806
2807#ifndef PRODUCT
2808  {
2809    ResourceMark rm;
2810    // Check the signatures.
2811    ciSignature* sig = callee()->signature();
2812#ifdef ASSERT
2813    BasicType rtype = sig->return_type()->basic_type();
2814    assert(rtype == T_VOID, "must return void");
2815    assert(sig->count() == 3, "has 3 arguments");
2816    assert(sig->type_at(0)->basic_type() == T_OBJECT, "base is object");
2817    assert(sig->type_at(1)->basic_type() == T_LONG, "offset is long");
2818#endif // ASSERT
2819  }
2820#endif //PRODUCT
2821
2822  C->set_has_unsafe_access(true);  // Mark eventual nmethod as "unsafe".
2823
2824  // Get arguments:
2825  Node* receiver = argument(0);  // type: oop
2826  Node* base     = argument(1);  // type: oop
2827  Node* offset   = argument(2);  // type: long
2828  Node* val      = argument(4);  // type: oop, int, or long
2829
2830  // Null check receiver.
2831  receiver = null_check(receiver);
2832  if (stopped()) {
2833    return true;
2834  }
2835
2836  // Build field offset expression.
2837  assert(Unsafe_field_offset_to_byte_offset(11) == 11, "fieldOffset must be byte-scaled");
2838  // 32-bit machines ignore the high half of long offsets
2839  offset = ConvL2X(offset);
2840  Node* adr = make_unsafe_address(base, offset);
2841  const TypePtr *adr_type = _gvn.type(adr)->isa_ptr();
2842  const Type *value_type = Type::get_const_basic_type(type);
2843  Compile::AliasType* alias_type = C->alias_type(adr_type);
2844
2845  insert_mem_bar(Op_MemBarRelease);
2846  insert_mem_bar(Op_MemBarCPUOrder);
2847  // Ensure that the store is atomic for longs:
2848  const bool require_atomic_access = true;
2849  Node* store;
2850  if (type == T_OBJECT) // reference stores need a store barrier.
2851    store = store_oop_to_unknown(control(), base, adr, adr_type, val, type);
2852  else {
2853    store = store_to_memory(control(), adr, val, type, adr_type, require_atomic_access);
2854  }
2855  insert_mem_bar(Op_MemBarCPUOrder);
2856  return true;
2857}
2858
2859bool LibraryCallKit::inline_unsafe_fence(vmIntrinsics::ID id) {
2860  // Regardless of form, don't allow previous ld/st to move down,
2861  // then issue acquire, release, or volatile mem_bar.
2862  insert_mem_bar(Op_MemBarCPUOrder);
2863  switch(id) {
2864    case vmIntrinsics::_loadFence:
2865      insert_mem_bar(Op_MemBarAcquire);
2866      return true;
2867    case vmIntrinsics::_storeFence:
2868      insert_mem_bar(Op_MemBarRelease);
2869      return true;
2870    case vmIntrinsics::_fullFence:
2871      insert_mem_bar(Op_MemBarVolatile);
2872      return true;
2873    default:
2874      fatal_unexpected_iid(id);
2875      return false;
2876  }
2877}
2878
2879//----------------------------inline_unsafe_allocate---------------------------
2880// public native Object sun.mics.Unsafe.allocateInstance(Class<?> cls);
2881bool LibraryCallKit::inline_unsafe_allocate() {
2882  if (callee()->is_static())  return false;  // caller must have the capability!
2883
2884  null_check_receiver();  // null-check, then ignore
2885  Node* cls = null_check(argument(1));
2886  if (stopped())  return true;
2887
2888  Node* kls = load_klass_from_mirror(cls, false, NULL, 0);
2889  kls = null_check(kls);
2890  if (stopped())  return true;  // argument was like int.class
2891
2892  // Note:  The argument might still be an illegal value like
2893  // Serializable.class or Object[].class.   The runtime will handle it.
2894  // But we must make an explicit check for initialization.
2895  Node* insp = basic_plus_adr(kls, in_bytes(InstanceKlass::init_state_offset()));
2896  // Use T_BOOLEAN for InstanceKlass::_init_state so the compiler
2897  // can generate code to load it as unsigned byte.
2898  Node* inst = make_load(NULL, insp, TypeInt::UBYTE, T_BOOLEAN);
2899  Node* bits = intcon(InstanceKlass::fully_initialized);
2900  Node* test = _gvn.transform(new (C) SubINode(inst, bits));
2901  // The 'test' is non-zero if we need to take a slow path.
2902
2903  Node* obj = new_instance(kls, test);
2904  set_result(obj);
2905  return true;
2906}
2907
2908#ifdef TRACE_HAVE_INTRINSICS
2909/*
2910 * oop -> myklass
2911 * myklass->trace_id |= USED
2912 * return myklass->trace_id & ~0x3
2913 */
2914bool LibraryCallKit::inline_native_classID() {
2915  null_check_receiver();  // null-check, then ignore
2916  Node* cls = null_check(argument(1), T_OBJECT);
2917  Node* kls = load_klass_from_mirror(cls, false, NULL, 0);
2918  kls = null_check(kls, T_OBJECT);
2919  ByteSize offset = TRACE_ID_OFFSET;
2920  Node* insp = basic_plus_adr(kls, in_bytes(offset));
2921  Node* tvalue = make_load(NULL, insp, TypeLong::LONG, T_LONG);
2922  Node* bits = longcon(~0x03l); // ignore bit 0 & 1
2923  Node* andl = _gvn.transform(new (C) AndLNode(tvalue, bits));
2924  Node* clsused = longcon(0x01l); // set the class bit
2925  Node* orl = _gvn.transform(new (C) OrLNode(tvalue, clsused));
2926
2927  const TypePtr *adr_type = _gvn.type(insp)->isa_ptr();
2928  store_to_memory(control(), insp, orl, T_LONG, adr_type);
2929  set_result(andl);
2930  return true;
2931}
2932
2933bool LibraryCallKit::inline_native_threadID() {
2934  Node* tls_ptr = NULL;
2935  Node* cur_thr = generate_current_thread(tls_ptr);
2936  Node* p = basic_plus_adr(top()/*!oop*/, tls_ptr, in_bytes(JavaThread::osthread_offset()));
2937  Node* osthread = make_load(NULL, p, TypeRawPtr::NOTNULL, T_ADDRESS);
2938  p = basic_plus_adr(top()/*!oop*/, osthread, in_bytes(OSThread::thread_id_offset()));
2939
2940  Node* threadid = NULL;
2941  size_t thread_id_size = OSThread::thread_id_size();
2942  if (thread_id_size == (size_t) BytesPerLong) {
2943    threadid = ConvL2I(make_load(control(), p, TypeLong::LONG, T_LONG));
2944  } else if (thread_id_size == (size_t) BytesPerInt) {
2945    threadid = make_load(control(), p, TypeInt::INT, T_INT);
2946  } else {
2947    ShouldNotReachHere();
2948  }
2949  set_result(threadid);
2950  return true;
2951}
2952#endif
2953
2954//------------------------inline_native_time_funcs--------------
2955// inline code for System.currentTimeMillis() and System.nanoTime()
2956// these have the same type and signature
2957bool LibraryCallKit::inline_native_time_funcs(address funcAddr, const char* funcName) {
2958  const TypeFunc* tf = OptoRuntime::void_long_Type();
2959  const TypePtr* no_memory_effects = NULL;
2960  Node* time = make_runtime_call(RC_LEAF, tf, funcAddr, funcName, no_memory_effects);
2961  Node* value = _gvn.transform(new (C) ProjNode(time, TypeFunc::Parms+0));
2962#ifdef ASSERT
2963  Node* value_top = _gvn.transform(new (C) ProjNode(time, TypeFunc::Parms+1));
2964  assert(value_top == top(), "second value must be top");
2965#endif
2966  set_result(value);
2967  return true;
2968}
2969
2970//------------------------inline_native_currentThread------------------
2971bool LibraryCallKit::inline_native_currentThread() {
2972  Node* junk = NULL;
2973  set_result(generate_current_thread(junk));
2974  return true;
2975}
2976
2977//------------------------inline_native_isInterrupted------------------
2978// private native boolean java.lang.Thread.isInterrupted(boolean ClearInterrupted);
2979bool LibraryCallKit::inline_native_isInterrupted() {
2980  // Add a fast path to t.isInterrupted(clear_int):
2981  //   (t == Thread.current() && (!TLS._osthread._interrupted || !clear_int))
2982  //   ? TLS._osthread._interrupted : /*slow path:*/ t.isInterrupted(clear_int)
2983  // So, in the common case that the interrupt bit is false,
2984  // we avoid making a call into the VM.  Even if the interrupt bit
2985  // is true, if the clear_int argument is false, we avoid the VM call.
2986  // However, if the receiver is not currentThread, we must call the VM,
2987  // because there must be some locking done around the operation.
2988
2989  // We only go to the fast case code if we pass two guards.
2990  // Paths which do not pass are accumulated in the slow_region.
2991
2992  enum {
2993    no_int_result_path   = 1, // t == Thread.current() && !TLS._osthread._interrupted
2994    no_clear_result_path = 2, // t == Thread.current() &&  TLS._osthread._interrupted && !clear_int
2995    slow_result_path     = 3, // slow path: t.isInterrupted(clear_int)
2996    PATH_LIMIT
2997  };
2998
2999  // Ensure that it's not possible to move the load of TLS._osthread._interrupted flag
3000  // out of the function.
3001  insert_mem_bar(Op_MemBarCPUOrder);
3002
3003  RegionNode* result_rgn = new (C) RegionNode(PATH_LIMIT);
3004  PhiNode*    result_val = new (C) PhiNode(result_rgn, TypeInt::BOOL);
3005
3006  RegionNode* slow_region = new (C) RegionNode(1);
3007  record_for_igvn(slow_region);
3008
3009  // (a) Receiving thread must be the current thread.
3010  Node* rec_thr = argument(0);
3011  Node* tls_ptr = NULL;
3012  Node* cur_thr = generate_current_thread(tls_ptr);
3013  Node* cmp_thr = _gvn.transform( new (C) CmpPNode(cur_thr, rec_thr) );
3014  Node* bol_thr = _gvn.transform( new (C) BoolNode(cmp_thr, BoolTest::ne) );
3015
3016  generate_slow_guard(bol_thr, slow_region);
3017
3018  // (b) Interrupt bit on TLS must be false.
3019  Node* p = basic_plus_adr(top()/*!oop*/, tls_ptr, in_bytes(JavaThread::osthread_offset()));
3020  Node* osthread = make_load(NULL, p, TypeRawPtr::NOTNULL, T_ADDRESS);
3021  p = basic_plus_adr(top()/*!oop*/, osthread, in_bytes(OSThread::interrupted_offset()));
3022
3023  // Set the control input on the field _interrupted read to prevent it floating up.
3024  Node* int_bit = make_load(control(), p, TypeInt::BOOL, T_INT);
3025  Node* cmp_bit = _gvn.transform( new (C) CmpINode(int_bit, intcon(0)) );
3026  Node* bol_bit = _gvn.transform( new (C) BoolNode(cmp_bit, BoolTest::ne) );
3027
3028  IfNode* iff_bit = create_and_map_if(control(), bol_bit, PROB_UNLIKELY_MAG(3), COUNT_UNKNOWN);
3029
3030  // First fast path:  if (!TLS._interrupted) return false;
3031  Node* false_bit = _gvn.transform( new (C) IfFalseNode(iff_bit) );
3032  result_rgn->init_req(no_int_result_path, false_bit);
3033  result_val->init_req(no_int_result_path, intcon(0));
3034
3035  // drop through to next case
3036  set_control( _gvn.transform(new (C) IfTrueNode(iff_bit)) );
3037
3038  // (c) Or, if interrupt bit is set and clear_int is false, use 2nd fast path.
3039  Node* clr_arg = argument(1);
3040  Node* cmp_arg = _gvn.transform( new (C) CmpINode(clr_arg, intcon(0)) );
3041  Node* bol_arg = _gvn.transform( new (C) BoolNode(cmp_arg, BoolTest::ne) );
3042  IfNode* iff_arg = create_and_map_if(control(), bol_arg, PROB_FAIR, COUNT_UNKNOWN);
3043
3044  // Second fast path:  ... else if (!clear_int) return true;
3045  Node* false_arg = _gvn.transform( new (C) IfFalseNode(iff_arg) );
3046  result_rgn->init_req(no_clear_result_path, false_arg);
3047  result_val->init_req(no_clear_result_path, intcon(1));
3048
3049  // drop through to next case
3050  set_control( _gvn.transform(new (C) IfTrueNode(iff_arg)) );
3051
3052  // (d) Otherwise, go to the slow path.
3053  slow_region->add_req(control());
3054  set_control( _gvn.transform(slow_region) );
3055
3056  if (stopped()) {
3057    // There is no slow path.
3058    result_rgn->init_req(slow_result_path, top());
3059    result_val->init_req(slow_result_path, top());
3060  } else {
3061    // non-virtual because it is a private non-static
3062    CallJavaNode* slow_call = generate_method_call(vmIntrinsics::_isInterrupted);
3063
3064    Node* slow_val = set_results_for_java_call(slow_call);
3065    // this->control() comes from set_results_for_java_call
3066
3067    Node* fast_io  = slow_call->in(TypeFunc::I_O);
3068    Node* fast_mem = slow_call->in(TypeFunc::Memory);
3069
3070    // These two phis are pre-filled with copies of of the fast IO and Memory
3071    PhiNode* result_mem  = PhiNode::make(result_rgn, fast_mem, Type::MEMORY, TypePtr::BOTTOM);
3072    PhiNode* result_io   = PhiNode::make(result_rgn, fast_io,  Type::ABIO);
3073
3074    result_rgn->init_req(slow_result_path, control());
3075    result_io ->init_req(slow_result_path, i_o());
3076    result_mem->init_req(slow_result_path, reset_memory());
3077    result_val->init_req(slow_result_path, slow_val);
3078
3079    set_all_memory(_gvn.transform(result_mem));
3080    set_i_o(       _gvn.transform(result_io));
3081  }
3082
3083  C->set_has_split_ifs(true); // Has chance for split-if optimization
3084  set_result(result_rgn, result_val);
3085  return true;
3086}
3087
3088//---------------------------load_mirror_from_klass----------------------------
3089// Given a klass oop, load its java mirror (a java.lang.Class oop).
3090Node* LibraryCallKit::load_mirror_from_klass(Node* klass) {
3091  Node* p = basic_plus_adr(klass, in_bytes(Klass::java_mirror_offset()));
3092  return make_load(NULL, p, TypeInstPtr::MIRROR, T_OBJECT);
3093}
3094
3095//-----------------------load_klass_from_mirror_common-------------------------
3096// Given a java mirror (a java.lang.Class oop), load its corresponding klass oop.
3097// Test the klass oop for null (signifying a primitive Class like Integer.TYPE),
3098// and branch to the given path on the region.
3099// If never_see_null, take an uncommon trap on null, so we can optimistically
3100// compile for the non-null case.
3101// If the region is NULL, force never_see_null = true.
3102Node* LibraryCallKit::load_klass_from_mirror_common(Node* mirror,
3103                                                    bool never_see_null,
3104                                                    RegionNode* region,
3105                                                    int null_path,
3106                                                    int offset) {
3107  if (region == NULL)  never_see_null = true;
3108  Node* p = basic_plus_adr(mirror, offset);
3109  const TypeKlassPtr*  kls_type = TypeKlassPtr::OBJECT_OR_NULL;
3110  Node* kls = _gvn.transform( LoadKlassNode::make(_gvn, immutable_memory(), p, TypeRawPtr::BOTTOM, kls_type) );
3111  Node* null_ctl = top();
3112  kls = null_check_oop(kls, &null_ctl, never_see_null);
3113  if (region != NULL) {
3114    // Set region->in(null_path) if the mirror is a primitive (e.g, int.class).
3115    region->init_req(null_path, null_ctl);
3116  } else {
3117    assert(null_ctl == top(), "no loose ends");
3118  }
3119  return kls;
3120}
3121
3122//--------------------(inline_native_Class_query helpers)---------------------
3123// Use this for JVM_ACC_INTERFACE, JVM_ACC_IS_CLONEABLE, JVM_ACC_HAS_FINALIZER.
3124// Fall through if (mods & mask) == bits, take the guard otherwise.
3125Node* LibraryCallKit::generate_access_flags_guard(Node* kls, int modifier_mask, int modifier_bits, RegionNode* region) {
3126  // Branch around if the given klass has the given modifier bit set.
3127  // Like generate_guard, adds a new path onto the region.
3128  Node* modp = basic_plus_adr(kls, in_bytes(Klass::access_flags_offset()));
3129  Node* mods = make_load(NULL, modp, TypeInt::INT, T_INT);
3130  Node* mask = intcon(modifier_mask);
3131  Node* bits = intcon(modifier_bits);
3132  Node* mbit = _gvn.transform( new (C) AndINode(mods, mask) );
3133  Node* cmp  = _gvn.transform( new (C) CmpINode(mbit, bits) );
3134  Node* bol  = _gvn.transform( new (C) BoolNode(cmp, BoolTest::ne) );
3135  return generate_fair_guard(bol, region);
3136}
3137Node* LibraryCallKit::generate_interface_guard(Node* kls, RegionNode* region) {
3138  return generate_access_flags_guard(kls, JVM_ACC_INTERFACE, 0, region);
3139}
3140
3141//-------------------------inline_native_Class_query-------------------
3142bool LibraryCallKit::inline_native_Class_query(vmIntrinsics::ID id) {
3143  const Type* return_type = TypeInt::BOOL;
3144  Node* prim_return_value = top();  // what happens if it's a primitive class?
3145  bool never_see_null = !too_many_traps(Deoptimization::Reason_null_check);
3146  bool expect_prim = false;     // most of these guys expect to work on refs
3147
3148  enum { _normal_path = 1, _prim_path = 2, PATH_LIMIT };
3149
3150  Node* mirror = argument(0);
3151  Node* obj    = top();
3152
3153  switch (id) {
3154  case vmIntrinsics::_isInstance:
3155    // nothing is an instance of a primitive type
3156    prim_return_value = intcon(0);
3157    obj = argument(1);
3158    break;
3159  case vmIntrinsics::_getModifiers:
3160    prim_return_value = intcon(JVM_ACC_ABSTRACT | JVM_ACC_FINAL | JVM_ACC_PUBLIC);
3161    assert(is_power_of_2((int)JVM_ACC_WRITTEN_FLAGS+1), "change next line");
3162    return_type = TypeInt::make(0, JVM_ACC_WRITTEN_FLAGS, Type::WidenMin);
3163    break;
3164  case vmIntrinsics::_isInterface:
3165    prim_return_value = intcon(0);
3166    break;
3167  case vmIntrinsics::_isArray:
3168    prim_return_value = intcon(0);
3169    expect_prim = true;  // cf. ObjectStreamClass.getClassSignature
3170    break;
3171  case vmIntrinsics::_isPrimitive:
3172    prim_return_value = intcon(1);
3173    expect_prim = true;  // obviously
3174    break;
3175  case vmIntrinsics::_getSuperclass:
3176    prim_return_value = null();
3177    return_type = TypeInstPtr::MIRROR->cast_to_ptr_type(TypePtr::BotPTR);
3178    break;
3179  case vmIntrinsics::_getComponentType:
3180    prim_return_value = null();
3181    return_type = TypeInstPtr::MIRROR->cast_to_ptr_type(TypePtr::BotPTR);
3182    break;
3183  case vmIntrinsics::_getClassAccessFlags:
3184    prim_return_value = intcon(JVM_ACC_ABSTRACT | JVM_ACC_FINAL | JVM_ACC_PUBLIC);
3185    return_type = TypeInt::INT;  // not bool!  6297094
3186    break;
3187  default:
3188    fatal_unexpected_iid(id);
3189    break;
3190  }
3191
3192  const TypeInstPtr* mirror_con = _gvn.type(mirror)->isa_instptr();
3193  if (mirror_con == NULL)  return false;  // cannot happen?
3194
3195#ifndef PRODUCT
3196  if (PrintIntrinsics || PrintInlining || PrintOptoInlining) {
3197    ciType* k = mirror_con->java_mirror_type();
3198    if (k) {
3199      tty->print("Inlining %s on constant Class ", vmIntrinsics::name_at(intrinsic_id()));
3200      k->print_name();
3201      tty->cr();
3202    }
3203  }
3204#endif
3205
3206  // Null-check the mirror, and the mirror's klass ptr (in case it is a primitive).
3207  RegionNode* region = new (C) RegionNode(PATH_LIMIT);
3208  record_for_igvn(region);
3209  PhiNode* phi = new (C) PhiNode(region, return_type);
3210
3211  // The mirror will never be null of Reflection.getClassAccessFlags, however
3212  // it may be null for Class.isInstance or Class.getModifiers. Throw a NPE
3213  // if it is. See bug 4774291.
3214
3215  // For Reflection.getClassAccessFlags(), the null check occurs in
3216  // the wrong place; see inline_unsafe_access(), above, for a similar
3217  // situation.
3218  mirror = null_check(mirror);
3219  // If mirror or obj is dead, only null-path is taken.
3220  if (stopped())  return true;
3221
3222  if (expect_prim)  never_see_null = false;  // expect nulls (meaning prims)
3223
3224  // Now load the mirror's klass metaobject, and null-check it.
3225  // Side-effects region with the control path if the klass is null.
3226  Node* kls = load_klass_from_mirror(mirror, never_see_null, region, _prim_path);
3227  // If kls is null, we have a primitive mirror.
3228  phi->init_req(_prim_path, prim_return_value);
3229  if (stopped()) { set_result(region, phi); return true; }
3230
3231  Node* p;  // handy temp
3232  Node* null_ctl;
3233
3234  // Now that we have the non-null klass, we can perform the real query.
3235  // For constant classes, the query will constant-fold in LoadNode::Value.
3236  Node* query_value = top();
3237  switch (id) {
3238  case vmIntrinsics::_isInstance:
3239    // nothing is an instance of a primitive type
3240    query_value = gen_instanceof(obj, kls);
3241    break;
3242
3243  case vmIntrinsics::_getModifiers:
3244    p = basic_plus_adr(kls, in_bytes(Klass::modifier_flags_offset()));
3245    query_value = make_load(NULL, p, TypeInt::INT, T_INT);
3246    break;
3247
3248  case vmIntrinsics::_isInterface:
3249    // (To verify this code sequence, check the asserts in JVM_IsInterface.)
3250    if (generate_interface_guard(kls, region) != NULL)
3251      // A guard was added.  If the guard is taken, it was an interface.
3252      phi->add_req(intcon(1));
3253    // If we fall through, it's a plain class.
3254    query_value = intcon(0);
3255    break;
3256
3257  case vmIntrinsics::_isArray:
3258    // (To verify this code sequence, check the asserts in JVM_IsArrayClass.)
3259    if (generate_array_guard(kls, region) != NULL)
3260      // A guard was added.  If the guard is taken, it was an array.
3261      phi->add_req(intcon(1));
3262    // If we fall through, it's a plain class.
3263    query_value = intcon(0);
3264    break;
3265
3266  case vmIntrinsics::_isPrimitive:
3267    query_value = intcon(0); // "normal" path produces false
3268    break;
3269
3270  case vmIntrinsics::_getSuperclass:
3271    // The rules here are somewhat unfortunate, but we can still do better
3272    // with random logic than with a JNI call.
3273    // Interfaces store null or Object as _super, but must report null.
3274    // Arrays store an intermediate super as _super, but must report Object.
3275    // Other types can report the actual _super.
3276    // (To verify this code sequence, check the asserts in JVM_IsInterface.)
3277    if (generate_interface_guard(kls, region) != NULL)
3278      // A guard was added.  If the guard is taken, it was an interface.
3279      phi->add_req(null());
3280    if (generate_array_guard(kls, region) != NULL)
3281      // A guard was added.  If the guard is taken, it was an array.
3282      phi->add_req(makecon(TypeInstPtr::make(env()->Object_klass()->java_mirror())));
3283    // If we fall through, it's a plain class.  Get its _super.
3284    p = basic_plus_adr(kls, in_bytes(Klass::super_offset()));
3285    kls = _gvn.transform( LoadKlassNode::make(_gvn, immutable_memory(), p, TypeRawPtr::BOTTOM, TypeKlassPtr::OBJECT_OR_NULL) );
3286    null_ctl = top();
3287    kls = null_check_oop(kls, &null_ctl);
3288    if (null_ctl != top()) {
3289      // If the guard is taken, Object.superClass is null (both klass and mirror).
3290      region->add_req(null_ctl);
3291      phi   ->add_req(null());
3292    }
3293    if (!stopped()) {
3294      query_value = load_mirror_from_klass(kls);
3295    }
3296    break;
3297
3298  case vmIntrinsics::_getComponentType:
3299    if (generate_array_guard(kls, region) != NULL) {
3300      // Be sure to pin the oop load to the guard edge just created:
3301      Node* is_array_ctrl = region->in(region->req()-1);
3302      Node* cma = basic_plus_adr(kls, in_bytes(ArrayKlass::component_mirror_offset()));
3303      Node* cmo = make_load(is_array_ctrl, cma, TypeInstPtr::MIRROR, T_OBJECT);
3304      phi->add_req(cmo);
3305    }
3306    query_value = null();  // non-array case is null
3307    break;
3308
3309  case vmIntrinsics::_getClassAccessFlags:
3310    p = basic_plus_adr(kls, in_bytes(Klass::access_flags_offset()));
3311    query_value = make_load(NULL, p, TypeInt::INT, T_INT);
3312    break;
3313
3314  default:
3315    fatal_unexpected_iid(id);
3316    break;
3317  }
3318
3319  // Fall-through is the normal case of a query to a real class.
3320  phi->init_req(1, query_value);
3321  region->init_req(1, control());
3322
3323  C->set_has_split_ifs(true); // Has chance for split-if optimization
3324  set_result(region, phi);
3325  return true;
3326}
3327
3328//--------------------------inline_native_subtype_check------------------------
3329// This intrinsic takes the JNI calls out of the heart of
3330// UnsafeFieldAccessorImpl.set, which improves Field.set, readObject, etc.
3331bool LibraryCallKit::inline_native_subtype_check() {
3332  // Pull both arguments off the stack.
3333  Node* args[2];                // two java.lang.Class mirrors: superc, subc
3334  args[0] = argument(0);
3335  args[1] = argument(1);
3336  Node* klasses[2];             // corresponding Klasses: superk, subk
3337  klasses[0] = klasses[1] = top();
3338
3339  enum {
3340    // A full decision tree on {superc is prim, subc is prim}:
3341    _prim_0_path = 1,           // {P,N} => false
3342                                // {P,P} & superc!=subc => false
3343    _prim_same_path,            // {P,P} & superc==subc => true
3344    _prim_1_path,               // {N,P} => false
3345    _ref_subtype_path,          // {N,N} & subtype check wins => true
3346    _both_ref_path,             // {N,N} & subtype check loses => false
3347    PATH_LIMIT
3348  };
3349
3350  RegionNode* region = new (C) RegionNode(PATH_LIMIT);
3351  Node*       phi    = new (C) PhiNode(region, TypeInt::BOOL);
3352  record_for_igvn(region);
3353
3354  const TypePtr* adr_type = TypeRawPtr::BOTTOM;   // memory type of loads
3355  const TypeKlassPtr* kls_type = TypeKlassPtr::OBJECT_OR_NULL;
3356  int class_klass_offset = java_lang_Class::klass_offset_in_bytes();
3357
3358  // First null-check both mirrors and load each mirror's klass metaobject.
3359  int which_arg;
3360  for (which_arg = 0; which_arg <= 1; which_arg++) {
3361    Node* arg = args[which_arg];
3362    arg = null_check(arg);
3363    if (stopped())  break;
3364    args[which_arg] = arg;
3365
3366    Node* p = basic_plus_adr(arg, class_klass_offset);
3367    Node* kls = LoadKlassNode::make(_gvn, immutable_memory(), p, adr_type, kls_type);
3368    klasses[which_arg] = _gvn.transform(kls);
3369  }
3370
3371  // Having loaded both klasses, test each for null.
3372  bool never_see_null = !too_many_traps(Deoptimization::Reason_null_check);
3373  for (which_arg = 0; which_arg <= 1; which_arg++) {
3374    Node* kls = klasses[which_arg];
3375    Node* null_ctl = top();
3376    kls = null_check_oop(kls, &null_ctl, never_see_null);
3377    int prim_path = (which_arg == 0 ? _prim_0_path : _prim_1_path);
3378    region->init_req(prim_path, null_ctl);
3379    if (stopped())  break;
3380    klasses[which_arg] = kls;
3381  }
3382
3383  if (!stopped()) {
3384    // now we have two reference types, in klasses[0..1]
3385    Node* subk   = klasses[1];  // the argument to isAssignableFrom
3386    Node* superk = klasses[0];  // the receiver
3387    region->set_req(_both_ref_path, gen_subtype_check(subk, superk));
3388    // now we have a successful reference subtype check
3389    region->set_req(_ref_subtype_path, control());
3390  }
3391
3392  // If both operands are primitive (both klasses null), then
3393  // we must return true when they are identical primitives.
3394  // It is convenient to test this after the first null klass check.
3395  set_control(region->in(_prim_0_path)); // go back to first null check
3396  if (!stopped()) {
3397    // Since superc is primitive, make a guard for the superc==subc case.
3398    Node* cmp_eq = _gvn.transform( new (C) CmpPNode(args[0], args[1]) );
3399    Node* bol_eq = _gvn.transform( new (C) BoolNode(cmp_eq, BoolTest::eq) );
3400    generate_guard(bol_eq, region, PROB_FAIR);
3401    if (region->req() == PATH_LIMIT+1) {
3402      // A guard was added.  If the added guard is taken, superc==subc.
3403      region->swap_edges(PATH_LIMIT, _prim_same_path);
3404      region->del_req(PATH_LIMIT);
3405    }
3406    region->set_req(_prim_0_path, control()); // Not equal after all.
3407  }
3408
3409  // these are the only paths that produce 'true':
3410  phi->set_req(_prim_same_path,   intcon(1));
3411  phi->set_req(_ref_subtype_path, intcon(1));
3412
3413  // pull together the cases:
3414  assert(region->req() == PATH_LIMIT, "sane region");
3415  for (uint i = 1; i < region->req(); i++) {
3416    Node* ctl = region->in(i);
3417    if (ctl == NULL || ctl == top()) {
3418      region->set_req(i, top());
3419      phi   ->set_req(i, top());
3420    } else if (phi->in(i) == NULL) {
3421      phi->set_req(i, intcon(0)); // all other paths produce 'false'
3422    }
3423  }
3424
3425  set_control(_gvn.transform(region));
3426  set_result(_gvn.transform(phi));
3427  return true;
3428}
3429
3430//---------------------generate_array_guard_common------------------------
3431Node* LibraryCallKit::generate_array_guard_common(Node* kls, RegionNode* region,
3432                                                  bool obj_array, bool not_array) {
3433  // If obj_array/non_array==false/false:
3434  // Branch around if the given klass is in fact an array (either obj or prim).
3435  // If obj_array/non_array==false/true:
3436  // Branch around if the given klass is not an array klass of any kind.
3437  // If obj_array/non_array==true/true:
3438  // Branch around if the kls is not an oop array (kls is int[], String, etc.)
3439  // If obj_array/non_array==true/false:
3440  // Branch around if the kls is an oop array (Object[] or subtype)
3441  //
3442  // Like generate_guard, adds a new path onto the region.
3443  jint  layout_con = 0;
3444  Node* layout_val = get_layout_helper(kls, layout_con);
3445  if (layout_val == NULL) {
3446    bool query = (obj_array
3447                  ? Klass::layout_helper_is_objArray(layout_con)
3448                  : Klass::layout_helper_is_array(layout_con));
3449    if (query == not_array) {
3450      return NULL;                       // never a branch
3451    } else {                             // always a branch
3452      Node* always_branch = control();
3453      if (region != NULL)
3454        region->add_req(always_branch);
3455      set_control(top());
3456      return always_branch;
3457    }
3458  }
3459  // Now test the correct condition.
3460  jint  nval = (obj_array
3461                ? ((jint)Klass::_lh_array_tag_type_value
3462                   <<    Klass::_lh_array_tag_shift)
3463                : Klass::_lh_neutral_value);
3464  Node* cmp = _gvn.transform( new(C) CmpINode(layout_val, intcon(nval)) );
3465  BoolTest::mask btest = BoolTest::lt;  // correct for testing is_[obj]array
3466  // invert the test if we are looking for a non-array
3467  if (not_array)  btest = BoolTest(btest).negate();
3468  Node* bol = _gvn.transform( new(C) BoolNode(cmp, btest) );
3469  return generate_fair_guard(bol, region);
3470}
3471
3472
3473//-----------------------inline_native_newArray--------------------------
3474// private static native Object java.lang.reflect.newArray(Class<?> componentType, int length);
3475bool LibraryCallKit::inline_native_newArray() {
3476  Node* mirror    = argument(0);
3477  Node* count_val = argument(1);
3478
3479  mirror = null_check(mirror);
3480  // If mirror or obj is dead, only null-path is taken.
3481  if (stopped())  return true;
3482
3483  enum { _normal_path = 1, _slow_path = 2, PATH_LIMIT };
3484  RegionNode* result_reg = new(C) RegionNode(PATH_LIMIT);
3485  PhiNode*    result_val = new(C) PhiNode(result_reg,
3486                                          TypeInstPtr::NOTNULL);
3487  PhiNode*    result_io  = new(C) PhiNode(result_reg, Type::ABIO);
3488  PhiNode*    result_mem = new(C) PhiNode(result_reg, Type::MEMORY,
3489                                          TypePtr::BOTTOM);
3490
3491  bool never_see_null = !too_many_traps(Deoptimization::Reason_null_check);
3492  Node* klass_node = load_array_klass_from_mirror(mirror, never_see_null,
3493                                                  result_reg, _slow_path);
3494  Node* normal_ctl   = control();
3495  Node* no_array_ctl = result_reg->in(_slow_path);
3496
3497  // Generate code for the slow case.  We make a call to newArray().
3498  set_control(no_array_ctl);
3499  if (!stopped()) {
3500    // Either the input type is void.class, or else the
3501    // array klass has not yet been cached.  Either the
3502    // ensuing call will throw an exception, or else it
3503    // will cache the array klass for next time.
3504    PreserveJVMState pjvms(this);
3505    CallJavaNode* slow_call = generate_method_call_static(vmIntrinsics::_newArray);
3506    Node* slow_result = set_results_for_java_call(slow_call);
3507    // this->control() comes from set_results_for_java_call
3508    result_reg->set_req(_slow_path, control());
3509    result_val->set_req(_slow_path, slow_result);
3510    result_io ->set_req(_slow_path, i_o());
3511    result_mem->set_req(_slow_path, reset_memory());
3512  }
3513
3514  set_control(normal_ctl);
3515  if (!stopped()) {
3516    // Normal case:  The array type has been cached in the java.lang.Class.
3517    // The following call works fine even if the array type is polymorphic.
3518    // It could be a dynamic mix of int[], boolean[], Object[], etc.
3519    Node* obj = new_array(klass_node, count_val, 0);  // no arguments to push
3520    result_reg->init_req(_normal_path, control());
3521    result_val->init_req(_normal_path, obj);
3522    result_io ->init_req(_normal_path, i_o());
3523    result_mem->init_req(_normal_path, reset_memory());
3524  }
3525
3526  // Return the combined state.
3527  set_i_o(        _gvn.transform(result_io)  );
3528  set_all_memory( _gvn.transform(result_mem) );
3529
3530  C->set_has_split_ifs(true); // Has chance for split-if optimization
3531  set_result(result_reg, result_val);
3532  return true;
3533}
3534
3535//----------------------inline_native_getLength--------------------------
3536// public static native int java.lang.reflect.Array.getLength(Object array);
3537bool LibraryCallKit::inline_native_getLength() {
3538  if (too_many_traps(Deoptimization::Reason_intrinsic))  return false;
3539
3540  Node* array = null_check(argument(0));
3541  // If array is dead, only null-path is taken.
3542  if (stopped())  return true;
3543
3544  // Deoptimize if it is a non-array.
3545  Node* non_array = generate_non_array_guard(load_object_klass(array), NULL);
3546
3547  if (non_array != NULL) {
3548    PreserveJVMState pjvms(this);
3549    set_control(non_array);
3550    uncommon_trap(Deoptimization::Reason_intrinsic,
3551                  Deoptimization::Action_maybe_recompile);
3552  }
3553
3554  // If control is dead, only non-array-path is taken.
3555  if (stopped())  return true;
3556
3557  // The works fine even if the array type is polymorphic.
3558  // It could be a dynamic mix of int[], boolean[], Object[], etc.
3559  Node* result = load_array_length(array);
3560
3561  C->set_has_split_ifs(true);  // Has chance for split-if optimization
3562  set_result(result);
3563  return true;
3564}
3565
3566//------------------------inline_array_copyOf----------------------------
3567// public static <T,U> T[] java.util.Arrays.copyOf(     U[] original, int newLength,         Class<? extends T[]> newType);
3568// public static <T,U> T[] java.util.Arrays.copyOfRange(U[] original, int from,      int to, Class<? extends T[]> newType);
3569bool LibraryCallKit::inline_array_copyOf(bool is_copyOfRange) {
3570  if (too_many_traps(Deoptimization::Reason_intrinsic))  return false;
3571
3572  // Get the arguments.
3573  Node* original          = argument(0);
3574  Node* start             = is_copyOfRange? argument(1): intcon(0);
3575  Node* end               = is_copyOfRange? argument(2): argument(1);
3576  Node* array_type_mirror = is_copyOfRange? argument(3): argument(2);
3577
3578  Node* newcopy;
3579
3580  // Set the original stack and the reexecute bit for the interpreter to reexecute
3581  // the bytecode that invokes Arrays.copyOf if deoptimization happens.
3582  { PreserveReexecuteState preexecs(this);
3583    jvms()->set_should_reexecute(true);
3584
3585    array_type_mirror = null_check(array_type_mirror);
3586    original          = null_check(original);
3587
3588    // Check if a null path was taken unconditionally.
3589    if (stopped())  return true;
3590
3591    Node* orig_length = load_array_length(original);
3592
3593    Node* klass_node = load_klass_from_mirror(array_type_mirror, false, NULL, 0);
3594    klass_node = null_check(klass_node);
3595
3596    RegionNode* bailout = new (C) RegionNode(1);
3597    record_for_igvn(bailout);
3598
3599    // Despite the generic type of Arrays.copyOf, the mirror might be int, int[], etc.
3600    // Bail out if that is so.
3601    Node* not_objArray = generate_non_objArray_guard(klass_node, bailout);
3602    if (not_objArray != NULL) {
3603      // Improve the klass node's type from the new optimistic assumption:
3604      ciKlass* ak = ciArrayKlass::make(env()->Object_klass());
3605      const Type* akls = TypeKlassPtr::make(TypePtr::NotNull, ak, 0/*offset*/);
3606      Node* cast = new (C) CastPPNode(klass_node, akls);
3607      cast->init_req(0, control());
3608      klass_node = _gvn.transform(cast);
3609    }
3610
3611    // Bail out if either start or end is negative.
3612    generate_negative_guard(start, bailout, &start);
3613    generate_negative_guard(end,   bailout, &end);
3614
3615    Node* length = end;
3616    if (_gvn.type(start) != TypeInt::ZERO) {
3617      length = _gvn.transform(new (C) SubINode(end, start));
3618    }
3619
3620    // Bail out if length is negative.
3621    // Without this the new_array would throw
3622    // NegativeArraySizeException but IllegalArgumentException is what
3623    // should be thrown
3624    generate_negative_guard(length, bailout, &length);
3625
3626    if (bailout->req() > 1) {
3627      PreserveJVMState pjvms(this);
3628      set_control(_gvn.transform(bailout));
3629      uncommon_trap(Deoptimization::Reason_intrinsic,
3630                    Deoptimization::Action_maybe_recompile);
3631    }
3632
3633    if (!stopped()) {
3634      // How many elements will we copy from the original?
3635      // The answer is MinI(orig_length - start, length).
3636      Node* orig_tail = _gvn.transform(new (C) SubINode(orig_length, start));
3637      Node* moved = generate_min_max(vmIntrinsics::_min, orig_tail, length);
3638
3639      newcopy = new_array(klass_node, length, 0);  // no argments to push
3640
3641      // Generate a direct call to the right arraycopy function(s).
3642      // We know the copy is disjoint but we might not know if the
3643      // oop stores need checking.
3644      // Extreme case:  Arrays.copyOf((Integer[])x, 10, String[].class).
3645      // This will fail a store-check if x contains any non-nulls.
3646      bool disjoint_bases = true;
3647      // if start > orig_length then the length of the copy may be
3648      // negative.
3649      bool length_never_negative = !is_copyOfRange;
3650      generate_arraycopy(TypeAryPtr::OOPS, T_OBJECT,
3651                         original, start, newcopy, intcon(0), moved,
3652                         disjoint_bases, length_never_negative);
3653    }
3654  } // original reexecute is set back here
3655
3656  C->set_has_split_ifs(true); // Has chance for split-if optimization
3657  if (!stopped()) {
3658    set_result(newcopy);
3659  }
3660  return true;
3661}
3662
3663
3664//----------------------generate_virtual_guard---------------------------
3665// Helper for hashCode and clone.  Peeks inside the vtable to avoid a call.
3666Node* LibraryCallKit::generate_virtual_guard(Node* obj_klass,
3667                                             RegionNode* slow_region) {
3668  ciMethod* method = callee();
3669  int vtable_index = method->vtable_index();
3670  // Get the Method* out of the appropriate vtable entry.
3671  int entry_offset  = (InstanceKlass::vtable_start_offset() +
3672                     vtable_index*vtableEntry::size()) * wordSize +
3673                     vtableEntry::method_offset_in_bytes();
3674  Node* entry_addr  = basic_plus_adr(obj_klass, entry_offset);
3675  Node* target_call = make_load(NULL, entry_addr, TypePtr::NOTNULL, T_ADDRESS);
3676
3677  // Compare the target method with the expected method (e.g., Object.hashCode).
3678  const TypePtr* native_call_addr = TypeMetadataPtr::make(method);
3679
3680  Node* native_call = makecon(native_call_addr);
3681  Node* chk_native  = _gvn.transform( new(C) CmpPNode(target_call, native_call) );
3682  Node* test_native = _gvn.transform( new(C) BoolNode(chk_native, BoolTest::ne) );
3683
3684  return generate_slow_guard(test_native, slow_region);
3685}
3686
3687//-----------------------generate_method_call----------------------------
3688// Use generate_method_call to make a slow-call to the real
3689// method if the fast path fails.  An alternative would be to
3690// use a stub like OptoRuntime::slow_arraycopy_Java.
3691// This only works for expanding the current library call,
3692// not another intrinsic.  (E.g., don't use this for making an
3693// arraycopy call inside of the copyOf intrinsic.)
3694CallJavaNode*
3695LibraryCallKit::generate_method_call(vmIntrinsics::ID method_id, bool is_virtual, bool is_static) {
3696  // When compiling the intrinsic method itself, do not use this technique.
3697  guarantee(callee() != C->method(), "cannot make slow-call to self");
3698
3699  ciMethod* method = callee();
3700  // ensure the JVMS we have will be correct for this call
3701  guarantee(method_id == method->intrinsic_id(), "must match");
3702
3703  const TypeFunc* tf = TypeFunc::make(method);
3704  CallJavaNode* slow_call;
3705  if (is_static) {
3706    assert(!is_virtual, "");
3707    slow_call = new(C) CallStaticJavaNode(C, tf,
3708                           SharedRuntime::get_resolve_static_call_stub(),
3709                           method, bci());
3710  } else if (is_virtual) {
3711    null_check_receiver();
3712    int vtable_index = Method::invalid_vtable_index;
3713    if (UseInlineCaches) {
3714      // Suppress the vtable call
3715    } else {
3716      // hashCode and clone are not a miranda methods,
3717      // so the vtable index is fixed.
3718      // No need to use the linkResolver to get it.
3719       vtable_index = method->vtable_index();
3720    }
3721    slow_call = new(C) CallDynamicJavaNode(tf,
3722                          SharedRuntime::get_resolve_virtual_call_stub(),
3723                          method, vtable_index, bci());
3724  } else {  // neither virtual nor static:  opt_virtual
3725    null_check_receiver();
3726    slow_call = new(C) CallStaticJavaNode(C, tf,
3727                                SharedRuntime::get_resolve_opt_virtual_call_stub(),
3728                                method, bci());
3729    slow_call->set_optimized_virtual(true);
3730  }
3731  set_arguments_for_java_call(slow_call);
3732  set_edges_for_java_call(slow_call);
3733  return slow_call;
3734}
3735
3736
3737//------------------------------inline_native_hashcode--------------------
3738// Build special case code for calls to hashCode on an object.
3739bool LibraryCallKit::inline_native_hashcode(bool is_virtual, bool is_static) {
3740  assert(is_static == callee()->is_static(), "correct intrinsic selection");
3741  assert(!(is_virtual && is_static), "either virtual, special, or static");
3742
3743  enum { _slow_path = 1, _fast_path, _null_path, PATH_LIMIT };
3744
3745  RegionNode* result_reg = new(C) RegionNode(PATH_LIMIT);
3746  PhiNode*    result_val = new(C) PhiNode(result_reg,
3747                                          TypeInt::INT);
3748  PhiNode*    result_io  = new(C) PhiNode(result_reg, Type::ABIO);
3749  PhiNode*    result_mem = new(C) PhiNode(result_reg, Type::MEMORY,
3750                                          TypePtr::BOTTOM);
3751  Node* obj = NULL;
3752  if (!is_static) {
3753    // Check for hashing null object
3754    obj = null_check_receiver();
3755    if (stopped())  return true;        // unconditionally null
3756    result_reg->init_req(_null_path, top());
3757    result_val->init_req(_null_path, top());
3758  } else {
3759    // Do a null check, and return zero if null.
3760    // System.identityHashCode(null) == 0
3761    obj = argument(0);
3762    Node* null_ctl = top();
3763    obj = null_check_oop(obj, &null_ctl);
3764    result_reg->init_req(_null_path, null_ctl);
3765    result_val->init_req(_null_path, _gvn.intcon(0));
3766  }
3767
3768  // Unconditionally null?  Then return right away.
3769  if (stopped()) {
3770    set_control( result_reg->in(_null_path));
3771    if (!stopped())
3772      set_result(result_val->in(_null_path));
3773    return true;
3774  }
3775
3776  // After null check, get the object's klass.
3777  Node* obj_klass = load_object_klass(obj);
3778
3779  // This call may be virtual (invokevirtual) or bound (invokespecial).
3780  // For each case we generate slightly different code.
3781
3782  // We only go to the fast case code if we pass a number of guards.  The
3783  // paths which do not pass are accumulated in the slow_region.
3784  RegionNode* slow_region = new (C) RegionNode(1);
3785  record_for_igvn(slow_region);
3786
3787  // If this is a virtual call, we generate a funny guard.  We pull out
3788  // the vtable entry corresponding to hashCode() from the target object.
3789  // If the target method which we are calling happens to be the native
3790  // Object hashCode() method, we pass the guard.  We do not need this
3791  // guard for non-virtual calls -- the caller is known to be the native
3792  // Object hashCode().
3793  if (is_virtual) {
3794    generate_virtual_guard(obj_klass, slow_region);
3795  }
3796
3797  // Get the header out of the object, use LoadMarkNode when available
3798  Node* header_addr = basic_plus_adr(obj, oopDesc::mark_offset_in_bytes());
3799  Node* header = make_load(control(), header_addr, TypeX_X, TypeX_X->basic_type());
3800
3801  // Test the header to see if it is unlocked.
3802  Node *lock_mask      = _gvn.MakeConX(markOopDesc::biased_lock_mask_in_place);
3803  Node *lmasked_header = _gvn.transform( new (C) AndXNode(header, lock_mask) );
3804  Node *unlocked_val   = _gvn.MakeConX(markOopDesc::unlocked_value);
3805  Node *chk_unlocked   = _gvn.transform( new (C) CmpXNode( lmasked_header, unlocked_val));
3806  Node *test_unlocked  = _gvn.transform( new (C) BoolNode( chk_unlocked, BoolTest::ne) );
3807
3808  generate_slow_guard(test_unlocked, slow_region);
3809
3810  // Get the hash value and check to see that it has been properly assigned.
3811  // We depend on hash_mask being at most 32 bits and avoid the use of
3812  // hash_mask_in_place because it could be larger than 32 bits in a 64-bit
3813  // vm: see markOop.hpp.
3814  Node *hash_mask      = _gvn.intcon(markOopDesc::hash_mask);
3815  Node *hash_shift     = _gvn.intcon(markOopDesc::hash_shift);
3816  Node *hshifted_header= _gvn.transform( new (C) URShiftXNode(header, hash_shift) );
3817  // This hack lets the hash bits live anywhere in the mark object now, as long
3818  // as the shift drops the relevant bits into the low 32 bits.  Note that
3819  // Java spec says that HashCode is an int so there's no point in capturing
3820  // an 'X'-sized hashcode (32 in 32-bit build or 64 in 64-bit build).
3821  hshifted_header      = ConvX2I(hshifted_header);
3822  Node *hash_val       = _gvn.transform( new (C) AndINode(hshifted_header, hash_mask) );
3823
3824  Node *no_hash_val    = _gvn.intcon(markOopDesc::no_hash);
3825  Node *chk_assigned   = _gvn.transform( new (C) CmpINode( hash_val, no_hash_val));
3826  Node *test_assigned  = _gvn.transform( new (C) BoolNode( chk_assigned, BoolTest::eq) );
3827
3828  generate_slow_guard(test_assigned, slow_region);
3829
3830  Node* init_mem = reset_memory();
3831  // fill in the rest of the null path:
3832  result_io ->init_req(_null_path, i_o());
3833  result_mem->init_req(_null_path, init_mem);
3834
3835  result_val->init_req(_fast_path, hash_val);
3836  result_reg->init_req(_fast_path, control());
3837  result_io ->init_req(_fast_path, i_o());
3838  result_mem->init_req(_fast_path, init_mem);
3839
3840  // Generate code for the slow case.  We make a call to hashCode().
3841  set_control(_gvn.transform(slow_region));
3842  if (!stopped()) {
3843    // No need for PreserveJVMState, because we're using up the present state.
3844    set_all_memory(init_mem);
3845    vmIntrinsics::ID hashCode_id = is_static ? vmIntrinsics::_identityHashCode : vmIntrinsics::_hashCode;
3846    CallJavaNode* slow_call = generate_method_call(hashCode_id, is_virtual, is_static);
3847    Node* slow_result = set_results_for_java_call(slow_call);
3848    // this->control() comes from set_results_for_java_call
3849    result_reg->init_req(_slow_path, control());
3850    result_val->init_req(_slow_path, slow_result);
3851    result_io  ->set_req(_slow_path, i_o());
3852    result_mem ->set_req(_slow_path, reset_memory());
3853  }
3854
3855  // Return the combined state.
3856  set_i_o(        _gvn.transform(result_io)  );
3857  set_all_memory( _gvn.transform(result_mem) );
3858
3859  set_result(result_reg, result_val);
3860  return true;
3861}
3862
3863//---------------------------inline_native_getClass----------------------------
3864// public final native Class<?> java.lang.Object.getClass();
3865//
3866// Build special case code for calls to getClass on an object.
3867bool LibraryCallKit::inline_native_getClass() {
3868  Node* obj = null_check_receiver();
3869  if (stopped())  return true;
3870  set_result(load_mirror_from_klass(load_object_klass(obj)));
3871  return true;
3872}
3873
3874//-----------------inline_native_Reflection_getCallerClass---------------------
3875// public static native Class<?> sun.reflect.Reflection.getCallerClass();
3876//
3877// In the presence of deep enough inlining, getCallerClass() becomes a no-op.
3878//
3879// NOTE: This code must perform the same logic as JVM_GetCallerClass
3880// in that it must skip particular security frames and checks for
3881// caller sensitive methods.
3882bool LibraryCallKit::inline_native_Reflection_getCallerClass() {
3883#ifndef PRODUCT
3884  if ((PrintIntrinsics || PrintInlining || PrintOptoInlining) && Verbose) {
3885    tty->print_cr("Attempting to inline sun.reflect.Reflection.getCallerClass");
3886  }
3887#endif
3888
3889  if (!jvms()->has_method()) {
3890#ifndef PRODUCT
3891    if ((PrintIntrinsics || PrintInlining || PrintOptoInlining) && Verbose) {
3892      tty->print_cr("  Bailing out because intrinsic was inlined at top level");
3893    }
3894#endif
3895    return false;
3896  }
3897
3898  // Walk back up the JVM state to find the caller at the required
3899  // depth.
3900  JVMState* caller_jvms = jvms();
3901
3902  // Cf. JVM_GetCallerClass
3903  // NOTE: Start the loop at depth 1 because the current JVM state does
3904  // not include the Reflection.getCallerClass() frame.
3905  for (int n = 1; caller_jvms != NULL; caller_jvms = caller_jvms->caller(), n++) {
3906    ciMethod* m = caller_jvms->method();
3907    switch (n) {
3908    case 0:
3909      fatal("current JVM state does not include the Reflection.getCallerClass frame");
3910      break;
3911    case 1:
3912      // Frame 0 and 1 must be caller sensitive (see JVM_GetCallerClass).
3913      if (!m->caller_sensitive()) {
3914#ifndef PRODUCT
3915        if ((PrintIntrinsics || PrintInlining || PrintOptoInlining) && Verbose) {
3916          tty->print_cr("  Bailing out: CallerSensitive annotation expected at frame %d", n);
3917        }
3918#endif
3919        return false;  // bail-out; let JVM_GetCallerClass do the work
3920      }
3921      break;
3922    default:
3923      if (!m->is_ignored_by_security_stack_walk()) {
3924        // We have reached the desired frame; return the holder class.
3925        // Acquire method holder as java.lang.Class and push as constant.
3926        ciInstanceKlass* caller_klass = caller_jvms->method()->holder();
3927        ciInstance* caller_mirror = caller_klass->java_mirror();
3928        set_result(makecon(TypeInstPtr::make(caller_mirror)));
3929
3930#ifndef PRODUCT
3931        if ((PrintIntrinsics || PrintInlining || PrintOptoInlining) && Verbose) {
3932          tty->print_cr("  Succeeded: caller = %d) %s.%s, JVMS depth = %d", n, caller_klass->name()->as_utf8(), caller_jvms->method()->name()->as_utf8(), jvms()->depth());
3933          tty->print_cr("  JVM state at this point:");
3934          for (int i = jvms()->depth(), n = 1; i >= 1; i--, n++) {
3935            ciMethod* m = jvms()->of_depth(i)->method();
3936            tty->print_cr("   %d) %s.%s", n, m->holder()->name()->as_utf8(), m->name()->as_utf8());
3937          }
3938        }
3939#endif
3940        return true;
3941      }
3942      break;
3943    }
3944  }
3945
3946#ifndef PRODUCT
3947  if ((PrintIntrinsics || PrintInlining || PrintOptoInlining) && Verbose) {
3948    tty->print_cr("  Bailing out because caller depth exceeded inlining depth = %d", jvms()->depth());
3949    tty->print_cr("  JVM state at this point:");
3950    for (int i = jvms()->depth(), n = 1; i >= 1; i--, n++) {
3951      ciMethod* m = jvms()->of_depth(i)->method();
3952      tty->print_cr("   %d) %s.%s", n, m->holder()->name()->as_utf8(), m->name()->as_utf8());
3953    }
3954  }
3955#endif
3956
3957  return false;  // bail-out; let JVM_GetCallerClass do the work
3958}
3959
3960bool LibraryCallKit::inline_fp_conversions(vmIntrinsics::ID id) {
3961  Node* arg = argument(0);
3962  Node* result;
3963
3964  switch (id) {
3965  case vmIntrinsics::_floatToRawIntBits:    result = new (C) MoveF2INode(arg);  break;
3966  case vmIntrinsics::_intBitsToFloat:       result = new (C) MoveI2FNode(arg);  break;
3967  case vmIntrinsics::_doubleToRawLongBits:  result = new (C) MoveD2LNode(arg);  break;
3968  case vmIntrinsics::_longBitsToDouble:     result = new (C) MoveL2DNode(arg);  break;
3969
3970  case vmIntrinsics::_doubleToLongBits: {
3971    // two paths (plus control) merge in a wood
3972    RegionNode *r = new (C) RegionNode(3);
3973    Node *phi = new (C) PhiNode(r, TypeLong::LONG);
3974
3975    Node *cmpisnan = _gvn.transform(new (C) CmpDNode(arg, arg));
3976    // Build the boolean node
3977    Node *bolisnan = _gvn.transform(new (C) BoolNode(cmpisnan, BoolTest::ne));
3978
3979    // Branch either way.
3980    // NaN case is less traveled, which makes all the difference.
3981    IfNode *ifisnan = create_and_xform_if(control(), bolisnan, PROB_STATIC_FREQUENT, COUNT_UNKNOWN);
3982    Node *opt_isnan = _gvn.transform(ifisnan);
3983    assert( opt_isnan->is_If(), "Expect an IfNode");
3984    IfNode *opt_ifisnan = (IfNode*)opt_isnan;
3985    Node *iftrue = _gvn.transform( new (C) IfTrueNode(opt_ifisnan) );
3986
3987    set_control(iftrue);
3988
3989    static const jlong nan_bits = CONST64(0x7ff8000000000000);
3990    Node *slow_result = longcon(nan_bits); // return NaN
3991    phi->init_req(1, _gvn.transform( slow_result ));
3992    r->init_req(1, iftrue);
3993
3994    // Else fall through
3995    Node *iffalse = _gvn.transform(new (C) IfFalseNode(opt_ifisnan));
3996    set_control(iffalse);
3997
3998    phi->init_req(2, _gvn.transform(new (C) MoveD2LNode(arg)));
3999    r->init_req(2, iffalse);
4000
4001    // Post merge
4002    set_control(_gvn.transform(r));
4003    record_for_igvn(r);
4004
4005    C->set_has_split_ifs(true); // Has chance for split-if optimization
4006    result = phi;
4007    assert(result->bottom_type()->isa_long(), "must be");
4008    break;
4009  }
4010
4011  case vmIntrinsics::_floatToIntBits: {
4012    // two paths (plus control) merge in a wood
4013    RegionNode *r = new (C) RegionNode(3);
4014    Node *phi = new (C) PhiNode(r, TypeInt::INT);
4015
4016    Node *cmpisnan = _gvn.transform(new (C) CmpFNode(arg, arg));
4017    // Build the boolean node
4018    Node *bolisnan = _gvn.transform(new (C) BoolNode(cmpisnan, BoolTest::ne));
4019
4020    // Branch either way.
4021    // NaN case is less traveled, which makes all the difference.
4022    IfNode *ifisnan = create_and_xform_if(control(), bolisnan, PROB_STATIC_FREQUENT, COUNT_UNKNOWN);
4023    Node *opt_isnan = _gvn.transform(ifisnan);
4024    assert( opt_isnan->is_If(), "Expect an IfNode");
4025    IfNode *opt_ifisnan = (IfNode*)opt_isnan;
4026    Node *iftrue = _gvn.transform( new (C) IfTrueNode(opt_ifisnan) );
4027
4028    set_control(iftrue);
4029
4030    static const jint nan_bits = 0x7fc00000;
4031    Node *slow_result = makecon(TypeInt::make(nan_bits)); // return NaN
4032    phi->init_req(1, _gvn.transform( slow_result ));
4033    r->init_req(1, iftrue);
4034
4035    // Else fall through
4036    Node *iffalse = _gvn.transform(new (C) IfFalseNode(opt_ifisnan));
4037    set_control(iffalse);
4038
4039    phi->init_req(2, _gvn.transform(new (C) MoveF2INode(arg)));
4040    r->init_req(2, iffalse);
4041
4042    // Post merge
4043    set_control(_gvn.transform(r));
4044    record_for_igvn(r);
4045
4046    C->set_has_split_ifs(true); // Has chance for split-if optimization
4047    result = phi;
4048    assert(result->bottom_type()->isa_int(), "must be");
4049    break;
4050  }
4051
4052  default:
4053    fatal_unexpected_iid(id);
4054    break;
4055  }
4056  set_result(_gvn.transform(result));
4057  return true;
4058}
4059
4060#ifdef _LP64
4061#define XTOP ,top() /*additional argument*/
4062#else  //_LP64
4063#define XTOP        /*no additional argument*/
4064#endif //_LP64
4065
4066//----------------------inline_unsafe_copyMemory-------------------------
4067// public native void sun.misc.Unsafe.copyMemory(Object srcBase, long srcOffset, Object destBase, long destOffset, long bytes);
4068bool LibraryCallKit::inline_unsafe_copyMemory() {
4069  if (callee()->is_static())  return false;  // caller must have the capability!
4070  null_check_receiver();  // null-check receiver
4071  if (stopped())  return true;
4072
4073  C->set_has_unsafe_access(true);  // Mark eventual nmethod as "unsafe".
4074
4075  Node* src_ptr =         argument(1);   // type: oop
4076  Node* src_off = ConvL2X(argument(2));  // type: long
4077  Node* dst_ptr =         argument(4);   // type: oop
4078  Node* dst_off = ConvL2X(argument(5));  // type: long
4079  Node* size    = ConvL2X(argument(7));  // type: long
4080
4081  assert(Unsafe_field_offset_to_byte_offset(11) == 11,
4082         "fieldOffset must be byte-scaled");
4083
4084  Node* src = make_unsafe_address(src_ptr, src_off);
4085  Node* dst = make_unsafe_address(dst_ptr, dst_off);
4086
4087  // Conservatively insert a memory barrier on all memory slices.
4088  // Do not let writes of the copy source or destination float below the copy.
4089  insert_mem_bar(Op_MemBarCPUOrder);
4090
4091  // Call it.  Note that the length argument is not scaled.
4092  make_runtime_call(RC_LEAF|RC_NO_FP,
4093                    OptoRuntime::fast_arraycopy_Type(),
4094                    StubRoutines::unsafe_arraycopy(),
4095                    "unsafe_arraycopy",
4096                    TypeRawPtr::BOTTOM,
4097                    src, dst, size XTOP);
4098
4099  // Do not let reads of the copy destination float above the copy.
4100  insert_mem_bar(Op_MemBarCPUOrder);
4101
4102  return true;
4103}
4104
4105//------------------------clone_coping-----------------------------------
4106// Helper function for inline_native_clone.
4107void LibraryCallKit::copy_to_clone(Node* obj, Node* alloc_obj, Node* obj_size, bool is_array, bool card_mark) {
4108  assert(obj_size != NULL, "");
4109  Node* raw_obj = alloc_obj->in(1);
4110  assert(alloc_obj->is_CheckCastPP() && raw_obj->is_Proj() && raw_obj->in(0)->is_Allocate(), "");
4111
4112  AllocateNode* alloc = NULL;
4113  if (ReduceBulkZeroing) {
4114    // We will be completely responsible for initializing this object -
4115    // mark Initialize node as complete.
4116    alloc = AllocateNode::Ideal_allocation(alloc_obj, &_gvn);
4117    // The object was just allocated - there should be no any stores!
4118    guarantee(alloc != NULL && alloc->maybe_set_complete(&_gvn), "");
4119    // Mark as complete_with_arraycopy so that on AllocateNode
4120    // expansion, we know this AllocateNode is initialized by an array
4121    // copy and a StoreStore barrier exists after the array copy.
4122    alloc->initialization()->set_complete_with_arraycopy();
4123  }
4124
4125  // Copy the fastest available way.
4126  // TODO: generate fields copies for small objects instead.
4127  Node* src  = obj;
4128  Node* dest = alloc_obj;
4129  Node* size = _gvn.transform(obj_size);
4130
4131  // Exclude the header but include array length to copy by 8 bytes words.
4132  // Can't use base_offset_in_bytes(bt) since basic type is unknown.
4133  int base_off = is_array ? arrayOopDesc::length_offset_in_bytes() :
4134                            instanceOopDesc::base_offset_in_bytes();
4135  // base_off:
4136  // 8  - 32-bit VM
4137  // 12 - 64-bit VM, compressed klass
4138  // 16 - 64-bit VM, normal klass
4139  if (base_off % BytesPerLong != 0) {
4140    assert(UseCompressedKlassPointers, "");
4141    if (is_array) {
4142      // Exclude length to copy by 8 bytes words.
4143      base_off += sizeof(int);
4144    } else {
4145      // Include klass to copy by 8 bytes words.
4146      base_off = instanceOopDesc::klass_offset_in_bytes();
4147    }
4148    assert(base_off % BytesPerLong == 0, "expect 8 bytes alignment");
4149  }
4150  src  = basic_plus_adr(src,  base_off);
4151  dest = basic_plus_adr(dest, base_off);
4152
4153  // Compute the length also, if needed:
4154  Node* countx = size;
4155  countx = _gvn.transform( new (C) SubXNode(countx, MakeConX(base_off)) );
4156  countx = _gvn.transform( new (C) URShiftXNode(countx, intcon(LogBytesPerLong) ));
4157
4158  const TypePtr* raw_adr_type = TypeRawPtr::BOTTOM;
4159  bool disjoint_bases = true;
4160  generate_unchecked_arraycopy(raw_adr_type, T_LONG, disjoint_bases,
4161                               src, NULL, dest, NULL, countx,
4162                               /*dest_uninitialized*/true);
4163
4164  // If necessary, emit some card marks afterwards.  (Non-arrays only.)
4165  if (card_mark) {
4166    assert(!is_array, "");
4167    // Put in store barrier for any and all oops we are sticking
4168    // into this object.  (We could avoid this if we could prove
4169    // that the object type contains no oop fields at all.)
4170    Node* no_particular_value = NULL;
4171    Node* no_particular_field = NULL;
4172    int raw_adr_idx = Compile::AliasIdxRaw;
4173    post_barrier(control(),
4174                 memory(raw_adr_type),
4175                 alloc_obj,
4176                 no_particular_field,
4177                 raw_adr_idx,
4178                 no_particular_value,
4179                 T_OBJECT,
4180                 false);
4181  }
4182
4183  // Do not let reads from the cloned object float above the arraycopy.
4184  if (alloc != NULL) {
4185    // Do not let stores that initialize this object be reordered with
4186    // a subsequent store that would make this object accessible by
4187    // other threads.
4188    // Record what AllocateNode this StoreStore protects so that
4189    // escape analysis can go from the MemBarStoreStoreNode to the
4190    // AllocateNode and eliminate the MemBarStoreStoreNode if possible
4191    // based on the escape status of the AllocateNode.
4192    insert_mem_bar(Op_MemBarStoreStore, alloc->proj_out(AllocateNode::RawAddress));
4193  } else {
4194    insert_mem_bar(Op_MemBarCPUOrder);
4195  }
4196}
4197
4198//------------------------inline_native_clone----------------------------
4199// protected native Object java.lang.Object.clone();
4200//
4201// Here are the simple edge cases:
4202//  null receiver => normal trap
4203//  virtual and clone was overridden => slow path to out-of-line clone
4204//  not cloneable or finalizer => slow path to out-of-line Object.clone
4205//
4206// The general case has two steps, allocation and copying.
4207// Allocation has two cases, and uses GraphKit::new_instance or new_array.
4208//
4209// Copying also has two cases, oop arrays and everything else.
4210// Oop arrays use arrayof_oop_arraycopy (same as System.arraycopy).
4211// Everything else uses the tight inline loop supplied by CopyArrayNode.
4212//
4213// These steps fold up nicely if and when the cloned object's klass
4214// can be sharply typed as an object array, a type array, or an instance.
4215//
4216bool LibraryCallKit::inline_native_clone(bool is_virtual) {
4217  PhiNode* result_val;
4218
4219  // Set the reexecute bit for the interpreter to reexecute
4220  // the bytecode that invokes Object.clone if deoptimization happens.
4221  { PreserveReexecuteState preexecs(this);
4222    jvms()->set_should_reexecute(true);
4223
4224    Node* obj = null_check_receiver();
4225    if (stopped())  return true;
4226
4227    Node* obj_klass = load_object_klass(obj);
4228    const TypeKlassPtr* tklass = _gvn.type(obj_klass)->isa_klassptr();
4229    const TypeOopPtr*   toop   = ((tklass != NULL)
4230                                ? tklass->as_instance_type()
4231                                : TypeInstPtr::NOTNULL);
4232
4233    // Conservatively insert a memory barrier on all memory slices.
4234    // Do not let writes into the original float below the clone.
4235    insert_mem_bar(Op_MemBarCPUOrder);
4236
4237    // paths into result_reg:
4238    enum {
4239      _slow_path = 1,     // out-of-line call to clone method (virtual or not)
4240      _objArray_path,     // plain array allocation, plus arrayof_oop_arraycopy
4241      _array_path,        // plain array allocation, plus arrayof_long_arraycopy
4242      _instance_path,     // plain instance allocation, plus arrayof_long_arraycopy
4243      PATH_LIMIT
4244    };
4245    RegionNode* result_reg = new(C) RegionNode(PATH_LIMIT);
4246    result_val             = new(C) PhiNode(result_reg,
4247                                            TypeInstPtr::NOTNULL);
4248    PhiNode*    result_i_o = new(C) PhiNode(result_reg, Type::ABIO);
4249    PhiNode*    result_mem = new(C) PhiNode(result_reg, Type::MEMORY,
4250                                            TypePtr::BOTTOM);
4251    record_for_igvn(result_reg);
4252
4253    const TypePtr* raw_adr_type = TypeRawPtr::BOTTOM;
4254    int raw_adr_idx = Compile::AliasIdxRaw;
4255
4256    Node* array_ctl = generate_array_guard(obj_klass, (RegionNode*)NULL);
4257    if (array_ctl != NULL) {
4258      // It's an array.
4259      PreserveJVMState pjvms(this);
4260      set_control(array_ctl);
4261      Node* obj_length = load_array_length(obj);
4262      Node* obj_size  = NULL;
4263      Node* alloc_obj = new_array(obj_klass, obj_length, 0, &obj_size);  // no arguments to push
4264
4265      if (!use_ReduceInitialCardMarks()) {
4266        // If it is an oop array, it requires very special treatment,
4267        // because card marking is required on each card of the array.
4268        Node* is_obja = generate_objArray_guard(obj_klass, (RegionNode*)NULL);
4269        if (is_obja != NULL) {
4270          PreserveJVMState pjvms2(this);
4271          set_control(is_obja);
4272          // Generate a direct call to the right arraycopy function(s).
4273          bool disjoint_bases = true;
4274          bool length_never_negative = true;
4275          generate_arraycopy(TypeAryPtr::OOPS, T_OBJECT,
4276                             obj, intcon(0), alloc_obj, intcon(0),
4277                             obj_length,
4278                             disjoint_bases, length_never_negative);
4279          result_reg->init_req(_objArray_path, control());
4280          result_val->init_req(_objArray_path, alloc_obj);
4281          result_i_o ->set_req(_objArray_path, i_o());
4282          result_mem ->set_req(_objArray_path, reset_memory());
4283        }
4284      }
4285      // Otherwise, there are no card marks to worry about.
4286      // (We can dispense with card marks if we know the allocation
4287      //  comes out of eden (TLAB)...  In fact, ReduceInitialCardMarks
4288      //  causes the non-eden paths to take compensating steps to
4289      //  simulate a fresh allocation, so that no further
4290      //  card marks are required in compiled code to initialize
4291      //  the object.)
4292
4293      if (!stopped()) {
4294        copy_to_clone(obj, alloc_obj, obj_size, true, false);
4295
4296        // Present the results of the copy.
4297        result_reg->init_req(_array_path, control());
4298        result_val->init_req(_array_path, alloc_obj);
4299        result_i_o ->set_req(_array_path, i_o());
4300        result_mem ->set_req(_array_path, reset_memory());
4301      }
4302    }
4303
4304    // We only go to the instance fast case code if we pass a number of guards.
4305    // The paths which do not pass are accumulated in the slow_region.
4306    RegionNode* slow_region = new (C) RegionNode(1);
4307    record_for_igvn(slow_region);
4308    if (!stopped()) {
4309      // It's an instance (we did array above).  Make the slow-path tests.
4310      // If this is a virtual call, we generate a funny guard.  We grab
4311      // the vtable entry corresponding to clone() from the target object.
4312      // If the target method which we are calling happens to be the
4313      // Object clone() method, we pass the guard.  We do not need this
4314      // guard for non-virtual calls; the caller is known to be the native
4315      // Object clone().
4316      if (is_virtual) {
4317        generate_virtual_guard(obj_klass, slow_region);
4318      }
4319
4320      // The object must be cloneable and must not have a finalizer.
4321      // Both of these conditions may be checked in a single test.
4322      // We could optimize the cloneable test further, but we don't care.
4323      generate_access_flags_guard(obj_klass,
4324                                  // Test both conditions:
4325                                  JVM_ACC_IS_CLONEABLE | JVM_ACC_HAS_FINALIZER,
4326                                  // Must be cloneable but not finalizer:
4327                                  JVM_ACC_IS_CLONEABLE,
4328                                  slow_region);
4329    }
4330
4331    if (!stopped()) {
4332      // It's an instance, and it passed the slow-path tests.
4333      PreserveJVMState pjvms(this);
4334      Node* obj_size  = NULL;
4335      Node* alloc_obj = new_instance(obj_klass, NULL, &obj_size);
4336
4337      copy_to_clone(obj, alloc_obj, obj_size, false, !use_ReduceInitialCardMarks());
4338
4339      // Present the results of the slow call.
4340      result_reg->init_req(_instance_path, control());
4341      result_val->init_req(_instance_path, alloc_obj);
4342      result_i_o ->set_req(_instance_path, i_o());
4343      result_mem ->set_req(_instance_path, reset_memory());
4344    }
4345
4346    // Generate code for the slow case.  We make a call to clone().
4347    set_control(_gvn.transform(slow_region));
4348    if (!stopped()) {
4349      PreserveJVMState pjvms(this);
4350      CallJavaNode* slow_call = generate_method_call(vmIntrinsics::_clone, is_virtual);
4351      Node* slow_result = set_results_for_java_call(slow_call);
4352      // this->control() comes from set_results_for_java_call
4353      result_reg->init_req(_slow_path, control());
4354      result_val->init_req(_slow_path, slow_result);
4355      result_i_o ->set_req(_slow_path, i_o());
4356      result_mem ->set_req(_slow_path, reset_memory());
4357    }
4358
4359    // Return the combined state.
4360    set_control(    _gvn.transform(result_reg) );
4361    set_i_o(        _gvn.transform(result_i_o) );
4362    set_all_memory( _gvn.transform(result_mem) );
4363  } // original reexecute is set back here
4364
4365  set_result(_gvn.transform(result_val));
4366  return true;
4367}
4368
4369//------------------------------basictype2arraycopy----------------------------
4370address LibraryCallKit::basictype2arraycopy(BasicType t,
4371                                            Node* src_offset,
4372                                            Node* dest_offset,
4373                                            bool disjoint_bases,
4374                                            const char* &name,
4375                                            bool dest_uninitialized) {
4376  const TypeInt* src_offset_inttype  = gvn().find_int_type(src_offset);;
4377  const TypeInt* dest_offset_inttype = gvn().find_int_type(dest_offset);;
4378
4379  bool aligned = false;
4380  bool disjoint = disjoint_bases;
4381
4382  // if the offsets are the same, we can treat the memory regions as
4383  // disjoint, because either the memory regions are in different arrays,
4384  // or they are identical (which we can treat as disjoint.)  We can also
4385  // treat a copy with a destination index  less that the source index
4386  // as disjoint since a low->high copy will work correctly in this case.
4387  if (src_offset_inttype != NULL && src_offset_inttype->is_con() &&
4388      dest_offset_inttype != NULL && dest_offset_inttype->is_con()) {
4389    // both indices are constants
4390    int s_offs = src_offset_inttype->get_con();
4391    int d_offs = dest_offset_inttype->get_con();
4392    int element_size = type2aelembytes(t);
4393    aligned = ((arrayOopDesc::base_offset_in_bytes(t) + s_offs * element_size) % HeapWordSize == 0) &&
4394              ((arrayOopDesc::base_offset_in_bytes(t) + d_offs * element_size) % HeapWordSize == 0);
4395    if (s_offs >= d_offs)  disjoint = true;
4396  } else if (src_offset == dest_offset && src_offset != NULL) {
4397    // This can occur if the offsets are identical non-constants.
4398    disjoint = true;
4399  }
4400
4401  return StubRoutines::select_arraycopy_function(t, aligned, disjoint, name, dest_uninitialized);
4402}
4403
4404
4405//------------------------------inline_arraycopy-----------------------
4406// public static native void java.lang.System.arraycopy(Object src,  int  srcPos,
4407//                                                      Object dest, int destPos,
4408//                                                      int length);
4409bool LibraryCallKit::inline_arraycopy() {
4410  // Get the arguments.
4411  Node* src         = argument(0);  // type: oop
4412  Node* src_offset  = argument(1);  // type: int
4413  Node* dest        = argument(2);  // type: oop
4414  Node* dest_offset = argument(3);  // type: int
4415  Node* length      = argument(4);  // type: int
4416
4417  // Compile time checks.  If any of these checks cannot be verified at compile time,
4418  // we do not make a fast path for this call.  Instead, we let the call remain as it
4419  // is.  The checks we choose to mandate at compile time are:
4420  //
4421  // (1) src and dest are arrays.
4422  const Type* src_type  = src->Value(&_gvn);
4423  const Type* dest_type = dest->Value(&_gvn);
4424  const TypeAryPtr* top_src  = src_type->isa_aryptr();
4425  const TypeAryPtr* top_dest = dest_type->isa_aryptr();
4426  if (top_src  == NULL || top_src->klass()  == NULL ||
4427      top_dest == NULL || top_dest->klass() == NULL) {
4428    // Conservatively insert a memory barrier on all memory slices.
4429    // Do not let writes into the source float below the arraycopy.
4430    insert_mem_bar(Op_MemBarCPUOrder);
4431
4432    // Call StubRoutines::generic_arraycopy stub.
4433    generate_arraycopy(TypeRawPtr::BOTTOM, T_CONFLICT,
4434                       src, src_offset, dest, dest_offset, length);
4435
4436    // Do not let reads from the destination float above the arraycopy.
4437    // Since we cannot type the arrays, we don't know which slices
4438    // might be affected.  We could restrict this barrier only to those
4439    // memory slices which pertain to array elements--but don't bother.
4440    if (!InsertMemBarAfterArraycopy)
4441      // (If InsertMemBarAfterArraycopy, there is already one in place.)
4442      insert_mem_bar(Op_MemBarCPUOrder);
4443    return true;
4444  }
4445
4446  // (2) src and dest arrays must have elements of the same BasicType
4447  // Figure out the size and type of the elements we will be copying.
4448  BasicType src_elem  =  top_src->klass()->as_array_klass()->element_type()->basic_type();
4449  BasicType dest_elem = top_dest->klass()->as_array_klass()->element_type()->basic_type();
4450  if (src_elem  == T_ARRAY)  src_elem  = T_OBJECT;
4451  if (dest_elem == T_ARRAY)  dest_elem = T_OBJECT;
4452
4453  if (src_elem != dest_elem || dest_elem == T_VOID) {
4454    // The component types are not the same or are not recognized.  Punt.
4455    // (But, avoid the native method wrapper to JVM_ArrayCopy.)
4456    generate_slow_arraycopy(TypePtr::BOTTOM,
4457                            src, src_offset, dest, dest_offset, length,
4458                            /*dest_uninitialized*/false);
4459    return true;
4460  }
4461
4462  //---------------------------------------------------------------------------
4463  // We will make a fast path for this call to arraycopy.
4464
4465  // We have the following tests left to perform:
4466  //
4467  // (3) src and dest must not be null.
4468  // (4) src_offset must not be negative.
4469  // (5) dest_offset must not be negative.
4470  // (6) length must not be negative.
4471  // (7) src_offset + length must not exceed length of src.
4472  // (8) dest_offset + length must not exceed length of dest.
4473  // (9) each element of an oop array must be assignable
4474
4475  RegionNode* slow_region = new (C) RegionNode(1);
4476  record_for_igvn(slow_region);
4477
4478  // (3) operands must not be null
4479  // We currently perform our null checks with the null_check routine.
4480  // This means that the null exceptions will be reported in the caller
4481  // rather than (correctly) reported inside of the native arraycopy call.
4482  // This should be corrected, given time.  We do our null check with the
4483  // stack pointer restored.
4484  src  = null_check(src,  T_ARRAY);
4485  dest = null_check(dest, T_ARRAY);
4486
4487  // (4) src_offset must not be negative.
4488  generate_negative_guard(src_offset, slow_region);
4489
4490  // (5) dest_offset must not be negative.
4491  generate_negative_guard(dest_offset, slow_region);
4492
4493  // (6) length must not be negative (moved to generate_arraycopy()).
4494  // generate_negative_guard(length, slow_region);
4495
4496  // (7) src_offset + length must not exceed length of src.
4497  generate_limit_guard(src_offset, length,
4498                       load_array_length(src),
4499                       slow_region);
4500
4501  // (8) dest_offset + length must not exceed length of dest.
4502  generate_limit_guard(dest_offset, length,
4503                       load_array_length(dest),
4504                       slow_region);
4505
4506  // (9) each element of an oop array must be assignable
4507  // The generate_arraycopy subroutine checks this.
4508
4509  // This is where the memory effects are placed:
4510  const TypePtr* adr_type = TypeAryPtr::get_array_body_type(dest_elem);
4511  generate_arraycopy(adr_type, dest_elem,
4512                     src, src_offset, dest, dest_offset, length,
4513                     false, false, slow_region);
4514
4515  return true;
4516}
4517
4518//-----------------------------generate_arraycopy----------------------
4519// Generate an optimized call to arraycopy.
4520// Caller must guard against non-arrays.
4521// Caller must determine a common array basic-type for both arrays.
4522// Caller must validate offsets against array bounds.
4523// The slow_region has already collected guard failure paths
4524// (such as out of bounds length or non-conformable array types).
4525// The generated code has this shape, in general:
4526//
4527//     if (length == 0)  return   // via zero_path
4528//     slowval = -1
4529//     if (types unknown) {
4530//       slowval = call generic copy loop
4531//       if (slowval == 0)  return  // via checked_path
4532//     } else if (indexes in bounds) {
4533//       if ((is object array) && !(array type check)) {
4534//         slowval = call checked copy loop
4535//         if (slowval == 0)  return  // via checked_path
4536//       } else {
4537//         call bulk copy loop
4538//         return  // via fast_path
4539//       }
4540//     }
4541//     // adjust params for remaining work:
4542//     if (slowval != -1) {
4543//       n = -1^slowval; src_offset += n; dest_offset += n; length -= n
4544//     }
4545//   slow_region:
4546//     call slow arraycopy(src, src_offset, dest, dest_offset, length)
4547//     return  // via slow_call_path
4548//
4549// This routine is used from several intrinsics:  System.arraycopy,
4550// Object.clone (the array subcase), and Arrays.copyOf[Range].
4551//
4552void
4553LibraryCallKit::generate_arraycopy(const TypePtr* adr_type,
4554                                   BasicType basic_elem_type,
4555                                   Node* src,  Node* src_offset,
4556                                   Node* dest, Node* dest_offset,
4557                                   Node* copy_length,
4558                                   bool disjoint_bases,
4559                                   bool length_never_negative,
4560                                   RegionNode* slow_region) {
4561
4562  if (slow_region == NULL) {
4563    slow_region = new(C) RegionNode(1);
4564    record_for_igvn(slow_region);
4565  }
4566
4567  Node* original_dest      = dest;
4568  AllocateArrayNode* alloc = NULL;  // used for zeroing, if needed
4569  bool  dest_uninitialized = false;
4570
4571  // See if this is the initialization of a newly-allocated array.
4572  // If so, we will take responsibility here for initializing it to zero.
4573  // (Note:  Because tightly_coupled_allocation performs checks on the
4574  // out-edges of the dest, we need to avoid making derived pointers
4575  // from it until we have checked its uses.)
4576  if (ReduceBulkZeroing
4577      && !ZeroTLAB              // pointless if already zeroed
4578      && basic_elem_type != T_CONFLICT // avoid corner case
4579      && !src->eqv_uncast(dest)
4580      && ((alloc = tightly_coupled_allocation(dest, slow_region))
4581          != NULL)
4582      && _gvn.find_int_con(alloc->in(AllocateNode::ALength), 1) > 0
4583      && alloc->maybe_set_complete(&_gvn)) {
4584    // "You break it, you buy it."
4585    InitializeNode* init = alloc->initialization();
4586    assert(init->is_complete(), "we just did this");
4587    init->set_complete_with_arraycopy();
4588    assert(dest->is_CheckCastPP(), "sanity");
4589    assert(dest->in(0)->in(0) == init, "dest pinned");
4590    adr_type = TypeRawPtr::BOTTOM;  // all initializations are into raw memory
4591    // From this point on, every exit path is responsible for
4592    // initializing any non-copied parts of the object to zero.
4593    // Also, if this flag is set we make sure that arraycopy interacts properly
4594    // with G1, eliding pre-barriers. See CR 6627983.
4595    dest_uninitialized = true;
4596  } else {
4597    // No zeroing elimination here.
4598    alloc             = NULL;
4599    //original_dest   = dest;
4600    //dest_uninitialized = false;
4601  }
4602
4603  // Results are placed here:
4604  enum { fast_path        = 1,  // normal void-returning assembly stub
4605         checked_path     = 2,  // special assembly stub with cleanup
4606         slow_call_path   = 3,  // something went wrong; call the VM
4607         zero_path        = 4,  // bypass when length of copy is zero
4608         bcopy_path       = 5,  // copy primitive array by 64-bit blocks
4609         PATH_LIMIT       = 6
4610  };
4611  RegionNode* result_region = new(C) RegionNode(PATH_LIMIT);
4612  PhiNode*    result_i_o    = new(C) PhiNode(result_region, Type::ABIO);
4613  PhiNode*    result_memory = new(C) PhiNode(result_region, Type::MEMORY, adr_type);
4614  record_for_igvn(result_region);
4615  _gvn.set_type_bottom(result_i_o);
4616  _gvn.set_type_bottom(result_memory);
4617  assert(adr_type != TypePtr::BOTTOM, "must be RawMem or a T[] slice");
4618
4619  // The slow_control path:
4620  Node* slow_control;
4621  Node* slow_i_o = i_o();
4622  Node* slow_mem = memory(adr_type);
4623  debug_only(slow_control = (Node*) badAddress);
4624
4625  // Checked control path:
4626  Node* checked_control = top();
4627  Node* checked_mem     = NULL;
4628  Node* checked_i_o     = NULL;
4629  Node* checked_value   = NULL;
4630
4631  if (basic_elem_type == T_CONFLICT) {
4632    assert(!dest_uninitialized, "");
4633    Node* cv = generate_generic_arraycopy(adr_type,
4634                                          src, src_offset, dest, dest_offset,
4635                                          copy_length, dest_uninitialized);
4636    if (cv == NULL)  cv = intcon(-1);  // failure (no stub available)
4637    checked_control = control();
4638    checked_i_o     = i_o();
4639    checked_mem     = memory(adr_type);
4640    checked_value   = cv;
4641    set_control(top());         // no fast path
4642  }
4643
4644  Node* not_pos = generate_nonpositive_guard(copy_length, length_never_negative);
4645  if (not_pos != NULL) {
4646    PreserveJVMState pjvms(this);
4647    set_control(not_pos);
4648
4649    // (6) length must not be negative.
4650    if (!length_never_negative) {
4651      generate_negative_guard(copy_length, slow_region);
4652    }
4653
4654    // copy_length is 0.
4655    if (!stopped() && dest_uninitialized) {
4656      Node* dest_length = alloc->in(AllocateNode::ALength);
4657      if (copy_length->eqv_uncast(dest_length)
4658          || _gvn.find_int_con(dest_length, 1) <= 0) {
4659        // There is no zeroing to do. No need for a secondary raw memory barrier.
4660      } else {
4661        // Clear the whole thing since there are no source elements to copy.
4662        generate_clear_array(adr_type, dest, basic_elem_type,
4663                             intcon(0), NULL,
4664                             alloc->in(AllocateNode::AllocSize));
4665        // Use a secondary InitializeNode as raw memory barrier.
4666        // Currently it is needed only on this path since other
4667        // paths have stub or runtime calls as raw memory barriers.
4668        InitializeNode* init = insert_mem_bar_volatile(Op_Initialize,
4669                                                       Compile::AliasIdxRaw,
4670                                                       top())->as_Initialize();
4671        init->set_complete(&_gvn);  // (there is no corresponding AllocateNode)
4672      }
4673    }
4674
4675    // Present the results of the fast call.
4676    result_region->init_req(zero_path, control());
4677    result_i_o   ->init_req(zero_path, i_o());
4678    result_memory->init_req(zero_path, memory(adr_type));
4679  }
4680
4681  if (!stopped() && dest_uninitialized) {
4682    // We have to initialize the *uncopied* part of the array to zero.
4683    // The copy destination is the slice dest[off..off+len].  The other slices
4684    // are dest_head = dest[0..off] and dest_tail = dest[off+len..dest.length].
4685    Node* dest_size   = alloc->in(AllocateNode::AllocSize);
4686    Node* dest_length = alloc->in(AllocateNode::ALength);
4687    Node* dest_tail   = _gvn.transform( new(C) AddINode(dest_offset,
4688                                                          copy_length) );
4689
4690    // If there is a head section that needs zeroing, do it now.
4691    if (find_int_con(dest_offset, -1) != 0) {
4692      generate_clear_array(adr_type, dest, basic_elem_type,
4693                           intcon(0), dest_offset,
4694                           NULL);
4695    }
4696
4697    // Next, perform a dynamic check on the tail length.
4698    // It is often zero, and we can win big if we prove this.
4699    // There are two wins:  Avoid generating the ClearArray
4700    // with its attendant messy index arithmetic, and upgrade
4701    // the copy to a more hardware-friendly word size of 64 bits.
4702    Node* tail_ctl = NULL;
4703    if (!stopped() && !dest_tail->eqv_uncast(dest_length)) {
4704      Node* cmp_lt   = _gvn.transform( new(C) CmpINode(dest_tail, dest_length) );
4705      Node* bol_lt   = _gvn.transform( new(C) BoolNode(cmp_lt, BoolTest::lt) );
4706      tail_ctl = generate_slow_guard(bol_lt, NULL);
4707      assert(tail_ctl != NULL || !stopped(), "must be an outcome");
4708    }
4709
4710    // At this point, let's assume there is no tail.
4711    if (!stopped() && alloc != NULL && basic_elem_type != T_OBJECT) {
4712      // There is no tail.  Try an upgrade to a 64-bit copy.
4713      bool didit = false;
4714      { PreserveJVMState pjvms(this);
4715        didit = generate_block_arraycopy(adr_type, basic_elem_type, alloc,
4716                                         src, src_offset, dest, dest_offset,
4717                                         dest_size, dest_uninitialized);
4718        if (didit) {
4719          // Present the results of the block-copying fast call.
4720          result_region->init_req(bcopy_path, control());
4721          result_i_o   ->init_req(bcopy_path, i_o());
4722          result_memory->init_req(bcopy_path, memory(adr_type));
4723        }
4724      }
4725      if (didit)
4726        set_control(top());     // no regular fast path
4727    }
4728
4729    // Clear the tail, if any.
4730    if (tail_ctl != NULL) {
4731      Node* notail_ctl = stopped() ? NULL : control();
4732      set_control(tail_ctl);
4733      if (notail_ctl == NULL) {
4734        generate_clear_array(adr_type, dest, basic_elem_type,
4735                             dest_tail, NULL,
4736                             dest_size);
4737      } else {
4738        // Make a local merge.
4739        Node* done_ctl = new(C) RegionNode(3);
4740        Node* done_mem = new(C) PhiNode(done_ctl, Type::MEMORY, adr_type);
4741        done_ctl->init_req(1, notail_ctl);
4742        done_mem->init_req(1, memory(adr_type));
4743        generate_clear_array(adr_type, dest, basic_elem_type,
4744                             dest_tail, NULL,
4745                             dest_size);
4746        done_ctl->init_req(2, control());
4747        done_mem->init_req(2, memory(adr_type));
4748        set_control( _gvn.transform(done_ctl) );
4749        set_memory(  _gvn.transform(done_mem), adr_type );
4750      }
4751    }
4752  }
4753
4754  BasicType copy_type = basic_elem_type;
4755  assert(basic_elem_type != T_ARRAY, "caller must fix this");
4756  if (!stopped() && copy_type == T_OBJECT) {
4757    // If src and dest have compatible element types, we can copy bits.
4758    // Types S[] and D[] are compatible if D is a supertype of S.
4759    //
4760    // If they are not, we will use checked_oop_disjoint_arraycopy,
4761    // which performs a fast optimistic per-oop check, and backs off
4762    // further to JVM_ArrayCopy on the first per-oop check that fails.
4763    // (Actually, we don't move raw bits only; the GC requires card marks.)
4764
4765    // Get the Klass* for both src and dest
4766    Node* src_klass  = load_object_klass(src);
4767    Node* dest_klass = load_object_klass(dest);
4768
4769    // Generate the subtype check.
4770    // This might fold up statically, or then again it might not.
4771    //
4772    // Non-static example:  Copying List<String>.elements to a new String[].
4773    // The backing store for a List<String> is always an Object[],
4774    // but its elements are always type String, if the generic types
4775    // are correct at the source level.
4776    //
4777    // Test S[] against D[], not S against D, because (probably)
4778    // the secondary supertype cache is less busy for S[] than S.
4779    // This usually only matters when D is an interface.
4780    Node* not_subtype_ctrl = gen_subtype_check(src_klass, dest_klass);
4781    // Plug failing path into checked_oop_disjoint_arraycopy
4782    if (not_subtype_ctrl != top()) {
4783      PreserveJVMState pjvms(this);
4784      set_control(not_subtype_ctrl);
4785      // (At this point we can assume disjoint_bases, since types differ.)
4786      int ek_offset = in_bytes(ObjArrayKlass::element_klass_offset());
4787      Node* p1 = basic_plus_adr(dest_klass, ek_offset);
4788      Node* n1 = LoadKlassNode::make(_gvn, immutable_memory(), p1, TypeRawPtr::BOTTOM);
4789      Node* dest_elem_klass = _gvn.transform(n1);
4790      Node* cv = generate_checkcast_arraycopy(adr_type,
4791                                              dest_elem_klass,
4792                                              src, src_offset, dest, dest_offset,
4793                                              ConvI2X(copy_length), dest_uninitialized);
4794      if (cv == NULL)  cv = intcon(-1);  // failure (no stub available)
4795      checked_control = control();
4796      checked_i_o     = i_o();
4797      checked_mem     = memory(adr_type);
4798      checked_value   = cv;
4799    }
4800    // At this point we know we do not need type checks on oop stores.
4801
4802    // Let's see if we need card marks:
4803    if (alloc != NULL && use_ReduceInitialCardMarks()) {
4804      // If we do not need card marks, copy using the jint or jlong stub.
4805      copy_type = LP64_ONLY(UseCompressedOops ? T_INT : T_LONG) NOT_LP64(T_INT);
4806      assert(type2aelembytes(basic_elem_type) == type2aelembytes(copy_type),
4807             "sizes agree");
4808    }
4809  }
4810
4811  if (!stopped()) {
4812    // Generate the fast path, if possible.
4813    PreserveJVMState pjvms(this);
4814    generate_unchecked_arraycopy(adr_type, copy_type, disjoint_bases,
4815                                 src, src_offset, dest, dest_offset,
4816                                 ConvI2X(copy_length), dest_uninitialized);
4817
4818    // Present the results of the fast call.
4819    result_region->init_req(fast_path, control());
4820    result_i_o   ->init_req(fast_path, i_o());
4821    result_memory->init_req(fast_path, memory(adr_type));
4822  }
4823
4824  // Here are all the slow paths up to this point, in one bundle:
4825  slow_control = top();
4826  if (slow_region != NULL)
4827    slow_control = _gvn.transform(slow_region);
4828  DEBUG_ONLY(slow_region = (RegionNode*)badAddress);
4829
4830  set_control(checked_control);
4831  if (!stopped()) {
4832    // Clean up after the checked call.
4833    // The returned value is either 0 or -1^K,
4834    // where K = number of partially transferred array elements.
4835    Node* cmp = _gvn.transform( new(C) CmpINode(checked_value, intcon(0)) );
4836    Node* bol = _gvn.transform( new(C) BoolNode(cmp, BoolTest::eq) );
4837    IfNode* iff = create_and_map_if(control(), bol, PROB_MAX, COUNT_UNKNOWN);
4838
4839    // If it is 0, we are done, so transfer to the end.
4840    Node* checks_done = _gvn.transform( new(C) IfTrueNode(iff) );
4841    result_region->init_req(checked_path, checks_done);
4842    result_i_o   ->init_req(checked_path, checked_i_o);
4843    result_memory->init_req(checked_path, checked_mem);
4844
4845    // If it is not zero, merge into the slow call.
4846    set_control( _gvn.transform( new(C) IfFalseNode(iff) ));
4847    RegionNode* slow_reg2 = new(C) RegionNode(3);
4848    PhiNode*    slow_i_o2 = new(C) PhiNode(slow_reg2, Type::ABIO);
4849    PhiNode*    slow_mem2 = new(C) PhiNode(slow_reg2, Type::MEMORY, adr_type);
4850    record_for_igvn(slow_reg2);
4851    slow_reg2  ->init_req(1, slow_control);
4852    slow_i_o2  ->init_req(1, slow_i_o);
4853    slow_mem2  ->init_req(1, slow_mem);
4854    slow_reg2  ->init_req(2, control());
4855    slow_i_o2  ->init_req(2, checked_i_o);
4856    slow_mem2  ->init_req(2, checked_mem);
4857
4858    slow_control = _gvn.transform(slow_reg2);
4859    slow_i_o     = _gvn.transform(slow_i_o2);
4860    slow_mem     = _gvn.transform(slow_mem2);
4861
4862    if (alloc != NULL) {
4863      // We'll restart from the very beginning, after zeroing the whole thing.
4864      // This can cause double writes, but that's OK since dest is brand new.
4865      // So we ignore the low 31 bits of the value returned from the stub.
4866    } else {
4867      // We must continue the copy exactly where it failed, or else
4868      // another thread might see the wrong number of writes to dest.
4869      Node* checked_offset = _gvn.transform( new(C) XorINode(checked_value, intcon(-1)) );
4870      Node* slow_offset    = new(C) PhiNode(slow_reg2, TypeInt::INT);
4871      slow_offset->init_req(1, intcon(0));
4872      slow_offset->init_req(2, checked_offset);
4873      slow_offset  = _gvn.transform(slow_offset);
4874
4875      // Adjust the arguments by the conditionally incoming offset.
4876      Node* src_off_plus  = _gvn.transform( new(C) AddINode(src_offset,  slow_offset) );
4877      Node* dest_off_plus = _gvn.transform( new(C) AddINode(dest_offset, slow_offset) );
4878      Node* length_minus  = _gvn.transform( new(C) SubINode(copy_length, slow_offset) );
4879
4880      // Tweak the node variables to adjust the code produced below:
4881      src_offset  = src_off_plus;
4882      dest_offset = dest_off_plus;
4883      copy_length = length_minus;
4884    }
4885  }
4886
4887  set_control(slow_control);
4888  if (!stopped()) {
4889    // Generate the slow path, if needed.
4890    PreserveJVMState pjvms(this);   // replace_in_map may trash the map
4891
4892    set_memory(slow_mem, adr_type);
4893    set_i_o(slow_i_o);
4894
4895    if (dest_uninitialized) {
4896      generate_clear_array(adr_type, dest, basic_elem_type,
4897                           intcon(0), NULL,
4898                           alloc->in(AllocateNode::AllocSize));
4899    }
4900
4901    generate_slow_arraycopy(adr_type,
4902                            src, src_offset, dest, dest_offset,
4903                            copy_length, /*dest_uninitialized*/false);
4904
4905    result_region->init_req(slow_call_path, control());
4906    result_i_o   ->init_req(slow_call_path, i_o());
4907    result_memory->init_req(slow_call_path, memory(adr_type));
4908  }
4909
4910  // Remove unused edges.
4911  for (uint i = 1; i < result_region->req(); i++) {
4912    if (result_region->in(i) == NULL)
4913      result_region->init_req(i, top());
4914  }
4915
4916  // Finished; return the combined state.
4917  set_control( _gvn.transform(result_region) );
4918  set_i_o(     _gvn.transform(result_i_o)    );
4919  set_memory(  _gvn.transform(result_memory), adr_type );
4920
4921  // The memory edges above are precise in order to model effects around
4922  // array copies accurately to allow value numbering of field loads around
4923  // arraycopy.  Such field loads, both before and after, are common in Java
4924  // collections and similar classes involving header/array data structures.
4925  //
4926  // But with low number of register or when some registers are used or killed
4927  // by arraycopy calls it causes registers spilling on stack. See 6544710.
4928  // The next memory barrier is added to avoid it. If the arraycopy can be
4929  // optimized away (which it can, sometimes) then we can manually remove
4930  // the membar also.
4931  //
4932  // Do not let reads from the cloned object float above the arraycopy.
4933  if (alloc != NULL) {
4934    // Do not let stores that initialize this object be reordered with
4935    // a subsequent store that would make this object accessible by
4936    // other threads.
4937    // Record what AllocateNode this StoreStore protects so that
4938    // escape analysis can go from the MemBarStoreStoreNode to the
4939    // AllocateNode and eliminate the MemBarStoreStoreNode if possible
4940    // based on the escape status of the AllocateNode.
4941    insert_mem_bar(Op_MemBarStoreStore, alloc->proj_out(AllocateNode::RawAddress));
4942  } else if (InsertMemBarAfterArraycopy)
4943    insert_mem_bar(Op_MemBarCPUOrder);
4944}
4945
4946
4947// Helper function which determines if an arraycopy immediately follows
4948// an allocation, with no intervening tests or other escapes for the object.
4949AllocateArrayNode*
4950LibraryCallKit::tightly_coupled_allocation(Node* ptr,
4951                                           RegionNode* slow_region) {
4952  if (stopped())             return NULL;  // no fast path
4953  if (C->AliasLevel() == 0)  return NULL;  // no MergeMems around
4954
4955  AllocateArrayNode* alloc = AllocateArrayNode::Ideal_array_allocation(ptr, &_gvn);
4956  if (alloc == NULL)  return NULL;
4957
4958  Node* rawmem = memory(Compile::AliasIdxRaw);
4959  // Is the allocation's memory state untouched?
4960  if (!(rawmem->is_Proj() && rawmem->in(0)->is_Initialize())) {
4961    // Bail out if there have been raw-memory effects since the allocation.
4962    // (Example:  There might have been a call or safepoint.)
4963    return NULL;
4964  }
4965  rawmem = rawmem->in(0)->as_Initialize()->memory(Compile::AliasIdxRaw);
4966  if (!(rawmem->is_Proj() && rawmem->in(0) == alloc)) {
4967    return NULL;
4968  }
4969
4970  // There must be no unexpected observers of this allocation.
4971  for (DUIterator_Fast imax, i = ptr->fast_outs(imax); i < imax; i++) {
4972    Node* obs = ptr->fast_out(i);
4973    if (obs != this->map()) {
4974      return NULL;
4975    }
4976  }
4977
4978  // This arraycopy must unconditionally follow the allocation of the ptr.
4979  Node* alloc_ctl = ptr->in(0);
4980  assert(just_allocated_object(alloc_ctl) == ptr, "most recent allo");
4981
4982  Node* ctl = control();
4983  while (ctl != alloc_ctl) {
4984    // There may be guards which feed into the slow_region.
4985    // Any other control flow means that we might not get a chance
4986    // to finish initializing the allocated object.
4987    if ((ctl->is_IfFalse() || ctl->is_IfTrue()) && ctl->in(0)->is_If()) {
4988      IfNode* iff = ctl->in(0)->as_If();
4989      Node* not_ctl = iff->proj_out(1 - ctl->as_Proj()->_con);
4990      assert(not_ctl != NULL && not_ctl != ctl, "found alternate");
4991      if (slow_region != NULL && slow_region->find_edge(not_ctl) >= 1) {
4992        ctl = iff->in(0);       // This test feeds the known slow_region.
4993        continue;
4994      }
4995      // One more try:  Various low-level checks bottom out in
4996      // uncommon traps.  If the debug-info of the trap omits
4997      // any reference to the allocation, as we've already
4998      // observed, then there can be no objection to the trap.
4999      bool found_trap = false;
5000      for (DUIterator_Fast jmax, j = not_ctl->fast_outs(jmax); j < jmax; j++) {
5001        Node* obs = not_ctl->fast_out(j);
5002        if (obs->in(0) == not_ctl && obs->is_Call() &&
5003            (obs->as_Call()->entry_point() == SharedRuntime::uncommon_trap_blob()->entry_point())) {
5004          found_trap = true; break;
5005        }
5006      }
5007      if (found_trap) {
5008        ctl = iff->in(0);       // This test feeds a harmless uncommon trap.
5009        continue;
5010      }
5011    }
5012    return NULL;
5013  }
5014
5015  // If we get this far, we have an allocation which immediately
5016  // precedes the arraycopy, and we can take over zeroing the new object.
5017  // The arraycopy will finish the initialization, and provide
5018  // a new control state to which we will anchor the destination pointer.
5019
5020  return alloc;
5021}
5022
5023// Helper for initialization of arrays, creating a ClearArray.
5024// It writes zero bits in [start..end), within the body of an array object.
5025// The memory effects are all chained onto the 'adr_type' alias category.
5026//
5027// Since the object is otherwise uninitialized, we are free
5028// to put a little "slop" around the edges of the cleared area,
5029// as long as it does not go back into the array's header,
5030// or beyond the array end within the heap.
5031//
5032// The lower edge can be rounded down to the nearest jint and the
5033// upper edge can be rounded up to the nearest MinObjAlignmentInBytes.
5034//
5035// Arguments:
5036//   adr_type           memory slice where writes are generated
5037//   dest               oop of the destination array
5038//   basic_elem_type    element type of the destination
5039//   slice_idx          array index of first element to store
5040//   slice_len          number of elements to store (or NULL)
5041//   dest_size          total size in bytes of the array object
5042//
5043// Exactly one of slice_len or dest_size must be non-NULL.
5044// If dest_size is non-NULL, zeroing extends to the end of the object.
5045// If slice_len is non-NULL, the slice_idx value must be a constant.
5046void
5047LibraryCallKit::generate_clear_array(const TypePtr* adr_type,
5048                                     Node* dest,
5049                                     BasicType basic_elem_type,
5050                                     Node* slice_idx,
5051                                     Node* slice_len,
5052                                     Node* dest_size) {
5053  // one or the other but not both of slice_len and dest_size:
5054  assert((slice_len != NULL? 1: 0) + (dest_size != NULL? 1: 0) == 1, "");
5055  if (slice_len == NULL)  slice_len = top();
5056  if (dest_size == NULL)  dest_size = top();
5057
5058  // operate on this memory slice:
5059  Node* mem = memory(adr_type); // memory slice to operate on
5060
5061  // scaling and rounding of indexes:
5062  int scale = exact_log2(type2aelembytes(basic_elem_type));
5063  int abase = arrayOopDesc::base_offset_in_bytes(basic_elem_type);
5064  int clear_low = (-1 << scale) & (BytesPerInt  - 1);
5065  int bump_bit  = (-1 << scale) & BytesPerInt;
5066
5067  // determine constant starts and ends
5068  const intptr_t BIG_NEG = -128;
5069  assert(BIG_NEG + 2*abase < 0, "neg enough");
5070  intptr_t slice_idx_con = (intptr_t) find_int_con(slice_idx, BIG_NEG);
5071  intptr_t slice_len_con = (intptr_t) find_int_con(slice_len, BIG_NEG);
5072  if (slice_len_con == 0) {
5073    return;                     // nothing to do here
5074  }
5075  intptr_t start_con = (abase + (slice_idx_con << scale)) & ~clear_low;
5076  intptr_t end_con   = find_intptr_t_con(dest_size, -1);
5077  if (slice_idx_con >= 0 && slice_len_con >= 0) {
5078    assert(end_con < 0, "not two cons");
5079    end_con = round_to(abase + ((slice_idx_con + slice_len_con) << scale),
5080                       BytesPerLong);
5081  }
5082
5083  if (start_con >= 0 && end_con >= 0) {
5084    // Constant start and end.  Simple.
5085    mem = ClearArrayNode::clear_memory(control(), mem, dest,
5086                                       start_con, end_con, &_gvn);
5087  } else if (start_con >= 0 && dest_size != top()) {
5088    // Constant start, pre-rounded end after the tail of the array.
5089    Node* end = dest_size;
5090    mem = ClearArrayNode::clear_memory(control(), mem, dest,
5091                                       start_con, end, &_gvn);
5092  } else if (start_con >= 0 && slice_len != top()) {
5093    // Constant start, non-constant end.  End needs rounding up.
5094    // End offset = round_up(abase + ((slice_idx_con + slice_len) << scale), 8)
5095    intptr_t end_base  = abase + (slice_idx_con << scale);
5096    int      end_round = (-1 << scale) & (BytesPerLong  - 1);
5097    Node*    end       = ConvI2X(slice_len);
5098    if (scale != 0)
5099      end = _gvn.transform( new(C) LShiftXNode(end, intcon(scale) ));
5100    end_base += end_round;
5101    end = _gvn.transform( new(C) AddXNode(end, MakeConX(end_base)) );
5102    end = _gvn.transform( new(C) AndXNode(end, MakeConX(~end_round)) );
5103    mem = ClearArrayNode::clear_memory(control(), mem, dest,
5104                                       start_con, end, &_gvn);
5105  } else if (start_con < 0 && dest_size != top()) {
5106    // Non-constant start, pre-rounded end after the tail of the array.
5107    // This is almost certainly a "round-to-end" operation.
5108    Node* start = slice_idx;
5109    start = ConvI2X(start);
5110    if (scale != 0)
5111      start = _gvn.transform( new(C) LShiftXNode( start, intcon(scale) ));
5112    start = _gvn.transform( new(C) AddXNode(start, MakeConX(abase)) );
5113    if ((bump_bit | clear_low) != 0) {
5114      int to_clear = (bump_bit | clear_low);
5115      // Align up mod 8, then store a jint zero unconditionally
5116      // just before the mod-8 boundary.
5117      if (((abase + bump_bit) & ~to_clear) - bump_bit
5118          < arrayOopDesc::length_offset_in_bytes() + BytesPerInt) {
5119        bump_bit = 0;
5120        assert((abase & to_clear) == 0, "array base must be long-aligned");
5121      } else {
5122        // Bump 'start' up to (or past) the next jint boundary:
5123        start = _gvn.transform( new(C) AddXNode(start, MakeConX(bump_bit)) );
5124        assert((abase & clear_low) == 0, "array base must be int-aligned");
5125      }
5126      // Round bumped 'start' down to jlong boundary in body of array.
5127      start = _gvn.transform( new(C) AndXNode(start, MakeConX(~to_clear)) );
5128      if (bump_bit != 0) {
5129        // Store a zero to the immediately preceding jint:
5130        Node* x1 = _gvn.transform( new(C) AddXNode(start, MakeConX(-bump_bit)) );
5131        Node* p1 = basic_plus_adr(dest, x1);
5132        mem = StoreNode::make(_gvn, control(), mem, p1, adr_type, intcon(0), T_INT);
5133        mem = _gvn.transform(mem);
5134      }
5135    }
5136    Node* end = dest_size; // pre-rounded
5137    mem = ClearArrayNode::clear_memory(control(), mem, dest,
5138                                       start, end, &_gvn);
5139  } else {
5140    // Non-constant start, unrounded non-constant end.
5141    // (Nobody zeroes a random midsection of an array using this routine.)
5142    ShouldNotReachHere();       // fix caller
5143  }
5144
5145  // Done.
5146  set_memory(mem, adr_type);
5147}
5148
5149
5150bool
5151LibraryCallKit::generate_block_arraycopy(const TypePtr* adr_type,
5152                                         BasicType basic_elem_type,
5153                                         AllocateNode* alloc,
5154                                         Node* src,  Node* src_offset,
5155                                         Node* dest, Node* dest_offset,
5156                                         Node* dest_size, bool dest_uninitialized) {
5157  // See if there is an advantage from block transfer.
5158  int scale = exact_log2(type2aelembytes(basic_elem_type));
5159  if (scale >= LogBytesPerLong)
5160    return false;               // it is already a block transfer
5161
5162  // Look at the alignment of the starting offsets.
5163  int abase = arrayOopDesc::base_offset_in_bytes(basic_elem_type);
5164
5165  intptr_t src_off_con  = (intptr_t) find_int_con(src_offset, -1);
5166  intptr_t dest_off_con = (intptr_t) find_int_con(dest_offset, -1);
5167  if (src_off_con < 0 || dest_off_con < 0)
5168    // At present, we can only understand constants.
5169    return false;
5170
5171  intptr_t src_off  = abase + (src_off_con  << scale);
5172  intptr_t dest_off = abase + (dest_off_con << scale);
5173
5174  if (((src_off | dest_off) & (BytesPerLong-1)) != 0) {
5175    // Non-aligned; too bad.
5176    // One more chance:  Pick off an initial 32-bit word.
5177    // This is a common case, since abase can be odd mod 8.
5178    if (((src_off | dest_off) & (BytesPerLong-1)) == BytesPerInt &&
5179        ((src_off ^ dest_off) & (BytesPerLong-1)) == 0) {
5180      Node* sptr = basic_plus_adr(src,  src_off);
5181      Node* dptr = basic_plus_adr(dest, dest_off);
5182      Node* sval = make_load(control(), sptr, TypeInt::INT, T_INT, adr_type);
5183      store_to_memory(control(), dptr, sval, T_INT, adr_type);
5184      src_off += BytesPerInt;
5185      dest_off += BytesPerInt;
5186    } else {
5187      return false;
5188    }
5189  }
5190  assert(src_off % BytesPerLong == 0, "");
5191  assert(dest_off % BytesPerLong == 0, "");
5192
5193  // Do this copy by giant steps.
5194  Node* sptr  = basic_plus_adr(src,  src_off);
5195  Node* dptr  = basic_plus_adr(dest, dest_off);
5196  Node* countx = dest_size;
5197  countx = _gvn.transform( new (C) SubXNode(countx, MakeConX(dest_off)) );
5198  countx = _gvn.transform( new (C) URShiftXNode(countx, intcon(LogBytesPerLong)) );
5199
5200  bool disjoint_bases = true;   // since alloc != NULL
5201  generate_unchecked_arraycopy(adr_type, T_LONG, disjoint_bases,
5202                               sptr, NULL, dptr, NULL, countx, dest_uninitialized);
5203
5204  return true;
5205}
5206
5207
5208// Helper function; generates code for the slow case.
5209// We make a call to a runtime method which emulates the native method,
5210// but without the native wrapper overhead.
5211void
5212LibraryCallKit::generate_slow_arraycopy(const TypePtr* adr_type,
5213                                        Node* src,  Node* src_offset,
5214                                        Node* dest, Node* dest_offset,
5215                                        Node* copy_length, bool dest_uninitialized) {
5216  assert(!dest_uninitialized, "Invariant");
5217  Node* call = make_runtime_call(RC_NO_LEAF | RC_UNCOMMON,
5218                                 OptoRuntime::slow_arraycopy_Type(),
5219                                 OptoRuntime::slow_arraycopy_Java(),
5220                                 "slow_arraycopy", adr_type,
5221                                 src, src_offset, dest, dest_offset,
5222                                 copy_length);
5223
5224  // Handle exceptions thrown by this fellow:
5225  make_slow_call_ex(call, env()->Throwable_klass(), false);
5226}
5227
5228// Helper function; generates code for cases requiring runtime checks.
5229Node*
5230LibraryCallKit::generate_checkcast_arraycopy(const TypePtr* adr_type,
5231                                             Node* dest_elem_klass,
5232                                             Node* src,  Node* src_offset,
5233                                             Node* dest, Node* dest_offset,
5234                                             Node* copy_length, bool dest_uninitialized) {
5235  if (stopped())  return NULL;
5236
5237  address copyfunc_addr = StubRoutines::checkcast_arraycopy(dest_uninitialized);
5238  if (copyfunc_addr == NULL) { // Stub was not generated, go slow path.
5239    return NULL;
5240  }
5241
5242  // Pick out the parameters required to perform a store-check
5243  // for the target array.  This is an optimistic check.  It will
5244  // look in each non-null element's class, at the desired klass's
5245  // super_check_offset, for the desired klass.
5246  int sco_offset = in_bytes(Klass::super_check_offset_offset());
5247  Node* p3 = basic_plus_adr(dest_elem_klass, sco_offset);
5248  Node* n3 = new(C) LoadINode(NULL, memory(p3), p3, _gvn.type(p3)->is_ptr());
5249  Node* check_offset = ConvI2X(_gvn.transform(n3));
5250  Node* check_value  = dest_elem_klass;
5251
5252  Node* src_start  = array_element_address(src,  src_offset,  T_OBJECT);
5253  Node* dest_start = array_element_address(dest, dest_offset, T_OBJECT);
5254
5255  // (We know the arrays are never conjoint, because their types differ.)
5256  Node* call = make_runtime_call(RC_LEAF|RC_NO_FP,
5257                                 OptoRuntime::checkcast_arraycopy_Type(),
5258                                 copyfunc_addr, "checkcast_arraycopy", adr_type,
5259                                 // five arguments, of which two are
5260                                 // intptr_t (jlong in LP64)
5261                                 src_start, dest_start,
5262                                 copy_length XTOP,
5263                                 check_offset XTOP,
5264                                 check_value);
5265
5266  return _gvn.transform(new (C) ProjNode(call, TypeFunc::Parms));
5267}
5268
5269
5270// Helper function; generates code for cases requiring runtime checks.
5271Node*
5272LibraryCallKit::generate_generic_arraycopy(const TypePtr* adr_type,
5273                                           Node* src,  Node* src_offset,
5274                                           Node* dest, Node* dest_offset,
5275                                           Node* copy_length, bool dest_uninitialized) {
5276  assert(!dest_uninitialized, "Invariant");
5277  if (stopped())  return NULL;
5278  address copyfunc_addr = StubRoutines::generic_arraycopy();
5279  if (copyfunc_addr == NULL) { // Stub was not generated, go slow path.
5280    return NULL;
5281  }
5282
5283  Node* call = make_runtime_call(RC_LEAF|RC_NO_FP,
5284                    OptoRuntime::generic_arraycopy_Type(),
5285                    copyfunc_addr, "generic_arraycopy", adr_type,
5286                    src, src_offset, dest, dest_offset, copy_length);
5287
5288  return _gvn.transform(new (C) ProjNode(call, TypeFunc::Parms));
5289}
5290
5291// Helper function; generates the fast out-of-line call to an arraycopy stub.
5292void
5293LibraryCallKit::generate_unchecked_arraycopy(const TypePtr* adr_type,
5294                                             BasicType basic_elem_type,
5295                                             bool disjoint_bases,
5296                                             Node* src,  Node* src_offset,
5297                                             Node* dest, Node* dest_offset,
5298                                             Node* copy_length, bool dest_uninitialized) {
5299  if (stopped())  return;               // nothing to do
5300
5301  Node* src_start  = src;
5302  Node* dest_start = dest;
5303  if (src_offset != NULL || dest_offset != NULL) {
5304    assert(src_offset != NULL && dest_offset != NULL, "");
5305    src_start  = array_element_address(src,  src_offset,  basic_elem_type);
5306    dest_start = array_element_address(dest, dest_offset, basic_elem_type);
5307  }
5308
5309  // Figure out which arraycopy runtime method to call.
5310  const char* copyfunc_name = "arraycopy";
5311  address     copyfunc_addr =
5312      basictype2arraycopy(basic_elem_type, src_offset, dest_offset,
5313                          disjoint_bases, copyfunc_name, dest_uninitialized);
5314
5315  // Call it.  Note that the count_ix value is not scaled to a byte-size.
5316  make_runtime_call(RC_LEAF|RC_NO_FP,
5317                    OptoRuntime::fast_arraycopy_Type(),
5318                    copyfunc_addr, copyfunc_name, adr_type,
5319                    src_start, dest_start, copy_length XTOP);
5320}
5321
5322//-------------inline_encodeISOArray-----------------------------------
5323// encode char[] to byte[] in ISO_8859_1
5324bool LibraryCallKit::inline_encodeISOArray() {
5325  assert(callee()->signature()->size() == 5, "encodeISOArray has 5 parameters");
5326  // no receiver since it is static method
5327  Node *src         = argument(0);
5328  Node *src_offset  = argument(1);
5329  Node *dst         = argument(2);
5330  Node *dst_offset  = argument(3);
5331  Node *length      = argument(4);
5332
5333  const Type* src_type = src->Value(&_gvn);
5334  const Type* dst_type = dst->Value(&_gvn);
5335  const TypeAryPtr* top_src = src_type->isa_aryptr();
5336  const TypeAryPtr* top_dest = dst_type->isa_aryptr();
5337  if (top_src  == NULL || top_src->klass()  == NULL ||
5338      top_dest == NULL || top_dest->klass() == NULL) {
5339    // failed array check
5340    return false;
5341  }
5342
5343  // Figure out the size and type of the elements we will be copying.
5344  BasicType src_elem = src_type->isa_aryptr()->klass()->as_array_klass()->element_type()->basic_type();
5345  BasicType dst_elem = dst_type->isa_aryptr()->klass()->as_array_klass()->element_type()->basic_type();
5346  if (src_elem != T_CHAR || dst_elem != T_BYTE) {
5347    return false;
5348  }
5349  Node* src_start = array_element_address(src, src_offset, src_elem);
5350  Node* dst_start = array_element_address(dst, dst_offset, dst_elem);
5351  // 'src_start' points to src array + scaled offset
5352  // 'dst_start' points to dst array + scaled offset
5353
5354  const TypeAryPtr* mtype = TypeAryPtr::BYTES;
5355  Node* enc = new (C) EncodeISOArrayNode(control(), memory(mtype), src_start, dst_start, length);
5356  enc = _gvn.transform(enc);
5357  Node* res_mem = _gvn.transform(new (C) SCMemProjNode(enc));
5358  set_memory(res_mem, mtype);
5359  set_result(enc);
5360  return true;
5361}
5362
5363//----------------------------inline_reference_get----------------------------
5364// public T java.lang.ref.Reference.get();
5365bool LibraryCallKit::inline_reference_get() {
5366  const int referent_offset = java_lang_ref_Reference::referent_offset;
5367  guarantee(referent_offset > 0, "should have already been set");
5368
5369  // Get the argument:
5370  Node* reference_obj = null_check_receiver();
5371  if (stopped()) return true;
5372
5373  Node* adr = basic_plus_adr(reference_obj, reference_obj, referent_offset);
5374
5375  ciInstanceKlass* klass = env()->Object_klass();
5376  const TypeOopPtr* object_type = TypeOopPtr::make_from_klass(klass);
5377
5378  Node* no_ctrl = NULL;
5379  Node* result = make_load(no_ctrl, adr, object_type, T_OBJECT);
5380
5381  // Use the pre-barrier to record the value in the referent field
5382  pre_barrier(false /* do_load */,
5383              control(),
5384              NULL /* obj */, NULL /* adr */, max_juint /* alias_idx */, NULL /* val */, NULL /* val_type */,
5385              result /* pre_val */,
5386              T_OBJECT);
5387
5388  // Add memory barrier to prevent commoning reads from this field
5389  // across safepoint since GC can change its value.
5390  insert_mem_bar(Op_MemBarCPUOrder);
5391
5392  set_result(result);
5393  return true;
5394}
5395
5396
5397Node * LibraryCallKit::load_field_from_object(Node * fromObj, const char * fieldName, const char * fieldTypeString,
5398                                              bool is_exact=true, bool is_static=false) {
5399
5400  const TypeInstPtr* tinst = _gvn.type(fromObj)->isa_instptr();
5401  assert(tinst != NULL, "obj is null");
5402  assert(tinst->klass()->is_loaded(), "obj is not loaded");
5403  assert(!is_exact || tinst->klass_is_exact(), "klass not exact");
5404
5405  ciField* field = tinst->klass()->as_instance_klass()->get_field_by_name(ciSymbol::make(fieldName),
5406                                                                          ciSymbol::make(fieldTypeString),
5407                                                                          is_static);
5408  if (field == NULL) return (Node *) NULL;
5409  assert (field != NULL, "undefined field");
5410
5411  // Next code  copied from Parse::do_get_xxx():
5412
5413  // Compute address and memory type.
5414  int offset  = field->offset_in_bytes();
5415  bool is_vol = field->is_volatile();
5416  ciType* field_klass = field->type();
5417  assert(field_klass->is_loaded(), "should be loaded");
5418  const TypePtr* adr_type = C->alias_type(field)->adr_type();
5419  Node *adr = basic_plus_adr(fromObj, fromObj, offset);
5420  BasicType bt = field->layout_type();
5421
5422  // Build the resultant type of the load
5423  const Type *type = TypeOopPtr::make_from_klass(field_klass->as_klass());
5424
5425  // Build the load.
5426  Node* loadedField = make_load(NULL, adr, type, bt, adr_type, is_vol);
5427  return loadedField;
5428}
5429
5430
5431//------------------------------inline_aescrypt_Block-----------------------
5432bool LibraryCallKit::inline_aescrypt_Block(vmIntrinsics::ID id) {
5433  address stubAddr;
5434  const char *stubName;
5435  assert(UseAES, "need AES instruction support");
5436
5437  switch(id) {
5438  case vmIntrinsics::_aescrypt_encryptBlock:
5439    stubAddr = StubRoutines::aescrypt_encryptBlock();
5440    stubName = "aescrypt_encryptBlock";
5441    break;
5442  case vmIntrinsics::_aescrypt_decryptBlock:
5443    stubAddr = StubRoutines::aescrypt_decryptBlock();
5444    stubName = "aescrypt_decryptBlock";
5445    break;
5446  }
5447  if (stubAddr == NULL) return false;
5448
5449  Node* aescrypt_object = argument(0);
5450  Node* src             = argument(1);
5451  Node* src_offset      = argument(2);
5452  Node* dest            = argument(3);
5453  Node* dest_offset     = argument(4);
5454
5455  // (1) src and dest are arrays.
5456  const Type* src_type = src->Value(&_gvn);
5457  const Type* dest_type = dest->Value(&_gvn);
5458  const TypeAryPtr* top_src = src_type->isa_aryptr();
5459  const TypeAryPtr* top_dest = dest_type->isa_aryptr();
5460  assert (top_src  != NULL && top_src->klass()  != NULL &&  top_dest != NULL && top_dest->klass() != NULL, "args are strange");
5461
5462  // for the quick and dirty code we will skip all the checks.
5463  // we are just trying to get the call to be generated.
5464  Node* src_start  = src;
5465  Node* dest_start = dest;
5466  if (src_offset != NULL || dest_offset != NULL) {
5467    assert(src_offset != NULL && dest_offset != NULL, "");
5468    src_start  = array_element_address(src,  src_offset,  T_BYTE);
5469    dest_start = array_element_address(dest, dest_offset, T_BYTE);
5470  }
5471
5472  // now need to get the start of its expanded key array
5473  // this requires a newer class file that has this array as littleEndian ints, otherwise we revert to java
5474  Node* k_start = get_key_start_from_aescrypt_object(aescrypt_object);
5475  if (k_start == NULL) return false;
5476
5477  // Call the stub.
5478  make_runtime_call(RC_LEAF|RC_NO_FP, OptoRuntime::aescrypt_block_Type(),
5479                    stubAddr, stubName, TypePtr::BOTTOM,
5480                    src_start, dest_start, k_start);
5481
5482  return true;
5483}
5484
5485//------------------------------inline_cipherBlockChaining_AESCrypt-----------------------
5486bool LibraryCallKit::inline_cipherBlockChaining_AESCrypt(vmIntrinsics::ID id) {
5487  address stubAddr;
5488  const char *stubName;
5489
5490  assert(UseAES, "need AES instruction support");
5491
5492  switch(id) {
5493  case vmIntrinsics::_cipherBlockChaining_encryptAESCrypt:
5494    stubAddr = StubRoutines::cipherBlockChaining_encryptAESCrypt();
5495    stubName = "cipherBlockChaining_encryptAESCrypt";
5496    break;
5497  case vmIntrinsics::_cipherBlockChaining_decryptAESCrypt:
5498    stubAddr = StubRoutines::cipherBlockChaining_decryptAESCrypt();
5499    stubName = "cipherBlockChaining_decryptAESCrypt";
5500    break;
5501  }
5502  if (stubAddr == NULL) return false;
5503
5504  Node* cipherBlockChaining_object = argument(0);
5505  Node* src                        = argument(1);
5506  Node* src_offset                 = argument(2);
5507  Node* len                        = argument(3);
5508  Node* dest                       = argument(4);
5509  Node* dest_offset                = argument(5);
5510
5511  // (1) src and dest are arrays.
5512  const Type* src_type = src->Value(&_gvn);
5513  const Type* dest_type = dest->Value(&_gvn);
5514  const TypeAryPtr* top_src = src_type->isa_aryptr();
5515  const TypeAryPtr* top_dest = dest_type->isa_aryptr();
5516  assert (top_src  != NULL && top_src->klass()  != NULL
5517          &&  top_dest != NULL && top_dest->klass() != NULL, "args are strange");
5518
5519  // checks are the responsibility of the caller
5520  Node* src_start  = src;
5521  Node* dest_start = dest;
5522  if (src_offset != NULL || dest_offset != NULL) {
5523    assert(src_offset != NULL && dest_offset != NULL, "");
5524    src_start  = array_element_address(src,  src_offset,  T_BYTE);
5525    dest_start = array_element_address(dest, dest_offset, T_BYTE);
5526  }
5527
5528  // if we are in this set of code, we "know" the embeddedCipher is an AESCrypt object
5529  // (because of the predicated logic executed earlier).
5530  // so we cast it here safely.
5531  // this requires a newer class file that has this array as littleEndian ints, otherwise we revert to java
5532
5533  Node* embeddedCipherObj = load_field_from_object(cipherBlockChaining_object, "embeddedCipher", "Lcom/sun/crypto/provider/SymmetricCipher;", /*is_exact*/ false);
5534  if (embeddedCipherObj == NULL) return false;
5535
5536  // cast it to what we know it will be at runtime
5537  const TypeInstPtr* tinst = _gvn.type(cipherBlockChaining_object)->isa_instptr();
5538  assert(tinst != NULL, "CBC obj is null");
5539  assert(tinst->klass()->is_loaded(), "CBC obj is not loaded");
5540  ciKlass* klass_AESCrypt = tinst->klass()->as_instance_klass()->find_klass(ciSymbol::make("com/sun/crypto/provider/AESCrypt"));
5541  if (!klass_AESCrypt->is_loaded()) return false;
5542
5543  ciInstanceKlass* instklass_AESCrypt = klass_AESCrypt->as_instance_klass();
5544  const TypeKlassPtr* aklass = TypeKlassPtr::make(instklass_AESCrypt);
5545  const TypeOopPtr* xtype = aklass->as_instance_type();
5546  Node* aescrypt_object = new(C) CheckCastPPNode(control(), embeddedCipherObj, xtype);
5547  aescrypt_object = _gvn.transform(aescrypt_object);
5548
5549  // we need to get the start of the aescrypt_object's expanded key array
5550  Node* k_start = get_key_start_from_aescrypt_object(aescrypt_object);
5551  if (k_start == NULL) return false;
5552
5553  // similarly, get the start address of the r vector
5554  Node* objRvec = load_field_from_object(cipherBlockChaining_object, "r", "[B", /*is_exact*/ false);
5555  if (objRvec == NULL) return false;
5556  Node* r_start = array_element_address(objRvec, intcon(0), T_BYTE);
5557
5558  // Call the stub, passing src_start, dest_start, k_start, r_start and src_len
5559  make_runtime_call(RC_LEAF|RC_NO_FP,
5560                    OptoRuntime::cipherBlockChaining_aescrypt_Type(),
5561                    stubAddr, stubName, TypePtr::BOTTOM,
5562                    src_start, dest_start, k_start, r_start, len);
5563
5564  // return is void so no result needs to be pushed
5565
5566  return true;
5567}
5568
5569//------------------------------get_key_start_from_aescrypt_object-----------------------
5570Node * LibraryCallKit::get_key_start_from_aescrypt_object(Node *aescrypt_object) {
5571  Node* objAESCryptKey = load_field_from_object(aescrypt_object, "K", "[I", /*is_exact*/ false);
5572  assert (objAESCryptKey != NULL, "wrong version of com.sun.crypto.provider.AESCrypt");
5573  if (objAESCryptKey == NULL) return (Node *) NULL;
5574
5575  // now have the array, need to get the start address of the K array
5576  Node* k_start = array_element_address(objAESCryptKey, intcon(0), T_INT);
5577  return k_start;
5578}
5579
5580//----------------------------inline_cipherBlockChaining_AESCrypt_predicate----------------------------
5581// Return node representing slow path of predicate check.
5582// the pseudo code we want to emulate with this predicate is:
5583// for encryption:
5584//    if (embeddedCipherObj instanceof AESCrypt) do_intrinsic, else do_javapath
5585// for decryption:
5586//    if ((embeddedCipherObj instanceof AESCrypt) && (cipher!=plain)) do_intrinsic, else do_javapath
5587//    note cipher==plain is more conservative than the original java code but that's OK
5588//
5589Node* LibraryCallKit::inline_cipherBlockChaining_AESCrypt_predicate(bool decrypting) {
5590  // First, check receiver for NULL since it is virtual method.
5591  Node* objCBC = argument(0);
5592  objCBC = null_check(objCBC);
5593
5594  if (stopped()) return NULL; // Always NULL
5595
5596  // Load embeddedCipher field of CipherBlockChaining object.
5597  Node* embeddedCipherObj = load_field_from_object(objCBC, "embeddedCipher", "Lcom/sun/crypto/provider/SymmetricCipher;", /*is_exact*/ false);
5598
5599  // get AESCrypt klass for instanceOf check
5600  // AESCrypt might not be loaded yet if some other SymmetricCipher got us to this compile point
5601  // will have same classloader as CipherBlockChaining object
5602  const TypeInstPtr* tinst = _gvn.type(objCBC)->isa_instptr();
5603  assert(tinst != NULL, "CBCobj is null");
5604  assert(tinst->klass()->is_loaded(), "CBCobj is not loaded");
5605
5606  // we want to do an instanceof comparison against the AESCrypt class
5607  ciKlass* klass_AESCrypt = tinst->klass()->as_instance_klass()->find_klass(ciSymbol::make("com/sun/crypto/provider/AESCrypt"));
5608  if (!klass_AESCrypt->is_loaded()) {
5609    // if AESCrypt is not even loaded, we never take the intrinsic fast path
5610    Node* ctrl = control();
5611    set_control(top()); // no regular fast path
5612    return ctrl;
5613  }
5614  ciInstanceKlass* instklass_AESCrypt = klass_AESCrypt->as_instance_klass();
5615
5616  Node* instof = gen_instanceof(embeddedCipherObj, makecon(TypeKlassPtr::make(instklass_AESCrypt)));
5617  Node* cmp_instof  = _gvn.transform(new (C) CmpINode(instof, intcon(1)));
5618  Node* bool_instof  = _gvn.transform(new (C) BoolNode(cmp_instof, BoolTest::ne));
5619
5620  Node* instof_false = generate_guard(bool_instof, NULL, PROB_MIN);
5621
5622  // for encryption, we are done
5623  if (!decrypting)
5624    return instof_false;  // even if it is NULL
5625
5626  // for decryption, we need to add a further check to avoid
5627  // taking the intrinsic path when cipher and plain are the same
5628  // see the original java code for why.
5629  RegionNode* region = new(C) RegionNode(3);
5630  region->init_req(1, instof_false);
5631  Node* src = argument(1);
5632  Node* dest = argument(4);
5633  Node* cmp_src_dest = _gvn.transform(new (C) CmpPNode(src, dest));
5634  Node* bool_src_dest = _gvn.transform(new (C) BoolNode(cmp_src_dest, BoolTest::eq));
5635  Node* src_dest_conjoint = generate_guard(bool_src_dest, NULL, PROB_MIN);
5636  region->init_req(2, src_dest_conjoint);
5637
5638  record_for_igvn(region);
5639  return _gvn.transform(region);
5640}
5641