library_call.cpp revision 5983:2113136690bc 
1/*
2 * Copyright (c) 1999, 2013, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25#include "precompiled.hpp"
26#include "classfile/systemDictionary.hpp"
27#include "classfile/vmSymbols.hpp"
28#include "compiler/compileBroker.hpp"
29#include "compiler/compileLog.hpp"
30#include "oops/objArrayKlass.hpp"
31#include "opto/addnode.hpp"
32#include "opto/callGenerator.hpp"
33#include "opto/cfgnode.hpp"
34#include "opto/idealKit.hpp"
35#include "opto/mathexactnode.hpp"
36#include "opto/mulnode.hpp"
37#include "opto/parse.hpp"
38#include "opto/runtime.hpp"
39#include "opto/subnode.hpp"
40#include "prims/nativeLookup.hpp"
41#include "runtime/sharedRuntime.hpp"
42#include "trace/traceMacros.hpp"
43
44class LibraryIntrinsic : public InlineCallGenerator {
45  // Extend the set of intrinsics known to the runtime:
46 public:
47 private:
48  bool             _is_virtual;
49  bool             _is_predicted;
50  bool             _does_virtual_dispatch;
51  vmIntrinsics::ID _intrinsic_id;
52
53 public:
54  LibraryIntrinsic(ciMethod* m, bool is_virtual, bool is_predicted, bool does_virtual_dispatch, vmIntrinsics::ID id)
55    : InlineCallGenerator(m),
56      _is_virtual(is_virtual),
57      _is_predicted(is_predicted),
58      _does_virtual_dispatch(does_virtual_dispatch),
59      _intrinsic_id(id)
60  {
61  }
62  virtual bool is_intrinsic() const { return true; }
63  virtual bool is_virtual()   const { return _is_virtual; }
64  virtual bool is_predicted()   const { return _is_predicted; }
65  virtual bool does_virtual_dispatch()   const { return _does_virtual_dispatch; }
66  virtual JVMState* generate(JVMState* jvms, Parse* parent_parser);
67  virtual Node* generate_predicate(JVMState* jvms);
68  vmIntrinsics::ID intrinsic_id() const { return _intrinsic_id; }
69};
70
71
72// Local helper class for LibraryIntrinsic:
73class LibraryCallKit : public GraphKit {
74 private:
75  LibraryIntrinsic* _intrinsic;     // the library intrinsic being called
76  Node*             _result;        // the result node, if any
77  int               _reexecute_sp;  // the stack pointer when bytecode needs to be reexecuted
78
79  const TypeOopPtr* sharpen_unsafe_type(Compile::AliasType* alias_type, const TypePtr *adr_type, bool is_native_ptr = false);
80
81 public:
82  LibraryCallKit(JVMState* jvms, LibraryIntrinsic* intrinsic)
83    : GraphKit(jvms),
84      _intrinsic(intrinsic),
85      _result(NULL)
86  {
87    // Check if this is a root compile.  In that case we don't have a caller.
88    if (!jvms->has_method()) {
89      _reexecute_sp = sp();
90    } else {
91      // Find out how many arguments the interpreter needs when deoptimizing
92      // and save the stack pointer value so it can used by uncommon_trap.
93      // We find the argument count by looking at the declared signature.
94      bool ignored_will_link;
95      ciSignature* declared_signature = NULL;
96      ciMethod* ignored_callee = caller()->get_method_at_bci(bci(), ignored_will_link, &declared_signature);
97      const int nargs = declared_signature->arg_size_for_bc(caller()->java_code_at_bci(bci()));
98      _reexecute_sp = sp() + nargs;  // "push" arguments back on stack
99    }
100  }
101
102  virtual LibraryCallKit* is_LibraryCallKit() const { return (LibraryCallKit*)this; }
103
104  ciMethod*         caller()    const    { return jvms()->method(); }
105  int               bci()       const    { return jvms()->bci(); }
106  LibraryIntrinsic* intrinsic() const    { return _intrinsic; }
107  vmIntrinsics::ID  intrinsic_id() const { return _intrinsic->intrinsic_id(); }
108  ciMethod*         callee()    const    { return _intrinsic->method(); }
109
110  bool try_to_inline();
111  Node* try_to_predicate();
112
113  void push_result() {
114    // Push the result onto the stack.
115    if (!stopped() && result() != NULL) {
116      BasicType bt = result()->bottom_type()->basic_type();
117      push_node(bt, result());
118    }
119  }
120
121 private:
122  void fatal_unexpected_iid(vmIntrinsics::ID iid) {
123    fatal(err_msg_res("unexpected intrinsic %d: %s", iid, vmIntrinsics::name_at(iid)));
124  }
125
126  void  set_result(Node* n) { assert(_result == NULL, "only set once"); _result = n; }
127  void  set_result(RegionNode* region, PhiNode* value);
128  Node*     result() { return _result; }
129
130  virtual int reexecute_sp() { return _reexecute_sp; }
131
132  // Helper functions to inline natives
133  Node* generate_guard(Node* test, RegionNode* region, float true_prob);
134  Node* generate_slow_guard(Node* test, RegionNode* region);
135  Node* generate_fair_guard(Node* test, RegionNode* region);
136  Node* generate_negative_guard(Node* index, RegionNode* region,
137                                // resulting CastII of index:
138                                Node* *pos_index = NULL);
139  Node* generate_nonpositive_guard(Node* index, bool never_negative,
140                                   // resulting CastII of index:
141                                   Node* *pos_index = NULL);
142  Node* generate_limit_guard(Node* offset, Node* subseq_length,
143                             Node* array_length,
144                             RegionNode* region);
145  Node* generate_current_thread(Node* &tls_output);
146  address basictype2arraycopy(BasicType t, Node *src_offset, Node *dest_offset,
147                              bool disjoint_bases, const char* &name, bool dest_uninitialized);
148  Node* load_mirror_from_klass(Node* klass);
149  Node* load_klass_from_mirror_common(Node* mirror, bool never_see_null,
150                                      RegionNode* region, int null_path,
151                                      int offset);
152  Node* load_klass_from_mirror(Node* mirror, bool never_see_null,
153                               RegionNode* region, int null_path) {
154    int offset = java_lang_Class::klass_offset_in_bytes();
155    return load_klass_from_mirror_common(mirror, never_see_null,
156                                         region, null_path,
157                                         offset);
158  }
159  Node* load_array_klass_from_mirror(Node* mirror, bool never_see_null,
160                                     RegionNode* region, int null_path) {
161    int offset = java_lang_Class::array_klass_offset_in_bytes();
162    return load_klass_from_mirror_common(mirror, never_see_null,
163                                         region, null_path,
164                                         offset);
165  }
166  Node* generate_access_flags_guard(Node* kls,
167                                    int modifier_mask, int modifier_bits,
168                                    RegionNode* region);
169  Node* generate_interface_guard(Node* kls, RegionNode* region);
170  Node* generate_array_guard(Node* kls, RegionNode* region) {
171    return generate_array_guard_common(kls, region, false, false);
172  }
173  Node* generate_non_array_guard(Node* kls, RegionNode* region) {
174    return generate_array_guard_common(kls, region, false, true);
175  }
176  Node* generate_objArray_guard(Node* kls, RegionNode* region) {
177    return generate_array_guard_common(kls, region, true, false);
178  }
179  Node* generate_non_objArray_guard(Node* kls, RegionNode* region) {
180    return generate_array_guard_common(kls, region, true, true);
181  }
182  Node* generate_array_guard_common(Node* kls, RegionNode* region,
183                                    bool obj_array, bool not_array);
184  Node* generate_virtual_guard(Node* obj_klass, RegionNode* slow_region);
185  CallJavaNode* generate_method_call(vmIntrinsics::ID method_id,
186                                     bool is_virtual = false, bool is_static = false);
187  CallJavaNode* generate_method_call_static(vmIntrinsics::ID method_id) {
188    return generate_method_call(method_id, false, true);
189  }
190  CallJavaNode* generate_method_call_virtual(vmIntrinsics::ID method_id) {
191    return generate_method_call(method_id, true, false);
192  }
193  Node * load_field_from_object(Node * fromObj, const char * fieldName, const char * fieldTypeString, bool is_exact, bool is_static);
194
195  Node* make_string_method_node(int opcode, Node* str1_start, Node* cnt1, Node* str2_start, Node* cnt2);
196  Node* make_string_method_node(int opcode, Node* str1, Node* str2);
197  bool inline_string_compareTo();
198  bool inline_string_indexOf();
199  Node* string_indexOf(Node* string_object, ciTypeArray* target_array, jint offset, jint cache_i, jint md2_i);
200  bool inline_string_equals();
201  Node* round_double_node(Node* n);
202  bool runtime_math(const TypeFunc* call_type, address funcAddr, const char* funcName);
203  bool inline_math_native(vmIntrinsics::ID id);
204  bool inline_trig(vmIntrinsics::ID id);
205  bool inline_math(vmIntrinsics::ID id);
206  void inline_math_mathExact(Node* math);
207  bool inline_math_addExactI(bool is_increment);
208  bool inline_math_addExactL(bool is_increment);
209  bool inline_math_multiplyExactI();
210  bool inline_math_multiplyExactL();
211  bool inline_math_negateExactI();
212  bool inline_math_negateExactL();
213  bool inline_math_subtractExactI(bool is_decrement);
214  bool inline_math_subtractExactL(bool is_decrement);
215  bool inline_exp();
216  bool inline_pow();
217  void finish_pow_exp(Node* result, Node* x, Node* y, const TypeFunc* call_type, address funcAddr, const char* funcName);
218  bool inline_min_max(vmIntrinsics::ID id);
219  Node* generate_min_max(vmIntrinsics::ID id, Node* x, Node* y);
220  // This returns Type::AnyPtr, RawPtr, or OopPtr.
221  int classify_unsafe_addr(Node* &base, Node* &offset);
222  Node* make_unsafe_address(Node* base, Node* offset);
223  // Helper for inline_unsafe_access.
224  // Generates the guards that check whether the result of
225  // Unsafe.getObject should be recorded in an SATB log buffer.
226  void insert_pre_barrier(Node* base_oop, Node* offset, Node* pre_val, bool need_mem_bar);
227  bool inline_unsafe_access(bool is_native_ptr, bool is_store, BasicType type, bool is_volatile);
228  bool inline_unsafe_prefetch(bool is_native_ptr, bool is_store, bool is_static);
229  static bool klass_needs_init_guard(Node* kls);
230  bool inline_unsafe_allocate();
231  bool inline_unsafe_copyMemory();
232  bool inline_native_currentThread();
233#ifdef TRACE_HAVE_INTRINSICS
234  bool inline_native_classID();
235  bool inline_native_threadID();
236#endif
237  bool inline_native_time_funcs(address method, const char* funcName);
238  bool inline_native_isInterrupted();
239  bool inline_native_Class_query(vmIntrinsics::ID id);
240  bool inline_native_subtype_check();
241
242  bool inline_native_newArray();
243  bool inline_native_getLength();
244  bool inline_array_copyOf(bool is_copyOfRange);
245  bool inline_array_equals();
246  void copy_to_clone(Node* obj, Node* alloc_obj, Node* obj_size, bool is_array, bool card_mark);
247  bool inline_native_clone(bool is_virtual);
248  bool inline_native_Reflection_getCallerClass();
249  // Helper function for inlining native object hash method
250  bool inline_native_hashcode(bool is_virtual, bool is_static);
251  bool inline_native_getClass();
252
253  // Helper functions for inlining arraycopy
254  bool inline_arraycopy();
255  void generate_arraycopy(const TypePtr* adr_type,
256                          BasicType basic_elem_type,
257                          Node* src,  Node* src_offset,
258                          Node* dest, Node* dest_offset,
259                          Node* copy_length,
260                          bool disjoint_bases = false,
261                          bool length_never_negative = false,
262                          RegionNode* slow_region = NULL);
263  AllocateArrayNode* tightly_coupled_allocation(Node* ptr,
264                                                RegionNode* slow_region);
265  void generate_clear_array(const TypePtr* adr_type,
266                            Node* dest,
267                            BasicType basic_elem_type,
268                            Node* slice_off,
269                            Node* slice_len,
270                            Node* slice_end);
271  bool generate_block_arraycopy(const TypePtr* adr_type,
272                                BasicType basic_elem_type,
273                                AllocateNode* alloc,
274                                Node* src,  Node* src_offset,
275                                Node* dest, Node* dest_offset,
276                                Node* dest_size, bool dest_uninitialized);
277  void generate_slow_arraycopy(const TypePtr* adr_type,
278                               Node* src,  Node* src_offset,
279                               Node* dest, Node* dest_offset,
280                               Node* copy_length, bool dest_uninitialized);
281  Node* generate_checkcast_arraycopy(const TypePtr* adr_type,
282                                     Node* dest_elem_klass,
283                                     Node* src,  Node* src_offset,
284                                     Node* dest, Node* dest_offset,
285                                     Node* copy_length, bool dest_uninitialized);
286  Node* generate_generic_arraycopy(const TypePtr* adr_type,
287                                   Node* src,  Node* src_offset,
288                                   Node* dest, Node* dest_offset,
289                                   Node* copy_length, bool dest_uninitialized);
290  void generate_unchecked_arraycopy(const TypePtr* adr_type,
291                                    BasicType basic_elem_type,
292                                    bool disjoint_bases,
293                                    Node* src,  Node* src_offset,
294                                    Node* dest, Node* dest_offset,
295                                    Node* copy_length, bool dest_uninitialized);
296  typedef enum { LS_xadd, LS_xchg, LS_cmpxchg } LoadStoreKind;
297  bool inline_unsafe_load_store(BasicType type,  LoadStoreKind kind);
298  bool inline_unsafe_ordered_store(BasicType type);
299  bool inline_unsafe_fence(vmIntrinsics::ID id);
300  bool inline_fp_conversions(vmIntrinsics::ID id);
301  bool inline_number_methods(vmIntrinsics::ID id);
302  bool inline_reference_get();
303  bool inline_aescrypt_Block(vmIntrinsics::ID id);
304  bool inline_cipherBlockChaining_AESCrypt(vmIntrinsics::ID id);
305  Node* inline_cipherBlockChaining_AESCrypt_predicate(bool decrypting);
306  Node* get_key_start_from_aescrypt_object(Node* aescrypt_object);
307  bool inline_encodeISOArray();
308  bool inline_updateCRC32();
309  bool inline_updateBytesCRC32();
310  bool inline_updateByteBufferCRC32();
311};
312
313
314//---------------------------make_vm_intrinsic----------------------------
315CallGenerator* Compile::make_vm_intrinsic(ciMethod* m, bool is_virtual) {
316  vmIntrinsics::ID id = m->intrinsic_id();
317  assert(id != vmIntrinsics::_none, "must be a VM intrinsic");
318
319  if (DisableIntrinsic[0] != '\0'
320      && strstr(DisableIntrinsic, vmIntrinsics::name_at(id)) != NULL) {
321    // disabled by a user request on the command line:
322    // example: -XX:DisableIntrinsic=_hashCode,_getClass
323    return NULL;
324  }
325
326  if (!m->is_loaded()) {
327    // do not attempt to inline unloaded methods
328    return NULL;
329  }
330
331  // Only a few intrinsics implement a virtual dispatch.
332  // They are expensive calls which are also frequently overridden.
333  if (is_virtual) {
334    switch (id) {
335    case vmIntrinsics::_hashCode:
336    case vmIntrinsics::_clone:
337      // OK, Object.hashCode and Object.clone intrinsics come in both flavors
338      break;
339    default:
340      return NULL;
341    }
342  }
343
344  // -XX:-InlineNatives disables nearly all intrinsics:
345  if (!InlineNatives) {
346    switch (id) {
347    case vmIntrinsics::_indexOf:
348    case vmIntrinsics::_compareTo:
349    case vmIntrinsics::_equals:
350    case vmIntrinsics::_equalsC:
351    case vmIntrinsics::_getAndAddInt:
352    case vmIntrinsics::_getAndAddLong:
353    case vmIntrinsics::_getAndSetInt:
354    case vmIntrinsics::_getAndSetLong:
355    case vmIntrinsics::_getAndSetObject:
356    case vmIntrinsics::_loadFence:
357    case vmIntrinsics::_storeFence:
358    case vmIntrinsics::_fullFence:
359      break;  // InlineNatives does not control String.compareTo
360    case vmIntrinsics::_Reference_get:
361      break;  // InlineNatives does not control Reference.get
362    default:
363      return NULL;
364    }
365  }
366
367  bool is_predicted = false;
368  bool does_virtual_dispatch = false;
369
370  switch (id) {
371  case vmIntrinsics::_compareTo:
372    if (!SpecialStringCompareTo)  return NULL;
373    if (!Matcher::match_rule_supported(Op_StrComp))  return NULL;
374    break;
375  case vmIntrinsics::_indexOf:
376    if (!SpecialStringIndexOf)  return NULL;
377    break;
378  case vmIntrinsics::_equals:
379    if (!SpecialStringEquals)  return NULL;
380    if (!Matcher::match_rule_supported(Op_StrEquals))  return NULL;
381    break;
382  case vmIntrinsics::_equalsC:
383    if (!SpecialArraysEquals)  return NULL;
384    if (!Matcher::match_rule_supported(Op_AryEq))  return NULL;
385    break;
386  case vmIntrinsics::_arraycopy:
387    if (!InlineArrayCopy)  return NULL;
388    break;
389  case vmIntrinsics::_copyMemory:
390    if (StubRoutines::unsafe_arraycopy() == NULL)  return NULL;
391    if (!InlineArrayCopy)  return NULL;
392    break;
393  case vmIntrinsics::_hashCode:
394    if (!InlineObjectHash)  return NULL;
395    does_virtual_dispatch = true;
396    break;
397  case vmIntrinsics::_clone:
398    does_virtual_dispatch = true;
399  case vmIntrinsics::_copyOf:
400  case vmIntrinsics::_copyOfRange:
401    if (!InlineObjectCopy)  return NULL;
402    // These also use the arraycopy intrinsic mechanism:
403    if (!InlineArrayCopy)  return NULL;
404    break;
405  case vmIntrinsics::_encodeISOArray:
406    if (!SpecialEncodeISOArray)  return NULL;
407    if (!Matcher::match_rule_supported(Op_EncodeISOArray))  return NULL;
408    break;
409  case vmIntrinsics::_checkIndex:
410    // We do not intrinsify this.  The optimizer does fine with it.
411    return NULL;
412
413  case vmIntrinsics::_getCallerClass:
414    if (!UseNewReflection)  return NULL;
415    if (!InlineReflectionGetCallerClass)  return NULL;
416    if (SystemDictionary::reflect_CallerSensitive_klass() == NULL)  return NULL;
417    break;
418
419  case vmIntrinsics::_bitCount_i:
420    if (!Matcher::match_rule_supported(Op_PopCountI)) return NULL;
421    break;
422
423  case vmIntrinsics::_bitCount_l:
424    if (!Matcher::match_rule_supported(Op_PopCountL)) return NULL;
425    break;
426
427  case vmIntrinsics::_numberOfLeadingZeros_i:
428    if (!Matcher::match_rule_supported(Op_CountLeadingZerosI)) return NULL;
429    break;
430
431  case vmIntrinsics::_numberOfLeadingZeros_l:
432    if (!Matcher::match_rule_supported(Op_CountLeadingZerosL)) return NULL;
433    break;
434
435  case vmIntrinsics::_numberOfTrailingZeros_i:
436    if (!Matcher::match_rule_supported(Op_CountTrailingZerosI)) return NULL;
437    break;
438
439  case vmIntrinsics::_numberOfTrailingZeros_l:
440    if (!Matcher::match_rule_supported(Op_CountTrailingZerosL)) return NULL;
441    break;
442
443  case vmIntrinsics::_reverseBytes_c:
444    if (!Matcher::match_rule_supported(Op_ReverseBytesUS)) return NULL;
445    break;
446  case vmIntrinsics::_reverseBytes_s:
447    if (!Matcher::match_rule_supported(Op_ReverseBytesS))  return NULL;
448    break;
449  case vmIntrinsics::_reverseBytes_i:
450    if (!Matcher::match_rule_supported(Op_ReverseBytesI))  return NULL;
451    break;
452  case vmIntrinsics::_reverseBytes_l:
453    if (!Matcher::match_rule_supported(Op_ReverseBytesL))  return NULL;
454    break;
455
456  case vmIntrinsics::_Reference_get:
457    // Use the intrinsic version of Reference.get() so that the value in
458    // the referent field can be registered by the G1 pre-barrier code.
459    // Also add memory barrier to prevent commoning reads from this field
460    // across safepoint since GC can change it value.
461    break;
462
463  case vmIntrinsics::_compareAndSwapObject:
464#ifdef _LP64
465    if (!UseCompressedOops && !Matcher::match_rule_supported(Op_CompareAndSwapP)) return NULL;
466#endif
467    break;
468
469  case vmIntrinsics::_compareAndSwapLong:
470    if (!Matcher::match_rule_supported(Op_CompareAndSwapL)) return NULL;
471    break;
472
473  case vmIntrinsics::_getAndAddInt:
474    if (!Matcher::match_rule_supported(Op_GetAndAddI)) return NULL;
475    break;
476
477  case vmIntrinsics::_getAndAddLong:
478    if (!Matcher::match_rule_supported(Op_GetAndAddL)) return NULL;
479    break;
480
481  case vmIntrinsics::_getAndSetInt:
482    if (!Matcher::match_rule_supported(Op_GetAndSetI)) return NULL;
483    break;
484
485  case vmIntrinsics::_getAndSetLong:
486    if (!Matcher::match_rule_supported(Op_GetAndSetL)) return NULL;
487    break;
488
489  case vmIntrinsics::_getAndSetObject:
490#ifdef _LP64
491    if (!UseCompressedOops && !Matcher::match_rule_supported(Op_GetAndSetP)) return NULL;
492    if (UseCompressedOops && !Matcher::match_rule_supported(Op_GetAndSetN)) return NULL;
493    break;
494#else
495    if (!Matcher::match_rule_supported(Op_GetAndSetP)) return NULL;
496    break;
497#endif
498
499  case vmIntrinsics::_aescrypt_encryptBlock:
500  case vmIntrinsics::_aescrypt_decryptBlock:
501    if (!UseAESIntrinsics) return NULL;
502    break;
503
504  case vmIntrinsics::_cipherBlockChaining_encryptAESCrypt:
505  case vmIntrinsics::_cipherBlockChaining_decryptAESCrypt:
506    if (!UseAESIntrinsics) return NULL;
507    // these two require the predicated logic
508    is_predicted = true;
509    break;
510
511  case vmIntrinsics::_updateCRC32:
512  case vmIntrinsics::_updateBytesCRC32:
513  case vmIntrinsics::_updateByteBufferCRC32:
514    if (!UseCRC32Intrinsics) return NULL;
515    break;
516
517  case vmIntrinsics::_incrementExactI:
518  case vmIntrinsics::_addExactI:
519    if (!Matcher::match_rule_supported(Op_AddExactI) || !UseMathExactIntrinsics) return NULL;
520    break;
521  case vmIntrinsics::_incrementExactL:
522  case vmIntrinsics::_addExactL:
523    if (!Matcher::match_rule_supported(Op_AddExactL) || !UseMathExactIntrinsics) return NULL;
524    break;
525  case vmIntrinsics::_decrementExactI:
526  case vmIntrinsics::_subtractExactI:
527    if (!Matcher::match_rule_supported(Op_SubExactI) || !UseMathExactIntrinsics) return NULL;
528    break;
529  case vmIntrinsics::_decrementExactL:
530  case vmIntrinsics::_subtractExactL:
531    if (!Matcher::match_rule_supported(Op_SubExactL) || !UseMathExactIntrinsics) return NULL;
532    break;
533  case vmIntrinsics::_negateExactI:
534    if (!Matcher::match_rule_supported(Op_NegExactI) || !UseMathExactIntrinsics) return NULL;
535    break;
536  case vmIntrinsics::_negateExactL:
537    if (!Matcher::match_rule_supported(Op_NegExactL) || !UseMathExactIntrinsics) return NULL;
538    break;
539  case vmIntrinsics::_multiplyExactI:
540    if (!Matcher::match_rule_supported(Op_MulExactI) || !UseMathExactIntrinsics) return NULL;
541    break;
542  case vmIntrinsics::_multiplyExactL:
543    if (!Matcher::match_rule_supported(Op_MulExactL) || !UseMathExactIntrinsics) return NULL;
544    break;
545
546 default:
547    assert(id <= vmIntrinsics::LAST_COMPILER_INLINE, "caller responsibility");
548    assert(id != vmIntrinsics::_Object_init && id != vmIntrinsics::_invoke, "enum out of order?");
549    break;
550  }
551
552  // -XX:-InlineClassNatives disables natives from the Class class.
553  // The flag applies to all reflective calls, notably Array.newArray
554  // (visible to Java programmers as Array.newInstance).
555  if (m->holder()->name() == ciSymbol::java_lang_Class() ||
556      m->holder()->name() == ciSymbol::java_lang_reflect_Array()) {
557    if (!InlineClassNatives)  return NULL;
558  }
559
560  // -XX:-InlineThreadNatives disables natives from the Thread class.
561  if (m->holder()->name() == ciSymbol::java_lang_Thread()) {
562    if (!InlineThreadNatives)  return NULL;
563  }
564
565  // -XX:-InlineMathNatives disables natives from the Math,Float and Double classes.
566  if (m->holder()->name() == ciSymbol::java_lang_Math() ||
567      m->holder()->name() == ciSymbol::java_lang_Float() ||
568      m->holder()->name() == ciSymbol::java_lang_Double()) {
569    if (!InlineMathNatives)  return NULL;
570  }
571
572  // -XX:-InlineUnsafeOps disables natives from the Unsafe class.
573  if (m->holder()->name() == ciSymbol::sun_misc_Unsafe()) {
574    if (!InlineUnsafeOps)  return NULL;
575  }
576
577  return new LibraryIntrinsic(m, is_virtual, is_predicted, does_virtual_dispatch, (vmIntrinsics::ID) id);
578}
579
580//----------------------register_library_intrinsics-----------------------
581// Initialize this file's data structures, for each Compile instance.
582void Compile::register_library_intrinsics() {
583  // Nothing to do here.
584}
585
586JVMState* LibraryIntrinsic::generate(JVMState* jvms, Parse* parent_parser) {
587  LibraryCallKit kit(jvms, this);
588  Compile* C = kit.C;
589  int nodes = C->unique();
590#ifndef PRODUCT
591  if ((C->print_intrinsics() || C->print_inlining()) && Verbose) {
592    char buf[1000];
593    const char* str = vmIntrinsics::short_name_as_C_string(intrinsic_id(), buf, sizeof(buf));
594    tty->print_cr("Intrinsic %s", str);
595  }
596#endif
597  ciMethod* callee = kit.callee();
598  const int bci    = kit.bci();
599
600  // Try to inline the intrinsic.
601  if (kit.try_to_inline()) {
602    if (C->print_intrinsics() || C->print_inlining()) {
603      C->print_inlining(callee, jvms->depth() - 1, bci, is_virtual() ? "(intrinsic, virtual)" : "(intrinsic)");
604    }
605    C->gather_intrinsic_statistics(intrinsic_id(), is_virtual(), Compile::_intrinsic_worked);
606    if (C->log()) {
607      C->log()->elem("intrinsic id='%s'%s nodes='%d'",
608                     vmIntrinsics::name_at(intrinsic_id()),
609                     (is_virtual() ? " virtual='1'" : ""),
610                     C->unique() - nodes);
611    }
612    // Push the result from the inlined method onto the stack.
613    kit.push_result();
614    return kit.transfer_exceptions_into_jvms();
615  }
616
617  // The intrinsic bailed out
618  if (C->print_intrinsics() || C->print_inlining()) {
619    if (jvms->has_method()) {
620      // Not a root compile.
621      const char* msg = is_virtual() ? "failed to inline (intrinsic, virtual)" : "failed to inline (intrinsic)";
622      C->print_inlining(callee, jvms->depth() - 1, bci, msg);
623    } else {
624      // Root compile
625      tty->print("Did not generate intrinsic %s%s at bci:%d in",
626               vmIntrinsics::name_at(intrinsic_id()),
627               (is_virtual() ? " (virtual)" : ""), bci);
628    }
629  }
630  C->gather_intrinsic_statistics(intrinsic_id(), is_virtual(), Compile::_intrinsic_failed);
631  return NULL;
632}
633
634Node* LibraryIntrinsic::generate_predicate(JVMState* jvms) {
635  LibraryCallKit kit(jvms, this);
636  Compile* C = kit.C;
637  int nodes = C->unique();
638#ifndef PRODUCT
639  assert(is_predicted(), "sanity");
640  if ((C->print_intrinsics() || C->print_inlining()) && Verbose) {
641    char buf[1000];
642    const char* str = vmIntrinsics::short_name_as_C_string(intrinsic_id(), buf, sizeof(buf));
643    tty->print_cr("Predicate for intrinsic %s", str);
644  }
645#endif
646  ciMethod* callee = kit.callee();
647  const int bci    = kit.bci();
648
649  Node* slow_ctl = kit.try_to_predicate();
650  if (!kit.failing()) {
651    if (C->print_intrinsics() || C->print_inlining()) {
652      C->print_inlining(callee, jvms->depth() - 1, bci, is_virtual() ? "(intrinsic, virtual)" : "(intrinsic)");
653    }
654    C->gather_intrinsic_statistics(intrinsic_id(), is_virtual(), Compile::_intrinsic_worked);
655    if (C->log()) {
656      C->log()->elem("predicate_intrinsic id='%s'%s nodes='%d'",
657                     vmIntrinsics::name_at(intrinsic_id()),
658                     (is_virtual() ? " virtual='1'" : ""),
659                     C->unique() - nodes);
660    }
661    return slow_ctl; // Could be NULL if the check folds.
662  }
663
664  // The intrinsic bailed out
665  if (C->print_intrinsics() || C->print_inlining()) {
666    if (jvms->has_method()) {
667      // Not a root compile.
668      const char* msg = "failed to generate predicate for intrinsic";
669      C->print_inlining(kit.callee(), jvms->depth() - 1, bci, msg);
670    } else {
671      // Root compile
672      C->print_inlining_stream()->print("Did not generate predicate for intrinsic %s%s at bci:%d in",
673                                        vmIntrinsics::name_at(intrinsic_id()),
674                                        (is_virtual() ? " (virtual)" : ""), bci);
675    }
676  }
677  C->gather_intrinsic_statistics(intrinsic_id(), is_virtual(), Compile::_intrinsic_failed);
678  return NULL;
679}
680
681bool LibraryCallKit::try_to_inline() {
682  // Handle symbolic names for otherwise undistinguished boolean switches:
683  const bool is_store       = true;
684  const bool is_native_ptr  = true;
685  const bool is_static      = true;
686  const bool is_volatile    = true;
687
688  if (!jvms()->has_method()) {
689    // Root JVMState has a null method.
690    assert(map()->memory()->Opcode() == Op_Parm, "");
691    // Insert the memory aliasing node
692    set_all_memory(reset_memory());
693  }
694  assert(merged_memory(), "");
695
696
697  switch (intrinsic_id()) {
698  case vmIntrinsics::_hashCode:                 return inline_native_hashcode(intrinsic()->is_virtual(), !is_static);
699  case vmIntrinsics::_identityHashCode:         return inline_native_hashcode(/*!virtual*/ false,         is_static);
700  case vmIntrinsics::_getClass:                 return inline_native_getClass();
701
702  case vmIntrinsics::_dsin:
703  case vmIntrinsics::_dcos:
704  case vmIntrinsics::_dtan:
705  case vmIntrinsics::_dabs:
706  case vmIntrinsics::_datan2:
707  case vmIntrinsics::_dsqrt:
708  case vmIntrinsics::_dexp:
709  case vmIntrinsics::_dlog:
710  case vmIntrinsics::_dlog10:
711  case vmIntrinsics::_dpow:                     return inline_math_native(intrinsic_id());
712
713  case vmIntrinsics::_min:
714  case vmIntrinsics::_max:                      return inline_min_max(intrinsic_id());
715
716  case vmIntrinsics::_addExactI:                return inline_math_addExactI(false /* add */);
717  case vmIntrinsics::_addExactL:                return inline_math_addExactL(false /* add */);
718  case vmIntrinsics::_decrementExactI:          return inline_math_subtractExactI(true /* decrement */);
719  case vmIntrinsics::_decrementExactL:          return inline_math_subtractExactL(true /* decrement */);
720  case vmIntrinsics::_incrementExactI:          return inline_math_addExactI(true /* increment */);
721  case vmIntrinsics::_incrementExactL:          return inline_math_addExactL(true /* increment */);
722  case vmIntrinsics::_multiplyExactI:           return inline_math_multiplyExactI();
723  case vmIntrinsics::_multiplyExactL:           return inline_math_multiplyExactL();
724  case vmIntrinsics::_negateExactI:             return inline_math_negateExactI();
725  case vmIntrinsics::_negateExactL:             return inline_math_negateExactL();
726  case vmIntrinsics::_subtractExactI:           return inline_math_subtractExactI(false /* subtract */);
727  case vmIntrinsics::_subtractExactL:           return inline_math_subtractExactL(false /* subtract */);
728
729  case vmIntrinsics::_arraycopy:                return inline_arraycopy();
730
731  case vmIntrinsics::_compareTo:                return inline_string_compareTo();
732  case vmIntrinsics::_indexOf:                  return inline_string_indexOf();
733  case vmIntrinsics::_equals:                   return inline_string_equals();
734
735  case vmIntrinsics::_getObject:                return inline_unsafe_access(!is_native_ptr, !is_store, T_OBJECT,  !is_volatile);
736  case vmIntrinsics::_getBoolean:               return inline_unsafe_access(!is_native_ptr, !is_store, T_BOOLEAN, !is_volatile);
737  case vmIntrinsics::_getByte:                  return inline_unsafe_access(!is_native_ptr, !is_store, T_BYTE,    !is_volatile);
738  case vmIntrinsics::_getShort:                 return inline_unsafe_access(!is_native_ptr, !is_store, T_SHORT,   !is_volatile);
739  case vmIntrinsics::_getChar:                  return inline_unsafe_access(!is_native_ptr, !is_store, T_CHAR,    !is_volatile);
740  case vmIntrinsics::_getInt:                   return inline_unsafe_access(!is_native_ptr, !is_store, T_INT,     !is_volatile);
741  case vmIntrinsics::_getLong:                  return inline_unsafe_access(!is_native_ptr, !is_store, T_LONG,    !is_volatile);
742  case vmIntrinsics::_getFloat:                 return inline_unsafe_access(!is_native_ptr, !is_store, T_FLOAT,   !is_volatile);
743  case vmIntrinsics::_getDouble:                return inline_unsafe_access(!is_native_ptr, !is_store, T_DOUBLE,  !is_volatile);
744
745  case vmIntrinsics::_putObject:                return inline_unsafe_access(!is_native_ptr,  is_store, T_OBJECT,  !is_volatile);
746  case vmIntrinsics::_putBoolean:               return inline_unsafe_access(!is_native_ptr,  is_store, T_BOOLEAN, !is_volatile);
747  case vmIntrinsics::_putByte:                  return inline_unsafe_access(!is_native_ptr,  is_store, T_BYTE,    !is_volatile);
748  case vmIntrinsics::_putShort:                 return inline_unsafe_access(!is_native_ptr,  is_store, T_SHORT,   !is_volatile);
749  case vmIntrinsics::_putChar:                  return inline_unsafe_access(!is_native_ptr,  is_store, T_CHAR,    !is_volatile);
750  case vmIntrinsics::_putInt:                   return inline_unsafe_access(!is_native_ptr,  is_store, T_INT,     !is_volatile);
751  case vmIntrinsics::_putLong:                  return inline_unsafe_access(!is_native_ptr,  is_store, T_LONG,    !is_volatile);
752  case vmIntrinsics::_putFloat:                 return inline_unsafe_access(!is_native_ptr,  is_store, T_FLOAT,   !is_volatile);
753  case vmIntrinsics::_putDouble:                return inline_unsafe_access(!is_native_ptr,  is_store, T_DOUBLE,  !is_volatile);
754
755  case vmIntrinsics::_getByte_raw:              return inline_unsafe_access( is_native_ptr, !is_store, T_BYTE,    !is_volatile);
756  case vmIntrinsics::_getShort_raw:             return inline_unsafe_access( is_native_ptr, !is_store, T_SHORT,   !is_volatile);
757  case vmIntrinsics::_getChar_raw:              return inline_unsafe_access( is_native_ptr, !is_store, T_CHAR,    !is_volatile);
758  case vmIntrinsics::_getInt_raw:               return inline_unsafe_access( is_native_ptr, !is_store, T_INT,     !is_volatile);
759  case vmIntrinsics::_getLong_raw:              return inline_unsafe_access( is_native_ptr, !is_store, T_LONG,    !is_volatile);
760  case vmIntrinsics::_getFloat_raw:             return inline_unsafe_access( is_native_ptr, !is_store, T_FLOAT,   !is_volatile);
761  case vmIntrinsics::_getDouble_raw:            return inline_unsafe_access( is_native_ptr, !is_store, T_DOUBLE,  !is_volatile);
762  case vmIntrinsics::_getAddress_raw:           return inline_unsafe_access( is_native_ptr, !is_store, T_ADDRESS, !is_volatile);
763
764  case vmIntrinsics::_putByte_raw:              return inline_unsafe_access( is_native_ptr,  is_store, T_BYTE,    !is_volatile);
765  case vmIntrinsics::_putShort_raw:             return inline_unsafe_access( is_native_ptr,  is_store, T_SHORT,   !is_volatile);
766  case vmIntrinsics::_putChar_raw:              return inline_unsafe_access( is_native_ptr,  is_store, T_CHAR,    !is_volatile);
767  case vmIntrinsics::_putInt_raw:               return inline_unsafe_access( is_native_ptr,  is_store, T_INT,     !is_volatile);
768  case vmIntrinsics::_putLong_raw:              return inline_unsafe_access( is_native_ptr,  is_store, T_LONG,    !is_volatile);
769  case vmIntrinsics::_putFloat_raw:             return inline_unsafe_access( is_native_ptr,  is_store, T_FLOAT,   !is_volatile);
770  case vmIntrinsics::_putDouble_raw:            return inline_unsafe_access( is_native_ptr,  is_store, T_DOUBLE,  !is_volatile);
771  case vmIntrinsics::_putAddress_raw:           return inline_unsafe_access( is_native_ptr,  is_store, T_ADDRESS, !is_volatile);
772
773  case vmIntrinsics::_getObjectVolatile:        return inline_unsafe_access(!is_native_ptr, !is_store, T_OBJECT,   is_volatile);
774  case vmIntrinsics::_getBooleanVolatile:       return inline_unsafe_access(!is_native_ptr, !is_store, T_BOOLEAN,  is_volatile);
775  case vmIntrinsics::_getByteVolatile:          return inline_unsafe_access(!is_native_ptr, !is_store, T_BYTE,     is_volatile);
776  case vmIntrinsics::_getShortVolatile:         return inline_unsafe_access(!is_native_ptr, !is_store, T_SHORT,    is_volatile);
777  case vmIntrinsics::_getCharVolatile:          return inline_unsafe_access(!is_native_ptr, !is_store, T_CHAR,     is_volatile);
778  case vmIntrinsics::_getIntVolatile:           return inline_unsafe_access(!is_native_ptr, !is_store, T_INT,      is_volatile);
779  case vmIntrinsics::_getLongVolatile:          return inline_unsafe_access(!is_native_ptr, !is_store, T_LONG,     is_volatile);
780  case vmIntrinsics::_getFloatVolatile:         return inline_unsafe_access(!is_native_ptr, !is_store, T_FLOAT,    is_volatile);
781  case vmIntrinsics::_getDoubleVolatile:        return inline_unsafe_access(!is_native_ptr, !is_store, T_DOUBLE,   is_volatile);
782
783  case vmIntrinsics::_putObjectVolatile:        return inline_unsafe_access(!is_native_ptr,  is_store, T_OBJECT,   is_volatile);
784  case vmIntrinsics::_putBooleanVolatile:       return inline_unsafe_access(!is_native_ptr,  is_store, T_BOOLEAN,  is_volatile);
785  case vmIntrinsics::_putByteVolatile:          return inline_unsafe_access(!is_native_ptr,  is_store, T_BYTE,     is_volatile);
786  case vmIntrinsics::_putShortVolatile:         return inline_unsafe_access(!is_native_ptr,  is_store, T_SHORT,    is_volatile);
787  case vmIntrinsics::_putCharVolatile:          return inline_unsafe_access(!is_native_ptr,  is_store, T_CHAR,     is_volatile);
788  case vmIntrinsics::_putIntVolatile:           return inline_unsafe_access(!is_native_ptr,  is_store, T_INT,      is_volatile);
789  case vmIntrinsics::_putLongVolatile:          return inline_unsafe_access(!is_native_ptr,  is_store, T_LONG,     is_volatile);
790  case vmIntrinsics::_putFloatVolatile:         return inline_unsafe_access(!is_native_ptr,  is_store, T_FLOAT,    is_volatile);
791  case vmIntrinsics::_putDoubleVolatile:        return inline_unsafe_access(!is_native_ptr,  is_store, T_DOUBLE,   is_volatile);
792
793  case vmIntrinsics::_prefetchRead:             return inline_unsafe_prefetch(!is_native_ptr, !is_store, !is_static);
794  case vmIntrinsics::_prefetchWrite:            return inline_unsafe_prefetch(!is_native_ptr,  is_store, !is_static);
795  case vmIntrinsics::_prefetchReadStatic:       return inline_unsafe_prefetch(!is_native_ptr, !is_store,  is_static);
796  case vmIntrinsics::_prefetchWriteStatic:      return inline_unsafe_prefetch(!is_native_ptr,  is_store,  is_static);
797
798  case vmIntrinsics::_compareAndSwapObject:     return inline_unsafe_load_store(T_OBJECT, LS_cmpxchg);
799  case vmIntrinsics::_compareAndSwapInt:        return inline_unsafe_load_store(T_INT,    LS_cmpxchg);
800  case vmIntrinsics::_compareAndSwapLong:       return inline_unsafe_load_store(T_LONG,   LS_cmpxchg);
801
802  case vmIntrinsics::_putOrderedObject:         return inline_unsafe_ordered_store(T_OBJECT);
803  case vmIntrinsics::_putOrderedInt:            return inline_unsafe_ordered_store(T_INT);
804  case vmIntrinsics::_putOrderedLong:           return inline_unsafe_ordered_store(T_LONG);
805
806  case vmIntrinsics::_getAndAddInt:             return inline_unsafe_load_store(T_INT,    LS_xadd);
807  case vmIntrinsics::_getAndAddLong:            return inline_unsafe_load_store(T_LONG,   LS_xadd);
808  case vmIntrinsics::_getAndSetInt:             return inline_unsafe_load_store(T_INT,    LS_xchg);
809  case vmIntrinsics::_getAndSetLong:            return inline_unsafe_load_store(T_LONG,   LS_xchg);
810  case vmIntrinsics::_getAndSetObject:          return inline_unsafe_load_store(T_OBJECT, LS_xchg);
811
812  case vmIntrinsics::_loadFence:
813  case vmIntrinsics::_storeFence:
814  case vmIntrinsics::_fullFence:                return inline_unsafe_fence(intrinsic_id());
815
816  case vmIntrinsics::_currentThread:            return inline_native_currentThread();
817  case vmIntrinsics::_isInterrupted:            return inline_native_isInterrupted();
818
819#ifdef TRACE_HAVE_INTRINSICS
820  case vmIntrinsics::_classID:                  return inline_native_classID();
821  case vmIntrinsics::_threadID:                 return inline_native_threadID();
822  case vmIntrinsics::_counterTime:              return inline_native_time_funcs(CAST_FROM_FN_PTR(address, TRACE_TIME_METHOD), "counterTime");
823#endif
824  case vmIntrinsics::_currentTimeMillis:        return inline_native_time_funcs(CAST_FROM_FN_PTR(address, os::javaTimeMillis), "currentTimeMillis");
825  case vmIntrinsics::_nanoTime:                 return inline_native_time_funcs(CAST_FROM_FN_PTR(address, os::javaTimeNanos), "nanoTime");
826  case vmIntrinsics::_allocateInstance:         return inline_unsafe_allocate();
827  case vmIntrinsics::_copyMemory:               return inline_unsafe_copyMemory();
828  case vmIntrinsics::_newArray:                 return inline_native_newArray();
829  case vmIntrinsics::_getLength:                return inline_native_getLength();
830  case vmIntrinsics::_copyOf:                   return inline_array_copyOf(false);
831  case vmIntrinsics::_copyOfRange:              return inline_array_copyOf(true);
832  case vmIntrinsics::_equalsC:                  return inline_array_equals();
833  case vmIntrinsics::_clone:                    return inline_native_clone(intrinsic()->is_virtual());
834
835  case vmIntrinsics::_isAssignableFrom:         return inline_native_subtype_check();
836
837  case vmIntrinsics::_isInstance:
838  case vmIntrinsics::_getModifiers:
839  case vmIntrinsics::_isInterface:
840  case vmIntrinsics::_isArray:
841  case vmIntrinsics::_isPrimitive:
842  case vmIntrinsics::_getSuperclass:
843  case vmIntrinsics::_getComponentType:
844  case vmIntrinsics::_getClassAccessFlags:      return inline_native_Class_query(intrinsic_id());
845
846  case vmIntrinsics::_floatToRawIntBits:
847  case vmIntrinsics::_floatToIntBits:
848  case vmIntrinsics::_intBitsToFloat:
849  case vmIntrinsics::_doubleToRawLongBits:
850  case vmIntrinsics::_doubleToLongBits:
851  case vmIntrinsics::_longBitsToDouble:         return inline_fp_conversions(intrinsic_id());
852
853  case vmIntrinsics::_numberOfLeadingZeros_i:
854  case vmIntrinsics::_numberOfLeadingZeros_l:
855  case vmIntrinsics::_numberOfTrailingZeros_i:
856  case vmIntrinsics::_numberOfTrailingZeros_l:
857  case vmIntrinsics::_bitCount_i:
858  case vmIntrinsics::_bitCount_l:
859  case vmIntrinsics::_reverseBytes_i:
860  case vmIntrinsics::_reverseBytes_l:
861  case vmIntrinsics::_reverseBytes_s:
862  case vmIntrinsics::_reverseBytes_c:           return inline_number_methods(intrinsic_id());
863
864  case vmIntrinsics::_getCallerClass:           return inline_native_Reflection_getCallerClass();
865
866  case vmIntrinsics::_Reference_get:            return inline_reference_get();
867
868  case vmIntrinsics::_aescrypt_encryptBlock:
869  case vmIntrinsics::_aescrypt_decryptBlock:    return inline_aescrypt_Block(intrinsic_id());
870
871  case vmIntrinsics::_cipherBlockChaining_encryptAESCrypt:
872  case vmIntrinsics::_cipherBlockChaining_decryptAESCrypt:
873    return inline_cipherBlockChaining_AESCrypt(intrinsic_id());
874
875  case vmIntrinsics::_encodeISOArray:
876    return inline_encodeISOArray();
877
878  case vmIntrinsics::_updateCRC32:
879    return inline_updateCRC32();
880  case vmIntrinsics::_updateBytesCRC32:
881    return inline_updateBytesCRC32();
882  case vmIntrinsics::_updateByteBufferCRC32:
883    return inline_updateByteBufferCRC32();
884
885  default:
886    // If you get here, it may be that someone has added a new intrinsic
887    // to the list in vmSymbols.hpp without implementing it here.
888#ifndef PRODUCT
889    if ((PrintMiscellaneous && (Verbose || WizardMode)) || PrintOpto) {
890      tty->print_cr("*** Warning: Unimplemented intrinsic %s(%d)",
891                    vmIntrinsics::name_at(intrinsic_id()), intrinsic_id());
892    }
893#endif
894    return false;
895  }
896}
897
898Node* LibraryCallKit::try_to_predicate() {
899  if (!jvms()->has_method()) {
900    // Root JVMState has a null method.
901    assert(map()->memory()->Opcode() == Op_Parm, "");
902    // Insert the memory aliasing node
903    set_all_memory(reset_memory());
904  }
905  assert(merged_memory(), "");
906
907  switch (intrinsic_id()) {
908  case vmIntrinsics::_cipherBlockChaining_encryptAESCrypt:
909    return inline_cipherBlockChaining_AESCrypt_predicate(false);
910  case vmIntrinsics::_cipherBlockChaining_decryptAESCrypt:
911    return inline_cipherBlockChaining_AESCrypt_predicate(true);
912
913  default:
914    // If you get here, it may be that someone has added a new intrinsic
915    // to the list in vmSymbols.hpp without implementing it here.
916#ifndef PRODUCT
917    if ((PrintMiscellaneous && (Verbose || WizardMode)) || PrintOpto) {
918      tty->print_cr("*** Warning: Unimplemented predicate for intrinsic %s(%d)",
919                    vmIntrinsics::name_at(intrinsic_id()), intrinsic_id());
920    }
921#endif
922    Node* slow_ctl = control();
923    set_control(top()); // No fast path instrinsic
924    return slow_ctl;
925  }
926}
927
928//------------------------------set_result-------------------------------
929// Helper function for finishing intrinsics.
930void LibraryCallKit::set_result(RegionNode* region, PhiNode* value) {
931  record_for_igvn(region);
932  set_control(_gvn.transform(region));
933  set_result( _gvn.transform(value));
934  assert(value->type()->basic_type() == result()->bottom_type()->basic_type(), "sanity");
935}
936
937//------------------------------generate_guard---------------------------
938// Helper function for generating guarded fast-slow graph structures.
939// The given 'test', if true, guards a slow path.  If the test fails
940// then a fast path can be taken.  (We generally hope it fails.)
941// In all cases, GraphKit::control() is updated to the fast path.
942// The returned value represents the control for the slow path.
943// The return value is never 'top'; it is either a valid control
944// or NULL if it is obvious that the slow path can never be taken.
945// Also, if region and the slow control are not NULL, the slow edge
946// is appended to the region.
947Node* LibraryCallKit::generate_guard(Node* test, RegionNode* region, float true_prob) {
948  if (stopped()) {
949    // Already short circuited.
950    return NULL;
951  }
952
953  // Build an if node and its projections.
954  // If test is true we take the slow path, which we assume is uncommon.
955  if (_gvn.type(test) == TypeInt::ZERO) {
956    // The slow branch is never taken.  No need to build this guard.
957    return NULL;
958  }
959
960  IfNode* iff = create_and_map_if(control(), test, true_prob, COUNT_UNKNOWN);
961
962  Node* if_slow = _gvn.transform(new (C) IfTrueNode(iff));
963  if (if_slow == top()) {
964    // The slow branch is never taken.  No need to build this guard.
965    return NULL;
966  }
967
968  if (region != NULL)
969    region->add_req(if_slow);
970
971  Node* if_fast = _gvn.transform(new (C) IfFalseNode(iff));
972  set_control(if_fast);
973
974  return if_slow;
975}
976
977inline Node* LibraryCallKit::generate_slow_guard(Node* test, RegionNode* region) {
978  return generate_guard(test, region, PROB_UNLIKELY_MAG(3));
979}
980inline Node* LibraryCallKit::generate_fair_guard(Node* test, RegionNode* region) {
981  return generate_guard(test, region, PROB_FAIR);
982}
983
984inline Node* LibraryCallKit::generate_negative_guard(Node* index, RegionNode* region,
985                                                     Node* *pos_index) {
986  if (stopped())
987    return NULL;                // already stopped
988  if (_gvn.type(index)->higher_equal(TypeInt::POS)) // [0,maxint]
989    return NULL;                // index is already adequately typed
990  Node* cmp_lt = _gvn.transform(new (C) CmpINode(index, intcon(0)));
991  Node* bol_lt = _gvn.transform(new (C) BoolNode(cmp_lt, BoolTest::lt));
992  Node* is_neg = generate_guard(bol_lt, region, PROB_MIN);
993  if (is_neg != NULL && pos_index != NULL) {
994    // Emulate effect of Parse::adjust_map_after_if.
995    Node* ccast = new (C) CastIINode(index, TypeInt::POS);
996    ccast->set_req(0, control());
997    (*pos_index) = _gvn.transform(ccast);
998  }
999  return is_neg;
1000}
1001
1002inline Node* LibraryCallKit::generate_nonpositive_guard(Node* index, bool never_negative,
1003                                                        Node* *pos_index) {
1004  if (stopped())
1005    return NULL;                // already stopped
1006  if (_gvn.type(index)->higher_equal(TypeInt::POS1)) // [1,maxint]
1007    return NULL;                // index is already adequately typed
1008  Node* cmp_le = _gvn.transform(new (C) CmpINode(index, intcon(0)));
1009  BoolTest::mask le_or_eq = (never_negative ? BoolTest::eq : BoolTest::le);
1010  Node* bol_le = _gvn.transform(new (C) BoolNode(cmp_le, le_or_eq));
1011  Node* is_notp = generate_guard(bol_le, NULL, PROB_MIN);
1012  if (is_notp != NULL && pos_index != NULL) {
1013    // Emulate effect of Parse::adjust_map_after_if.
1014    Node* ccast = new (C) CastIINode(index, TypeInt::POS1);
1015    ccast->set_req(0, control());
1016    (*pos_index) = _gvn.transform(ccast);
1017  }
1018  return is_notp;
1019}
1020
1021// Make sure that 'position' is a valid limit index, in [0..length].
1022// There are two equivalent plans for checking this:
1023//   A. (offset + copyLength)  unsigned<=  arrayLength
1024//   B. offset  <=  (arrayLength - copyLength)
1025// We require that all of the values above, except for the sum and
1026// difference, are already known to be non-negative.
1027// Plan A is robust in the face of overflow, if offset and copyLength
1028// are both hugely positive.
1029//
1030// Plan B is less direct and intuitive, but it does not overflow at
1031// all, since the difference of two non-negatives is always
1032// representable.  Whenever Java methods must perform the equivalent
1033// check they generally use Plan B instead of Plan A.
1034// For the moment we use Plan A.
1035inline Node* LibraryCallKit::generate_limit_guard(Node* offset,
1036                                                  Node* subseq_length,
1037                                                  Node* array_length,
1038                                                  RegionNode* region) {
1039  if (stopped())
1040    return NULL;                // already stopped
1041  bool zero_offset = _gvn.type(offset) == TypeInt::ZERO;
1042  if (zero_offset && subseq_length->eqv_uncast(array_length))
1043    return NULL;                // common case of whole-array copy
1044  Node* last = subseq_length;
1045  if (!zero_offset)             // last += offset
1046    last = _gvn.transform(new (C) AddINode(last, offset));
1047  Node* cmp_lt = _gvn.transform(new (C) CmpUNode(array_length, last));
1048  Node* bol_lt = _gvn.transform(new (C) BoolNode(cmp_lt, BoolTest::lt));
1049  Node* is_over = generate_guard(bol_lt, region, PROB_MIN);
1050  return is_over;
1051}
1052
1053
1054//--------------------------generate_current_thread--------------------
1055Node* LibraryCallKit::generate_current_thread(Node* &tls_output) {
1056  ciKlass*    thread_klass = env()->Thread_klass();
1057  const Type* thread_type  = TypeOopPtr::make_from_klass(thread_klass)->cast_to_ptr_type(TypePtr::NotNull);
1058  Node* thread = _gvn.transform(new (C) ThreadLocalNode());
1059  Node* p = basic_plus_adr(top()/*!oop*/, thread, in_bytes(JavaThread::threadObj_offset()));
1060  Node* threadObj = make_load(NULL, p, thread_type, T_OBJECT, MemNode::unordered);
1061  tls_output = thread;
1062  return threadObj;
1063}
1064
1065
1066//------------------------------make_string_method_node------------------------
1067// Helper method for String intrinsic functions. This version is called
1068// with str1 and str2 pointing to String object nodes.
1069//
1070Node* LibraryCallKit::make_string_method_node(int opcode, Node* str1, Node* str2) {
1071  Node* no_ctrl = NULL;
1072
1073  // Get start addr of string
1074  Node* str1_value   = load_String_value(no_ctrl, str1);
1075  Node* str1_offset  = load_String_offset(no_ctrl, str1);
1076  Node* str1_start   = array_element_address(str1_value, str1_offset, T_CHAR);
1077
1078  // Get length of string 1
1079  Node* str1_len  = load_String_length(no_ctrl, str1);
1080
1081  Node* str2_value   = load_String_value(no_ctrl, str2);
1082  Node* str2_offset  = load_String_offset(no_ctrl, str2);
1083  Node* str2_start   = array_element_address(str2_value, str2_offset, T_CHAR);
1084
1085  Node* str2_len = NULL;
1086  Node* result = NULL;
1087
1088  switch (opcode) {
1089  case Op_StrIndexOf:
1090    // Get length of string 2
1091    str2_len = load_String_length(no_ctrl, str2);
1092
1093    result = new (C) StrIndexOfNode(control(), memory(TypeAryPtr::CHARS),
1094                                 str1_start, str1_len, str2_start, str2_len);
1095    break;
1096  case Op_StrComp:
1097    // Get length of string 2
1098    str2_len = load_String_length(no_ctrl, str2);
1099
1100    result = new (C) StrCompNode(control(), memory(TypeAryPtr::CHARS),
1101                                 str1_start, str1_len, str2_start, str2_len);
1102    break;
1103  case Op_StrEquals:
1104    result = new (C) StrEqualsNode(control(), memory(TypeAryPtr::CHARS),
1105                               str1_start, str2_start, str1_len);
1106    break;
1107  default:
1108    ShouldNotReachHere();
1109    return NULL;
1110  }
1111
1112  // All these intrinsics have checks.
1113  C->set_has_split_ifs(true); // Has chance for split-if optimization
1114
1115  return _gvn.transform(result);
1116}
1117
1118// Helper method for String intrinsic functions. This version is called
1119// with str1 and str2 pointing to char[] nodes, with cnt1 and cnt2 pointing
1120// to Int nodes containing the lenghts of str1 and str2.
1121//
1122Node* LibraryCallKit::make_string_method_node(int opcode, Node* str1_start, Node* cnt1, Node* str2_start, Node* cnt2) {
1123  Node* result = NULL;
1124  switch (opcode) {
1125  case Op_StrIndexOf:
1126    result = new (C) StrIndexOfNode(control(), memory(TypeAryPtr::CHARS),
1127                                 str1_start, cnt1, str2_start, cnt2);
1128    break;
1129  case Op_StrComp:
1130    result = new (C) StrCompNode(control(), memory(TypeAryPtr::CHARS),
1131                                 str1_start, cnt1, str2_start, cnt2);
1132    break;
1133  case Op_StrEquals:
1134    result = new (C) StrEqualsNode(control(), memory(TypeAryPtr::CHARS),
1135                                 str1_start, str2_start, cnt1);
1136    break;
1137  default:
1138    ShouldNotReachHere();
1139    return NULL;
1140  }
1141
1142  // All these intrinsics have checks.
1143  C->set_has_split_ifs(true); // Has chance for split-if optimization
1144
1145  return _gvn.transform(result);
1146}
1147
1148//------------------------------inline_string_compareTo------------------------
1149// public int java.lang.String.compareTo(String anotherString);
1150bool LibraryCallKit::inline_string_compareTo() {
1151  Node* receiver = null_check(argument(0));
1152  Node* arg      = null_check(argument(1));
1153  if (stopped()) {
1154    return true;
1155  }
1156  set_result(make_string_method_node(Op_StrComp, receiver, arg));
1157  return true;
1158}
1159
1160//------------------------------inline_string_equals------------------------
1161bool LibraryCallKit::inline_string_equals() {
1162  Node* receiver = null_check_receiver();
1163  // NOTE: Do not null check argument for String.equals() because spec
1164  // allows to specify NULL as argument.
1165  Node* argument = this->argument(1);
1166  if (stopped()) {
1167    return true;
1168  }
1169
1170  // paths (plus control) merge
1171  RegionNode* region = new (C) RegionNode(5);
1172  Node* phi = new (C) PhiNode(region, TypeInt::BOOL);
1173
1174  // does source == target string?
1175  Node* cmp = _gvn.transform(new (C) CmpPNode(receiver, argument));
1176  Node* bol = _gvn.transform(new (C) BoolNode(cmp, BoolTest::eq));
1177
1178  Node* if_eq = generate_slow_guard(bol, NULL);
1179  if (if_eq != NULL) {
1180    // receiver == argument
1181    phi->init_req(2, intcon(1));
1182    region->init_req(2, if_eq);
1183  }
1184
1185  // get String klass for instanceOf
1186  ciInstanceKlass* klass = env()->String_klass();
1187
1188  if (!stopped()) {
1189    Node* inst = gen_instanceof(argument, makecon(TypeKlassPtr::make(klass)));
1190    Node* cmp  = _gvn.transform(new (C) CmpINode(inst, intcon(1)));
1191    Node* bol  = _gvn.transform(new (C) BoolNode(cmp, BoolTest::ne));
1192
1193    Node* inst_false = generate_guard(bol, NULL, PROB_MIN);
1194    //instanceOf == true, fallthrough
1195
1196    if (inst_false != NULL) {
1197      phi->init_req(3, intcon(0));
1198      region->init_req(3, inst_false);
1199    }
1200  }
1201
1202  if (!stopped()) {
1203    const TypeOopPtr* string_type = TypeOopPtr::make_from_klass(klass);
1204
1205    // Properly cast the argument to String
1206    argument = _gvn.transform(new (C) CheckCastPPNode(control(), argument, string_type));
1207    // This path is taken only when argument's type is String:NotNull.
1208    argument = cast_not_null(argument, false);
1209
1210    Node* no_ctrl = NULL;
1211
1212    // Get start addr of receiver
1213    Node* receiver_val    = load_String_value(no_ctrl, receiver);
1214    Node* receiver_offset = load_String_offset(no_ctrl, receiver);
1215    Node* receiver_start = array_element_address(receiver_val, receiver_offset, T_CHAR);
1216
1217    // Get length of receiver
1218    Node* receiver_cnt  = load_String_length(no_ctrl, receiver);
1219
1220    // Get start addr of argument
1221    Node* argument_val    = load_String_value(no_ctrl, argument);
1222    Node* argument_offset = load_String_offset(no_ctrl, argument);
1223    Node* argument_start = array_element_address(argument_val, argument_offset, T_CHAR);
1224
1225    // Get length of argument
1226    Node* argument_cnt  = load_String_length(no_ctrl, argument);
1227
1228    // Check for receiver count != argument count
1229    Node* cmp = _gvn.transform(new(C) CmpINode(receiver_cnt, argument_cnt));
1230    Node* bol = _gvn.transform(new(C) BoolNode(cmp, BoolTest::ne));
1231    Node* if_ne = generate_slow_guard(bol, NULL);
1232    if (if_ne != NULL) {
1233      phi->init_req(4, intcon(0));
1234      region->init_req(4, if_ne);
1235    }
1236
1237    // Check for count == 0 is done by assembler code for StrEquals.
1238
1239    if (!stopped()) {
1240      Node* equals = make_string_method_node(Op_StrEquals, receiver_start, receiver_cnt, argument_start, argument_cnt);
1241      phi->init_req(1, equals);
1242      region->init_req(1, control());
1243    }
1244  }
1245
1246  // post merge
1247  set_control(_gvn.transform(region));
1248  record_for_igvn(region);
1249
1250  set_result(_gvn.transform(phi));
1251  return true;
1252}
1253
1254//------------------------------inline_array_equals----------------------------
1255bool LibraryCallKit::inline_array_equals() {
1256  Node* arg1 = argument(0);
1257  Node* arg2 = argument(1);
1258  set_result(_gvn.transform(new (C) AryEqNode(control(), memory(TypeAryPtr::CHARS), arg1, arg2)));
1259  return true;
1260}
1261
1262// Java version of String.indexOf(constant string)
1263// class StringDecl {
1264//   StringDecl(char[] ca) {
1265//     offset = 0;
1266//     count = ca.length;
1267//     value = ca;
1268//   }
1269//   int offset;
1270//   int count;
1271//   char[] value;
1272// }
1273//
1274// static int string_indexOf_J(StringDecl string_object, char[] target_object,
1275//                             int targetOffset, int cache_i, int md2) {
1276//   int cache = cache_i;
1277//   int sourceOffset = string_object.offset;
1278//   int sourceCount = string_object.count;
1279//   int targetCount = target_object.length;
1280//
1281//   int targetCountLess1 = targetCount - 1;
1282//   int sourceEnd = sourceOffset + sourceCount - targetCountLess1;
1283//
1284//   char[] source = string_object.value;
1285//   char[] target = target_object;
1286//   int lastChar = target[targetCountLess1];
1287//
1288//  outer_loop:
1289//   for (int i = sourceOffset; i < sourceEnd; ) {
1290//     int src = source[i + targetCountLess1];
1291//     if (src == lastChar) {
1292//       // With random strings and a 4-character alphabet,
1293//       // reverse matching at this point sets up 0.8% fewer
1294//       // frames, but (paradoxically) makes 0.3% more probes.
1295//       // Since those probes are nearer the lastChar probe,
1296//       // there is may be a net D$ win with reverse matching.
1297//       // But, reversing loop inhibits unroll of inner loop
1298//       // for unknown reason.  So, does running outer loop from
1299//       // (sourceOffset - targetCountLess1) to (sourceOffset + sourceCount)
1300//       for (int j = 0; j < targetCountLess1; j++) {
1301//         if (target[targetOffset + j] != source[i+j]) {
1302//           if ((cache & (1 << source[i+j])) == 0) {
1303//             if (md2 < j+1) {
1304//               i += j+1;
1305//               continue outer_loop;
1306//             }
1307//           }
1308//           i += md2;
1309//           continue outer_loop;
1310//         }
1311//       }
1312//       return i - sourceOffset;
1313//     }
1314//     if ((cache & (1 << src)) == 0) {
1315//       i += targetCountLess1;
1316//     } // using "i += targetCount;" and an "else i++;" causes a jump to jump.
1317//     i++;
1318//   }
1319//   return -1;
1320// }
1321
1322//------------------------------string_indexOf------------------------
1323Node* LibraryCallKit::string_indexOf(Node* string_object, ciTypeArray* target_array, jint targetOffset_i,
1324                                     jint cache_i, jint md2_i) {
1325
1326  Node* no_ctrl  = NULL;
1327  float likely   = PROB_LIKELY(0.9);
1328  float unlikely = PROB_UNLIKELY(0.9);
1329
1330  const int nargs = 0; // no arguments to push back for uncommon trap in predicate
1331
1332  Node* source        = load_String_value(no_ctrl, string_object);
1333  Node* sourceOffset  = load_String_offset(no_ctrl, string_object);
1334  Node* sourceCount   = load_String_length(no_ctrl, string_object);
1335
1336  Node* target = _gvn.transform( makecon(TypeOopPtr::make_from_constant(target_array, true)));
1337  jint target_length = target_array->length();
1338  const TypeAry* target_array_type = TypeAry::make(TypeInt::CHAR, TypeInt::make(0, target_length, Type::WidenMin));
1339  const TypeAryPtr* target_type = TypeAryPtr::make(TypePtr::BotPTR, target_array_type, target_array->klass(), true, Type::OffsetBot);
1340
1341  // String.value field is known to be @Stable.
1342  if (UseImplicitStableValues) {
1343    target = cast_array_to_stable(target, target_type);
1344  }
1345
1346  IdealKit kit(this, false, true);
1347#define __ kit.
1348  Node* zero             = __ ConI(0);
1349  Node* one              = __ ConI(1);
1350  Node* cache            = __ ConI(cache_i);
1351  Node* md2              = __ ConI(md2_i);
1352  Node* lastChar         = __ ConI(target_array->char_at(target_length - 1));
1353  Node* targetCount      = __ ConI(target_length);
1354  Node* targetCountLess1 = __ ConI(target_length - 1);
1355  Node* targetOffset     = __ ConI(targetOffset_i);
1356  Node* sourceEnd        = __ SubI(__ AddI(sourceOffset, sourceCount), targetCountLess1);
1357
1358  IdealVariable rtn(kit), i(kit), j(kit); __ declarations_done();
1359  Node* outer_loop = __ make_label(2 /* goto */);
1360  Node* return_    = __ make_label(1);
1361
1362  __ set(rtn,__ ConI(-1));
1363  __ loop(this, nargs, i, sourceOffset, BoolTest::lt, sourceEnd); {
1364       Node* i2  = __ AddI(__ value(i), targetCountLess1);
1365       // pin to prohibit loading of "next iteration" value which may SEGV (rare)
1366       Node* src = load_array_element(__ ctrl(), source, i2, TypeAryPtr::CHARS);
1367       __ if_then(src, BoolTest::eq, lastChar, unlikely); {
1368         __ loop(this, nargs, j, zero, BoolTest::lt, targetCountLess1); {
1369              Node* tpj = __ AddI(targetOffset, __ value(j));
1370              Node* targ = load_array_element(no_ctrl, target, tpj, target_type);
1371              Node* ipj  = __ AddI(__ value(i), __ value(j));
1372              Node* src2 = load_array_element(no_ctrl, source, ipj, TypeAryPtr::CHARS);
1373              __ if_then(targ, BoolTest::ne, src2); {
1374                __ if_then(__ AndI(cache, __ LShiftI(one, src2)), BoolTest::eq, zero); {
1375                  __ if_then(md2, BoolTest::lt, __ AddI(__ value(j), one)); {
1376                    __ increment(i, __ AddI(__ value(j), one));
1377                    __ goto_(outer_loop);
1378                  } __ end_if(); __ dead(j);
1379                }__ end_if(); __ dead(j);
1380                __ increment(i, md2);
1381                __ goto_(outer_loop);
1382              }__ end_if();
1383              __ increment(j, one);
1384         }__ end_loop(); __ dead(j);
1385         __ set(rtn, __ SubI(__ value(i), sourceOffset)); __ dead(i);
1386         __ goto_(return_);
1387       }__ end_if();
1388       __ if_then(__ AndI(cache, __ LShiftI(one, src)), BoolTest::eq, zero, likely); {
1389         __ increment(i, targetCountLess1);
1390       }__ end_if();
1391       __ increment(i, one);
1392       __ bind(outer_loop);
1393  }__ end_loop(); __ dead(i);
1394  __ bind(return_);
1395
1396  // Final sync IdealKit and GraphKit.
1397  final_sync(kit);
1398  Node* result = __ value(rtn);
1399#undef __
1400  C->set_has_loops(true);
1401  return result;
1402}
1403
1404//------------------------------inline_string_indexOf------------------------
1405bool LibraryCallKit::inline_string_indexOf() {
1406  Node* receiver = argument(0);
1407  Node* arg      = argument(1);
1408
1409  Node* result;
1410  // Disable the use of pcmpestri until it can be guaranteed that
1411  // the load doesn't cross into the uncommited space.
1412  if (Matcher::has_match_rule(Op_StrIndexOf) &&
1413      UseSSE42Intrinsics) {
1414    // Generate SSE4.2 version of indexOf
1415    // We currently only have match rules that use SSE4.2
1416
1417    receiver = null_check(receiver);
1418    arg      = null_check(arg);
1419    if (stopped()) {
1420      return true;
1421    }
1422
1423    ciInstanceKlass* str_klass = env()->String_klass();
1424    const TypeOopPtr* string_type = TypeOopPtr::make_from_klass(str_klass);
1425
1426    // Make the merge point
1427    RegionNode* result_rgn = new (C) RegionNode(4);
1428    Node*       result_phi = new (C) PhiNode(result_rgn, TypeInt::INT);
1429    Node* no_ctrl  = NULL;
1430
1431    // Get start addr of source string
1432    Node* source = load_String_value(no_ctrl, receiver);
1433    Node* source_offset = load_String_offset(no_ctrl, receiver);
1434    Node* source_start = array_element_address(source, source_offset, T_CHAR);
1435
1436    // Get length of source string
1437    Node* source_cnt  = load_String_length(no_ctrl, receiver);
1438
1439    // Get start addr of substring
1440    Node* substr = load_String_value(no_ctrl, arg);
1441    Node* substr_offset = load_String_offset(no_ctrl, arg);
1442    Node* substr_start = array_element_address(substr, substr_offset, T_CHAR);
1443
1444    // Get length of source string
1445    Node* substr_cnt  = load_String_length(no_ctrl, arg);
1446
1447    // Check for substr count > string count
1448    Node* cmp = _gvn.transform(new(C) CmpINode(substr_cnt, source_cnt));
1449    Node* bol = _gvn.transform(new(C) BoolNode(cmp, BoolTest::gt));
1450    Node* if_gt = generate_slow_guard(bol, NULL);
1451    if (if_gt != NULL) {
1452      result_phi->init_req(2, intcon(-1));
1453      result_rgn->init_req(2, if_gt);
1454    }
1455
1456    if (!stopped()) {
1457      // Check for substr count == 0
1458      cmp = _gvn.transform(new(C) CmpINode(substr_cnt, intcon(0)));
1459      bol = _gvn.transform(new(C) BoolNode(cmp, BoolTest::eq));
1460      Node* if_zero = generate_slow_guard(bol, NULL);
1461      if (if_zero != NULL) {
1462        result_phi->init_req(3, intcon(0));
1463        result_rgn->init_req(3, if_zero);
1464      }
1465    }
1466
1467    if (!stopped()) {
1468      result = make_string_method_node(Op_StrIndexOf, source_start, source_cnt, substr_start, substr_cnt);
1469      result_phi->init_req(1, result);
1470      result_rgn->init_req(1, control());
1471    }
1472    set_control(_gvn.transform(result_rgn));
1473    record_for_igvn(result_rgn);
1474    result = _gvn.transform(result_phi);
1475
1476  } else { // Use LibraryCallKit::string_indexOf
1477    // don't intrinsify if argument isn't a constant string.
1478    if (!arg->is_Con()) {
1479     return false;
1480    }
1481    const TypeOopPtr* str_type = _gvn.type(arg)->isa_oopptr();
1482    if (str_type == NULL) {
1483      return false;
1484    }
1485    ciInstanceKlass* klass = env()->String_klass();
1486    ciObject* str_const = str_type->const_oop();
1487    if (str_const == NULL || str_const->klass() != klass) {
1488      return false;
1489    }
1490    ciInstance* str = str_const->as_instance();
1491    assert(str != NULL, "must be instance");
1492
1493    ciObject* v = str->field_value_by_offset(java_lang_String::value_offset_in_bytes()).as_object();
1494    ciTypeArray* pat = v->as_type_array(); // pattern (argument) character array
1495
1496    int o;
1497    int c;
1498    if (java_lang_String::has_offset_field()) {
1499      o = str->field_value_by_offset(java_lang_String::offset_offset_in_bytes()).as_int();
1500      c = str->field_value_by_offset(java_lang_String::count_offset_in_bytes()).as_int();
1501    } else {
1502      o = 0;
1503      c = pat->length();
1504    }
1505
1506    // constant strings have no offset and count == length which
1507    // simplifies the resulting code somewhat so lets optimize for that.
1508    if (o != 0 || c != pat->length()) {
1509     return false;
1510    }
1511
1512    receiver = null_check(receiver, T_OBJECT);
1513    // NOTE: No null check on the argument is needed since it's a constant String oop.
1514    if (stopped()) {
1515      return true;
1516    }
1517
1518    // The null string as a pattern always returns 0 (match at beginning of string)
1519    if (c == 0) {
1520      set_result(intcon(0));
1521      return true;
1522    }
1523
1524    // Generate default indexOf
1525    jchar lastChar = pat->char_at(o + (c - 1));
1526    int cache = 0;
1527    int i;
1528    for (i = 0; i < c - 1; i++) {
1529      assert(i < pat->length(), "out of range");
1530      cache |= (1 << (pat->char_at(o + i) & (sizeof(cache) * BitsPerByte - 1)));
1531    }
1532
1533    int md2 = c;
1534    for (i = 0; i < c - 1; i++) {
1535      assert(i < pat->length(), "out of range");
1536      if (pat->char_at(o + i) == lastChar) {
1537        md2 = (c - 1) - i;
1538      }
1539    }
1540
1541    result = string_indexOf(receiver, pat, o, cache, md2);
1542  }
1543  set_result(result);
1544  return true;
1545}
1546
1547//--------------------------round_double_node--------------------------------
1548// Round a double node if necessary.
1549Node* LibraryCallKit::round_double_node(Node* n) {
1550  if (Matcher::strict_fp_requires_explicit_rounding && UseSSE <= 1)
1551    n = _gvn.transform(new (C) RoundDoubleNode(0, n));
1552  return n;
1553}
1554
1555//------------------------------inline_math-----------------------------------
1556// public static double Math.abs(double)
1557// public static double Math.sqrt(double)
1558// public static double Math.log(double)
1559// public static double Math.log10(double)
1560bool LibraryCallKit::inline_math(vmIntrinsics::ID id) {
1561  Node* arg = round_double_node(argument(0));
1562  Node* n;
1563  switch (id) {
1564  case vmIntrinsics::_dabs:   n = new (C) AbsDNode(                arg);  break;
1565  case vmIntrinsics::_dsqrt:  n = new (C) SqrtDNode(C, control(),  arg);  break;
1566  case vmIntrinsics::_dlog:   n = new (C) LogDNode(C, control(),   arg);  break;
1567  case vmIntrinsics::_dlog10: n = new (C) Log10DNode(C, control(), arg);  break;
1568  default:  fatal_unexpected_iid(id);  break;
1569  }
1570  set_result(_gvn.transform(n));
1571  return true;
1572}
1573
1574//------------------------------inline_trig----------------------------------
1575// Inline sin/cos/tan instructions, if possible.  If rounding is required, do
1576// argument reduction which will turn into a fast/slow diamond.
1577bool LibraryCallKit::inline_trig(vmIntrinsics::ID id) {
1578  Node* arg = round_double_node(argument(0));
1579  Node* n = NULL;
1580
1581  switch (id) {
1582  case vmIntrinsics::_dsin:  n = new (C) SinDNode(C, control(), arg);  break;
1583  case vmIntrinsics::_dcos:  n = new (C) CosDNode(C, control(), arg);  break;
1584  case vmIntrinsics::_dtan:  n = new (C) TanDNode(C, control(), arg);  break;
1585  default:  fatal_unexpected_iid(id);  break;
1586  }
1587  n = _gvn.transform(n);
1588
1589  // Rounding required?  Check for argument reduction!
1590  if (Matcher::strict_fp_requires_explicit_rounding) {
1591    static const double     pi_4 =  0.7853981633974483;
1592    static const double neg_pi_4 = -0.7853981633974483;
1593    // pi/2 in 80-bit extended precision
1594    // static const unsigned char pi_2_bits_x[] = {0x35,0xc2,0x68,0x21,0xa2,0xda,0x0f,0xc9,0xff,0x3f,0x00,0x00,0x00,0x00,0x00,0x00};
1595    // -pi/2 in 80-bit extended precision
1596    // static const unsigned char neg_pi_2_bits_x[] = {0x35,0xc2,0x68,0x21,0xa2,0xda,0x0f,0xc9,0xff,0xbf,0x00,0x00,0x00,0x00,0x00,0x00};
1597    // Cutoff value for using this argument reduction technique
1598    //static const double    pi_2_minus_epsilon =  1.564660403643354;
1599    //static const double neg_pi_2_plus_epsilon = -1.564660403643354;
1600
1601    // Pseudocode for sin:
1602    // if (x <= Math.PI / 4.0) {
1603    //   if (x >= -Math.PI / 4.0) return  fsin(x);
1604    //   if (x >= -Math.PI / 2.0) return -fcos(x + Math.PI / 2.0);
1605    // } else {
1606    //   if (x <=  Math.PI / 2.0) return  fcos(x - Math.PI / 2.0);
1607    // }
1608    // return StrictMath.sin(x);
1609
1610    // Pseudocode for cos:
1611    // if (x <= Math.PI / 4.0) {
1612    //   if (x >= -Math.PI / 4.0) return  fcos(x);
1613    //   if (x >= -Math.PI / 2.0) return  fsin(x + Math.PI / 2.0);
1614    // } else {
1615    //   if (x <=  Math.PI / 2.0) return -fsin(x - Math.PI / 2.0);
1616    // }
1617    // return StrictMath.cos(x);
1618
1619    // Actually, sticking in an 80-bit Intel value into C2 will be tough; it
1620    // requires a special machine instruction to load it.  Instead we'll try
1621    // the 'easy' case.  If we really need the extra range +/- PI/2 we'll
1622    // probably do the math inside the SIN encoding.
1623
1624    // Make the merge point
1625    RegionNode* r = new (C) RegionNode(3);
1626    Node* phi = new (C) PhiNode(r, Type::DOUBLE);
1627
1628    // Flatten arg so we need only 1 test
1629    Node *abs = _gvn.transform(new (C) AbsDNode(arg));
1630    // Node for PI/4 constant
1631    Node *pi4 = makecon(TypeD::make(pi_4));
1632    // Check PI/4 : abs(arg)
1633    Node *cmp = _gvn.transform(new (C) CmpDNode(pi4,abs));
1634    // Check: If PI/4 < abs(arg) then go slow
1635    Node *bol = _gvn.transform(new (C) BoolNode( cmp, BoolTest::lt ));
1636    // Branch either way
1637    IfNode *iff = create_and_xform_if(control(),bol, PROB_STATIC_FREQUENT, COUNT_UNKNOWN);
1638    set_control(opt_iff(r,iff));
1639
1640    // Set fast path result
1641    phi->init_req(2, n);
1642
1643    // Slow path - non-blocking leaf call
1644    Node* call = NULL;
1645    switch (id) {
1646    case vmIntrinsics::_dsin:
1647      call = make_runtime_call(RC_LEAF, OptoRuntime::Math_D_D_Type(),
1648                               CAST_FROM_FN_PTR(address, SharedRuntime::dsin),
1649                               "Sin", NULL, arg, top());
1650      break;
1651    case vmIntrinsics::_dcos:
1652      call = make_runtime_call(RC_LEAF, OptoRuntime::Math_D_D_Type(),
1653                               CAST_FROM_FN_PTR(address, SharedRuntime::dcos),
1654                               "Cos", NULL, arg, top());
1655      break;
1656    case vmIntrinsics::_dtan:
1657      call = make_runtime_call(RC_LEAF, OptoRuntime::Math_D_D_Type(),
1658                               CAST_FROM_FN_PTR(address, SharedRuntime::dtan),
1659                               "Tan", NULL, arg, top());
1660      break;
1661    }
1662    assert(control()->in(0) == call, "");
1663    Node* slow_result = _gvn.transform(new (C) ProjNode(call, TypeFunc::Parms));
1664    r->init_req(1, control());
1665    phi->init_req(1, slow_result);
1666
1667    // Post-merge
1668    set_control(_gvn.transform(r));
1669    record_for_igvn(r);
1670    n = _gvn.transform(phi);
1671
1672    C->set_has_split_ifs(true); // Has chance for split-if optimization
1673  }
1674  set_result(n);
1675  return true;
1676}
1677
1678void LibraryCallKit::finish_pow_exp(Node* result, Node* x, Node* y, const TypeFunc* call_type, address funcAddr, const char* funcName) {
1679  //-------------------
1680  //result=(result.isNaN())? funcAddr():result;
1681  // Check: If isNaN() by checking result!=result? then either trap
1682  // or go to runtime
1683  Node* cmpisnan = _gvn.transform(new (C) CmpDNode(result, result));
1684  // Build the boolean node
1685  Node* bolisnum = _gvn.transform(new (C) BoolNode(cmpisnan, BoolTest::eq));
1686
1687  if (!too_many_traps(Deoptimization::Reason_intrinsic)) {
1688    { BuildCutout unless(this, bolisnum, PROB_STATIC_FREQUENT);
1689      // The pow or exp intrinsic returned a NaN, which requires a call
1690      // to the runtime.  Recompile with the runtime call.
1691      uncommon_trap(Deoptimization::Reason_intrinsic,
1692                    Deoptimization::Action_make_not_entrant);
1693    }
1694    set_result(result);
1695  } else {
1696    // If this inlining ever returned NaN in the past, we compile a call
1697    // to the runtime to properly handle corner cases
1698
1699    IfNode* iff = create_and_xform_if(control(), bolisnum, PROB_STATIC_FREQUENT, COUNT_UNKNOWN);
1700    Node* if_slow = _gvn.transform(new (C) IfFalseNode(iff));
1701    Node* if_fast = _gvn.transform(new (C) IfTrueNode(iff));
1702
1703    if (!if_slow->is_top()) {
1704      RegionNode* result_region = new (C) RegionNode(3);
1705      PhiNode*    result_val = new (C) PhiNode(result_region, Type::DOUBLE);
1706
1707      result_region->init_req(1, if_fast);
1708      result_val->init_req(1, result);
1709
1710      set_control(if_slow);
1711
1712      const TypePtr* no_memory_effects = NULL;
1713      Node* rt = make_runtime_call(RC_LEAF, call_type, funcAddr, funcName,
1714                                   no_memory_effects,
1715                                   x, top(), y, y ? top() : NULL);
1716      Node* value = _gvn.transform(new (C) ProjNode(rt, TypeFunc::Parms+0));
1717#ifdef ASSERT
1718      Node* value_top = _gvn.transform(new (C) ProjNode(rt, TypeFunc::Parms+1));
1719      assert(value_top == top(), "second value must be top");
1720#endif
1721
1722      result_region->init_req(2, control());
1723      result_val->init_req(2, value);
1724      set_result(result_region, result_val);
1725    } else {
1726      set_result(result);
1727    }
1728  }
1729}
1730
1731//------------------------------inline_exp-------------------------------------
1732// Inline exp instructions, if possible.  The Intel hardware only misses
1733// really odd corner cases (+/- Infinity).  Just uncommon-trap them.
1734bool LibraryCallKit::inline_exp() {
1735  Node* arg = round_double_node(argument(0));
1736  Node* n   = _gvn.transform(new (C) ExpDNode(C, control(), arg));
1737
1738  finish_pow_exp(n, arg, NULL, OptoRuntime::Math_D_D_Type(), CAST_FROM_FN_PTR(address, SharedRuntime::dexp), "EXP");
1739
1740  C->set_has_split_ifs(true); // Has chance for split-if optimization
1741  return true;
1742}
1743
1744//------------------------------inline_pow-------------------------------------
1745// Inline power instructions, if possible.
1746bool LibraryCallKit::inline_pow() {
1747  // Pseudocode for pow
1748  // if (x <= 0.0) {
1749  //   long longy = (long)y;
1750  //   if ((double)longy == y) { // if y is long
1751  //     if (y + 1 == y) longy = 0; // huge number: even
1752  //     result = ((1&longy) == 0)?-DPow(abs(x), y):DPow(abs(x), y);
1753  //   } else {
1754  //     result = NaN;
1755  //   }
1756  // } else {
1757  //   result = DPow(x,y);
1758  // }
1759  // if (result != result)?  {
1760  //   result = uncommon_trap() or runtime_call();
1761  // }
1762  // return result;
1763
1764  Node* x = round_double_node(argument(0));
1765  Node* y = round_double_node(argument(2));
1766
1767  Node* result = NULL;
1768
1769  if (!too_many_traps(Deoptimization::Reason_intrinsic)) {
1770    // Short form: skip the fancy tests and just check for NaN result.
1771    result = _gvn.transform(new (C) PowDNode(C, control(), x, y));
1772  } else {
1773    // If this inlining ever returned NaN in the past, include all
1774    // checks + call to the runtime.
1775
1776    // Set the merge point for If node with condition of (x <= 0.0)
1777    // There are four possible paths to region node and phi node
1778    RegionNode *r = new (C) RegionNode(4);
1779    Node *phi = new (C) PhiNode(r, Type::DOUBLE);
1780
1781    // Build the first if node: if (x <= 0.0)
1782    // Node for 0 constant
1783    Node *zeronode = makecon(TypeD::ZERO);
1784    // Check x:0
1785    Node *cmp = _gvn.transform(new (C) CmpDNode(x, zeronode));
1786    // Check: If (x<=0) then go complex path
1787    Node *bol1 = _gvn.transform(new (C) BoolNode( cmp, BoolTest::le ));
1788    // Branch either way
1789    IfNode *if1 = create_and_xform_if(control(),bol1, PROB_STATIC_INFREQUENT, COUNT_UNKNOWN);
1790    // Fast path taken; set region slot 3
1791    Node *fast_taken = _gvn.transform(new (C) IfFalseNode(if1));
1792    r->init_req(3,fast_taken); // Capture fast-control
1793
1794    // Fast path not-taken, i.e. slow path
1795    Node *complex_path = _gvn.transform(new (C) IfTrueNode(if1));
1796
1797    // Set fast path result
1798    Node *fast_result = _gvn.transform(new (C) PowDNode(C, control(), x, y));
1799    phi->init_req(3, fast_result);
1800
1801    // Complex path
1802    // Build the second if node (if y is long)
1803    // Node for (long)y
1804    Node *longy = _gvn.transform(new (C) ConvD2LNode(y));
1805    // Node for (double)((long) y)
1806    Node *doublelongy= _gvn.transform(new (C) ConvL2DNode(longy));
1807    // Check (double)((long) y) : y
1808    Node *cmplongy= _gvn.transform(new (C) CmpDNode(doublelongy, y));
1809    // Check if (y isn't long) then go to slow path
1810
1811    Node *bol2 = _gvn.transform(new (C) BoolNode( cmplongy, BoolTest::ne ));
1812    // Branch either way
1813    IfNode *if2 = create_and_xform_if(complex_path,bol2, PROB_STATIC_INFREQUENT, COUNT_UNKNOWN);
1814    Node* ylong_path = _gvn.transform(new (C) IfFalseNode(if2));
1815
1816    Node *slow_path = _gvn.transform(new (C) IfTrueNode(if2));
1817
1818    // Calculate DPow(abs(x), y)*(1 & (long)y)
1819    // Node for constant 1
1820    Node *conone = longcon(1);
1821    // 1& (long)y
1822    Node *signnode= _gvn.transform(new (C) AndLNode(conone, longy));
1823
1824    // A huge number is always even. Detect a huge number by checking
1825    // if y + 1 == y and set integer to be tested for parity to 0.
1826    // Required for corner case:
1827    // (long)9.223372036854776E18 = max_jlong
1828    // (double)(long)9.223372036854776E18 = 9.223372036854776E18
1829    // max_jlong is odd but 9.223372036854776E18 is even
1830    Node* yplus1 = _gvn.transform(new (C) AddDNode(y, makecon(TypeD::make(1))));
1831    Node *cmpyplus1= _gvn.transform(new (C) CmpDNode(yplus1, y));
1832    Node *bolyplus1 = _gvn.transform(new (C) BoolNode( cmpyplus1, BoolTest::eq ));
1833    Node* correctedsign = NULL;
1834    if (ConditionalMoveLimit != 0) {
1835      correctedsign = _gvn.transform( CMoveNode::make(C, NULL, bolyplus1, signnode, longcon(0), TypeLong::LONG));
1836    } else {
1837      IfNode *ifyplus1 = create_and_xform_if(ylong_path,bolyplus1, PROB_FAIR, COUNT_UNKNOWN);
1838      RegionNode *r = new (C) RegionNode(3);
1839      Node *phi = new (C) PhiNode(r, TypeLong::LONG);
1840      r->init_req(1, _gvn.transform(new (C) IfFalseNode(ifyplus1)));
1841      r->init_req(2, _gvn.transform(new (C) IfTrueNode(ifyplus1)));
1842      phi->init_req(1, signnode);
1843      phi->init_req(2, longcon(0));
1844      correctedsign = _gvn.transform(phi);
1845      ylong_path = _gvn.transform(r);
1846      record_for_igvn(r);
1847    }
1848
1849    // zero node
1850    Node *conzero = longcon(0);
1851    // Check (1&(long)y)==0?
1852    Node *cmpeq1 = _gvn.transform(new (C) CmpLNode(correctedsign, conzero));
1853    // Check if (1&(long)y)!=0?, if so the result is negative
1854    Node *bol3 = _gvn.transform(new (C) BoolNode( cmpeq1, BoolTest::ne ));
1855    // abs(x)
1856    Node *absx=_gvn.transform(new (C) AbsDNode(x));
1857    // abs(x)^y
1858    Node *absxpowy = _gvn.transform(new (C) PowDNode(C, control(), absx, y));
1859    // -abs(x)^y
1860    Node *negabsxpowy = _gvn.transform(new (C) NegDNode (absxpowy));
1861    // (1&(long)y)==1?-DPow(abs(x), y):DPow(abs(x), y)
1862    Node *signresult = NULL;
1863    if (ConditionalMoveLimit != 0) {
1864      signresult = _gvn.transform( CMoveNode::make(C, NULL, bol3, absxpowy, negabsxpowy, Type::DOUBLE));
1865    } else {
1866      IfNode *ifyeven = create_and_xform_if(ylong_path,bol3, PROB_FAIR, COUNT_UNKNOWN);
1867      RegionNode *r = new (C) RegionNode(3);
1868      Node *phi = new (C) PhiNode(r, Type::DOUBLE);
1869      r->init_req(1, _gvn.transform(new (C) IfFalseNode(ifyeven)));
1870      r->init_req(2, _gvn.transform(new (C) IfTrueNode(ifyeven)));
1871      phi->init_req(1, absxpowy);
1872      phi->init_req(2, negabsxpowy);
1873      signresult = _gvn.transform(phi);
1874      ylong_path = _gvn.transform(r);
1875      record_for_igvn(r);
1876    }
1877    // Set complex path fast result
1878    r->init_req(2, ylong_path);
1879    phi->init_req(2, signresult);
1880
1881    static const jlong nan_bits = CONST64(0x7ff8000000000000);
1882    Node *slow_result = makecon(TypeD::make(*(double*)&nan_bits)); // return NaN
1883    r->init_req(1,slow_path);
1884    phi->init_req(1,slow_result);
1885
1886    // Post merge
1887    set_control(_gvn.transform(r));
1888    record_for_igvn(r);
1889    result = _gvn.transform(phi);
1890  }
1891
1892  finish_pow_exp(result, x, y, OptoRuntime::Math_DD_D_Type(), CAST_FROM_FN_PTR(address, SharedRuntime::dpow), "POW");
1893
1894  C->set_has_split_ifs(true); // Has chance for split-if optimization
1895  return true;
1896}
1897
1898//------------------------------runtime_math-----------------------------
1899bool LibraryCallKit::runtime_math(const TypeFunc* call_type, address funcAddr, const char* funcName) {
1900  assert(call_type == OptoRuntime::Math_DD_D_Type() || call_type == OptoRuntime::Math_D_D_Type(),
1901         "must be (DD)D or (D)D type");
1902
1903  // Inputs
1904  Node* a = round_double_node(argument(0));
1905  Node* b = (call_type == OptoRuntime::Math_DD_D_Type()) ? round_double_node(argument(2)) : NULL;
1906
1907  const TypePtr* no_memory_effects = NULL;
1908  Node* trig = make_runtime_call(RC_LEAF, call_type, funcAddr, funcName,
1909                                 no_memory_effects,
1910                                 a, top(), b, b ? top() : NULL);
1911  Node* value = _gvn.transform(new (C) ProjNode(trig, TypeFunc::Parms+0));
1912#ifdef ASSERT
1913  Node* value_top = _gvn.transform(new (C) ProjNode(trig, TypeFunc::Parms+1));
1914  assert(value_top == top(), "second value must be top");
1915#endif
1916
1917  set_result(value);
1918  return true;
1919}
1920
1921//------------------------------inline_math_native-----------------------------
1922bool LibraryCallKit::inline_math_native(vmIntrinsics::ID id) {
1923#define FN_PTR(f) CAST_FROM_FN_PTR(address, f)
1924  switch (id) {
1925    // These intrinsics are not properly supported on all hardware
1926  case vmIntrinsics::_dcos:   return Matcher::has_match_rule(Op_CosD)   ? inline_trig(id) :
1927    runtime_math(OptoRuntime::Math_D_D_Type(), FN_PTR(SharedRuntime::dcos),   "COS");
1928  case vmIntrinsics::_dsin:   return Matcher::has_match_rule(Op_SinD)   ? inline_trig(id) :
1929    runtime_math(OptoRuntime::Math_D_D_Type(), FN_PTR(SharedRuntime::dsin),   "SIN");
1930  case vmIntrinsics::_dtan:   return Matcher::has_match_rule(Op_TanD)   ? inline_trig(id) :
1931    runtime_math(OptoRuntime::Math_D_D_Type(), FN_PTR(SharedRuntime::dtan),   "TAN");
1932
1933  case vmIntrinsics::_dlog:   return Matcher::has_match_rule(Op_LogD)   ? inline_math(id) :
1934    runtime_math(OptoRuntime::Math_D_D_Type(), FN_PTR(SharedRuntime::dlog),   "LOG");
1935  case vmIntrinsics::_dlog10: return Matcher::has_match_rule(Op_Log10D) ? inline_math(id) :
1936    runtime_math(OptoRuntime::Math_D_D_Type(), FN_PTR(SharedRuntime::dlog10), "LOG10");
1937
1938    // These intrinsics are supported on all hardware
1939  case vmIntrinsics::_dsqrt:  return Matcher::has_match_rule(Op_SqrtD)  ? inline_math(id) : false;
1940  case vmIntrinsics::_dabs:   return Matcher::has_match_rule(Op_AbsD)   ? inline_math(id) : false;
1941
1942  case vmIntrinsics::_dexp:   return Matcher::has_match_rule(Op_ExpD)   ? inline_exp()    :
1943    runtime_math(OptoRuntime::Math_D_D_Type(),  FN_PTR(SharedRuntime::dexp),  "EXP");
1944  case vmIntrinsics::_dpow:   return Matcher::has_match_rule(Op_PowD)   ? inline_pow()    :
1945    runtime_math(OptoRuntime::Math_DD_D_Type(), FN_PTR(SharedRuntime::dpow),  "POW");
1946#undef FN_PTR
1947
1948   // These intrinsics are not yet correctly implemented
1949  case vmIntrinsics::_datan2:
1950    return false;
1951
1952  default:
1953    fatal_unexpected_iid(id);
1954    return false;
1955  }
1956}
1957
1958static bool is_simple_name(Node* n) {
1959  return (n->req() == 1         // constant
1960          || (n->is_Type() && n->as_Type()->type()->singleton())
1961          || n->is_Proj()       // parameter or return value
1962          || n->is_Phi()        // local of some sort
1963          );
1964}
1965
1966//----------------------------inline_min_max-----------------------------------
1967bool LibraryCallKit::inline_min_max(vmIntrinsics::ID id) {
1968  set_result(generate_min_max(id, argument(0), argument(1)));
1969  return true;
1970}
1971
1972void LibraryCallKit::inline_math_mathExact(Node* math) {
1973  // If we didn't get the expected opcode it means we have optimized
1974  // the node to something else and don't need the exception edge.
1975  if (!math->is_MathExact()) {
1976    set_result(math);
1977    return;
1978  }
1979
1980  Node* result = _gvn.transform( new(C) ProjNode(math, MathExactNode::result_proj_node));
1981  Node* flags = _gvn.transform( new(C) FlagsProjNode(math, MathExactNode::flags_proj_node));
1982
1983  Node* bol = _gvn.transform( new (C) BoolNode(flags, BoolTest::overflow) );
1984  IfNode* check = create_and_map_if(control(), bol, PROB_UNLIKELY_MAG(3), COUNT_UNKNOWN);
1985  Node* fast_path = _gvn.transform( new (C) IfFalseNode(check));
1986  Node* slow_path = _gvn.transform( new (C) IfTrueNode(check) );
1987
1988  {
1989    PreserveJVMState pjvms(this);
1990    PreserveReexecuteState preexecs(this);
1991    jvms()->set_should_reexecute(true);
1992
1993    set_control(slow_path);
1994    set_i_o(i_o());
1995
1996    uncommon_trap(Deoptimization::Reason_intrinsic,
1997                  Deoptimization::Action_none);
1998  }
1999
2000  set_control(fast_path);
2001  set_result(result);
2002}
2003
2004bool LibraryCallKit::inline_math_addExactI(bool is_increment) {
2005  Node* arg1 = argument(0);
2006  Node* arg2 = NULL;
2007
2008  if (is_increment) {
2009    arg2 = intcon(1);
2010  } else {
2011    arg2 = argument(1);
2012  }
2013
2014  Node* add = _gvn.transform( new(C) AddExactINode(NULL, arg1, arg2) );
2015  inline_math_mathExact(add);
2016  return true;
2017}
2018
2019bool LibraryCallKit::inline_math_addExactL(bool is_increment) {
2020  Node* arg1 = argument(0); // type long
2021  // argument(1) == TOP
2022  Node* arg2 = NULL;
2023
2024  if (is_increment) {
2025    arg2 = longcon(1);
2026  } else {
2027    arg2 = argument(2); // type long
2028    // argument(3) == TOP
2029  }
2030
2031  Node* add = _gvn.transform(new(C) AddExactLNode(NULL, arg1, arg2));
2032  inline_math_mathExact(add);
2033  return true;
2034}
2035
2036bool LibraryCallKit::inline_math_subtractExactI(bool is_decrement) {
2037  Node* arg1 = argument(0);
2038  Node* arg2 = NULL;
2039
2040  if (is_decrement) {
2041    arg2 = intcon(1);
2042  } else {
2043    arg2 = argument(1);
2044  }
2045
2046  Node* sub = _gvn.transform(new(C) SubExactINode(NULL, arg1, arg2));
2047  inline_math_mathExact(sub);
2048  return true;
2049}
2050
2051bool LibraryCallKit::inline_math_subtractExactL(bool is_decrement) {
2052  Node* arg1 = argument(0); // type long
2053  // argument(1) == TOP
2054  Node* arg2 = NULL;
2055
2056  if (is_decrement) {
2057    arg2 = longcon(1);
2058  } else {
2059    arg2 = argument(2); // type long
2060    // argument(3) == TOP
2061  }
2062
2063  Node* sub = _gvn.transform(new(C) SubExactLNode(NULL, arg1, arg2));
2064  inline_math_mathExact(sub);
2065  return true;
2066}
2067
2068bool LibraryCallKit::inline_math_negateExactI() {
2069  Node* arg1 = argument(0);
2070
2071  Node* neg = _gvn.transform(new(C) NegExactINode(NULL, arg1));
2072  inline_math_mathExact(neg);
2073  return true;
2074}
2075
2076bool LibraryCallKit::inline_math_negateExactL() {
2077  Node* arg1 = argument(0);
2078  // argument(1) == TOP
2079
2080  Node* neg = _gvn.transform(new(C) NegExactLNode(NULL, arg1));
2081  inline_math_mathExact(neg);
2082  return true;
2083}
2084
2085bool LibraryCallKit::inline_math_multiplyExactI() {
2086  Node* arg1 = argument(0);
2087  Node* arg2 = argument(1);
2088
2089  Node* mul = _gvn.transform(new(C) MulExactINode(NULL, arg1, arg2));
2090  inline_math_mathExact(mul);
2091  return true;
2092}
2093
2094bool LibraryCallKit::inline_math_multiplyExactL() {
2095  Node* arg1 = argument(0);
2096  // argument(1) == TOP
2097  Node* arg2 = argument(2);
2098  // argument(3) == TOP
2099
2100  Node* mul = _gvn.transform(new(C) MulExactLNode(NULL, arg1, arg2));
2101  inline_math_mathExact(mul);
2102  return true;
2103}
2104
2105Node*
2106LibraryCallKit::generate_min_max(vmIntrinsics::ID id, Node* x0, Node* y0) {
2107  // These are the candidate return value:
2108  Node* xvalue = x0;
2109  Node* yvalue = y0;
2110
2111  if (xvalue == yvalue) {
2112    return xvalue;
2113  }
2114
2115  bool want_max = (id == vmIntrinsics::_max);
2116
2117  const TypeInt* txvalue = _gvn.type(xvalue)->isa_int();
2118  const TypeInt* tyvalue = _gvn.type(yvalue)->isa_int();
2119  if (txvalue == NULL || tyvalue == NULL)  return top();
2120  // This is not really necessary, but it is consistent with a
2121  // hypothetical MaxINode::Value method:
2122  int widen = MAX2(txvalue->_widen, tyvalue->_widen);
2123
2124  // %%% This folding logic should (ideally) be in a different place.
2125  // Some should be inside IfNode, and there to be a more reliable
2126  // transformation of ?: style patterns into cmoves.  We also want
2127  // more powerful optimizations around cmove and min/max.
2128
2129  // Try to find a dominating comparison of these guys.
2130  // It can simplify the index computation for Arrays.copyOf
2131  // and similar uses of System.arraycopy.
2132  // First, compute the normalized version of CmpI(x, y).
2133  int   cmp_op = Op_CmpI;
2134  Node* xkey = xvalue;
2135  Node* ykey = yvalue;
2136  Node* ideal_cmpxy = _gvn.transform(new(C) CmpINode(xkey, ykey));
2137  if (ideal_cmpxy->is_Cmp()) {
2138    // E.g., if we have CmpI(length - offset, count),
2139    // it might idealize to CmpI(length, count + offset)
2140    cmp_op = ideal_cmpxy->Opcode();
2141    xkey = ideal_cmpxy->in(1);
2142    ykey = ideal_cmpxy->in(2);
2143  }
2144
2145  // Start by locating any relevant comparisons.
2146  Node* start_from = (xkey->outcnt() < ykey->outcnt()) ? xkey : ykey;
2147  Node* cmpxy = NULL;
2148  Node* cmpyx = NULL;
2149  for (DUIterator_Fast kmax, k = start_from->fast_outs(kmax); k < kmax; k++) {
2150    Node* cmp = start_from->fast_out(k);
2151    if (cmp->outcnt() > 0 &&            // must have prior uses
2152        cmp->in(0) == NULL &&           // must be context-independent
2153        cmp->Opcode() == cmp_op) {      // right kind of compare
2154      if (cmp->in(1) == xkey && cmp->in(2) == ykey)  cmpxy = cmp;
2155      if (cmp->in(1) == ykey && cmp->in(2) == xkey)  cmpyx = cmp;
2156    }
2157  }
2158
2159  const int NCMPS = 2;
2160  Node* cmps[NCMPS] = { cmpxy, cmpyx };
2161  int cmpn;
2162  for (cmpn = 0; cmpn < NCMPS; cmpn++) {
2163    if (cmps[cmpn] != NULL)  break;     // find a result
2164  }
2165  if (cmpn < NCMPS) {
2166    // Look for a dominating test that tells us the min and max.
2167    int depth = 0;                // Limit search depth for speed
2168    Node* dom = control();
2169    for (; dom != NULL; dom = IfNode::up_one_dom(dom, true)) {
2170      if (++depth >= 100)  break;
2171      Node* ifproj = dom;
2172      if (!ifproj->is_Proj())  continue;
2173      Node* iff = ifproj->in(0);
2174      if (!iff->is_If())  continue;
2175      Node* bol = iff->in(1);
2176      if (!bol->is_Bool())  continue;
2177      Node* cmp = bol->in(1);
2178      if (cmp == NULL)  continue;
2179      for (cmpn = 0; cmpn < NCMPS; cmpn++)
2180        if (cmps[cmpn] == cmp)  break;
2181      if (cmpn == NCMPS)  continue;
2182      BoolTest::mask btest = bol->as_Bool()->_test._test;
2183      if (ifproj->is_IfFalse())  btest = BoolTest(btest).negate();
2184      if (cmp->in(1) == ykey)    btest = BoolTest(btest).commute();
2185      // At this point, we know that 'x btest y' is true.
2186      switch (btest) {
2187      case BoolTest::eq:
2188        // They are proven equal, so we can collapse the min/max.
2189        // Either value is the answer.  Choose the simpler.
2190        if (is_simple_name(yvalue) && !is_simple_name(xvalue))
2191          return yvalue;
2192        return xvalue;
2193      case BoolTest::lt:          // x < y
2194      case BoolTest::le:          // x <= y
2195        return (want_max ? yvalue : xvalue);
2196      case BoolTest::gt:          // x > y
2197      case BoolTest::ge:          // x >= y
2198        return (want_max ? xvalue : yvalue);
2199      }
2200    }
2201  }
2202
2203  // We failed to find a dominating test.
2204  // Let's pick a test that might GVN with prior tests.
2205  Node*          best_bol   = NULL;
2206  BoolTest::mask best_btest = BoolTest::illegal;
2207  for (cmpn = 0; cmpn < NCMPS; cmpn++) {
2208    Node* cmp = cmps[cmpn];
2209    if (cmp == NULL)  continue;
2210    for (DUIterator_Fast jmax, j = cmp->fast_outs(jmax); j < jmax; j++) {
2211      Node* bol = cmp->fast_out(j);
2212      if (!bol->is_Bool())  continue;
2213      BoolTest::mask btest = bol->as_Bool()->_test._test;
2214      if (btest == BoolTest::eq || btest == BoolTest::ne)  continue;
2215      if (cmp->in(1) == ykey)   btest = BoolTest(btest).commute();
2216      if (bol->outcnt() > (best_bol == NULL ? 0 : best_bol->outcnt())) {
2217        best_bol   = bol->as_Bool();
2218        best_btest = btest;
2219      }
2220    }
2221  }
2222
2223  Node* answer_if_true  = NULL;
2224  Node* answer_if_false = NULL;
2225  switch (best_btest) {
2226  default:
2227    if (cmpxy == NULL)
2228      cmpxy = ideal_cmpxy;
2229    best_bol = _gvn.transform(new(C) BoolNode(cmpxy, BoolTest::lt));
2230    // and fall through:
2231  case BoolTest::lt:          // x < y
2232  case BoolTest::le:          // x <= y
2233    answer_if_true  = (want_max ? yvalue : xvalue);
2234    answer_if_false = (want_max ? xvalue : yvalue);
2235    break;
2236  case BoolTest::gt:          // x > y
2237  case BoolTest::ge:          // x >= y
2238    answer_if_true  = (want_max ? xvalue : yvalue);
2239    answer_if_false = (want_max ? yvalue : xvalue);
2240    break;
2241  }
2242
2243  jint hi, lo;
2244  if (want_max) {
2245    // We can sharpen the minimum.
2246    hi = MAX2(txvalue->_hi, tyvalue->_hi);
2247    lo = MAX2(txvalue->_lo, tyvalue->_lo);
2248  } else {
2249    // We can sharpen the maximum.
2250    hi = MIN2(txvalue->_hi, tyvalue->_hi);
2251    lo = MIN2(txvalue->_lo, tyvalue->_lo);
2252  }
2253
2254  // Use a flow-free graph structure, to avoid creating excess control edges
2255  // which could hinder other optimizations.
2256  // Since Math.min/max is often used with arraycopy, we want
2257  // tightly_coupled_allocation to be able to see beyond min/max expressions.
2258  Node* cmov = CMoveNode::make(C, NULL, best_bol,
2259                               answer_if_false, answer_if_true,
2260                               TypeInt::make(lo, hi, widen));
2261
2262  return _gvn.transform(cmov);
2263
2264  /*
2265  // This is not as desirable as it may seem, since Min and Max
2266  // nodes do not have a full set of optimizations.
2267  // And they would interfere, anyway, with 'if' optimizations
2268  // and with CMoveI canonical forms.
2269  switch (id) {
2270  case vmIntrinsics::_min:
2271    result_val = _gvn.transform(new (C, 3) MinINode(x,y)); break;
2272  case vmIntrinsics::_max:
2273    result_val = _gvn.transform(new (C, 3) MaxINode(x,y)); break;
2274  default:
2275    ShouldNotReachHere();
2276  }
2277  */
2278}
2279
2280inline int
2281LibraryCallKit::classify_unsafe_addr(Node* &base, Node* &offset) {
2282  const TypePtr* base_type = TypePtr::NULL_PTR;
2283  if (base != NULL)  base_type = _gvn.type(base)->isa_ptr();
2284  if (base_type == NULL) {
2285    // Unknown type.
2286    return Type::AnyPtr;
2287  } else if (base_type == TypePtr::NULL_PTR) {
2288    // Since this is a NULL+long form, we have to switch to a rawptr.
2289    base   = _gvn.transform(new (C) CastX2PNode(offset));
2290    offset = MakeConX(0);
2291    return Type::RawPtr;
2292  } else if (base_type->base() == Type::RawPtr) {
2293    return Type::RawPtr;
2294  } else if (base_type->isa_oopptr()) {
2295    // Base is never null => always a heap address.
2296    if (base_type->ptr() == TypePtr::NotNull) {
2297      return Type::OopPtr;
2298    }
2299    // Offset is small => always a heap address.
2300    const TypeX* offset_type = _gvn.type(offset)->isa_intptr_t();
2301    if (offset_type != NULL &&
2302        base_type->offset() == 0 &&     // (should always be?)
2303        offset_type->_lo >= 0 &&
2304        !MacroAssembler::needs_explicit_null_check(offset_type->_hi)) {
2305      return Type::OopPtr;
2306    }
2307    // Otherwise, it might either be oop+off or NULL+addr.
2308    return Type::AnyPtr;
2309  } else {
2310    // No information:
2311    return Type::AnyPtr;
2312  }
2313}
2314
2315inline Node* LibraryCallKit::make_unsafe_address(Node* base, Node* offset) {
2316  int kind = classify_unsafe_addr(base, offset);
2317  if (kind == Type::RawPtr) {
2318    return basic_plus_adr(top(), base, offset);
2319  } else {
2320    return basic_plus_adr(base, offset);
2321  }
2322}
2323
2324//--------------------------inline_number_methods-----------------------------
2325// inline int     Integer.numberOfLeadingZeros(int)
2326// inline int        Long.numberOfLeadingZeros(long)
2327//
2328// inline int     Integer.numberOfTrailingZeros(int)
2329// inline int        Long.numberOfTrailingZeros(long)
2330//
2331// inline int     Integer.bitCount(int)
2332// inline int        Long.bitCount(long)
2333//
2334// inline char  Character.reverseBytes(char)
2335// inline short     Short.reverseBytes(short)
2336// inline int     Integer.reverseBytes(int)
2337// inline long       Long.reverseBytes(long)
2338bool LibraryCallKit::inline_number_methods(vmIntrinsics::ID id) {
2339  Node* arg = argument(0);
2340  Node* n;
2341  switch (id) {
2342  case vmIntrinsics::_numberOfLeadingZeros_i:   n = new (C) CountLeadingZerosINode( arg);  break;
2343  case vmIntrinsics::_numberOfLeadingZeros_l:   n = new (C) CountLeadingZerosLNode( arg);  break;
2344  case vmIntrinsics::_numberOfTrailingZeros_i:  n = new (C) CountTrailingZerosINode(arg);  break;
2345  case vmIntrinsics::_numberOfTrailingZeros_l:  n = new (C) CountTrailingZerosLNode(arg);  break;
2346  case vmIntrinsics::_bitCount_i:               n = new (C) PopCountINode(          arg);  break;
2347  case vmIntrinsics::_bitCount_l:               n = new (C) PopCountLNode(          arg);  break;
2348  case vmIntrinsics::_reverseBytes_c:           n = new (C) ReverseBytesUSNode(0,   arg);  break;
2349  case vmIntrinsics::_reverseBytes_s:           n = new (C) ReverseBytesSNode( 0,   arg);  break;
2350  case vmIntrinsics::_reverseBytes_i:           n = new (C) ReverseBytesINode( 0,   arg);  break;
2351  case vmIntrinsics::_reverseBytes_l:           n = new (C) ReverseBytesLNode( 0,   arg);  break;
2352  default:  fatal_unexpected_iid(id);  break;
2353  }
2354  set_result(_gvn.transform(n));
2355  return true;
2356}
2357
2358//----------------------------inline_unsafe_access----------------------------
2359
2360const static BasicType T_ADDRESS_HOLDER = T_LONG;
2361
2362// Helper that guards and inserts a pre-barrier.
2363void LibraryCallKit::insert_pre_barrier(Node* base_oop, Node* offset,
2364                                        Node* pre_val, bool need_mem_bar) {
2365  // We could be accessing the referent field of a reference object. If so, when G1
2366  // is enabled, we need to log the value in the referent field in an SATB buffer.
2367  // This routine performs some compile time filters and generates suitable
2368  // runtime filters that guard the pre-barrier code.
2369  // Also add memory barrier for non volatile load from the referent field
2370  // to prevent commoning of loads across safepoint.
2371  if (!UseG1GC && !need_mem_bar)
2372    return;
2373
2374  // Some compile time checks.
2375
2376  // If offset is a constant, is it java_lang_ref_Reference::_reference_offset?
2377  const TypeX* otype = offset->find_intptr_t_type();
2378  if (otype != NULL && otype->is_con() &&
2379      otype->get_con() != java_lang_ref_Reference::referent_offset) {
2380    // Constant offset but not the reference_offset so just return
2381    return;
2382  }
2383
2384  // We only need to generate the runtime guards for instances.
2385  const TypeOopPtr* btype = base_oop->bottom_type()->isa_oopptr();
2386  if (btype != NULL) {
2387    if (btype->isa_aryptr()) {
2388      // Array type so nothing to do
2389      return;
2390    }
2391
2392    const TypeInstPtr* itype = btype->isa_instptr();
2393    if (itype != NULL) {
2394      // Can the klass of base_oop be statically determined to be
2395      // _not_ a sub-class of Reference and _not_ Object?
2396      ciKlass* klass = itype->klass();
2397      if ( klass->is_loaded() &&
2398          !klass->is_subtype_of(env()->Reference_klass()) &&
2399          !env()->Object_klass()->is_subtype_of(klass)) {
2400        return;
2401      }
2402    }
2403  }
2404
2405  // The compile time filters did not reject base_oop/offset so
2406  // we need to generate the following runtime filters
2407  //
2408  // if (offset == java_lang_ref_Reference::_reference_offset) {
2409  //   if (instance_of(base, java.lang.ref.Reference)) {
2410  //     pre_barrier(_, pre_val, ...);
2411  //   }
2412  // }
2413
2414  float likely   = PROB_LIKELY(  0.999);
2415  float unlikely = PROB_UNLIKELY(0.999);
2416
2417  IdealKit ideal(this);
2418#define __ ideal.
2419
2420  Node* referent_off = __ ConX(java_lang_ref_Reference::referent_offset);
2421
2422  __ if_then(offset, BoolTest::eq, referent_off, unlikely); {
2423      // Update graphKit memory and control from IdealKit.
2424      sync_kit(ideal);
2425
2426      Node* ref_klass_con = makecon(TypeKlassPtr::make(env()->Reference_klass()));
2427      Node* is_instof = gen_instanceof(base_oop, ref_klass_con);
2428
2429      // Update IdealKit memory and control from graphKit.
2430      __ sync_kit(this);
2431
2432      Node* one = __ ConI(1);
2433      // is_instof == 0 if base_oop == NULL
2434      __ if_then(is_instof, BoolTest::eq, one, unlikely); {
2435
2436        // Update graphKit from IdeakKit.
2437        sync_kit(ideal);
2438
2439        // Use the pre-barrier to record the value in the referent field
2440        pre_barrier(false /* do_load */,
2441                    __ ctrl(),
2442                    NULL /* obj */, NULL /* adr */, max_juint /* alias_idx */, NULL /* val */, NULL /* val_type */,
2443                    pre_val /* pre_val */,
2444                    T_OBJECT);
2445        if (need_mem_bar) {
2446          // Add memory barrier to prevent commoning reads from this field
2447          // across safepoint since GC can change its value.
2448          insert_mem_bar(Op_MemBarCPUOrder);
2449        }
2450        // Update IdealKit from graphKit.
2451        __ sync_kit(this);
2452
2453      } __ end_if(); // _ref_type != ref_none
2454  } __ end_if(); // offset == referent_offset
2455
2456  // Final sync IdealKit and GraphKit.
2457  final_sync(ideal);
2458#undef __
2459}
2460
2461
2462// Interpret Unsafe.fieldOffset cookies correctly:
2463extern jlong Unsafe_field_offset_to_byte_offset(jlong field_offset);
2464
2465const TypeOopPtr* LibraryCallKit::sharpen_unsafe_type(Compile::AliasType* alias_type, const TypePtr *adr_type, bool is_native_ptr) {
2466  // Attempt to infer a sharper value type from the offset and base type.
2467  ciKlass* sharpened_klass = NULL;
2468
2469  // See if it is an instance field, with an object type.
2470  if (alias_type->field() != NULL) {
2471    assert(!is_native_ptr, "native pointer op cannot use a java address");
2472    if (alias_type->field()->type()->is_klass()) {
2473      sharpened_klass = alias_type->field()->type()->as_klass();
2474    }
2475  }
2476
2477  // See if it is a narrow oop array.
2478  if (adr_type->isa_aryptr()) {
2479    if (adr_type->offset() >= objArrayOopDesc::base_offset_in_bytes()) {
2480      const TypeOopPtr *elem_type = adr_type->is_aryptr()->elem()->isa_oopptr();
2481      if (elem_type != NULL) {
2482        sharpened_klass = elem_type->klass();
2483      }
2484    }
2485  }
2486
2487  // The sharpened class might be unloaded if there is no class loader
2488  // contraint in place.
2489  if (sharpened_klass != NULL && sharpened_klass->is_loaded()) {
2490    const TypeOopPtr* tjp = TypeOopPtr::make_from_klass(sharpened_klass);
2491
2492#ifndef PRODUCT
2493    if (C->print_intrinsics() || C->print_inlining()) {
2494      tty->print("  from base type: ");  adr_type->dump();
2495      tty->print("  sharpened value: ");  tjp->dump();
2496    }
2497#endif
2498    // Sharpen the value type.
2499    return tjp;
2500  }
2501  return NULL;
2502}
2503
2504bool LibraryCallKit::inline_unsafe_access(bool is_native_ptr, bool is_store, BasicType type, bool is_volatile) {
2505  if (callee()->is_static())  return false;  // caller must have the capability!
2506
2507#ifndef PRODUCT
2508  {
2509    ResourceMark rm;
2510    // Check the signatures.
2511    ciSignature* sig = callee()->signature();
2512#ifdef ASSERT
2513    if (!is_store) {
2514      // Object getObject(Object base, int/long offset), etc.
2515      BasicType rtype = sig->return_type()->basic_type();
2516      if (rtype == T_ADDRESS_HOLDER && callee()->name() == ciSymbol::getAddress_name())
2517          rtype = T_ADDRESS;  // it is really a C void*
2518      assert(rtype == type, "getter must return the expected value");
2519      if (!is_native_ptr) {
2520        assert(sig->count() == 2, "oop getter has 2 arguments");
2521        assert(sig->type_at(0)->basic_type() == T_OBJECT, "getter base is object");
2522        assert(sig->type_at(1)->basic_type() == T_LONG, "getter offset is correct");
2523      } else {
2524        assert(sig->count() == 1, "native getter has 1 argument");
2525        assert(sig->type_at(0)->basic_type() == T_LONG, "getter base is long");
2526      }
2527    } else {
2528      // void putObject(Object base, int/long offset, Object x), etc.
2529      assert(sig->return_type()->basic_type() == T_VOID, "putter must not return a value");
2530      if (!is_native_ptr) {
2531        assert(sig->count() == 3, "oop putter has 3 arguments");
2532        assert(sig->type_at(0)->basic_type() == T_OBJECT, "putter base is object");
2533        assert(sig->type_at(1)->basic_type() == T_LONG, "putter offset is correct");
2534      } else {
2535        assert(sig->count() == 2, "native putter has 2 arguments");
2536        assert(sig->type_at(0)->basic_type() == T_LONG, "putter base is long");
2537      }
2538      BasicType vtype = sig->type_at(sig->count()-1)->basic_type();
2539      if (vtype == T_ADDRESS_HOLDER && callee()->name() == ciSymbol::putAddress_name())
2540        vtype = T_ADDRESS;  // it is really a C void*
2541      assert(vtype == type, "putter must accept the expected value");
2542    }
2543#endif // ASSERT
2544 }
2545#endif //PRODUCT
2546
2547  C->set_has_unsafe_access(true);  // Mark eventual nmethod as "unsafe".
2548
2549  Node* receiver = argument(0);  // type: oop
2550
2551  // Build address expression.  See the code in inline_unsafe_prefetch.
2552  Node* adr;
2553  Node* heap_base_oop = top();
2554  Node* offset = top();
2555  Node* val;
2556
2557  if (!is_native_ptr) {
2558    // The base is either a Java object or a value produced by Unsafe.staticFieldBase
2559    Node* base = argument(1);  // type: oop
2560    // The offset is a value produced by Unsafe.staticFieldOffset or Unsafe.objectFieldOffset
2561    offset = argument(2);  // type: long
2562    // We currently rely on the cookies produced by Unsafe.xxxFieldOffset
2563    // to be plain byte offsets, which are also the same as those accepted
2564    // by oopDesc::field_base.
2565    assert(Unsafe_field_offset_to_byte_offset(11) == 11,
2566           "fieldOffset must be byte-scaled");
2567    // 32-bit machines ignore the high half!
2568    offset = ConvL2X(offset);
2569    adr = make_unsafe_address(base, offset);
2570    heap_base_oop = base;
2571    val = is_store ? argument(4) : NULL;
2572  } else {
2573    Node* ptr = argument(1);  // type: long
2574    ptr = ConvL2X(ptr);  // adjust Java long to machine word
2575    adr = make_unsafe_address(NULL, ptr);
2576    val = is_store ? argument(3) : NULL;
2577  }
2578
2579  const TypePtr *adr_type = _gvn.type(adr)->isa_ptr();
2580
2581  // First guess at the value type.
2582  const Type *value_type = Type::get_const_basic_type(type);
2583
2584  // Try to categorize the address.  If it comes up as TypeJavaPtr::BOTTOM,
2585  // there was not enough information to nail it down.
2586  Compile::AliasType* alias_type = C->alias_type(adr_type);
2587  assert(alias_type->index() != Compile::AliasIdxBot, "no bare pointers here");
2588
2589  // We will need memory barriers unless we can determine a unique
2590  // alias category for this reference.  (Note:  If for some reason
2591  // the barriers get omitted and the unsafe reference begins to "pollute"
2592  // the alias analysis of the rest of the graph, either Compile::can_alias
2593  // or Compile::must_alias will throw a diagnostic assert.)
2594  bool need_mem_bar = (alias_type->adr_type() == TypeOopPtr::BOTTOM);
2595
2596  // If we are reading the value of the referent field of a Reference
2597  // object (either by using Unsafe directly or through reflection)
2598  // then, if G1 is enabled, we need to record the referent in an
2599  // SATB log buffer using the pre-barrier mechanism.
2600  // Also we need to add memory barrier to prevent commoning reads
2601  // from this field across safepoint since GC can change its value.
2602  bool need_read_barrier = !is_native_ptr && !is_store &&
2603                           offset != top() && heap_base_oop != top();
2604
2605  if (!is_store && type == T_OBJECT) {
2606    const TypeOopPtr* tjp = sharpen_unsafe_type(alias_type, adr_type, is_native_ptr);
2607    if (tjp != NULL) {
2608      value_type = tjp;
2609    }
2610  }
2611
2612  receiver = null_check(receiver);
2613  if (stopped()) {
2614    return true;
2615  }
2616  // Heap pointers get a null-check from the interpreter,
2617  // as a courtesy.  However, this is not guaranteed by Unsafe,
2618  // and it is not possible to fully distinguish unintended nulls
2619  // from intended ones in this API.
2620
2621  if (is_volatile) {
2622    // We need to emit leading and trailing CPU membars (see below) in
2623    // addition to memory membars when is_volatile. This is a little
2624    // too strong, but avoids the need to insert per-alias-type
2625    // volatile membars (for stores; compare Parse::do_put_xxx), which
2626    // we cannot do effectively here because we probably only have a
2627    // rough approximation of type.
2628    need_mem_bar = true;
2629    // For Stores, place a memory ordering barrier now.
2630    if (is_store)
2631      insert_mem_bar(Op_MemBarRelease);
2632  }
2633
2634  // Memory barrier to prevent normal and 'unsafe' accesses from
2635  // bypassing each other.  Happens after null checks, so the
2636  // exception paths do not take memory state from the memory barrier,
2637  // so there's no problems making a strong assert about mixing users
2638  // of safe & unsafe memory.  Otherwise fails in a CTW of rt.jar
2639  // around 5701, class sun/reflect/UnsafeBooleanFieldAccessorImpl.
2640  if (need_mem_bar) insert_mem_bar(Op_MemBarCPUOrder);
2641
2642  if (!is_store) {
2643    Node* p = make_load(control(), adr, value_type, type, adr_type, MemNode::unordered, is_volatile);
2644    // load value
2645    switch (type) {
2646    case T_BOOLEAN:
2647    case T_CHAR:
2648    case T_BYTE:
2649    case T_SHORT:
2650    case T_INT:
2651    case T_LONG:
2652    case T_FLOAT:
2653    case T_DOUBLE:
2654      break;
2655    case T_OBJECT:
2656      if (need_read_barrier) {
2657        insert_pre_barrier(heap_base_oop, offset, p, !(is_volatile || need_mem_bar));
2658      }
2659      break;
2660    case T_ADDRESS:
2661      // Cast to an int type.
2662      p = _gvn.transform(new (C) CastP2XNode(NULL, p));
2663      p = ConvX2L(p);
2664      break;
2665    default:
2666      fatal(err_msg_res("unexpected type %d: %s", type, type2name(type)));
2667      break;
2668    }
2669    // The load node has the control of the preceding MemBarCPUOrder.  All
2670    // following nodes will have the control of the MemBarCPUOrder inserted at
2671    // the end of this method.  So, pushing the load onto the stack at a later
2672    // point is fine.
2673    set_result(p);
2674  } else {
2675    // place effect of store into memory
2676    switch (type) {
2677    case T_DOUBLE:
2678      val = dstore_rounding(val);
2679      break;
2680    case T_ADDRESS:
2681      // Repackage the long as a pointer.
2682      val = ConvL2X(val);
2683      val = _gvn.transform(new (C) CastX2PNode(val));
2684      break;
2685    }
2686
2687    MemNode::MemOrd mo = is_volatile ? MemNode::release : MemNode::unordered;
2688    if (type != T_OBJECT ) {
2689      (void) store_to_memory(control(), adr, val, type, adr_type, mo, is_volatile);
2690    } else {
2691      // Possibly an oop being stored to Java heap or native memory
2692      if (!TypePtr::NULL_PTR->higher_equal(_gvn.type(heap_base_oop))) {
2693        // oop to Java heap.
2694        (void) store_oop_to_unknown(control(), heap_base_oop, adr, adr_type, val, type, mo);
2695      } else {
2696        // We can't tell at compile time if we are storing in the Java heap or outside
2697        // of it. So we need to emit code to conditionally do the proper type of
2698        // store.
2699
2700        IdealKit ideal(this);
2701#define __ ideal.
2702        // QQQ who knows what probability is here??
2703        __ if_then(heap_base_oop, BoolTest::ne, null(), PROB_UNLIKELY(0.999)); {
2704          // Sync IdealKit and graphKit.
2705          sync_kit(ideal);
2706          Node* st = store_oop_to_unknown(control(), heap_base_oop, adr, adr_type, val, type, mo);
2707          // Update IdealKit memory.
2708          __ sync_kit(this);
2709        } __ else_(); {
2710          __ store(__ ctrl(), adr, val, type, alias_type->index(), mo, is_volatile);
2711        } __ end_if();
2712        // Final sync IdealKit and GraphKit.
2713        final_sync(ideal);
2714#undef __
2715      }
2716    }
2717  }
2718
2719  if (is_volatile) {
2720    if (!is_store)
2721      insert_mem_bar(Op_MemBarAcquire);
2722    else
2723      insert_mem_bar(Op_MemBarVolatile);
2724  }
2725
2726  if (need_mem_bar) insert_mem_bar(Op_MemBarCPUOrder);
2727
2728  return true;
2729}
2730
2731//----------------------------inline_unsafe_prefetch----------------------------
2732
2733bool LibraryCallKit::inline_unsafe_prefetch(bool is_native_ptr, bool is_store, bool is_static) {
2734#ifndef PRODUCT
2735  {
2736    ResourceMark rm;
2737    // Check the signatures.
2738    ciSignature* sig = callee()->signature();
2739#ifdef ASSERT
2740    // Object getObject(Object base, int/long offset), etc.
2741    BasicType rtype = sig->return_type()->basic_type();
2742    if (!is_native_ptr) {
2743      assert(sig->count() == 2, "oop prefetch has 2 arguments");
2744      assert(sig->type_at(0)->basic_type() == T_OBJECT, "prefetch base is object");
2745      assert(sig->type_at(1)->basic_type() == T_LONG, "prefetcha offset is correct");
2746    } else {
2747      assert(sig->count() == 1, "native prefetch has 1 argument");
2748      assert(sig->type_at(0)->basic_type() == T_LONG, "prefetch base is long");
2749    }
2750#endif // ASSERT
2751  }
2752#endif // !PRODUCT
2753
2754  C->set_has_unsafe_access(true);  // Mark eventual nmethod as "unsafe".
2755
2756  const int idx = is_static ? 0 : 1;
2757  if (!is_static) {
2758    null_check_receiver();
2759    if (stopped()) {
2760      return true;
2761    }
2762  }
2763
2764  // Build address expression.  See the code in inline_unsafe_access.
2765  Node *adr;
2766  if (!is_native_ptr) {
2767    // The base is either a Java object or a value produced by Unsafe.staticFieldBase
2768    Node* base   = argument(idx + 0);  // type: oop
2769    // The offset is a value produced by Unsafe.staticFieldOffset or Unsafe.objectFieldOffset
2770    Node* offset = argument(idx + 1);  // type: long
2771    // We currently rely on the cookies produced by Unsafe.xxxFieldOffset
2772    // to be plain byte offsets, which are also the same as those accepted
2773    // by oopDesc::field_base.
2774    assert(Unsafe_field_offset_to_byte_offset(11) == 11,
2775           "fieldOffset must be byte-scaled");
2776    // 32-bit machines ignore the high half!
2777    offset = ConvL2X(offset);
2778    adr = make_unsafe_address(base, offset);
2779  } else {
2780    Node* ptr = argument(idx + 0);  // type: long
2781    ptr = ConvL2X(ptr);  // adjust Java long to machine word
2782    adr = make_unsafe_address(NULL, ptr);
2783  }
2784
2785  // Generate the read or write prefetch
2786  Node *prefetch;
2787  if (is_store) {
2788    prefetch = new (C) PrefetchWriteNode(i_o(), adr);
2789  } else {
2790    prefetch = new (C) PrefetchReadNode(i_o(), adr);
2791  }
2792  prefetch->init_req(0, control());
2793  set_i_o(_gvn.transform(prefetch));
2794
2795  return true;
2796}
2797
2798//----------------------------inline_unsafe_load_store----------------------------
2799// This method serves a couple of different customers (depending on LoadStoreKind):
2800//
2801// LS_cmpxchg:
2802//   public final native boolean compareAndSwapObject(Object o, long offset, Object expected, Object x);
2803//   public final native boolean compareAndSwapInt(   Object o, long offset, int    expected, int    x);
2804//   public final native boolean compareAndSwapLong(  Object o, long offset, long   expected, long   x);
2805//
2806// LS_xadd:
2807//   public int  getAndAddInt( Object o, long offset, int  delta)
2808//   public long getAndAddLong(Object o, long offset, long delta)
2809//
2810// LS_xchg:
2811//   int    getAndSet(Object o, long offset, int    newValue)
2812//   long   getAndSet(Object o, long offset, long   newValue)
2813//   Object getAndSet(Object o, long offset, Object newValue)
2814//
2815bool LibraryCallKit::inline_unsafe_load_store(BasicType type, LoadStoreKind kind) {
2816  // This basic scheme here is the same as inline_unsafe_access, but
2817  // differs in enough details that combining them would make the code
2818  // overly confusing.  (This is a true fact! I originally combined
2819  // them, but even I was confused by it!) As much code/comments as
2820  // possible are retained from inline_unsafe_access though to make
2821  // the correspondences clearer. - dl
2822
2823  if (callee()->is_static())  return false;  // caller must have the capability!
2824
2825#ifndef PRODUCT
2826  BasicType rtype;
2827  {
2828    ResourceMark rm;
2829    // Check the signatures.
2830    ciSignature* sig = callee()->signature();
2831    rtype = sig->return_type()->basic_type();
2832    if (kind == LS_xadd || kind == LS_xchg) {
2833      // Check the signatures.
2834#ifdef ASSERT
2835      assert(rtype == type, "get and set must return the expected type");
2836      assert(sig->count() == 3, "get and set has 3 arguments");
2837      assert(sig->type_at(0)->basic_type() == T_OBJECT, "get and set base is object");
2838      assert(sig->type_at(1)->basic_type() == T_LONG, "get and set offset is long");
2839      assert(sig->type_at(2)->basic_type() == type, "get and set must take expected type as new value/delta");
2840#endif // ASSERT
2841    } else if (kind == LS_cmpxchg) {
2842      // Check the signatures.
2843#ifdef ASSERT
2844      assert(rtype == T_BOOLEAN, "CAS must return boolean");
2845      assert(sig->count() == 4, "CAS has 4 arguments");
2846      assert(sig->type_at(0)->basic_type() == T_OBJECT, "CAS base is object");
2847      assert(sig->type_at(1)->basic_type() == T_LONG, "CAS offset is long");
2848#endif // ASSERT
2849    } else {
2850      ShouldNotReachHere();
2851    }
2852  }
2853#endif //PRODUCT
2854
2855  C->set_has_unsafe_access(true);  // Mark eventual nmethod as "unsafe".
2856
2857  // Get arguments:
2858  Node* receiver = NULL;
2859  Node* base     = NULL;
2860  Node* offset   = NULL;
2861  Node* oldval   = NULL;
2862  Node* newval   = NULL;
2863  if (kind == LS_cmpxchg) {
2864    const bool two_slot_type = type2size[type] == 2;
2865    receiver = argument(0);  // type: oop
2866    base     = argument(1);  // type: oop
2867    offset   = argument(2);  // type: long
2868    oldval   = argument(4);  // type: oop, int, or long
2869    newval   = argument(two_slot_type ? 6 : 5);  // type: oop, int, or long
2870  } else if (kind == LS_xadd || kind == LS_xchg){
2871    receiver = argument(0);  // type: oop
2872    base     = argument(1);  // type: oop
2873    offset   = argument(2);  // type: long
2874    oldval   = NULL;
2875    newval   = argument(4);  // type: oop, int, or long
2876  }
2877
2878  // Null check receiver.
2879  receiver = null_check(receiver);
2880  if (stopped()) {
2881    return true;
2882  }
2883
2884  // Build field offset expression.
2885  // We currently rely on the cookies produced by Unsafe.xxxFieldOffset
2886  // to be plain byte offsets, which are also the same as those accepted
2887  // by oopDesc::field_base.
2888  assert(Unsafe_field_offset_to_byte_offset(11) == 11, "fieldOffset must be byte-scaled");
2889  // 32-bit machines ignore the high half of long offsets
2890  offset = ConvL2X(offset);
2891  Node* adr = make_unsafe_address(base, offset);
2892  const TypePtr *adr_type = _gvn.type(adr)->isa_ptr();
2893
2894  // For CAS, unlike inline_unsafe_access, there seems no point in
2895  // trying to refine types. Just use the coarse types here.
2896  const Type *value_type = Type::get_const_basic_type(type);
2897  Compile::AliasType* alias_type = C->alias_type(adr_type);
2898  assert(alias_type->index() != Compile::AliasIdxBot, "no bare pointers here");
2899
2900  if (kind == LS_xchg && type == T_OBJECT) {
2901    const TypeOopPtr* tjp = sharpen_unsafe_type(alias_type, adr_type);
2902    if (tjp != NULL) {
2903      value_type = tjp;
2904    }
2905  }
2906
2907  int alias_idx = C->get_alias_index(adr_type);
2908
2909  // Memory-model-wise, a LoadStore acts like a little synchronized
2910  // block, so needs barriers on each side.  These don't translate
2911  // into actual barriers on most machines, but we still need rest of
2912  // compiler to respect ordering.
2913
2914  insert_mem_bar(Op_MemBarRelease);
2915  insert_mem_bar(Op_MemBarCPUOrder);
2916
2917  // 4984716: MemBars must be inserted before this
2918  //          memory node in order to avoid a false
2919  //          dependency which will confuse the scheduler.
2920  Node *mem = memory(alias_idx);
2921
2922  // For now, we handle only those cases that actually exist: ints,
2923  // longs, and Object. Adding others should be straightforward.
2924  Node* load_store;
2925  switch(type) {
2926  case T_INT:
2927    if (kind == LS_xadd) {
2928      load_store = _gvn.transform(new (C) GetAndAddINode(control(), mem, adr, newval, adr_type));
2929    } else if (kind == LS_xchg) {
2930      load_store = _gvn.transform(new (C) GetAndSetINode(control(), mem, adr, newval, adr_type));
2931    } else if (kind == LS_cmpxchg) {
2932      load_store = _gvn.transform(new (C) CompareAndSwapINode(control(), mem, adr, newval, oldval));
2933    } else {
2934      ShouldNotReachHere();
2935    }
2936    break;
2937  case T_LONG:
2938    if (kind == LS_xadd) {
2939      load_store = _gvn.transform(new (C) GetAndAddLNode(control(), mem, adr, newval, adr_type));
2940    } else if (kind == LS_xchg) {
2941      load_store = _gvn.transform(new (C) GetAndSetLNode(control(), mem, adr, newval, adr_type));
2942    } else if (kind == LS_cmpxchg) {
2943      load_store = _gvn.transform(new (C) CompareAndSwapLNode(control(), mem, adr, newval, oldval));
2944    } else {
2945      ShouldNotReachHere();
2946    }
2947    break;
2948  case T_OBJECT:
2949    // Transformation of a value which could be NULL pointer (CastPP #NULL)
2950    // could be delayed during Parse (for example, in adjust_map_after_if()).
2951    // Execute transformation here to avoid barrier generation in such case.
2952    if (_gvn.type(newval) == TypePtr::NULL_PTR)
2953      newval = _gvn.makecon(TypePtr::NULL_PTR);
2954
2955    // Reference stores need a store barrier.
2956    if (kind == LS_xchg) {
2957      // If pre-barrier must execute before the oop store, old value will require do_load here.
2958      if (!can_move_pre_barrier()) {
2959        pre_barrier(true /* do_load*/,
2960                    control(), base, adr, alias_idx, newval, value_type->make_oopptr(),
2961                    NULL /* pre_val*/,
2962                    T_OBJECT);
2963      } // Else move pre_barrier to use load_store value, see below.
2964    } else if (kind == LS_cmpxchg) {
2965      // Same as for newval above:
2966      if (_gvn.type(oldval) == TypePtr::NULL_PTR) {
2967        oldval = _gvn.makecon(TypePtr::NULL_PTR);
2968      }
2969      // The only known value which might get overwritten is oldval.
2970      pre_barrier(false /* do_load */,
2971                  control(), NULL, NULL, max_juint, NULL, NULL,
2972                  oldval /* pre_val */,
2973                  T_OBJECT);
2974    } else {
2975      ShouldNotReachHere();
2976    }
2977
2978#ifdef _LP64
2979    if (adr->bottom_type()->is_ptr_to_narrowoop()) {
2980      Node *newval_enc = _gvn.transform(new (C) EncodePNode(newval, newval->bottom_type()->make_narrowoop()));
2981      if (kind == LS_xchg) {
2982        load_store = _gvn.transform(new (C) GetAndSetNNode(control(), mem, adr,
2983                                                           newval_enc, adr_type, value_type->make_narrowoop()));
2984      } else {
2985        assert(kind == LS_cmpxchg, "wrong LoadStore operation");
2986        Node *oldval_enc = _gvn.transform(new (C) EncodePNode(oldval, oldval->bottom_type()->make_narrowoop()));
2987        load_store = _gvn.transform(new (C) CompareAndSwapNNode(control(), mem, adr,
2988                                                                newval_enc, oldval_enc));
2989      }
2990    } else
2991#endif
2992    {
2993      if (kind == LS_xchg) {
2994        load_store = _gvn.transform(new (C) GetAndSetPNode(control(), mem, adr, newval, adr_type, value_type->is_oopptr()));
2995      } else {
2996        assert(kind == LS_cmpxchg, "wrong LoadStore operation");
2997        load_store = _gvn.transform(new (C) CompareAndSwapPNode(control(), mem, adr, newval, oldval));
2998      }
2999    }
3000    post_barrier(control(), load_store, base, adr, alias_idx, newval, T_OBJECT, true);
3001    break;
3002  default:
3003    fatal(err_msg_res("unexpected type %d: %s", type, type2name(type)));
3004    break;
3005  }
3006
3007  // SCMemProjNodes represent the memory state of a LoadStore. Their
3008  // main role is to prevent LoadStore nodes from being optimized away
3009  // when their results aren't used.
3010  Node* proj = _gvn.transform(new (C) SCMemProjNode(load_store));
3011  set_memory(proj, alias_idx);
3012
3013  if (type == T_OBJECT && kind == LS_xchg) {
3014#ifdef _LP64
3015    if (adr->bottom_type()->is_ptr_to_narrowoop()) {
3016      load_store = _gvn.transform(new (C) DecodeNNode(load_store, load_store->get_ptr_type()));
3017    }
3018#endif
3019    if (can_move_pre_barrier()) {
3020      // Don't need to load pre_val. The old value is returned by load_store.
3021      // The pre_barrier can execute after the xchg as long as no safepoint
3022      // gets inserted between them.
3023      pre_barrier(false /* do_load */,
3024                  control(), NULL, NULL, max_juint, NULL, NULL,
3025                  load_store /* pre_val */,
3026                  T_OBJECT);
3027    }
3028  }
3029
3030  // Add the trailing membar surrounding the access
3031  insert_mem_bar(Op_MemBarCPUOrder);
3032  insert_mem_bar(Op_MemBarAcquire);
3033
3034  assert(type2size[load_store->bottom_type()->basic_type()] == type2size[rtype], "result type should match");
3035  set_result(load_store);
3036  return true;
3037}
3038
3039//----------------------------inline_unsafe_ordered_store----------------------
3040// public native void sun.misc.Unsafe.putOrderedObject(Object o, long offset, Object x);
3041// public native void sun.misc.Unsafe.putOrderedInt(Object o, long offset, int x);
3042// public native void sun.misc.Unsafe.putOrderedLong(Object o, long offset, long x);
3043bool LibraryCallKit::inline_unsafe_ordered_store(BasicType type) {
3044  // This is another variant of inline_unsafe_access, differing in
3045  // that it always issues store-store ("release") barrier and ensures
3046  // store-atomicity (which only matters for "long").
3047
3048  if (callee()->is_static())  return false;  // caller must have the capability!
3049
3050#ifndef PRODUCT
3051  {
3052    ResourceMark rm;
3053    // Check the signatures.
3054    ciSignature* sig = callee()->signature();
3055#ifdef ASSERT
3056    BasicType rtype = sig->return_type()->basic_type();
3057    assert(rtype == T_VOID, "must return void");
3058    assert(sig->count() == 3, "has 3 arguments");
3059    assert(sig->type_at(0)->basic_type() == T_OBJECT, "base is object");
3060    assert(sig->type_at(1)->basic_type() == T_LONG, "offset is long");
3061#endif // ASSERT
3062  }
3063#endif //PRODUCT
3064
3065  C->set_has_unsafe_access(true);  // Mark eventual nmethod as "unsafe".
3066
3067  // Get arguments:
3068  Node* receiver = argument(0);  // type: oop
3069  Node* base     = argument(1);  // type: oop
3070  Node* offset   = argument(2);  // type: long
3071  Node* val      = argument(4);  // type: oop, int, or long
3072
3073  // Null check receiver.
3074  receiver = null_check(receiver);
3075  if (stopped()) {
3076    return true;
3077  }
3078
3079  // Build field offset expression.
3080  assert(Unsafe_field_offset_to_byte_offset(11) == 11, "fieldOffset must be byte-scaled");
3081  // 32-bit machines ignore the high half of long offsets
3082  offset = ConvL2X(offset);
3083  Node* adr = make_unsafe_address(base, offset);
3084  const TypePtr *adr_type = _gvn.type(adr)->isa_ptr();
3085  const Type *value_type = Type::get_const_basic_type(type);
3086  Compile::AliasType* alias_type = C->alias_type(adr_type);
3087
3088  insert_mem_bar(Op_MemBarRelease);
3089  insert_mem_bar(Op_MemBarCPUOrder);
3090  // Ensure that the store is atomic for longs:
3091  const bool require_atomic_access = true;
3092  Node* store;
3093  if (type == T_OBJECT) // reference stores need a store barrier.
3094    store = store_oop_to_unknown(control(), base, adr, adr_type, val, type, MemNode::release);
3095  else {
3096    store = store_to_memory(control(), adr, val, type, adr_type, MemNode::release, require_atomic_access);
3097  }
3098  insert_mem_bar(Op_MemBarCPUOrder);
3099  return true;
3100}
3101
3102bool LibraryCallKit::inline_unsafe_fence(vmIntrinsics::ID id) {
3103  // Regardless of form, don't allow previous ld/st to move down,
3104  // then issue acquire, release, or volatile mem_bar.
3105  insert_mem_bar(Op_MemBarCPUOrder);
3106  switch(id) {
3107    case vmIntrinsics::_loadFence:
3108      insert_mem_bar(Op_MemBarAcquire);
3109      return true;
3110    case vmIntrinsics::_storeFence:
3111      insert_mem_bar(Op_MemBarRelease);
3112      return true;
3113    case vmIntrinsics::_fullFence:
3114      insert_mem_bar(Op_MemBarVolatile);
3115      return true;
3116    default:
3117      fatal_unexpected_iid(id);
3118      return false;
3119  }
3120}
3121
3122bool LibraryCallKit::klass_needs_init_guard(Node* kls) {
3123  if (!kls->is_Con()) {
3124    return true;
3125  }
3126  const TypeKlassPtr* klsptr = kls->bottom_type()->isa_klassptr();
3127  if (klsptr == NULL) {
3128    return true;
3129  }
3130  ciInstanceKlass* ik = klsptr->klass()->as_instance_klass();
3131  // don't need a guard for a klass that is already initialized
3132  return !ik->is_initialized();
3133}
3134
3135//----------------------------inline_unsafe_allocate---------------------------
3136// public native Object sun.misc.Unsafe.allocateInstance(Class<?> cls);
3137bool LibraryCallKit::inline_unsafe_allocate() {
3138  if (callee()->is_static())  return false;  // caller must have the capability!
3139
3140  null_check_receiver();  // null-check, then ignore
3141  Node* cls = null_check(argument(1));
3142  if (stopped())  return true;
3143
3144  Node* kls = load_klass_from_mirror(cls, false, NULL, 0);
3145  kls = null_check(kls);
3146  if (stopped())  return true;  // argument was like int.class
3147
3148  Node* test = NULL;
3149  if (LibraryCallKit::klass_needs_init_guard(kls)) {
3150    // Note:  The argument might still be an illegal value like
3151    // Serializable.class or Object[].class.   The runtime will handle it.
3152    // But we must make an explicit check for initialization.
3153    Node* insp = basic_plus_adr(kls, in_bytes(InstanceKlass::init_state_offset()));
3154    // Use T_BOOLEAN for InstanceKlass::_init_state so the compiler
3155    // can generate code to load it as unsigned byte.
3156    Node* inst = make_load(NULL, insp, TypeInt::UBYTE, T_BOOLEAN, MemNode::unordered);
3157    Node* bits = intcon(InstanceKlass::fully_initialized);
3158    test = _gvn.transform(new (C) SubINode(inst, bits));
3159    // The 'test' is non-zero if we need to take a slow path.
3160  }
3161
3162  Node* obj = new_instance(kls, test);
3163  set_result(obj);
3164  return true;
3165}
3166
3167#ifdef TRACE_HAVE_INTRINSICS
3168/*
3169 * oop -> myklass
3170 * myklass->trace_id |= USED
3171 * return myklass->trace_id & ~0x3
3172 */
3173bool LibraryCallKit::inline_native_classID() {
3174  null_check_receiver();  // null-check, then ignore
3175  Node* cls = null_check(argument(1), T_OBJECT);
3176  Node* kls = load_klass_from_mirror(cls, false, NULL, 0);
3177  kls = null_check(kls, T_OBJECT);
3178  ByteSize offset = TRACE_ID_OFFSET;
3179  Node* insp = basic_plus_adr(kls, in_bytes(offset));
3180  Node* tvalue = make_load(NULL, insp, TypeLong::LONG, T_LONG, MemNode::unordered);
3181  Node* bits = longcon(~0x03l); // ignore bit 0 & 1
3182  Node* andl = _gvn.transform(new (C) AndLNode(tvalue, bits));
3183  Node* clsused = longcon(0x01l); // set the class bit
3184  Node* orl = _gvn.transform(new (C) OrLNode(tvalue, clsused));
3185
3186  const TypePtr *adr_type = _gvn.type(insp)->isa_ptr();
3187  store_to_memory(control(), insp, orl, T_LONG, adr_type, MemNode::unordered);
3188  set_result(andl);
3189  return true;
3190}
3191
3192bool LibraryCallKit::inline_native_threadID() {
3193  Node* tls_ptr = NULL;
3194  Node* cur_thr = generate_current_thread(tls_ptr);
3195  Node* p = basic_plus_adr(top()/*!oop*/, tls_ptr, in_bytes(JavaThread::osthread_offset()));
3196  Node* osthread = make_load(NULL, p, TypeRawPtr::NOTNULL, T_ADDRESS, MemNode::unordered);
3197  p = basic_plus_adr(top()/*!oop*/, osthread, in_bytes(OSThread::thread_id_offset()));
3198
3199  Node* threadid = NULL;
3200  size_t thread_id_size = OSThread::thread_id_size();
3201  if (thread_id_size == (size_t) BytesPerLong) {
3202    threadid = ConvL2I(make_load(control(), p, TypeLong::LONG, T_LONG, MemNode::unordered));
3203  } else if (thread_id_size == (size_t) BytesPerInt) {
3204    threadid = make_load(control(), p, TypeInt::INT, T_INT, MemNode::unordered);
3205  } else {
3206    ShouldNotReachHere();
3207  }
3208  set_result(threadid);
3209  return true;
3210}
3211#endif
3212
3213//------------------------inline_native_time_funcs--------------
3214// inline code for System.currentTimeMillis() and System.nanoTime()
3215// these have the same type and signature
3216bool LibraryCallKit::inline_native_time_funcs(address funcAddr, const char* funcName) {
3217  const TypeFunc* tf = OptoRuntime::void_long_Type();
3218  const TypePtr* no_memory_effects = NULL;
3219  Node* time = make_runtime_call(RC_LEAF, tf, funcAddr, funcName, no_memory_effects);
3220  Node* value = _gvn.transform(new (C) ProjNode(time, TypeFunc::Parms+0));
3221#ifdef ASSERT
3222  Node* value_top = _gvn.transform(new (C) ProjNode(time, TypeFunc::Parms+1));
3223  assert(value_top == top(), "second value must be top");
3224#endif
3225  set_result(value);
3226  return true;
3227}
3228
3229//------------------------inline_native_currentThread------------------
3230bool LibraryCallKit::inline_native_currentThread() {
3231  Node* junk = NULL;
3232  set_result(generate_current_thread(junk));
3233  return true;
3234}
3235
3236//------------------------inline_native_isInterrupted------------------
3237// private native boolean java.lang.Thread.isInterrupted(boolean ClearInterrupted);
3238bool LibraryCallKit::inline_native_isInterrupted() {
3239  // Add a fast path to t.isInterrupted(clear_int):
3240  //   (t == Thread.current() && (!TLS._osthread._interrupted || !clear_int))
3241  //   ? TLS._osthread._interrupted : /*slow path:*/ t.isInterrupted(clear_int)
3242  // So, in the common case that the interrupt bit is false,
3243  // we avoid making a call into the VM.  Even if the interrupt bit
3244  // is true, if the clear_int argument is false, we avoid the VM call.
3245  // However, if the receiver is not currentThread, we must call the VM,
3246  // because there must be some locking done around the operation.
3247
3248  // We only go to the fast case code if we pass two guards.
3249  // Paths which do not pass are accumulated in the slow_region.
3250
3251  enum {
3252    no_int_result_path   = 1, // t == Thread.current() && !TLS._osthread._interrupted
3253    no_clear_result_path = 2, // t == Thread.current() &&  TLS._osthread._interrupted && !clear_int
3254    slow_result_path     = 3, // slow path: t.isInterrupted(clear_int)
3255    PATH_LIMIT
3256  };
3257
3258  // Ensure that it's not possible to move the load of TLS._osthread._interrupted flag
3259  // out of the function.
3260  insert_mem_bar(Op_MemBarCPUOrder);
3261
3262  RegionNode* result_rgn = new (C) RegionNode(PATH_LIMIT);
3263  PhiNode*    result_val = new (C) PhiNode(result_rgn, TypeInt::BOOL);
3264
3265  RegionNode* slow_region = new (C) RegionNode(1);
3266  record_for_igvn(slow_region);
3267
3268  // (a) Receiving thread must be the current thread.
3269  Node* rec_thr = argument(0);
3270  Node* tls_ptr = NULL;
3271  Node* cur_thr = generate_current_thread(tls_ptr);
3272  Node* cmp_thr = _gvn.transform(new (C) CmpPNode(cur_thr, rec_thr));
3273  Node* bol_thr = _gvn.transform(new (C) BoolNode(cmp_thr, BoolTest::ne));
3274
3275  generate_slow_guard(bol_thr, slow_region);
3276
3277  // (b) Interrupt bit on TLS must be false.
3278  Node* p = basic_plus_adr(top()/*!oop*/, tls_ptr, in_bytes(JavaThread::osthread_offset()));
3279  Node* osthread = make_load(NULL, p, TypeRawPtr::NOTNULL, T_ADDRESS, MemNode::unordered);
3280  p = basic_plus_adr(top()/*!oop*/, osthread, in_bytes(OSThread::interrupted_offset()));
3281
3282  // Set the control input on the field _interrupted read to prevent it floating up.
3283  Node* int_bit = make_load(control(), p, TypeInt::BOOL, T_INT, MemNode::unordered);
3284  Node* cmp_bit = _gvn.transform(new (C) CmpINode(int_bit, intcon(0)));
3285  Node* bol_bit = _gvn.transform(new (C) BoolNode(cmp_bit, BoolTest::ne));
3286
3287  IfNode* iff_bit = create_and_map_if(control(), bol_bit, PROB_UNLIKELY_MAG(3), COUNT_UNKNOWN);
3288
3289  // First fast path:  if (!TLS._interrupted) return false;
3290  Node* false_bit = _gvn.transform(new (C) IfFalseNode(iff_bit));
3291  result_rgn->init_req(no_int_result_path, false_bit);
3292  result_val->init_req(no_int_result_path, intcon(0));
3293
3294  // drop through to next case
3295  set_control( _gvn.transform(new (C) IfTrueNode(iff_bit)));
3296
3297  // (c) Or, if interrupt bit is set and clear_int is false, use 2nd fast path.
3298  Node* clr_arg = argument(1);
3299  Node* cmp_arg = _gvn.transform(new (C) CmpINode(clr_arg, intcon(0)));
3300  Node* bol_arg = _gvn.transform(new (C) BoolNode(cmp_arg, BoolTest::ne));
3301  IfNode* iff_arg = create_and_map_if(control(), bol_arg, PROB_FAIR, COUNT_UNKNOWN);
3302
3303  // Second fast path:  ... else if (!clear_int) return true;
3304  Node* false_arg = _gvn.transform(new (C) IfFalseNode(iff_arg));
3305  result_rgn->init_req(no_clear_result_path, false_arg);
3306  result_val->init_req(no_clear_result_path, intcon(1));
3307
3308  // drop through to next case
3309  set_control( _gvn.transform(new (C) IfTrueNode(iff_arg)));
3310
3311  // (d) Otherwise, go to the slow path.
3312  slow_region->add_req(control());
3313  set_control( _gvn.transform(slow_region));
3314
3315  if (stopped()) {
3316    // There is no slow path.
3317    result_rgn->init_req(slow_result_path, top());
3318    result_val->init_req(slow_result_path, top());
3319  } else {
3320    // non-virtual because it is a private non-static
3321    CallJavaNode* slow_call = generate_method_call(vmIntrinsics::_isInterrupted);
3322
3323    Node* slow_val = set_results_for_java_call(slow_call);
3324    // this->control() comes from set_results_for_java_call
3325
3326    Node* fast_io  = slow_call->in(TypeFunc::I_O);
3327    Node* fast_mem = slow_call->in(TypeFunc::Memory);
3328
3329    // These two phis are pre-filled with copies of of the fast IO and Memory
3330    PhiNode* result_mem  = PhiNode::make(result_rgn, fast_mem, Type::MEMORY, TypePtr::BOTTOM);
3331    PhiNode* result_io   = PhiNode::make(result_rgn, fast_io,  Type::ABIO);
3332
3333    result_rgn->init_req(slow_result_path, control());
3334    result_io ->init_req(slow_result_path, i_o());
3335    result_mem->init_req(slow_result_path, reset_memory());
3336    result_val->init_req(slow_result_path, slow_val);
3337
3338    set_all_memory(_gvn.transform(result_mem));
3339    set_i_o(       _gvn.transform(result_io));
3340  }
3341
3342  C->set_has_split_ifs(true); // Has chance for split-if optimization
3343  set_result(result_rgn, result_val);
3344  return true;
3345}
3346
3347//---------------------------load_mirror_from_klass----------------------------
3348// Given a klass oop, load its java mirror (a java.lang.Class oop).
3349Node* LibraryCallKit::load_mirror_from_klass(Node* klass) {
3350  Node* p = basic_plus_adr(klass, in_bytes(Klass::java_mirror_offset()));
3351  return make_load(NULL, p, TypeInstPtr::MIRROR, T_OBJECT, MemNode::unordered);
3352}
3353
3354//-----------------------load_klass_from_mirror_common-------------------------
3355// Given a java mirror (a java.lang.Class oop), load its corresponding klass oop.
3356// Test the klass oop for null (signifying a primitive Class like Integer.TYPE),
3357// and branch to the given path on the region.
3358// If never_see_null, take an uncommon trap on null, so we can optimistically
3359// compile for the non-null case.
3360// If the region is NULL, force never_see_null = true.
3361Node* LibraryCallKit::load_klass_from_mirror_common(Node* mirror,
3362                                                    bool never_see_null,
3363                                                    RegionNode* region,
3364                                                    int null_path,
3365                                                    int offset) {
3366  if (region == NULL)  never_see_null = true;
3367  Node* p = basic_plus_adr(mirror, offset);
3368  const TypeKlassPtr*  kls_type = TypeKlassPtr::OBJECT_OR_NULL;
3369  Node* kls = _gvn.transform( LoadKlassNode::make(_gvn, immutable_memory(), p, TypeRawPtr::BOTTOM, kls_type));
3370  Node* null_ctl = top();
3371  kls = null_check_oop(kls, &null_ctl, never_see_null);
3372  if (region != NULL) {
3373    // Set region->in(null_path) if the mirror is a primitive (e.g, int.class).
3374    region->init_req(null_path, null_ctl);
3375  } else {
3376    assert(null_ctl == top(), "no loose ends");
3377  }
3378  return kls;
3379}
3380
3381//--------------------(inline_native_Class_query helpers)---------------------
3382// Use this for JVM_ACC_INTERFACE, JVM_ACC_IS_CLONEABLE, JVM_ACC_HAS_FINALIZER.
3383// Fall through if (mods & mask) == bits, take the guard otherwise.
3384Node* LibraryCallKit::generate_access_flags_guard(Node* kls, int modifier_mask, int modifier_bits, RegionNode* region) {
3385  // Branch around if the given klass has the given modifier bit set.
3386  // Like generate_guard, adds a new path onto the region.
3387  Node* modp = basic_plus_adr(kls, in_bytes(Klass::access_flags_offset()));
3388  Node* mods = make_load(NULL, modp, TypeInt::INT, T_INT, MemNode::unordered);
3389  Node* mask = intcon(modifier_mask);
3390  Node* bits = intcon(modifier_bits);
3391  Node* mbit = _gvn.transform(new (C) AndINode(mods, mask));
3392  Node* cmp  = _gvn.transform(new (C) CmpINode(mbit, bits));
3393  Node* bol  = _gvn.transform(new (C) BoolNode(cmp, BoolTest::ne));
3394  return generate_fair_guard(bol, region);
3395}
3396Node* LibraryCallKit::generate_interface_guard(Node* kls, RegionNode* region) {
3397  return generate_access_flags_guard(kls, JVM_ACC_INTERFACE, 0, region);
3398}
3399
3400//-------------------------inline_native_Class_query-------------------
3401bool LibraryCallKit::inline_native_Class_query(vmIntrinsics::ID id) {
3402  const Type* return_type = TypeInt::BOOL;
3403  Node* prim_return_value = top();  // what happens if it's a primitive class?
3404  bool never_see_null = !too_many_traps(Deoptimization::Reason_null_check);
3405  bool expect_prim = false;     // most of these guys expect to work on refs
3406
3407  enum { _normal_path = 1, _prim_path = 2, PATH_LIMIT };
3408
3409  Node* mirror = argument(0);
3410  Node* obj    = top();
3411
3412  switch (id) {
3413  case vmIntrinsics::_isInstance:
3414    // nothing is an instance of a primitive type
3415    prim_return_value = intcon(0);
3416    obj = argument(1);
3417    break;
3418  case vmIntrinsics::_getModifiers:
3419    prim_return_value = intcon(JVM_ACC_ABSTRACT | JVM_ACC_FINAL | JVM_ACC_PUBLIC);
3420    assert(is_power_of_2((int)JVM_ACC_WRITTEN_FLAGS+1), "change next line");
3421    return_type = TypeInt::make(0, JVM_ACC_WRITTEN_FLAGS, Type::WidenMin);
3422    break;
3423  case vmIntrinsics::_isInterface:
3424    prim_return_value = intcon(0);
3425    break;
3426  case vmIntrinsics::_isArray:
3427    prim_return_value = intcon(0);
3428    expect_prim = true;  // cf. ObjectStreamClass.getClassSignature
3429    break;
3430  case vmIntrinsics::_isPrimitive:
3431    prim_return_value = intcon(1);
3432    expect_prim = true;  // obviously
3433    break;
3434  case vmIntrinsics::_getSuperclass:
3435    prim_return_value = null();
3436    return_type = TypeInstPtr::MIRROR->cast_to_ptr_type(TypePtr::BotPTR);
3437    break;
3438  case vmIntrinsics::_getComponentType:
3439    prim_return_value = null();
3440    return_type = TypeInstPtr::MIRROR->cast_to_ptr_type(TypePtr::BotPTR);
3441    break;
3442  case vmIntrinsics::_getClassAccessFlags:
3443    prim_return_value = intcon(JVM_ACC_ABSTRACT | JVM_ACC_FINAL | JVM_ACC_PUBLIC);
3444    return_type = TypeInt::INT;  // not bool!  6297094
3445    break;
3446  default:
3447    fatal_unexpected_iid(id);
3448    break;
3449  }
3450
3451  const TypeInstPtr* mirror_con = _gvn.type(mirror)->isa_instptr();
3452  if (mirror_con == NULL)  return false;  // cannot happen?
3453
3454#ifndef PRODUCT
3455  if (C->print_intrinsics() || C->print_inlining()) {
3456    ciType* k = mirror_con->java_mirror_type();
3457    if (k) {
3458      tty->print("Inlining %s on constant Class ", vmIntrinsics::name_at(intrinsic_id()));
3459      k->print_name();
3460      tty->cr();
3461    }
3462  }
3463#endif
3464
3465  // Null-check the mirror, and the mirror's klass ptr (in case it is a primitive).
3466  RegionNode* region = new (C) RegionNode(PATH_LIMIT);
3467  record_for_igvn(region);
3468  PhiNode* phi = new (C) PhiNode(region, return_type);
3469
3470  // The mirror will never be null of Reflection.getClassAccessFlags, however
3471  // it may be null for Class.isInstance or Class.getModifiers. Throw a NPE
3472  // if it is. See bug 4774291.
3473
3474  // For Reflection.getClassAccessFlags(), the null check occurs in
3475  // the wrong place; see inline_unsafe_access(), above, for a similar
3476  // situation.
3477  mirror = null_check(mirror);
3478  // If mirror or obj is dead, only null-path is taken.
3479  if (stopped())  return true;
3480
3481  if (expect_prim)  never_see_null = false;  // expect nulls (meaning prims)
3482
3483  // Now load the mirror's klass metaobject, and null-check it.
3484  // Side-effects region with the control path if the klass is null.
3485  Node* kls = load_klass_from_mirror(mirror, never_see_null, region, _prim_path);
3486  // If kls is null, we have a primitive mirror.
3487  phi->init_req(_prim_path, prim_return_value);
3488  if (stopped()) { set_result(region, phi); return true; }
3489  bool safe_for_replace = (region->in(_prim_path) == top());
3490
3491  Node* p;  // handy temp
3492  Node* null_ctl;
3493
3494  // Now that we have the non-null klass, we can perform the real query.
3495  // For constant classes, the query will constant-fold in LoadNode::Value.
3496  Node* query_value = top();
3497  switch (id) {
3498  case vmIntrinsics::_isInstance:
3499    // nothing is an instance of a primitive type
3500    query_value = gen_instanceof(obj, kls, safe_for_replace);
3501    break;
3502
3503  case vmIntrinsics::_getModifiers:
3504    p = basic_plus_adr(kls, in_bytes(Klass::modifier_flags_offset()));
3505    query_value = make_load(NULL, p, TypeInt::INT, T_INT, MemNode::unordered);
3506    break;
3507
3508  case vmIntrinsics::_isInterface:
3509    // (To verify this code sequence, check the asserts in JVM_IsInterface.)
3510    if (generate_interface_guard(kls, region) != NULL)
3511      // A guard was added.  If the guard is taken, it was an interface.
3512      phi->add_req(intcon(1));
3513    // If we fall through, it's a plain class.
3514    query_value = intcon(0);
3515    break;
3516
3517  case vmIntrinsics::_isArray:
3518    // (To verify this code sequence, check the asserts in JVM_IsArrayClass.)
3519    if (generate_array_guard(kls, region) != NULL)
3520      // A guard was added.  If the guard is taken, it was an array.
3521      phi->add_req(intcon(1));
3522    // If we fall through, it's a plain class.
3523    query_value = intcon(0);
3524    break;
3525
3526  case vmIntrinsics::_isPrimitive:
3527    query_value = intcon(0); // "normal" path produces false
3528    break;
3529
3530  case vmIntrinsics::_getSuperclass:
3531    // The rules here are somewhat unfortunate, but we can still do better
3532    // with random logic than with a JNI call.
3533    // Interfaces store null or Object as _super, but must report null.
3534    // Arrays store an intermediate super as _super, but must report Object.
3535    // Other types can report the actual _super.
3536    // (To verify this code sequence, check the asserts in JVM_IsInterface.)
3537    if (generate_interface_guard(kls, region) != NULL)
3538      // A guard was added.  If the guard is taken, it was an interface.
3539      phi->add_req(null());
3540    if (generate_array_guard(kls, region) != NULL)
3541      // A guard was added.  If the guard is taken, it was an array.
3542      phi->add_req(makecon(TypeInstPtr::make(env()->Object_klass()->java_mirror())));
3543    // If we fall through, it's a plain class.  Get its _super.
3544    p = basic_plus_adr(kls, in_bytes(Klass::super_offset()));
3545    kls = _gvn.transform( LoadKlassNode::make(_gvn, immutable_memory(), p, TypeRawPtr::BOTTOM, TypeKlassPtr::OBJECT_OR_NULL));
3546    null_ctl = top();
3547    kls = null_check_oop(kls, &null_ctl);
3548    if (null_ctl != top()) {
3549      // If the guard is taken, Object.superClass is null (both klass and mirror).
3550      region->add_req(null_ctl);
3551      phi   ->add_req(null());
3552    }
3553    if (!stopped()) {
3554      query_value = load_mirror_from_klass(kls);
3555    }
3556    break;
3557
3558  case vmIntrinsics::_getComponentType:
3559    if (generate_array_guard(kls, region) != NULL) {
3560      // Be sure to pin the oop load to the guard edge just created:
3561      Node* is_array_ctrl = region->in(region->req()-1);
3562      Node* cma = basic_plus_adr(kls, in_bytes(ArrayKlass::component_mirror_offset()));
3563      Node* cmo = make_load(is_array_ctrl, cma, TypeInstPtr::MIRROR, T_OBJECT, MemNode::unordered);
3564      phi->add_req(cmo);
3565    }
3566    query_value = null();  // non-array case is null
3567    break;
3568
3569  case vmIntrinsics::_getClassAccessFlags:
3570    p = basic_plus_adr(kls, in_bytes(Klass::access_flags_offset()));
3571    query_value = make_load(NULL, p, TypeInt::INT, T_INT, MemNode::unordered);
3572    break;
3573
3574  default:
3575    fatal_unexpected_iid(id);
3576    break;
3577  }
3578
3579  // Fall-through is the normal case of a query to a real class.
3580  phi->init_req(1, query_value);
3581  region->init_req(1, control());
3582
3583  C->set_has_split_ifs(true); // Has chance for split-if optimization
3584  set_result(region, phi);
3585  return true;
3586}
3587
3588//--------------------------inline_native_subtype_check------------------------
3589// This intrinsic takes the JNI calls out of the heart of
3590// UnsafeFieldAccessorImpl.set, which improves Field.set, readObject, etc.
3591bool LibraryCallKit::inline_native_subtype_check() {
3592  // Pull both arguments off the stack.
3593  Node* args[2];                // two java.lang.Class mirrors: superc, subc
3594  args[0] = argument(0);
3595  args[1] = argument(1);
3596  Node* klasses[2];             // corresponding Klasses: superk, subk
3597  klasses[0] = klasses[1] = top();
3598
3599  enum {
3600    // A full decision tree on {superc is prim, subc is prim}:
3601    _prim_0_path = 1,           // {P,N} => false
3602                                // {P,P} & superc!=subc => false
3603    _prim_same_path,            // {P,P} & superc==subc => true
3604    _prim_1_path,               // {N,P} => false
3605    _ref_subtype_path,          // {N,N} & subtype check wins => true
3606    _both_ref_path,             // {N,N} & subtype check loses => false
3607    PATH_LIMIT
3608  };
3609
3610  RegionNode* region = new (C) RegionNode(PATH_LIMIT);
3611  Node*       phi    = new (C) PhiNode(region, TypeInt::BOOL);
3612  record_for_igvn(region);
3613
3614  const TypePtr* adr_type = TypeRawPtr::BOTTOM;   // memory type of loads
3615  const TypeKlassPtr* kls_type = TypeKlassPtr::OBJECT_OR_NULL;
3616  int class_klass_offset = java_lang_Class::klass_offset_in_bytes();
3617
3618  // First null-check both mirrors and load each mirror's klass metaobject.
3619  int which_arg;
3620  for (which_arg = 0; which_arg <= 1; which_arg++) {
3621    Node* arg = args[which_arg];
3622    arg = null_check(arg);
3623    if (stopped())  break;
3624    args[which_arg] = arg;
3625
3626    Node* p = basic_plus_adr(arg, class_klass_offset);
3627    Node* kls = LoadKlassNode::make(_gvn, immutable_memory(), p, adr_type, kls_type);
3628    klasses[which_arg] = _gvn.transform(kls);
3629  }
3630
3631  // Having loaded both klasses, test each for null.
3632  bool never_see_null = !too_many_traps(Deoptimization::Reason_null_check);
3633  for (which_arg = 0; which_arg <= 1; which_arg++) {
3634    Node* kls = klasses[which_arg];
3635    Node* null_ctl = top();
3636    kls = null_check_oop(kls, &null_ctl, never_see_null);
3637    int prim_path = (which_arg == 0 ? _prim_0_path : _prim_1_path);
3638    region->init_req(prim_path, null_ctl);
3639    if (stopped())  break;
3640    klasses[which_arg] = kls;
3641  }
3642
3643  if (!stopped()) {
3644    // now we have two reference types, in klasses[0..1]
3645    Node* subk   = klasses[1];  // the argument to isAssignableFrom
3646    Node* superk = klasses[0];  // the receiver
3647    region->set_req(_both_ref_path, gen_subtype_check(subk, superk));
3648    // now we have a successful reference subtype check
3649    region->set_req(_ref_subtype_path, control());
3650  }
3651
3652  // If both operands are primitive (both klasses null), then
3653  // we must return true when they are identical primitives.
3654  // It is convenient to test this after the first null klass check.
3655  set_control(region->in(_prim_0_path)); // go back to first null check
3656  if (!stopped()) {
3657    // Since superc is primitive, make a guard for the superc==subc case.
3658    Node* cmp_eq = _gvn.transform(new (C) CmpPNode(args[0], args[1]));
3659    Node* bol_eq = _gvn.transform(new (C) BoolNode(cmp_eq, BoolTest::eq));
3660    generate_guard(bol_eq, region, PROB_FAIR);
3661    if (region->req() == PATH_LIMIT+1) {
3662      // A guard was added.  If the added guard is taken, superc==subc.
3663      region->swap_edges(PATH_LIMIT, _prim_same_path);
3664      region->del_req(PATH_LIMIT);
3665    }
3666    region->set_req(_prim_0_path, control()); // Not equal after all.
3667  }
3668
3669  // these are the only paths that produce 'true':
3670  phi->set_req(_prim_same_path,   intcon(1));
3671  phi->set_req(_ref_subtype_path, intcon(1));
3672
3673  // pull together the cases:
3674  assert(region->req() == PATH_LIMIT, "sane region");
3675  for (uint i = 1; i < region->req(); i++) {
3676    Node* ctl = region->in(i);
3677    if (ctl == NULL || ctl == top()) {
3678      region->set_req(i, top());
3679      phi   ->set_req(i, top());
3680    } else if (phi->in(i) == NULL) {
3681      phi->set_req(i, intcon(0)); // all other paths produce 'false'
3682    }
3683  }
3684
3685  set_control(_gvn.transform(region));
3686  set_result(_gvn.transform(phi));
3687  return true;
3688}
3689
3690//---------------------generate_array_guard_common------------------------
3691Node* LibraryCallKit::generate_array_guard_common(Node* kls, RegionNode* region,
3692                                                  bool obj_array, bool not_array) {
3693  // If obj_array/non_array==false/false:
3694  // Branch around if the given klass is in fact an array (either obj or prim).
3695  // If obj_array/non_array==false/true:
3696  // Branch around if the given klass is not an array klass of any kind.
3697  // If obj_array/non_array==true/true:
3698  // Branch around if the kls is not an oop array (kls is int[], String, etc.)
3699  // If obj_array/non_array==true/false:
3700  // Branch around if the kls is an oop array (Object[] or subtype)
3701  //
3702  // Like generate_guard, adds a new path onto the region.
3703  jint  layout_con = 0;
3704  Node* layout_val = get_layout_helper(kls, layout_con);
3705  if (layout_val == NULL) {
3706    bool query = (obj_array
3707                  ? Klass::layout_helper_is_objArray(layout_con)
3708                  : Klass::layout_helper_is_array(layout_con));
3709    if (query == not_array) {
3710      return NULL;                       // never a branch
3711    } else {                             // always a branch
3712      Node* always_branch = control();
3713      if (region != NULL)
3714        region->add_req(always_branch);
3715      set_control(top());
3716      return always_branch;
3717    }
3718  }
3719  // Now test the correct condition.
3720  jint  nval = (obj_array
3721                ? ((jint)Klass::_lh_array_tag_type_value
3722                   <<    Klass::_lh_array_tag_shift)
3723                : Klass::_lh_neutral_value);
3724  Node* cmp = _gvn.transform(new(C) CmpINode(layout_val, intcon(nval)));
3725  BoolTest::mask btest = BoolTest::lt;  // correct for testing is_[obj]array
3726  // invert the test if we are looking for a non-array
3727  if (not_array)  btest = BoolTest(btest).negate();
3728  Node* bol = _gvn.transform(new(C) BoolNode(cmp, btest));
3729  return generate_fair_guard(bol, region);
3730}
3731
3732
3733//-----------------------inline_native_newArray--------------------------
3734// private static native Object java.lang.reflect.newArray(Class<?> componentType, int length);
3735bool LibraryCallKit::inline_native_newArray() {
3736  Node* mirror    = argument(0);
3737  Node* count_val = argument(1);
3738
3739  mirror = null_check(mirror);
3740  // If mirror or obj is dead, only null-path is taken.
3741  if (stopped())  return true;
3742
3743  enum { _normal_path = 1, _slow_path = 2, PATH_LIMIT };
3744  RegionNode* result_reg = new(C) RegionNode(PATH_LIMIT);
3745  PhiNode*    result_val = new(C) PhiNode(result_reg,
3746                                          TypeInstPtr::NOTNULL);
3747  PhiNode*    result_io  = new(C) PhiNode(result_reg, Type::ABIO);
3748  PhiNode*    result_mem = new(C) PhiNode(result_reg, Type::MEMORY,
3749                                          TypePtr::BOTTOM);
3750
3751  bool never_see_null = !too_many_traps(Deoptimization::Reason_null_check);
3752  Node* klass_node = load_array_klass_from_mirror(mirror, never_see_null,
3753                                                  result_reg, _slow_path);
3754  Node* normal_ctl   = control();
3755  Node* no_array_ctl = result_reg->in(_slow_path);
3756
3757  // Generate code for the slow case.  We make a call to newArray().
3758  set_control(no_array_ctl);
3759  if (!stopped()) {
3760    // Either the input type is void.class, or else the
3761    // array klass has not yet been cached.  Either the
3762    // ensuing call will throw an exception, or else it
3763    // will cache the array klass for next time.
3764    PreserveJVMState pjvms(this);
3765    CallJavaNode* slow_call = generate_method_call_static(vmIntrinsics::_newArray);
3766    Node* slow_result = set_results_for_java_call(slow_call);
3767    // this->control() comes from set_results_for_java_call
3768    result_reg->set_req(_slow_path, control());
3769    result_val->set_req(_slow_path, slow_result);
3770    result_io ->set_req(_slow_path, i_o());
3771    result_mem->set_req(_slow_path, reset_memory());
3772  }
3773
3774  set_control(normal_ctl);
3775  if (!stopped()) {
3776    // Normal case:  The array type has been cached in the java.lang.Class.
3777    // The following call works fine even if the array type is polymorphic.
3778    // It could be a dynamic mix of int[], boolean[], Object[], etc.
3779    Node* obj = new_array(klass_node, count_val, 0);  // no arguments to push
3780    result_reg->init_req(_normal_path, control());
3781    result_val->init_req(_normal_path, obj);
3782    result_io ->init_req(_normal_path, i_o());
3783    result_mem->init_req(_normal_path, reset_memory());
3784  }
3785
3786  // Return the combined state.
3787  set_i_o(        _gvn.transform(result_io)  );
3788  set_all_memory( _gvn.transform(result_mem));
3789
3790  C->set_has_split_ifs(true); // Has chance for split-if optimization
3791  set_result(result_reg, result_val);
3792  return true;
3793}
3794
3795//----------------------inline_native_getLength--------------------------
3796// public static native int java.lang.reflect.Array.getLength(Object array);
3797bool LibraryCallKit::inline_native_getLength() {
3798  if (too_many_traps(Deoptimization::Reason_intrinsic))  return false;
3799
3800  Node* array = null_check(argument(0));
3801  // If array is dead, only null-path is taken.
3802  if (stopped())  return true;
3803
3804  // Deoptimize if it is a non-array.
3805  Node* non_array = generate_non_array_guard(load_object_klass(array), NULL);
3806
3807  if (non_array != NULL) {
3808    PreserveJVMState pjvms(this);
3809    set_control(non_array);
3810    uncommon_trap(Deoptimization::Reason_intrinsic,
3811                  Deoptimization::Action_maybe_recompile);
3812  }
3813
3814  // If control is dead, only non-array-path is taken.
3815  if (stopped())  return true;
3816
3817  // The works fine even if the array type is polymorphic.
3818  // It could be a dynamic mix of int[], boolean[], Object[], etc.
3819  Node* result = load_array_length(array);
3820
3821  C->set_has_split_ifs(true);  // Has chance for split-if optimization
3822  set_result(result);
3823  return true;
3824}
3825
3826//------------------------inline_array_copyOf----------------------------
3827// public static <T,U> T[] java.util.Arrays.copyOf(     U[] original, int newLength,         Class<? extends T[]> newType);
3828// public static <T,U> T[] java.util.Arrays.copyOfRange(U[] original, int from,      int to, Class<? extends T[]> newType);
3829bool LibraryCallKit::inline_array_copyOf(bool is_copyOfRange) {
3830  if (too_many_traps(Deoptimization::Reason_intrinsic))  return false;
3831
3832  // Get the arguments.
3833  Node* original          = argument(0);
3834  Node* start             = is_copyOfRange? argument(1): intcon(0);
3835  Node* end               = is_copyOfRange? argument(2): argument(1);
3836  Node* array_type_mirror = is_copyOfRange? argument(3): argument(2);
3837
3838  Node* newcopy;
3839
3840  // Set the original stack and the reexecute bit for the interpreter to reexecute
3841  // the bytecode that invokes Arrays.copyOf if deoptimization happens.
3842  { PreserveReexecuteState preexecs(this);
3843    jvms()->set_should_reexecute(true);
3844
3845    array_type_mirror = null_check(array_type_mirror);
3846    original          = null_check(original);
3847
3848    // Check if a null path was taken unconditionally.
3849    if (stopped())  return true;
3850
3851    Node* orig_length = load_array_length(original);
3852
3853    Node* klass_node = load_klass_from_mirror(array_type_mirror, false, NULL, 0);
3854    klass_node = null_check(klass_node);
3855
3856    RegionNode* bailout = new (C) RegionNode(1);
3857    record_for_igvn(bailout);
3858
3859    // Despite the generic type of Arrays.copyOf, the mirror might be int, int[], etc.
3860    // Bail out if that is so.
3861    Node* not_objArray = generate_non_objArray_guard(klass_node, bailout);
3862    if (not_objArray != NULL) {
3863      // Improve the klass node's type from the new optimistic assumption:
3864      ciKlass* ak = ciArrayKlass::make(env()->Object_klass());
3865      const Type* akls = TypeKlassPtr::make(TypePtr::NotNull, ak, 0/*offset*/);
3866      Node* cast = new (C) CastPPNode(klass_node, akls);
3867      cast->init_req(0, control());
3868      klass_node = _gvn.transform(cast);
3869    }
3870
3871    // Bail out if either start or end is negative.
3872    generate_negative_guard(start, bailout, &start);
3873    generate_negative_guard(end,   bailout, &end);
3874
3875    Node* length = end;
3876    if (_gvn.type(start) != TypeInt::ZERO) {
3877      length = _gvn.transform(new (C) SubINode(end, start));
3878    }
3879
3880    // Bail out if length is negative.
3881    // Without this the new_array would throw
3882    // NegativeArraySizeException but IllegalArgumentException is what
3883    // should be thrown
3884    generate_negative_guard(length, bailout, &length);
3885
3886    if (bailout->req() > 1) {
3887      PreserveJVMState pjvms(this);
3888      set_control(_gvn.transform(bailout));
3889      uncommon_trap(Deoptimization::Reason_intrinsic,
3890                    Deoptimization::Action_maybe_recompile);
3891    }
3892
3893    if (!stopped()) {
3894      // How many elements will we copy from the original?
3895      // The answer is MinI(orig_length - start, length).
3896      Node* orig_tail = _gvn.transform(new (C) SubINode(orig_length, start));
3897      Node* moved = generate_min_max(vmIntrinsics::_min, orig_tail, length);
3898
3899      newcopy = new_array(klass_node, length, 0);  // no argments to push
3900
3901      // Generate a direct call to the right arraycopy function(s).
3902      // We know the copy is disjoint but we might not know if the
3903      // oop stores need checking.
3904      // Extreme case:  Arrays.copyOf((Integer[])x, 10, String[].class).
3905      // This will fail a store-check if x contains any non-nulls.
3906      bool disjoint_bases = true;
3907      // if start > orig_length then the length of the copy may be
3908      // negative.
3909      bool length_never_negative = !is_copyOfRange;
3910      generate_arraycopy(TypeAryPtr::OOPS, T_OBJECT,
3911                         original, start, newcopy, intcon(0), moved,
3912                         disjoint_bases, length_never_negative);
3913    }
3914  } // original reexecute is set back here
3915
3916  C->set_has_split_ifs(true); // Has chance for split-if optimization
3917  if (!stopped()) {
3918    set_result(newcopy);
3919  }
3920  return true;
3921}
3922
3923
3924//----------------------generate_virtual_guard---------------------------
3925// Helper for hashCode and clone.  Peeks inside the vtable to avoid a call.
3926Node* LibraryCallKit::generate_virtual_guard(Node* obj_klass,
3927                                             RegionNode* slow_region) {
3928  ciMethod* method = callee();
3929  int vtable_index = method->vtable_index();
3930  assert(vtable_index >= 0 || vtable_index == Method::nonvirtual_vtable_index,
3931         err_msg_res("bad index %d", vtable_index));
3932  // Get the Method* out of the appropriate vtable entry.
3933  int entry_offset  = (InstanceKlass::vtable_start_offset() +
3934                     vtable_index*vtableEntry::size()) * wordSize +
3935                     vtableEntry::method_offset_in_bytes();
3936  Node* entry_addr  = basic_plus_adr(obj_klass, entry_offset);
3937  Node* target_call = make_load(NULL, entry_addr, TypePtr::NOTNULL, T_ADDRESS, MemNode::unordered);
3938
3939  // Compare the target method with the expected method (e.g., Object.hashCode).
3940  const TypePtr* native_call_addr = TypeMetadataPtr::make(method);
3941
3942  Node* native_call = makecon(native_call_addr);
3943  Node* chk_native  = _gvn.transform(new(C) CmpPNode(target_call, native_call));
3944  Node* test_native = _gvn.transform(new(C) BoolNode(chk_native, BoolTest::ne));
3945
3946  return generate_slow_guard(test_native, slow_region);
3947}
3948
3949//-----------------------generate_method_call----------------------------
3950// Use generate_method_call to make a slow-call to the real
3951// method if the fast path fails.  An alternative would be to
3952// use a stub like OptoRuntime::slow_arraycopy_Java.
3953// This only works for expanding the current library call,
3954// not another intrinsic.  (E.g., don't use this for making an
3955// arraycopy call inside of the copyOf intrinsic.)
3956CallJavaNode*
3957LibraryCallKit::generate_method_call(vmIntrinsics::ID method_id, bool is_virtual, bool is_static) {
3958  // When compiling the intrinsic method itself, do not use this technique.
3959  guarantee(callee() != C->method(), "cannot make slow-call to self");
3960
3961  ciMethod* method = callee();
3962  // ensure the JVMS we have will be correct for this call
3963  guarantee(method_id == method->intrinsic_id(), "must match");
3964
3965  const TypeFunc* tf = TypeFunc::make(method);
3966  CallJavaNode* slow_call;
3967  if (is_static) {
3968    assert(!is_virtual, "");
3969    slow_call = new(C) CallStaticJavaNode(C, tf,
3970                           SharedRuntime::get_resolve_static_call_stub(),
3971                           method, bci());
3972  } else if (is_virtual) {
3973    null_check_receiver();
3974    int vtable_index = Method::invalid_vtable_index;
3975    if (UseInlineCaches) {
3976      // Suppress the vtable call
3977    } else {
3978      // hashCode and clone are not a miranda methods,
3979      // so the vtable index is fixed.
3980      // No need to use the linkResolver to get it.
3981       vtable_index = method->vtable_index();
3982       assert(vtable_index >= 0 || vtable_index == Method::nonvirtual_vtable_index,
3983              err_msg_res("bad index %d", vtable_index));
3984    }
3985    slow_call = new(C) CallDynamicJavaNode(tf,
3986                          SharedRuntime::get_resolve_virtual_call_stub(),
3987                          method, vtable_index, bci());
3988  } else {  // neither virtual nor static:  opt_virtual
3989    null_check_receiver();
3990    slow_call = new(C) CallStaticJavaNode(C, tf,
3991                                SharedRuntime::get_resolve_opt_virtual_call_stub(),
3992                                method, bci());
3993    slow_call->set_optimized_virtual(true);
3994  }
3995  set_arguments_for_java_call(slow_call);
3996  set_edges_for_java_call(slow_call);
3997  return slow_call;
3998}
3999
4000
4001//------------------------------inline_native_hashcode--------------------
4002// Build special case code for calls to hashCode on an object.
4003bool LibraryCallKit::inline_native_hashcode(bool is_virtual, bool is_static) {
4004  assert(is_static == callee()->is_static(), "correct intrinsic selection");
4005  assert(!(is_virtual && is_static), "either virtual, special, or static");
4006
4007  enum { _slow_path = 1, _fast_path, _null_path, PATH_LIMIT };
4008
4009  RegionNode* result_reg = new(C) RegionNode(PATH_LIMIT);
4010  PhiNode*    result_val = new(C) PhiNode(result_reg,
4011                                          TypeInt::INT);
4012  PhiNode*    result_io  = new(C) PhiNode(result_reg, Type::ABIO);
4013  PhiNode*    result_mem = new(C) PhiNode(result_reg, Type::MEMORY,
4014                                          TypePtr::BOTTOM);
4015  Node* obj = NULL;
4016  if (!is_static) {
4017    // Check for hashing null object
4018    obj = null_check_receiver();
4019    if (stopped())  return true;        // unconditionally null
4020    result_reg->init_req(_null_path, top());
4021    result_val->init_req(_null_path, top());
4022  } else {
4023    // Do a null check, and return zero if null.
4024    // System.identityHashCode(null) == 0
4025    obj = argument(0);
4026    Node* null_ctl = top();
4027    obj = null_check_oop(obj, &null_ctl);
4028    result_reg->init_req(_null_path, null_ctl);
4029    result_val->init_req(_null_path, _gvn.intcon(0));
4030  }
4031
4032  // Unconditionally null?  Then return right away.
4033  if (stopped()) {
4034    set_control( result_reg->in(_null_path));
4035    if (!stopped())
4036      set_result(result_val->in(_null_path));
4037    return true;
4038  }
4039
4040  // After null check, get the object's klass.
4041  Node* obj_klass = load_object_klass(obj);
4042
4043  // This call may be virtual (invokevirtual) or bound (invokespecial).
4044  // For each case we generate slightly different code.
4045
4046  // We only go to the fast case code if we pass a number of guards.  The
4047  // paths which do not pass are accumulated in the slow_region.
4048  RegionNode* slow_region = new (C) RegionNode(1);
4049  record_for_igvn(slow_region);
4050
4051  // If this is a virtual call, we generate a funny guard.  We pull out
4052  // the vtable entry corresponding to hashCode() from the target object.
4053  // If the target method which we are calling happens to be the native
4054  // Object hashCode() method, we pass the guard.  We do not need this
4055  // guard for non-virtual calls -- the caller is known to be the native
4056  // Object hashCode().
4057  if (is_virtual) {
4058    generate_virtual_guard(obj_klass, slow_region);
4059  }
4060
4061  // Get the header out of the object, use LoadMarkNode when available
4062  Node* header_addr = basic_plus_adr(obj, oopDesc::mark_offset_in_bytes());
4063  Node* header = make_load(control(), header_addr, TypeX_X, TypeX_X->basic_type(), MemNode::unordered);
4064
4065  // Test the header to see if it is unlocked.
4066  Node *lock_mask      = _gvn.MakeConX(markOopDesc::biased_lock_mask_in_place);
4067  Node *lmasked_header = _gvn.transform(new (C) AndXNode(header, lock_mask));
4068  Node *unlocked_val   = _gvn.MakeConX(markOopDesc::unlocked_value);
4069  Node *chk_unlocked   = _gvn.transform(new (C) CmpXNode( lmasked_header, unlocked_val));
4070  Node *test_unlocked  = _gvn.transform(new (C) BoolNode( chk_unlocked, BoolTest::ne));
4071
4072  generate_slow_guard(test_unlocked, slow_region);
4073
4074  // Get the hash value and check to see that it has been properly assigned.
4075  // We depend on hash_mask being at most 32 bits and avoid the use of
4076  // hash_mask_in_place because it could be larger than 32 bits in a 64-bit
4077  // vm: see markOop.hpp.
4078  Node *hash_mask      = _gvn.intcon(markOopDesc::hash_mask);
4079  Node *hash_shift     = _gvn.intcon(markOopDesc::hash_shift);
4080  Node *hshifted_header= _gvn.transform(new (C) URShiftXNode(header, hash_shift));
4081  // This hack lets the hash bits live anywhere in the mark object now, as long
4082  // as the shift drops the relevant bits into the low 32 bits.  Note that
4083  // Java spec says that HashCode is an int so there's no point in capturing
4084  // an 'X'-sized hashcode (32 in 32-bit build or 64 in 64-bit build).
4085  hshifted_header      = ConvX2I(hshifted_header);
4086  Node *hash_val       = _gvn.transform(new (C) AndINode(hshifted_header, hash_mask));
4087
4088  Node *no_hash_val    = _gvn.intcon(markOopDesc::no_hash);
4089  Node *chk_assigned   = _gvn.transform(new (C) CmpINode( hash_val, no_hash_val));
4090  Node *test_assigned  = _gvn.transform(new (C) BoolNode( chk_assigned, BoolTest::eq));
4091
4092  generate_slow_guard(test_assigned, slow_region);
4093
4094  Node* init_mem = reset_memory();
4095  // fill in the rest of the null path:
4096  result_io ->init_req(_null_path, i_o());
4097  result_mem->init_req(_null_path, init_mem);
4098
4099  result_val->init_req(_fast_path, hash_val);
4100  result_reg->init_req(_fast_path, control());
4101  result_io ->init_req(_fast_path, i_o());
4102  result_mem->init_req(_fast_path, init_mem);
4103
4104  // Generate code for the slow case.  We make a call to hashCode().
4105  set_control(_gvn.transform(slow_region));
4106  if (!stopped()) {
4107    // No need for PreserveJVMState, because we're using up the present state.
4108    set_all_memory(init_mem);
4109    vmIntrinsics::ID hashCode_id = is_static ? vmIntrinsics::_identityHashCode : vmIntrinsics::_hashCode;
4110    CallJavaNode* slow_call = generate_method_call(hashCode_id, is_virtual, is_static);
4111    Node* slow_result = set_results_for_java_call(slow_call);
4112    // this->control() comes from set_results_for_java_call
4113    result_reg->init_req(_slow_path, control());
4114    result_val->init_req(_slow_path, slow_result);
4115    result_io  ->set_req(_slow_path, i_o());
4116    result_mem ->set_req(_slow_path, reset_memory());
4117  }
4118
4119  // Return the combined state.
4120  set_i_o(        _gvn.transform(result_io)  );
4121  set_all_memory( _gvn.transform(result_mem));
4122
4123  set_result(result_reg, result_val);
4124  return true;
4125}
4126
4127//---------------------------inline_native_getClass----------------------------
4128// public final native Class<?> java.lang.Object.getClass();
4129//
4130// Build special case code for calls to getClass on an object.
4131bool LibraryCallKit::inline_native_getClass() {
4132  Node* obj = null_check_receiver();
4133  if (stopped())  return true;
4134  set_result(load_mirror_from_klass(load_object_klass(obj)));
4135  return true;
4136}
4137
4138//-----------------inline_native_Reflection_getCallerClass---------------------
4139// public static native Class<?> sun.reflect.Reflection.getCallerClass();
4140//
4141// In the presence of deep enough inlining, getCallerClass() becomes a no-op.
4142//
4143// NOTE: This code must perform the same logic as JVM_GetCallerClass
4144// in that it must skip particular security frames and checks for
4145// caller sensitive methods.
4146bool LibraryCallKit::inline_native_Reflection_getCallerClass() {
4147#ifndef PRODUCT
4148  if ((C->print_intrinsics() || C->print_inlining()) && Verbose) {
4149    tty->print_cr("Attempting to inline sun.reflect.Reflection.getCallerClass");
4150  }
4151#endif
4152
4153  if (!jvms()->has_method()) {
4154#ifndef PRODUCT
4155    if ((C->print_intrinsics() || C->print_inlining()) && Verbose) {
4156      tty->print_cr("  Bailing out because intrinsic was inlined at top level");
4157    }
4158#endif
4159    return false;
4160  }
4161
4162  // Walk back up the JVM state to find the caller at the required
4163  // depth.
4164  JVMState* caller_jvms = jvms();
4165
4166  // Cf. JVM_GetCallerClass
4167  // NOTE: Start the loop at depth 1 because the current JVM state does
4168  // not include the Reflection.getCallerClass() frame.
4169  for (int n = 1; caller_jvms != NULL; caller_jvms = caller_jvms->caller(), n++) {
4170    ciMethod* m = caller_jvms->method();
4171    switch (n) {
4172    case 0:
4173      fatal("current JVM state does not include the Reflection.getCallerClass frame");
4174      break;
4175    case 1:
4176      // Frame 0 and 1 must be caller sensitive (see JVM_GetCallerClass).
4177      if (!m->caller_sensitive()) {
4178#ifndef PRODUCT
4179        if ((C->print_intrinsics() || C->print_inlining()) && Verbose) {
4180          tty->print_cr("  Bailing out: CallerSensitive annotation expected at frame %d", n);
4181        }
4182#endif
4183        return false;  // bail-out; let JVM_GetCallerClass do the work
4184      }
4185      break;
4186    default:
4187      if (!m->is_ignored_by_security_stack_walk()) {
4188        // We have reached the desired frame; return the holder class.
4189        // Acquire method holder as java.lang.Class and push as constant.
4190        ciInstanceKlass* caller_klass = caller_jvms->method()->holder();
4191        ciInstance* caller_mirror = caller_klass->java_mirror();
4192        set_result(makecon(TypeInstPtr::make(caller_mirror)));
4193
4194#ifndef PRODUCT
4195        if ((C->print_intrinsics() || C->print_inlining()) && Verbose) {
4196          tty->print_cr("  Succeeded: caller = %d) %s.%s, JVMS depth = %d", n, caller_klass->name()->as_utf8(), caller_jvms->method()->name()->as_utf8(), jvms()->depth());
4197          tty->print_cr("  JVM state at this point:");
4198          for (int i = jvms()->depth(), n = 1; i >= 1; i--, n++) {
4199            ciMethod* m = jvms()->of_depth(i)->method();
4200            tty->print_cr("   %d) %s.%s", n, m->holder()->name()->as_utf8(), m->name()->as_utf8());
4201          }
4202        }
4203#endif
4204        return true;
4205      }
4206      break;
4207    }
4208  }
4209
4210#ifndef PRODUCT
4211  if ((C->print_intrinsics() || C->print_inlining()) && Verbose) {
4212    tty->print_cr("  Bailing out because caller depth exceeded inlining depth = %d", jvms()->depth());
4213    tty->print_cr("  JVM state at this point:");
4214    for (int i = jvms()->depth(), n = 1; i >= 1; i--, n++) {
4215      ciMethod* m = jvms()->of_depth(i)->method();
4216      tty->print_cr("   %d) %s.%s", n, m->holder()->name()->as_utf8(), m->name()->as_utf8());
4217    }
4218  }
4219#endif
4220
4221  return false;  // bail-out; let JVM_GetCallerClass do the work
4222}
4223
4224bool LibraryCallKit::inline_fp_conversions(vmIntrinsics::ID id) {
4225  Node* arg = argument(0);
4226  Node* result;
4227
4228  switch (id) {
4229  case vmIntrinsics::_floatToRawIntBits:    result = new (C) MoveF2INode(arg);  break;
4230  case vmIntrinsics::_intBitsToFloat:       result = new (C) MoveI2FNode(arg);  break;
4231  case vmIntrinsics::_doubleToRawLongBits:  result = new (C) MoveD2LNode(arg);  break;
4232  case vmIntrinsics::_longBitsToDouble:     result = new (C) MoveL2DNode(arg);  break;
4233
4234  case vmIntrinsics::_doubleToLongBits: {
4235    // two paths (plus control) merge in a wood
4236    RegionNode *r = new (C) RegionNode(3);
4237    Node *phi = new (C) PhiNode(r, TypeLong::LONG);
4238
4239    Node *cmpisnan = _gvn.transform(new (C) CmpDNode(arg, arg));
4240    // Build the boolean node
4241    Node *bolisnan = _gvn.transform(new (C) BoolNode(cmpisnan, BoolTest::ne));
4242
4243    // Branch either way.
4244    // NaN case is less traveled, which makes all the difference.
4245    IfNode *ifisnan = create_and_xform_if(control(), bolisnan, PROB_STATIC_FREQUENT, COUNT_UNKNOWN);
4246    Node *opt_isnan = _gvn.transform(ifisnan);
4247    assert( opt_isnan->is_If(), "Expect an IfNode");
4248    IfNode *opt_ifisnan = (IfNode*)opt_isnan;
4249    Node *iftrue = _gvn.transform(new (C) IfTrueNode(opt_ifisnan));
4250
4251    set_control(iftrue);
4252
4253    static const jlong nan_bits = CONST64(0x7ff8000000000000);
4254    Node *slow_result = longcon(nan_bits); // return NaN
4255    phi->init_req(1, _gvn.transform( slow_result ));
4256    r->init_req(1, iftrue);
4257
4258    // Else fall through
4259    Node *iffalse = _gvn.transform(new (C) IfFalseNode(opt_ifisnan));
4260    set_control(iffalse);
4261
4262    phi->init_req(2, _gvn.transform(new (C) MoveD2LNode(arg)));
4263    r->init_req(2, iffalse);
4264
4265    // Post merge
4266    set_control(_gvn.transform(r));
4267    record_for_igvn(r);
4268
4269    C->set_has_split_ifs(true); // Has chance for split-if optimization
4270    result = phi;
4271    assert(result->bottom_type()->isa_long(), "must be");
4272    break;
4273  }
4274
4275  case vmIntrinsics::_floatToIntBits: {
4276    // two paths (plus control) merge in a wood
4277    RegionNode *r = new (C) RegionNode(3);
4278    Node *phi = new (C) PhiNode(r, TypeInt::INT);
4279
4280    Node *cmpisnan = _gvn.transform(new (C) CmpFNode(arg, arg));
4281    // Build the boolean node
4282    Node *bolisnan = _gvn.transform(new (C) BoolNode(cmpisnan, BoolTest::ne));
4283
4284    // Branch either way.
4285    // NaN case is less traveled, which makes all the difference.
4286    IfNode *ifisnan = create_and_xform_if(control(), bolisnan, PROB_STATIC_FREQUENT, COUNT_UNKNOWN);
4287    Node *opt_isnan = _gvn.transform(ifisnan);
4288    assert( opt_isnan->is_If(), "Expect an IfNode");
4289    IfNode *opt_ifisnan = (IfNode*)opt_isnan;
4290    Node *iftrue = _gvn.transform(new (C) IfTrueNode(opt_ifisnan));
4291
4292    set_control(iftrue);
4293
4294    static const jint nan_bits = 0x7fc00000;
4295    Node *slow_result = makecon(TypeInt::make(nan_bits)); // return NaN
4296    phi->init_req(1, _gvn.transform( slow_result ));
4297    r->init_req(1, iftrue);
4298
4299    // Else fall through
4300    Node *iffalse = _gvn.transform(new (C) IfFalseNode(opt_ifisnan));
4301    set_control(iffalse);
4302
4303    phi->init_req(2, _gvn.transform(new (C) MoveF2INode(arg)));
4304    r->init_req(2, iffalse);
4305
4306    // Post merge
4307    set_control(_gvn.transform(r));
4308    record_for_igvn(r);
4309
4310    C->set_has_split_ifs(true); // Has chance for split-if optimization
4311    result = phi;
4312    assert(result->bottom_type()->isa_int(), "must be");
4313    break;
4314  }
4315
4316  default:
4317    fatal_unexpected_iid(id);
4318    break;
4319  }
4320  set_result(_gvn.transform(result));
4321  return true;
4322}
4323
4324#ifdef _LP64
4325#define XTOP ,top() /*additional argument*/
4326#else  //_LP64
4327#define XTOP        /*no additional argument*/
4328#endif //_LP64
4329
4330//----------------------inline_unsafe_copyMemory-------------------------
4331// public native void sun.misc.Unsafe.copyMemory(Object srcBase, long srcOffset, Object destBase, long destOffset, long bytes);
4332bool LibraryCallKit::inline_unsafe_copyMemory() {
4333  if (callee()->is_static())  return false;  // caller must have the capability!
4334  null_check_receiver();  // null-check receiver
4335  if (stopped())  return true;
4336
4337  C->set_has_unsafe_access(true);  // Mark eventual nmethod as "unsafe".
4338
4339  Node* src_ptr =         argument(1);   // type: oop
4340  Node* src_off = ConvL2X(argument(2));  // type: long
4341  Node* dst_ptr =         argument(4);   // type: oop
4342  Node* dst_off = ConvL2X(argument(5));  // type: long
4343  Node* size    = ConvL2X(argument(7));  // type: long
4344
4345  assert(Unsafe_field_offset_to_byte_offset(11) == 11,
4346         "fieldOffset must be byte-scaled");
4347
4348  Node* src = make_unsafe_address(src_ptr, src_off);
4349  Node* dst = make_unsafe_address(dst_ptr, dst_off);
4350
4351  // Conservatively insert a memory barrier on all memory slices.
4352  // Do not let writes of the copy source or destination float below the copy.
4353  insert_mem_bar(Op_MemBarCPUOrder);
4354
4355  // Call it.  Note that the length argument is not scaled.
4356  make_runtime_call(RC_LEAF|RC_NO_FP,
4357                    OptoRuntime::fast_arraycopy_Type(),
4358                    StubRoutines::unsafe_arraycopy(),
4359                    "unsafe_arraycopy",
4360                    TypeRawPtr::BOTTOM,
4361                    src, dst, size XTOP);
4362
4363  // Do not let reads of the copy destination float above the copy.
4364  insert_mem_bar(Op_MemBarCPUOrder);
4365
4366  return true;
4367}
4368
4369//------------------------clone_coping-----------------------------------
4370// Helper function for inline_native_clone.
4371void LibraryCallKit::copy_to_clone(Node* obj, Node* alloc_obj, Node* obj_size, bool is_array, bool card_mark) {
4372  assert(obj_size != NULL, "");
4373  Node* raw_obj = alloc_obj->in(1);
4374  assert(alloc_obj->is_CheckCastPP() && raw_obj->is_Proj() && raw_obj->in(0)->is_Allocate(), "");
4375
4376  AllocateNode* alloc = NULL;
4377  if (ReduceBulkZeroing) {
4378    // We will be completely responsible for initializing this object -
4379    // mark Initialize node as complete.
4380    alloc = AllocateNode::Ideal_allocation(alloc_obj, &_gvn);
4381    // The object was just allocated - there should be no any stores!
4382    guarantee(alloc != NULL && alloc->maybe_set_complete(&_gvn), "");
4383    // Mark as complete_with_arraycopy so that on AllocateNode
4384    // expansion, we know this AllocateNode is initialized by an array
4385    // copy and a StoreStore barrier exists after the array copy.
4386    alloc->initialization()->set_complete_with_arraycopy();
4387  }
4388
4389  // Copy the fastest available way.
4390  // TODO: generate fields copies for small objects instead.
4391  Node* src  = obj;
4392  Node* dest = alloc_obj;
4393  Node* size = _gvn.transform(obj_size);
4394
4395  // Exclude the header but include array length to copy by 8 bytes words.
4396  // Can't use base_offset_in_bytes(bt) since basic type is unknown.
4397  int base_off = is_array ? arrayOopDesc::length_offset_in_bytes() :
4398                            instanceOopDesc::base_offset_in_bytes();
4399  // base_off:
4400  // 8  - 32-bit VM
4401  // 12 - 64-bit VM, compressed klass
4402  // 16 - 64-bit VM, normal klass
4403  if (base_off % BytesPerLong != 0) {
4404    assert(UseCompressedClassPointers, "");
4405    if (is_array) {
4406      // Exclude length to copy by 8 bytes words.
4407      base_off += sizeof(int);
4408    } else {
4409      // Include klass to copy by 8 bytes words.
4410      base_off = instanceOopDesc::klass_offset_in_bytes();
4411    }
4412    assert(base_off % BytesPerLong == 0, "expect 8 bytes alignment");
4413  }
4414  src  = basic_plus_adr(src,  base_off);
4415  dest = basic_plus_adr(dest, base_off);
4416
4417  // Compute the length also, if needed:
4418  Node* countx = size;
4419  countx = _gvn.transform(new (C) SubXNode(countx, MakeConX(base_off)));
4420  countx = _gvn.transform(new (C) URShiftXNode(countx, intcon(LogBytesPerLong) ));
4421
4422  const TypePtr* raw_adr_type = TypeRawPtr::BOTTOM;
4423  bool disjoint_bases = true;
4424  generate_unchecked_arraycopy(raw_adr_type, T_LONG, disjoint_bases,
4425                               src, NULL, dest, NULL, countx,
4426                               /*dest_uninitialized*/true);
4427
4428  // If necessary, emit some card marks afterwards.  (Non-arrays only.)
4429  if (card_mark) {
4430    assert(!is_array, "");
4431    // Put in store barrier for any and all oops we are sticking
4432    // into this object.  (We could avoid this if we could prove
4433    // that the object type contains no oop fields at all.)
4434    Node* no_particular_value = NULL;
4435    Node* no_particular_field = NULL;
4436    int raw_adr_idx = Compile::AliasIdxRaw;
4437    post_barrier(control(),
4438                 memory(raw_adr_type),
4439                 alloc_obj,
4440                 no_particular_field,
4441                 raw_adr_idx,
4442                 no_particular_value,
4443                 T_OBJECT,
4444                 false);
4445  }
4446
4447  // Do not let reads from the cloned object float above the arraycopy.
4448  if (alloc != NULL) {
4449    // Do not let stores that initialize this object be reordered with
4450    // a subsequent store that would make this object accessible by
4451    // other threads.
4452    // Record what AllocateNode this StoreStore protects so that
4453    // escape analysis can go from the MemBarStoreStoreNode to the
4454    // AllocateNode and eliminate the MemBarStoreStoreNode if possible
4455    // based on the escape status of the AllocateNode.
4456    insert_mem_bar(Op_MemBarStoreStore, alloc->proj_out(AllocateNode::RawAddress));
4457  } else {
4458    insert_mem_bar(Op_MemBarCPUOrder);
4459  }
4460}
4461
4462//------------------------inline_native_clone----------------------------
4463// protected native Object java.lang.Object.clone();
4464//
4465// Here are the simple edge cases:
4466//  null receiver => normal trap
4467//  virtual and clone was overridden => slow path to out-of-line clone
4468//  not cloneable or finalizer => slow path to out-of-line Object.clone
4469//
4470// The general case has two steps, allocation and copying.
4471// Allocation has two cases, and uses GraphKit::new_instance or new_array.
4472//
4473// Copying also has two cases, oop arrays and everything else.
4474// Oop arrays use arrayof_oop_arraycopy (same as System.arraycopy).
4475// Everything else uses the tight inline loop supplied by CopyArrayNode.
4476//
4477// These steps fold up nicely if and when the cloned object's klass
4478// can be sharply typed as an object array, a type array, or an instance.
4479//
4480bool LibraryCallKit::inline_native_clone(bool is_virtual) {
4481  PhiNode* result_val;
4482
4483  // Set the reexecute bit for the interpreter to reexecute
4484  // the bytecode that invokes Object.clone if deoptimization happens.
4485  { PreserveReexecuteState preexecs(this);
4486    jvms()->set_should_reexecute(true);
4487
4488    Node* obj = null_check_receiver();
4489    if (stopped())  return true;
4490
4491    Node* obj_klass = load_object_klass(obj);
4492    const TypeKlassPtr* tklass = _gvn.type(obj_klass)->isa_klassptr();
4493    const TypeOopPtr*   toop   = ((tklass != NULL)
4494                                ? tklass->as_instance_type()
4495                                : TypeInstPtr::NOTNULL);
4496
4497    // Conservatively insert a memory barrier on all memory slices.
4498    // Do not let writes into the original float below the clone.
4499    insert_mem_bar(Op_MemBarCPUOrder);
4500
4501    // paths into result_reg:
4502    enum {
4503      _slow_path = 1,     // out-of-line call to clone method (virtual or not)
4504      _objArray_path,     // plain array allocation, plus arrayof_oop_arraycopy
4505      _array_path,        // plain array allocation, plus arrayof_long_arraycopy
4506      _instance_path,     // plain instance allocation, plus arrayof_long_arraycopy
4507      PATH_LIMIT
4508    };
4509    RegionNode* result_reg = new(C) RegionNode(PATH_LIMIT);
4510    result_val             = new(C) PhiNode(result_reg,
4511                                            TypeInstPtr::NOTNULL);
4512    PhiNode*    result_i_o = new(C) PhiNode(result_reg, Type::ABIO);
4513    PhiNode*    result_mem = new(C) PhiNode(result_reg, Type::MEMORY,
4514                                            TypePtr::BOTTOM);
4515    record_for_igvn(result_reg);
4516
4517    const TypePtr* raw_adr_type = TypeRawPtr::BOTTOM;
4518    int raw_adr_idx = Compile::AliasIdxRaw;
4519
4520    Node* array_ctl = generate_array_guard(obj_klass, (RegionNode*)NULL);
4521    if (array_ctl != NULL) {
4522      // It's an array.
4523      PreserveJVMState pjvms(this);
4524      set_control(array_ctl);
4525      Node* obj_length = load_array_length(obj);
4526      Node* obj_size  = NULL;
4527      Node* alloc_obj = new_array(obj_klass, obj_length, 0, &obj_size);  // no arguments to push
4528
4529      if (!use_ReduceInitialCardMarks()) {
4530        // If it is an oop array, it requires very special treatment,
4531        // because card marking is required on each card of the array.
4532        Node* is_obja = generate_objArray_guard(obj_klass, (RegionNode*)NULL);
4533        if (is_obja != NULL) {
4534          PreserveJVMState pjvms2(this);
4535          set_control(is_obja);
4536          // Generate a direct call to the right arraycopy function(s).
4537          bool disjoint_bases = true;
4538          bool length_never_negative = true;
4539          generate_arraycopy(TypeAryPtr::OOPS, T_OBJECT,
4540                             obj, intcon(0), alloc_obj, intcon(0),
4541                             obj_length,
4542                             disjoint_bases, length_never_negative);
4543          result_reg->init_req(_objArray_path, control());
4544          result_val->init_req(_objArray_path, alloc_obj);
4545          result_i_o ->set_req(_objArray_path, i_o());
4546          result_mem ->set_req(_objArray_path, reset_memory());
4547        }
4548      }
4549      // Otherwise, there are no card marks to worry about.
4550      // (We can dispense with card marks if we know the allocation
4551      //  comes out of eden (TLAB)...  In fact, ReduceInitialCardMarks
4552      //  causes the non-eden paths to take compensating steps to
4553      //  simulate a fresh allocation, so that no further
4554      //  card marks are required in compiled code to initialize
4555      //  the object.)
4556
4557      if (!stopped()) {
4558        copy_to_clone(obj, alloc_obj, obj_size, true, false);
4559
4560        // Present the results of the copy.
4561        result_reg->init_req(_array_path, control());
4562        result_val->init_req(_array_path, alloc_obj);
4563        result_i_o ->set_req(_array_path, i_o());
4564        result_mem ->set_req(_array_path, reset_memory());
4565      }
4566    }
4567
4568    // We only go to the instance fast case code if we pass a number of guards.
4569    // The paths which do not pass are accumulated in the slow_region.
4570    RegionNode* slow_region = new (C) RegionNode(1);
4571    record_for_igvn(slow_region);
4572    if (!stopped()) {
4573      // It's an instance (we did array above).  Make the slow-path tests.
4574      // If this is a virtual call, we generate a funny guard.  We grab
4575      // the vtable entry corresponding to clone() from the target object.
4576      // If the target method which we are calling happens to be the
4577      // Object clone() method, we pass the guard.  We do not need this
4578      // guard for non-virtual calls; the caller is known to be the native
4579      // Object clone().
4580      if (is_virtual) {
4581        generate_virtual_guard(obj_klass, slow_region);
4582      }
4583
4584      // The object must be cloneable and must not have a finalizer.
4585      // Both of these conditions may be checked in a single test.
4586      // We could optimize the cloneable test further, but we don't care.
4587      generate_access_flags_guard(obj_klass,
4588                                  // Test both conditions:
4589                                  JVM_ACC_IS_CLONEABLE | JVM_ACC_HAS_FINALIZER,
4590                                  // Must be cloneable but not finalizer:
4591                                  JVM_ACC_IS_CLONEABLE,
4592                                  slow_region);
4593    }
4594
4595    if (!stopped()) {
4596      // It's an instance, and it passed the slow-path tests.
4597      PreserveJVMState pjvms(this);
4598      Node* obj_size  = NULL;
4599      Node* alloc_obj = new_instance(obj_klass, NULL, &obj_size);
4600
4601      copy_to_clone(obj, alloc_obj, obj_size, false, !use_ReduceInitialCardMarks());
4602
4603      // Present the results of the slow call.
4604      result_reg->init_req(_instance_path, control());
4605      result_val->init_req(_instance_path, alloc_obj);
4606      result_i_o ->set_req(_instance_path, i_o());
4607      result_mem ->set_req(_instance_path, reset_memory());
4608    }
4609
4610    // Generate code for the slow case.  We make a call to clone().
4611    set_control(_gvn.transform(slow_region));
4612    if (!stopped()) {
4613      PreserveJVMState pjvms(this);
4614      CallJavaNode* slow_call = generate_method_call(vmIntrinsics::_clone, is_virtual);
4615      Node* slow_result = set_results_for_java_call(slow_call);
4616      // this->control() comes from set_results_for_java_call
4617      result_reg->init_req(_slow_path, control());
4618      result_val->init_req(_slow_path, slow_result);
4619      result_i_o ->set_req(_slow_path, i_o());
4620      result_mem ->set_req(_slow_path, reset_memory());
4621    }
4622
4623    // Return the combined state.
4624    set_control(    _gvn.transform(result_reg));
4625    set_i_o(        _gvn.transform(result_i_o));
4626    set_all_memory( _gvn.transform(result_mem));
4627  } // original reexecute is set back here
4628
4629  set_result(_gvn.transform(result_val));
4630  return true;
4631}
4632
4633//------------------------------basictype2arraycopy----------------------------
4634address LibraryCallKit::basictype2arraycopy(BasicType t,
4635                                            Node* src_offset,
4636                                            Node* dest_offset,
4637                                            bool disjoint_bases,
4638                                            const char* &name,
4639                                            bool dest_uninitialized) {
4640  const TypeInt* src_offset_inttype  = gvn().find_int_type(src_offset);;
4641  const TypeInt* dest_offset_inttype = gvn().find_int_type(dest_offset);;
4642
4643  bool aligned = false;
4644  bool disjoint = disjoint_bases;
4645
4646  // if the offsets are the same, we can treat the memory regions as
4647  // disjoint, because either the memory regions are in different arrays,
4648  // or they are identical (which we can treat as disjoint.)  We can also
4649  // treat a copy with a destination index  less that the source index
4650  // as disjoint since a low->high copy will work correctly in this case.
4651  if (src_offset_inttype != NULL && src_offset_inttype->is_con() &&
4652      dest_offset_inttype != NULL && dest_offset_inttype->is_con()) {
4653    // both indices are constants
4654    int s_offs = src_offset_inttype->get_con();
4655    int d_offs = dest_offset_inttype->get_con();
4656    int element_size = type2aelembytes(t);
4657    aligned = ((arrayOopDesc::base_offset_in_bytes(t) + s_offs * element_size) % HeapWordSize == 0) &&
4658              ((arrayOopDesc::base_offset_in_bytes(t) + d_offs * element_size) % HeapWordSize == 0);
4659    if (s_offs >= d_offs)  disjoint = true;
4660  } else if (src_offset == dest_offset && src_offset != NULL) {
4661    // This can occur if the offsets are identical non-constants.
4662    disjoint = true;
4663  }
4664
4665  return StubRoutines::select_arraycopy_function(t, aligned, disjoint, name, dest_uninitialized);
4666}
4667
4668
4669//------------------------------inline_arraycopy-----------------------
4670// public static native void java.lang.System.arraycopy(Object src,  int  srcPos,
4671//                                                      Object dest, int destPos,
4672//                                                      int length);
4673bool LibraryCallKit::inline_arraycopy() {
4674  // Get the arguments.
4675  Node* src         = argument(0);  // type: oop
4676  Node* src_offset  = argument(1);  // type: int
4677  Node* dest        = argument(2);  // type: oop
4678  Node* dest_offset = argument(3);  // type: int
4679  Node* length      = argument(4);  // type: int
4680
4681  // Compile time checks.  If any of these checks cannot be verified at compile time,
4682  // we do not make a fast path for this call.  Instead, we let the call remain as it
4683  // is.  The checks we choose to mandate at compile time are:
4684  //
4685  // (1) src and dest are arrays.
4686  const Type* src_type  = src->Value(&_gvn);
4687  const Type* dest_type = dest->Value(&_gvn);
4688  const TypeAryPtr* top_src  = src_type->isa_aryptr();
4689  const TypeAryPtr* top_dest = dest_type->isa_aryptr();
4690
4691  // Do we have the type of src?
4692  bool has_src = (top_src != NULL && top_src->klass() != NULL);
4693  // Do we have the type of dest?
4694  bool has_dest = (top_dest != NULL && top_dest->klass() != NULL);
4695  // Is the type for src from speculation?
4696  bool src_spec = false;
4697  // Is the type for dest from speculation?
4698  bool dest_spec = false;
4699
4700  if (!has_src || !has_dest) {
4701    // We don't have sufficient type information, let's see if
4702    // speculative types can help. We need to have types for both src
4703    // and dest so that it pays off.
4704
4705    // Do we already have or could we have type information for src
4706    bool could_have_src = has_src;
4707    // Do we already have or could we have type information for dest
4708    bool could_have_dest = has_dest;
4709
4710    ciKlass* src_k = NULL;
4711    if (!has_src) {
4712      src_k = src_type->speculative_type();
4713      if (src_k != NULL && src_k->is_array_klass()) {
4714        could_have_src = true;
4715      }
4716    }
4717
4718    ciKlass* dest_k = NULL;
4719    if (!has_dest) {
4720      dest_k = dest_type->speculative_type();
4721      if (dest_k != NULL && dest_k->is_array_klass()) {
4722        could_have_dest = true;
4723      }
4724    }
4725
4726    if (could_have_src && could_have_dest) {
4727      // This is going to pay off so emit the required guards
4728      if (!has_src) {
4729        src = maybe_cast_profiled_obj(src, src_k);
4730        src_type  = _gvn.type(src);
4731        top_src  = src_type->isa_aryptr();
4732        has_src = (top_src != NULL && top_src->klass() != NULL);
4733        src_spec = true;
4734      }
4735      if (!has_dest) {
4736        dest = maybe_cast_profiled_obj(dest, dest_k);
4737        dest_type  = _gvn.type(dest);
4738        top_dest  = dest_type->isa_aryptr();
4739        has_dest = (top_dest != NULL && top_dest->klass() != NULL);
4740        dest_spec = true;
4741      }
4742    }
4743  }
4744
4745  if (!has_src || !has_dest) {
4746    // Conservatively insert a memory barrier on all memory slices.
4747    // Do not let writes into the source float below the arraycopy.
4748    insert_mem_bar(Op_MemBarCPUOrder);
4749
4750    // Call StubRoutines::generic_arraycopy stub.
4751    generate_arraycopy(TypeRawPtr::BOTTOM, T_CONFLICT,
4752                       src, src_offset, dest, dest_offset, length);
4753
4754    // Do not let reads from the destination float above the arraycopy.
4755    // Since we cannot type the arrays, we don't know which slices
4756    // might be affected.  We could restrict this barrier only to those
4757    // memory slices which pertain to array elements--but don't bother.
4758    if (!InsertMemBarAfterArraycopy)
4759      // (If InsertMemBarAfterArraycopy, there is already one in place.)
4760      insert_mem_bar(Op_MemBarCPUOrder);
4761    return true;
4762  }
4763
4764  // (2) src and dest arrays must have elements of the same BasicType
4765  // Figure out the size and type of the elements we will be copying.
4766  BasicType src_elem  =  top_src->klass()->as_array_klass()->element_type()->basic_type();
4767  BasicType dest_elem = top_dest->klass()->as_array_klass()->element_type()->basic_type();
4768  if (src_elem  == T_ARRAY)  src_elem  = T_OBJECT;
4769  if (dest_elem == T_ARRAY)  dest_elem = T_OBJECT;
4770
4771  if (src_elem != dest_elem || dest_elem == T_VOID) {
4772    // The component types are not the same or are not recognized.  Punt.
4773    // (But, avoid the native method wrapper to JVM_ArrayCopy.)
4774    generate_slow_arraycopy(TypePtr::BOTTOM,
4775                            src, src_offset, dest, dest_offset, length,
4776                            /*dest_uninitialized*/false);
4777    return true;
4778  }
4779
4780  if (src_elem == T_OBJECT) {
4781    // If both arrays are object arrays then having the exact types
4782    // for both will remove the need for a subtype check at runtime
4783    // before the call and may make it possible to pick a faster copy
4784    // routine (without a subtype check on every element)
4785    // Do we have the exact type of src?
4786    bool could_have_src = src_spec;
4787    // Do we have the exact type of dest?
4788    bool could_have_dest = dest_spec;
4789    ciKlass* src_k = top_src->klass();
4790    ciKlass* dest_k = top_dest->klass();
4791    if (!src_spec) {
4792      src_k = src_type->speculative_type();
4793      if (src_k != NULL && src_k->is_array_klass()) {
4794          could_have_src = true;
4795      }
4796    }
4797    if (!dest_spec) {
4798      dest_k = dest_type->speculative_type();
4799      if (dest_k != NULL && dest_k->is_array_klass()) {
4800        could_have_dest = true;
4801      }
4802    }
4803    if (could_have_src && could_have_dest) {
4804      // If we can have both exact types, emit the missing guards
4805      if (could_have_src && !src_spec) {
4806        src = maybe_cast_profiled_obj(src, src_k);
4807      }
4808      if (could_have_dest && !dest_spec) {
4809        dest = maybe_cast_profiled_obj(dest, dest_k);
4810      }
4811    }
4812  }
4813
4814  //---------------------------------------------------------------------------
4815  // We will make a fast path for this call to arraycopy.
4816
4817  // We have the following tests left to perform:
4818  //
4819  // (3) src and dest must not be null.
4820  // (4) src_offset must not be negative.
4821  // (5) dest_offset must not be negative.
4822  // (6) length must not be negative.
4823  // (7) src_offset + length must not exceed length of src.
4824  // (8) dest_offset + length must not exceed length of dest.
4825  // (9) each element of an oop array must be assignable
4826
4827  RegionNode* slow_region = new (C) RegionNode(1);
4828  record_for_igvn(slow_region);
4829
4830  // (3) operands must not be null
4831  // We currently perform our null checks with the null_check routine.
4832  // This means that the null exceptions will be reported in the caller
4833  // rather than (correctly) reported inside of the native arraycopy call.
4834  // This should be corrected, given time.  We do our null check with the
4835  // stack pointer restored.
4836  src  = null_check(src,  T_ARRAY);
4837  dest = null_check(dest, T_ARRAY);
4838
4839  // (4) src_offset must not be negative.
4840  generate_negative_guard(src_offset, slow_region);
4841
4842  // (5) dest_offset must not be negative.
4843  generate_negative_guard(dest_offset, slow_region);
4844
4845  // (6) length must not be negative (moved to generate_arraycopy()).
4846  // generate_negative_guard(length, slow_region);
4847
4848  // (7) src_offset + length must not exceed length of src.
4849  generate_limit_guard(src_offset, length,
4850                       load_array_length(src),
4851                       slow_region);
4852
4853  // (8) dest_offset + length must not exceed length of dest.
4854  generate_limit_guard(dest_offset, length,
4855                       load_array_length(dest),
4856                       slow_region);
4857
4858  // (9) each element of an oop array must be assignable
4859  // The generate_arraycopy subroutine checks this.
4860
4861  // This is where the memory effects are placed:
4862  const TypePtr* adr_type = TypeAryPtr::get_array_body_type(dest_elem);
4863  generate_arraycopy(adr_type, dest_elem,
4864                     src, src_offset, dest, dest_offset, length,
4865                     false, false, slow_region);
4866
4867  return true;
4868}
4869
4870//-----------------------------generate_arraycopy----------------------
4871// Generate an optimized call to arraycopy.
4872// Caller must guard against non-arrays.
4873// Caller must determine a common array basic-type for both arrays.
4874// Caller must validate offsets against array bounds.
4875// The slow_region has already collected guard failure paths
4876// (such as out of bounds length or non-conformable array types).
4877// The generated code has this shape, in general:
4878//
4879//     if (length == 0)  return   // via zero_path
4880//     slowval = -1
4881//     if (types unknown) {
4882//       slowval = call generic copy loop
4883//       if (slowval == 0)  return  // via checked_path
4884//     } else if (indexes in bounds) {
4885//       if ((is object array) && !(array type check)) {
4886//         slowval = call checked copy loop
4887//         if (slowval == 0)  return  // via checked_path
4888//       } else {
4889//         call bulk copy loop
4890//         return  // via fast_path
4891//       }
4892//     }
4893//     // adjust params for remaining work:
4894//     if (slowval != -1) {
4895//       n = -1^slowval; src_offset += n; dest_offset += n; length -= n
4896//     }
4897//   slow_region:
4898//     call slow arraycopy(src, src_offset, dest, dest_offset, length)
4899//     return  // via slow_call_path
4900//
4901// This routine is used from several intrinsics:  System.arraycopy,
4902// Object.clone (the array subcase), and Arrays.copyOf[Range].
4903//
4904void
4905LibraryCallKit::generate_arraycopy(const TypePtr* adr_type,
4906                                   BasicType basic_elem_type,
4907                                   Node* src,  Node* src_offset,
4908                                   Node* dest, Node* dest_offset,
4909                                   Node* copy_length,
4910                                   bool disjoint_bases,
4911                                   bool length_never_negative,
4912                                   RegionNode* slow_region) {
4913
4914  if (slow_region == NULL) {
4915    slow_region = new(C) RegionNode(1);
4916    record_for_igvn(slow_region);
4917  }
4918
4919  Node* original_dest      = dest;
4920  AllocateArrayNode* alloc = NULL;  // used for zeroing, if needed
4921  bool  dest_uninitialized = false;
4922
4923  // See if this is the initialization of a newly-allocated array.
4924  // If so, we will take responsibility here for initializing it to zero.
4925  // (Note:  Because tightly_coupled_allocation performs checks on the
4926  // out-edges of the dest, we need to avoid making derived pointers
4927  // from it until we have checked its uses.)
4928  if (ReduceBulkZeroing
4929      && !ZeroTLAB              // pointless if already zeroed
4930      && basic_elem_type != T_CONFLICT // avoid corner case
4931      && !src->eqv_uncast(dest)
4932      && ((alloc = tightly_coupled_allocation(dest, slow_region))
4933          != NULL)
4934      && _gvn.find_int_con(alloc->in(AllocateNode::ALength), 1) > 0
4935      && alloc->maybe_set_complete(&_gvn)) {
4936    // "You break it, you buy it."
4937    InitializeNode* init = alloc->initialization();
4938    assert(init->is_complete(), "we just did this");
4939    init->set_complete_with_arraycopy();
4940    assert(dest->is_CheckCastPP(), "sanity");
4941    assert(dest->in(0)->in(0) == init, "dest pinned");
4942    adr_type = TypeRawPtr::BOTTOM;  // all initializations are into raw memory
4943    // From this point on, every exit path is responsible for
4944    // initializing any non-copied parts of the object to zero.
4945    // Also, if this flag is set we make sure that arraycopy interacts properly
4946    // with G1, eliding pre-barriers. See CR 6627983.
4947    dest_uninitialized = true;
4948  } else {
4949    // No zeroing elimination here.
4950    alloc             = NULL;
4951    //original_dest   = dest;
4952    //dest_uninitialized = false;
4953  }
4954
4955  // Results are placed here:
4956  enum { fast_path        = 1,  // normal void-returning assembly stub
4957         checked_path     = 2,  // special assembly stub with cleanup
4958         slow_call_path   = 3,  // something went wrong; call the VM
4959         zero_path        = 4,  // bypass when length of copy is zero
4960         bcopy_path       = 5,  // copy primitive array by 64-bit blocks
4961         PATH_LIMIT       = 6
4962  };
4963  RegionNode* result_region = new(C) RegionNode(PATH_LIMIT);
4964  PhiNode*    result_i_o    = new(C) PhiNode(result_region, Type::ABIO);
4965  PhiNode*    result_memory = new(C) PhiNode(result_region, Type::MEMORY, adr_type);
4966  record_for_igvn(result_region);
4967  _gvn.set_type_bottom(result_i_o);
4968  _gvn.set_type_bottom(result_memory);
4969  assert(adr_type != TypePtr::BOTTOM, "must be RawMem or a T[] slice");
4970
4971  // The slow_control path:
4972  Node* slow_control;
4973  Node* slow_i_o = i_o();
4974  Node* slow_mem = memory(adr_type);
4975  debug_only(slow_control = (Node*) badAddress);
4976
4977  // Checked control path:
4978  Node* checked_control = top();
4979  Node* checked_mem     = NULL;
4980  Node* checked_i_o     = NULL;
4981  Node* checked_value   = NULL;
4982
4983  if (basic_elem_type == T_CONFLICT) {
4984    assert(!dest_uninitialized, "");
4985    Node* cv = generate_generic_arraycopy(adr_type,
4986                                          src, src_offset, dest, dest_offset,
4987                                          copy_length, dest_uninitialized);
4988    if (cv == NULL)  cv = intcon(-1);  // failure (no stub available)
4989    checked_control = control();
4990    checked_i_o     = i_o();
4991    checked_mem     = memory(adr_type);
4992    checked_value   = cv;
4993    set_control(top());         // no fast path
4994  }
4995
4996  Node* not_pos = generate_nonpositive_guard(copy_length, length_never_negative);
4997  if (not_pos != NULL) {
4998    PreserveJVMState pjvms(this);
4999    set_control(not_pos);
5000
5001    // (6) length must not be negative.
5002    if (!length_never_negative) {
5003      generate_negative_guard(copy_length, slow_region);
5004    }
5005
5006    // copy_length is 0.
5007    if (!stopped() && dest_uninitialized) {
5008      Node* dest_length = alloc->in(AllocateNode::ALength);
5009      if (copy_length->eqv_uncast(dest_length)
5010          || _gvn.find_int_con(dest_length, 1) <= 0) {
5011        // There is no zeroing to do. No need for a secondary raw memory barrier.
5012      } else {
5013        // Clear the whole thing since there are no source elements to copy.
5014        generate_clear_array(adr_type, dest, basic_elem_type,
5015                             intcon(0), NULL,
5016                             alloc->in(AllocateNode::AllocSize));
5017        // Use a secondary InitializeNode as raw memory barrier.
5018        // Currently it is needed only on this path since other
5019        // paths have stub or runtime calls as raw memory barriers.
5020        InitializeNode* init = insert_mem_bar_volatile(Op_Initialize,
5021                                                       Compile::AliasIdxRaw,
5022                                                       top())->as_Initialize();
5023        init->set_complete(&_gvn);  // (there is no corresponding AllocateNode)
5024      }
5025    }
5026
5027    // Present the results of the fast call.
5028    result_region->init_req(zero_path, control());
5029    result_i_o   ->init_req(zero_path, i_o());
5030    result_memory->init_req(zero_path, memory(adr_type));
5031  }
5032
5033  if (!stopped() && dest_uninitialized) {
5034    // We have to initialize the *uncopied* part of the array to zero.
5035    // The copy destination is the slice dest[off..off+len].  The other slices
5036    // are dest_head = dest[0..off] and dest_tail = dest[off+len..dest.length].
5037    Node* dest_size   = alloc->in(AllocateNode::AllocSize);
5038    Node* dest_length = alloc->in(AllocateNode::ALength);
5039    Node* dest_tail   = _gvn.transform(new(C) AddINode(dest_offset,
5040                                                          copy_length));
5041
5042    // If there is a head section that needs zeroing, do it now.
5043    if (find_int_con(dest_offset, -1) != 0) {
5044      generate_clear_array(adr_type, dest, basic_elem_type,
5045                           intcon(0), dest_offset,
5046                           NULL);
5047    }
5048
5049    // Next, perform a dynamic check on the tail length.
5050    // It is often zero, and we can win big if we prove this.
5051    // There are two wins:  Avoid generating the ClearArray
5052    // with its attendant messy index arithmetic, and upgrade
5053    // the copy to a more hardware-friendly word size of 64 bits.
5054    Node* tail_ctl = NULL;
5055    if (!stopped() && !dest_tail->eqv_uncast(dest_length)) {
5056      Node* cmp_lt   = _gvn.transform(new(C) CmpINode(dest_tail, dest_length));
5057      Node* bol_lt   = _gvn.transform(new(C) BoolNode(cmp_lt, BoolTest::lt));
5058      tail_ctl = generate_slow_guard(bol_lt, NULL);
5059      assert(tail_ctl != NULL || !stopped(), "must be an outcome");
5060    }
5061
5062    // At this point, let's assume there is no tail.
5063    if (!stopped() && alloc != NULL && basic_elem_type != T_OBJECT) {
5064      // There is no tail.  Try an upgrade to a 64-bit copy.
5065      bool didit = false;
5066      { PreserveJVMState pjvms(this);
5067        didit = generate_block_arraycopy(adr_type, basic_elem_type, alloc,
5068                                         src, src_offset, dest, dest_offset,
5069                                         dest_size, dest_uninitialized);
5070        if (didit) {
5071          // Present the results of the block-copying fast call.
5072          result_region->init_req(bcopy_path, control());
5073          result_i_o   ->init_req(bcopy_path, i_o());
5074          result_memory->init_req(bcopy_path, memory(adr_type));
5075        }
5076      }
5077      if (didit)
5078        set_control(top());     // no regular fast path
5079    }
5080
5081    // Clear the tail, if any.
5082    if (tail_ctl != NULL) {
5083      Node* notail_ctl = stopped() ? NULL : control();
5084      set_control(tail_ctl);
5085      if (notail_ctl == NULL) {
5086        generate_clear_array(adr_type, dest, basic_elem_type,
5087                             dest_tail, NULL,
5088                             dest_size);
5089      } else {
5090        // Make a local merge.
5091        Node* done_ctl = new(C) RegionNode(3);
5092        Node* done_mem = new(C) PhiNode(done_ctl, Type::MEMORY, adr_type);
5093        done_ctl->init_req(1, notail_ctl);
5094        done_mem->init_req(1, memory(adr_type));
5095        generate_clear_array(adr_type, dest, basic_elem_type,
5096                             dest_tail, NULL,
5097                             dest_size);
5098        done_ctl->init_req(2, control());
5099        done_mem->init_req(2, memory(adr_type));
5100        set_control( _gvn.transform(done_ctl));
5101        set_memory(  _gvn.transform(done_mem), adr_type );
5102      }
5103    }
5104  }
5105
5106  BasicType copy_type = basic_elem_type;
5107  assert(basic_elem_type != T_ARRAY, "caller must fix this");
5108  if (!stopped() && copy_type == T_OBJECT) {
5109    // If src and dest have compatible element types, we can copy bits.
5110    // Types S[] and D[] are compatible if D is a supertype of S.
5111    //
5112    // If they are not, we will use checked_oop_disjoint_arraycopy,
5113    // which performs a fast optimistic per-oop check, and backs off
5114    // further to JVM_ArrayCopy on the first per-oop check that fails.
5115    // (Actually, we don't move raw bits only; the GC requires card marks.)
5116
5117    // Get the Klass* for both src and dest
5118    Node* src_klass  = load_object_klass(src);
5119    Node* dest_klass = load_object_klass(dest);
5120
5121    // Generate the subtype check.
5122    // This might fold up statically, or then again it might not.
5123    //
5124    // Non-static example:  Copying List<String>.elements to a new String[].
5125    // The backing store for a List<String> is always an Object[],
5126    // but its elements are always type String, if the generic types
5127    // are correct at the source level.
5128    //
5129    // Test S[] against D[], not S against D, because (probably)
5130    // the secondary supertype cache is less busy for S[] than S.
5131    // This usually only matters when D is an interface.
5132    Node* not_subtype_ctrl = gen_subtype_check(src_klass, dest_klass);
5133    // Plug failing path into checked_oop_disjoint_arraycopy
5134    if (not_subtype_ctrl != top()) {
5135      PreserveJVMState pjvms(this);
5136      set_control(not_subtype_ctrl);
5137      // (At this point we can assume disjoint_bases, since types differ.)
5138      int ek_offset = in_bytes(ObjArrayKlass::element_klass_offset());
5139      Node* p1 = basic_plus_adr(dest_klass, ek_offset);
5140      Node* n1 = LoadKlassNode::make(_gvn, immutable_memory(), p1, TypeRawPtr::BOTTOM);
5141      Node* dest_elem_klass = _gvn.transform(n1);
5142      Node* cv = generate_checkcast_arraycopy(adr_type,
5143                                              dest_elem_klass,
5144                                              src, src_offset, dest, dest_offset,
5145                                              ConvI2X(copy_length), dest_uninitialized);
5146      if (cv == NULL)  cv = intcon(-1);  // failure (no stub available)
5147      checked_control = control();
5148      checked_i_o     = i_o();
5149      checked_mem     = memory(adr_type);
5150      checked_value   = cv;
5151    }
5152    // At this point we know we do not need type checks on oop stores.
5153
5154    // Let's see if we need card marks:
5155    if (alloc != NULL && use_ReduceInitialCardMarks()) {
5156      // If we do not need card marks, copy using the jint or jlong stub.
5157      copy_type = LP64_ONLY(UseCompressedOops ? T_INT : T_LONG) NOT_LP64(T_INT);
5158      assert(type2aelembytes(basic_elem_type) == type2aelembytes(copy_type),
5159             "sizes agree");
5160    }
5161  }
5162
5163  if (!stopped()) {
5164    // Generate the fast path, if possible.
5165    PreserveJVMState pjvms(this);
5166    generate_unchecked_arraycopy(adr_type, copy_type, disjoint_bases,
5167                                 src, src_offset, dest, dest_offset,
5168                                 ConvI2X(copy_length), dest_uninitialized);
5169
5170    // Present the results of the fast call.
5171    result_region->init_req(fast_path, control());
5172    result_i_o   ->init_req(fast_path, i_o());
5173    result_memory->init_req(fast_path, memory(adr_type));
5174  }
5175
5176  // Here are all the slow paths up to this point, in one bundle:
5177  slow_control = top();
5178  if (slow_region != NULL)
5179    slow_control = _gvn.transform(slow_region);
5180  DEBUG_ONLY(slow_region = (RegionNode*)badAddress);
5181
5182  set_control(checked_control);
5183  if (!stopped()) {
5184    // Clean up after the checked call.
5185    // The returned value is either 0 or -1^K,
5186    // where K = number of partially transferred array elements.
5187    Node* cmp = _gvn.transform(new(C) CmpINode(checked_value, intcon(0)));
5188    Node* bol = _gvn.transform(new(C) BoolNode(cmp, BoolTest::eq));
5189    IfNode* iff = create_and_map_if(control(), bol, PROB_MAX, COUNT_UNKNOWN);
5190
5191    // If it is 0, we are done, so transfer to the end.
5192    Node* checks_done = _gvn.transform(new(C) IfTrueNode(iff));
5193    result_region->init_req(checked_path, checks_done);
5194    result_i_o   ->init_req(checked_path, checked_i_o);
5195    result_memory->init_req(checked_path, checked_mem);
5196
5197    // If it is not zero, merge into the slow call.
5198    set_control( _gvn.transform(new(C) IfFalseNode(iff) ));
5199    RegionNode* slow_reg2 = new(C) RegionNode(3);
5200    PhiNode*    slow_i_o2 = new(C) PhiNode(slow_reg2, Type::ABIO);
5201    PhiNode*    slow_mem2 = new(C) PhiNode(slow_reg2, Type::MEMORY, adr_type);
5202    record_for_igvn(slow_reg2);
5203    slow_reg2  ->init_req(1, slow_control);
5204    slow_i_o2  ->init_req(1, slow_i_o);
5205    slow_mem2  ->init_req(1, slow_mem);
5206    slow_reg2  ->init_req(2, control());
5207    slow_i_o2  ->init_req(2, checked_i_o);
5208    slow_mem2  ->init_req(2, checked_mem);
5209
5210    slow_control = _gvn.transform(slow_reg2);
5211    slow_i_o     = _gvn.transform(slow_i_o2);
5212    slow_mem     = _gvn.transform(slow_mem2);
5213
5214    if (alloc != NULL) {
5215      // We'll restart from the very beginning, after zeroing the whole thing.
5216      // This can cause double writes, but that's OK since dest is brand new.
5217      // So we ignore the low 31 bits of the value returned from the stub.
5218    } else {
5219      // We must continue the copy exactly where it failed, or else
5220      // another thread might see the wrong number of writes to dest.
5221      Node* checked_offset = _gvn.transform(new(C) XorINode(checked_value, intcon(-1)));
5222      Node* slow_offset    = new(C) PhiNode(slow_reg2, TypeInt::INT);
5223      slow_offset->init_req(1, intcon(0));
5224      slow_offset->init_req(2, checked_offset);
5225      slow_offset  = _gvn.transform(slow_offset);
5226
5227      // Adjust the arguments by the conditionally incoming offset.
5228      Node* src_off_plus  = _gvn.transform(new(C) AddINode(src_offset,  slow_offset));
5229      Node* dest_off_plus = _gvn.transform(new(C) AddINode(dest_offset, slow_offset));
5230      Node* length_minus  = _gvn.transform(new(C) SubINode(copy_length, slow_offset));
5231
5232      // Tweak the node variables to adjust the code produced below:
5233      src_offset  = src_off_plus;
5234      dest_offset = dest_off_plus;
5235      copy_length = length_minus;
5236    }
5237  }
5238
5239  set_control(slow_control);
5240  if (!stopped()) {
5241    // Generate the slow path, if needed.
5242    PreserveJVMState pjvms(this);   // replace_in_map may trash the map
5243
5244    set_memory(slow_mem, adr_type);
5245    set_i_o(slow_i_o);
5246
5247    if (dest_uninitialized) {
5248      generate_clear_array(adr_type, dest, basic_elem_type,
5249                           intcon(0), NULL,
5250                           alloc->in(AllocateNode::AllocSize));
5251    }
5252
5253    generate_slow_arraycopy(adr_type,
5254                            src, src_offset, dest, dest_offset,
5255                            copy_length, /*dest_uninitialized*/false);
5256
5257    result_region->init_req(slow_call_path, control());
5258    result_i_o   ->init_req(slow_call_path, i_o());
5259    result_memory->init_req(slow_call_path, memory(adr_type));
5260  }
5261
5262  // Remove unused edges.
5263  for (uint i = 1; i < result_region->req(); i++) {
5264    if (result_region->in(i) == NULL)
5265      result_region->init_req(i, top());
5266  }
5267
5268  // Finished; return the combined state.
5269  set_control( _gvn.transform(result_region));
5270  set_i_o(     _gvn.transform(result_i_o)    );
5271  set_memory(  _gvn.transform(result_memory), adr_type );
5272
5273  // The memory edges above are precise in order to model effects around
5274  // array copies accurately to allow value numbering of field loads around
5275  // arraycopy.  Such field loads, both before and after, are common in Java
5276  // collections and similar classes involving header/array data structures.
5277  //
5278  // But with low number of register or when some registers are used or killed
5279  // by arraycopy calls it causes registers spilling on stack. See 6544710.
5280  // The next memory barrier is added to avoid it. If the arraycopy can be
5281  // optimized away (which it can, sometimes) then we can manually remove
5282  // the membar also.
5283  //
5284  // Do not let reads from the cloned object float above the arraycopy.
5285  if (alloc != NULL) {
5286    // Do not let stores that initialize this object be reordered with
5287    // a subsequent store that would make this object accessible by
5288    // other threads.
5289    // Record what AllocateNode this StoreStore protects so that
5290    // escape analysis can go from the MemBarStoreStoreNode to the
5291    // AllocateNode and eliminate the MemBarStoreStoreNode if possible
5292    // based on the escape status of the AllocateNode.
5293    insert_mem_bar(Op_MemBarStoreStore, alloc->proj_out(AllocateNode::RawAddress));
5294  } else if (InsertMemBarAfterArraycopy)
5295    insert_mem_bar(Op_MemBarCPUOrder);
5296}
5297
5298
5299// Helper function which determines if an arraycopy immediately follows
5300// an allocation, with no intervening tests or other escapes for the object.
5301AllocateArrayNode*
5302LibraryCallKit::tightly_coupled_allocation(Node* ptr,
5303                                           RegionNode* slow_region) {
5304  if (stopped())             return NULL;  // no fast path
5305  if (C->AliasLevel() == 0)  return NULL;  // no MergeMems around
5306
5307  AllocateArrayNode* alloc = AllocateArrayNode::Ideal_array_allocation(ptr, &_gvn);
5308  if (alloc == NULL)  return NULL;
5309
5310  Node* rawmem = memory(Compile::AliasIdxRaw);
5311  // Is the allocation's memory state untouched?
5312  if (!(rawmem->is_Proj() && rawmem->in(0)->is_Initialize())) {
5313    // Bail out if there have been raw-memory effects since the allocation.
5314    // (Example:  There might have been a call or safepoint.)
5315    return NULL;
5316  }
5317  rawmem = rawmem->in(0)->as_Initialize()->memory(Compile::AliasIdxRaw);
5318  if (!(rawmem->is_Proj() && rawmem->in(0) == alloc)) {
5319    return NULL;
5320  }
5321
5322  // There must be no unexpected observers of this allocation.
5323  for (DUIterator_Fast imax, i = ptr->fast_outs(imax); i < imax; i++) {
5324    Node* obs = ptr->fast_out(i);
5325    if (obs != this->map()) {
5326      return NULL;
5327    }
5328  }
5329
5330  // This arraycopy must unconditionally follow the allocation of the ptr.
5331  Node* alloc_ctl = ptr->in(0);
5332  assert(just_allocated_object(alloc_ctl) == ptr, "most recent allo");
5333
5334  Node* ctl = control();
5335  while (ctl != alloc_ctl) {
5336    // There may be guards which feed into the slow_region.
5337    // Any other control flow means that we might not get a chance
5338    // to finish initializing the allocated object.
5339    if ((ctl->is_IfFalse() || ctl->is_IfTrue()) && ctl->in(0)->is_If()) {
5340      IfNode* iff = ctl->in(0)->as_If();
5341      Node* not_ctl = iff->proj_out(1 - ctl->as_Proj()->_con);
5342      assert(not_ctl != NULL && not_ctl != ctl, "found alternate");
5343      if (slow_region != NULL && slow_region->find_edge(not_ctl) >= 1) {
5344        ctl = iff->in(0);       // This test feeds the known slow_region.
5345        continue;
5346      }
5347      // One more try:  Various low-level checks bottom out in
5348      // uncommon traps.  If the debug-info of the trap omits
5349      // any reference to the allocation, as we've already
5350      // observed, then there can be no objection to the trap.
5351      bool found_trap = false;
5352      for (DUIterator_Fast jmax, j = not_ctl->fast_outs(jmax); j < jmax; j++) {
5353        Node* obs = not_ctl->fast_out(j);
5354        if (obs->in(0) == not_ctl && obs->is_Call() &&
5355            (obs->as_Call()->entry_point() == SharedRuntime::uncommon_trap_blob()->entry_point())) {
5356          found_trap = true; break;
5357        }
5358      }
5359      if (found_trap) {
5360        ctl = iff->in(0);       // This test feeds a harmless uncommon trap.
5361        continue;
5362      }
5363    }
5364    return NULL;
5365  }
5366
5367  // If we get this far, we have an allocation which immediately
5368  // precedes the arraycopy, and we can take over zeroing the new object.
5369  // The arraycopy will finish the initialization, and provide
5370  // a new control state to which we will anchor the destination pointer.
5371
5372  return alloc;
5373}
5374
5375// Helper for initialization of arrays, creating a ClearArray.
5376// It writes zero bits in [start..end), within the body of an array object.
5377// The memory effects are all chained onto the 'adr_type' alias category.
5378//
5379// Since the object is otherwise uninitialized, we are free
5380// to put a little "slop" around the edges of the cleared area,
5381// as long as it does not go back into the array's header,
5382// or beyond the array end within the heap.
5383//
5384// The lower edge can be rounded down to the nearest jint and the
5385// upper edge can be rounded up to the nearest MinObjAlignmentInBytes.
5386//
5387// Arguments:
5388//   adr_type           memory slice where writes are generated
5389//   dest               oop of the destination array
5390//   basic_elem_type    element type of the destination
5391//   slice_idx          array index of first element to store
5392//   slice_len          number of elements to store (or NULL)
5393//   dest_size          total size in bytes of the array object
5394//
5395// Exactly one of slice_len or dest_size must be non-NULL.
5396// If dest_size is non-NULL, zeroing extends to the end of the object.
5397// If slice_len is non-NULL, the slice_idx value must be a constant.
5398void
5399LibraryCallKit::generate_clear_array(const TypePtr* adr_type,
5400                                     Node* dest,
5401                                     BasicType basic_elem_type,
5402                                     Node* slice_idx,
5403                                     Node* slice_len,
5404                                     Node* dest_size) {
5405  // one or the other but not both of slice_len and dest_size:
5406  assert((slice_len != NULL? 1: 0) + (dest_size != NULL? 1: 0) == 1, "");
5407  if (slice_len == NULL)  slice_len = top();
5408  if (dest_size == NULL)  dest_size = top();
5409
5410  // operate on this memory slice:
5411  Node* mem = memory(adr_type); // memory slice to operate on
5412
5413  // scaling and rounding of indexes:
5414  int scale = exact_log2(type2aelembytes(basic_elem_type));
5415  int abase = arrayOopDesc::base_offset_in_bytes(basic_elem_type);
5416  int clear_low = (-1 << scale) & (BytesPerInt  - 1);
5417  int bump_bit  = (-1 << scale) & BytesPerInt;
5418
5419  // determine constant starts and ends
5420  const intptr_t BIG_NEG = -128;
5421  assert(BIG_NEG + 2*abase < 0, "neg enough");
5422  intptr_t slice_idx_con = (intptr_t) find_int_con(slice_idx, BIG_NEG);
5423  intptr_t slice_len_con = (intptr_t) find_int_con(slice_len, BIG_NEG);
5424  if (slice_len_con == 0) {
5425    return;                     // nothing to do here
5426  }
5427  intptr_t start_con = (abase + (slice_idx_con << scale)) & ~clear_low;
5428  intptr_t end_con   = find_intptr_t_con(dest_size, -1);
5429  if (slice_idx_con >= 0 && slice_len_con >= 0) {
5430    assert(end_con < 0, "not two cons");
5431    end_con = round_to(abase + ((slice_idx_con + slice_len_con) << scale),
5432                       BytesPerLong);
5433  }
5434
5435  if (start_con >= 0 && end_con >= 0) {
5436    // Constant start and end.  Simple.
5437    mem = ClearArrayNode::clear_memory(control(), mem, dest,
5438                                       start_con, end_con, &_gvn);
5439  } else if (start_con >= 0 && dest_size != top()) {
5440    // Constant start, pre-rounded end after the tail of the array.
5441    Node* end = dest_size;
5442    mem = ClearArrayNode::clear_memory(control(), mem, dest,
5443                                       start_con, end, &_gvn);
5444  } else if (start_con >= 0 && slice_len != top()) {
5445    // Constant start, non-constant end.  End needs rounding up.
5446    // End offset = round_up(abase + ((slice_idx_con + slice_len) << scale), 8)
5447    intptr_t end_base  = abase + (slice_idx_con << scale);
5448    int      end_round = (-1 << scale) & (BytesPerLong  - 1);
5449    Node*    end       = ConvI2X(slice_len);
5450    if (scale != 0)
5451      end = _gvn.transform(new(C) LShiftXNode(end, intcon(scale) ));
5452    end_base += end_round;
5453    end = _gvn.transform(new(C) AddXNode(end, MakeConX(end_base)));
5454    end = _gvn.transform(new(C) AndXNode(end, MakeConX(~end_round)));
5455    mem = ClearArrayNode::clear_memory(control(), mem, dest,
5456                                       start_con, end, &_gvn);
5457  } else if (start_con < 0 && dest_size != top()) {
5458    // Non-constant start, pre-rounded end after the tail of the array.
5459    // This is almost certainly a "round-to-end" operation.
5460    Node* start = slice_idx;
5461    start = ConvI2X(start);
5462    if (scale != 0)
5463      start = _gvn.transform(new(C) LShiftXNode( start, intcon(scale) ));
5464    start = _gvn.transform(new(C) AddXNode(start, MakeConX(abase)));
5465    if ((bump_bit | clear_low) != 0) {
5466      int to_clear = (bump_bit | clear_low);
5467      // Align up mod 8, then store a jint zero unconditionally
5468      // just before the mod-8 boundary.
5469      if (((abase + bump_bit) & ~to_clear) - bump_bit
5470          < arrayOopDesc::length_offset_in_bytes() + BytesPerInt) {
5471        bump_bit = 0;
5472        assert((abase & to_clear) == 0, "array base must be long-aligned");
5473      } else {
5474        // Bump 'start' up to (or past) the next jint boundary:
5475        start = _gvn.transform(new(C) AddXNode(start, MakeConX(bump_bit)));
5476        assert((abase & clear_low) == 0, "array base must be int-aligned");
5477      }
5478      // Round bumped 'start' down to jlong boundary in body of array.
5479      start = _gvn.transform(new(C) AndXNode(start, MakeConX(~to_clear)));
5480      if (bump_bit != 0) {
5481        // Store a zero to the immediately preceding jint:
5482        Node* x1 = _gvn.transform(new(C) AddXNode(start, MakeConX(-bump_bit)));
5483        Node* p1 = basic_plus_adr(dest, x1);
5484        mem = StoreNode::make(_gvn, control(), mem, p1, adr_type, intcon(0), T_INT, MemNode::unordered);
5485        mem = _gvn.transform(mem);
5486      }
5487    }
5488    Node* end = dest_size; // pre-rounded
5489    mem = ClearArrayNode::clear_memory(control(), mem, dest,
5490                                       start, end, &_gvn);
5491  } else {
5492    // Non-constant start, unrounded non-constant end.
5493    // (Nobody zeroes a random midsection of an array using this routine.)
5494    ShouldNotReachHere();       // fix caller
5495  }
5496
5497  // Done.
5498  set_memory(mem, adr_type);
5499}
5500
5501
5502bool
5503LibraryCallKit::generate_block_arraycopy(const TypePtr* adr_type,
5504                                         BasicType basic_elem_type,
5505                                         AllocateNode* alloc,
5506                                         Node* src,  Node* src_offset,
5507                                         Node* dest, Node* dest_offset,
5508                                         Node* dest_size, bool dest_uninitialized) {
5509  // See if there is an advantage from block transfer.
5510  int scale = exact_log2(type2aelembytes(basic_elem_type));
5511  if (scale >= LogBytesPerLong)
5512    return false;               // it is already a block transfer
5513
5514  // Look at the alignment of the starting offsets.
5515  int abase = arrayOopDesc::base_offset_in_bytes(basic_elem_type);
5516
5517  intptr_t src_off_con  = (intptr_t) find_int_con(src_offset, -1);
5518  intptr_t dest_off_con = (intptr_t) find_int_con(dest_offset, -1);
5519  if (src_off_con < 0 || dest_off_con < 0)
5520    // At present, we can only understand constants.
5521    return false;
5522
5523  intptr_t src_off  = abase + (src_off_con  << scale);
5524  intptr_t dest_off = abase + (dest_off_con << scale);
5525
5526  if (((src_off | dest_off) & (BytesPerLong-1)) != 0) {
5527    // Non-aligned; too bad.
5528    // One more chance:  Pick off an initial 32-bit word.
5529    // This is a common case, since abase can be odd mod 8.
5530    if (((src_off | dest_off) & (BytesPerLong-1)) == BytesPerInt &&
5531        ((src_off ^ dest_off) & (BytesPerLong-1)) == 0) {
5532      Node* sptr = basic_plus_adr(src,  src_off);
5533      Node* dptr = basic_plus_adr(dest, dest_off);
5534      Node* sval = make_load(control(), sptr, TypeInt::INT, T_INT, adr_type, MemNode::unordered);
5535      store_to_memory(control(), dptr, sval, T_INT, adr_type, MemNode::unordered);
5536      src_off += BytesPerInt;
5537      dest_off += BytesPerInt;
5538    } else {
5539      return false;
5540    }
5541  }
5542  assert(src_off % BytesPerLong == 0, "");
5543  assert(dest_off % BytesPerLong == 0, "");
5544
5545  // Do this copy by giant steps.
5546  Node* sptr  = basic_plus_adr(src,  src_off);
5547  Node* dptr  = basic_plus_adr(dest, dest_off);
5548  Node* countx = dest_size;
5549  countx = _gvn.transform(new (C) SubXNode(countx, MakeConX(dest_off)));
5550  countx = _gvn.transform(new (C) URShiftXNode(countx, intcon(LogBytesPerLong)));
5551
5552  bool disjoint_bases = true;   // since alloc != NULL
5553  generate_unchecked_arraycopy(adr_type, T_LONG, disjoint_bases,
5554                               sptr, NULL, dptr, NULL, countx, dest_uninitialized);
5555
5556  return true;
5557}
5558
5559
5560// Helper function; generates code for the slow case.
5561// We make a call to a runtime method which emulates the native method,
5562// but without the native wrapper overhead.
5563void
5564LibraryCallKit::generate_slow_arraycopy(const TypePtr* adr_type,
5565                                        Node* src,  Node* src_offset,
5566                                        Node* dest, Node* dest_offset,
5567                                        Node* copy_length, bool dest_uninitialized) {
5568  assert(!dest_uninitialized, "Invariant");
5569  Node* call = make_runtime_call(RC_NO_LEAF | RC_UNCOMMON,
5570                                 OptoRuntime::slow_arraycopy_Type(),
5571                                 OptoRuntime::slow_arraycopy_Java(),
5572                                 "slow_arraycopy", adr_type,
5573                                 src, src_offset, dest, dest_offset,
5574                                 copy_length);
5575
5576  // Handle exceptions thrown by this fellow:
5577  make_slow_call_ex(call, env()->Throwable_klass(), false);
5578}
5579
5580// Helper function; generates code for cases requiring runtime checks.
5581Node*
5582LibraryCallKit::generate_checkcast_arraycopy(const TypePtr* adr_type,
5583                                             Node* dest_elem_klass,
5584                                             Node* src,  Node* src_offset,
5585                                             Node* dest, Node* dest_offset,
5586                                             Node* copy_length, bool dest_uninitialized) {
5587  if (stopped())  return NULL;
5588
5589  address copyfunc_addr = StubRoutines::checkcast_arraycopy(dest_uninitialized);
5590  if (copyfunc_addr == NULL) { // Stub was not generated, go slow path.
5591    return NULL;
5592  }
5593
5594  // Pick out the parameters required to perform a store-check
5595  // for the target array.  This is an optimistic check.  It will
5596  // look in each non-null element's class, at the desired klass's
5597  // super_check_offset, for the desired klass.
5598  int sco_offset = in_bytes(Klass::super_check_offset_offset());
5599  Node* p3 = basic_plus_adr(dest_elem_klass, sco_offset);
5600  Node* n3 = new(C) LoadINode(NULL, memory(p3), p3, _gvn.type(p3)->is_ptr(), TypeInt::INT, MemNode::unordered);
5601  Node* check_offset = ConvI2X(_gvn.transform(n3));
5602  Node* check_value  = dest_elem_klass;
5603
5604  Node* src_start  = array_element_address(src,  src_offset,  T_OBJECT);
5605  Node* dest_start = array_element_address(dest, dest_offset, T_OBJECT);
5606
5607  // (We know the arrays are never conjoint, because their types differ.)
5608  Node* call = make_runtime_call(RC_LEAF|RC_NO_FP,
5609                                 OptoRuntime::checkcast_arraycopy_Type(),
5610                                 copyfunc_addr, "checkcast_arraycopy", adr_type,
5611                                 // five arguments, of which two are
5612                                 // intptr_t (jlong in LP64)
5613                                 src_start, dest_start,
5614                                 copy_length XTOP,
5615                                 check_offset XTOP,
5616                                 check_value);
5617
5618  return _gvn.transform(new (C) ProjNode(call, TypeFunc::Parms));
5619}
5620
5621
5622// Helper function; generates code for cases requiring runtime checks.
5623Node*
5624LibraryCallKit::generate_generic_arraycopy(const TypePtr* adr_type,
5625                                           Node* src,  Node* src_offset,
5626                                           Node* dest, Node* dest_offset,
5627                                           Node* copy_length, bool dest_uninitialized) {
5628  assert(!dest_uninitialized, "Invariant");
5629  if (stopped())  return NULL;
5630  address copyfunc_addr = StubRoutines::generic_arraycopy();
5631  if (copyfunc_addr == NULL) { // Stub was not generated, go slow path.
5632    return NULL;
5633  }
5634
5635  Node* call = make_runtime_call(RC_LEAF|RC_NO_FP,
5636                    OptoRuntime::generic_arraycopy_Type(),
5637                    copyfunc_addr, "generic_arraycopy", adr_type,
5638                    src, src_offset, dest, dest_offset, copy_length);
5639
5640  return _gvn.transform(new (C) ProjNode(call, TypeFunc::Parms));
5641}
5642
5643// Helper function; generates the fast out-of-line call to an arraycopy stub.
5644void
5645LibraryCallKit::generate_unchecked_arraycopy(const TypePtr* adr_type,
5646                                             BasicType basic_elem_type,
5647                                             bool disjoint_bases,
5648                                             Node* src,  Node* src_offset,
5649                                             Node* dest, Node* dest_offset,
5650                                             Node* copy_length, bool dest_uninitialized) {
5651  if (stopped())  return;               // nothing to do
5652
5653  Node* src_start  = src;
5654  Node* dest_start = dest;
5655  if (src_offset != NULL || dest_offset != NULL) {
5656    assert(src_offset != NULL && dest_offset != NULL, "");
5657    src_start  = array_element_address(src,  src_offset,  basic_elem_type);
5658    dest_start = array_element_address(dest, dest_offset, basic_elem_type);
5659  }
5660
5661  // Figure out which arraycopy runtime method to call.
5662  const char* copyfunc_name = "arraycopy";
5663  address     copyfunc_addr =
5664      basictype2arraycopy(basic_elem_type, src_offset, dest_offset,
5665                          disjoint_bases, copyfunc_name, dest_uninitialized);
5666
5667  // Call it.  Note that the count_ix value is not scaled to a byte-size.
5668  make_runtime_call(RC_LEAF|RC_NO_FP,
5669                    OptoRuntime::fast_arraycopy_Type(),
5670                    copyfunc_addr, copyfunc_name, adr_type,
5671                    src_start, dest_start, copy_length XTOP);
5672}
5673
5674//-------------inline_encodeISOArray-----------------------------------
5675// encode char[] to byte[] in ISO_8859_1
5676bool LibraryCallKit::inline_encodeISOArray() {
5677  assert(callee()->signature()->size() == 5, "encodeISOArray has 5 parameters");
5678  // no receiver since it is static method
5679  Node *src         = argument(0);
5680  Node *src_offset  = argument(1);
5681  Node *dst         = argument(2);
5682  Node *dst_offset  = argument(3);
5683  Node *length      = argument(4);
5684
5685  const Type* src_type = src->Value(&_gvn);
5686  const Type* dst_type = dst->Value(&_gvn);
5687  const TypeAryPtr* top_src = src_type->isa_aryptr();
5688  const TypeAryPtr* top_dest = dst_type->isa_aryptr();
5689  if (top_src  == NULL || top_src->klass()  == NULL ||
5690      top_dest == NULL || top_dest->klass() == NULL) {
5691    // failed array check
5692    return false;
5693  }
5694
5695  // Figure out the size and type of the elements we will be copying.
5696  BasicType src_elem = src_type->isa_aryptr()->klass()->as_array_klass()->element_type()->basic_type();
5697  BasicType dst_elem = dst_type->isa_aryptr()->klass()->as_array_klass()->element_type()->basic_type();
5698  if (src_elem != T_CHAR || dst_elem != T_BYTE) {
5699    return false;
5700  }
5701  Node* src_start = array_element_address(src, src_offset, src_elem);
5702  Node* dst_start = array_element_address(dst, dst_offset, dst_elem);
5703  // 'src_start' points to src array + scaled offset
5704  // 'dst_start' points to dst array + scaled offset
5705
5706  const TypeAryPtr* mtype = TypeAryPtr::BYTES;
5707  Node* enc = new (C) EncodeISOArrayNode(control(), memory(mtype), src_start, dst_start, length);
5708  enc = _gvn.transform(enc);
5709  Node* res_mem = _gvn.transform(new (C) SCMemProjNode(enc));
5710  set_memory(res_mem, mtype);
5711  set_result(enc);
5712  return true;
5713}
5714
5715/**
5716 * Calculate CRC32 for byte.
5717 * int java.util.zip.CRC32.update(int crc, int b)
5718 */
5719bool LibraryCallKit::inline_updateCRC32() {
5720  assert(UseCRC32Intrinsics, "need AVX and LCMUL instructions support");
5721  assert(callee()->signature()->size() == 2, "update has 2 parameters");
5722  // no receiver since it is static method
5723  Node* crc  = argument(0); // type: int
5724  Node* b    = argument(1); // type: int
5725
5726  /*
5727   *    int c = ~ crc;
5728   *    b = timesXtoThe32[(b ^ c) & 0xFF];
5729   *    b = b ^ (c >>> 8);
5730   *    crc = ~b;
5731   */
5732
5733  Node* M1 = intcon(-1);
5734  crc = _gvn.transform(new (C) XorINode(crc, M1));
5735  Node* result = _gvn.transform(new (C) XorINode(crc, b));
5736  result = _gvn.transform(new (C) AndINode(result, intcon(0xFF)));
5737
5738  Node* base = makecon(TypeRawPtr::make(StubRoutines::crc_table_addr()));
5739  Node* offset = _gvn.transform(new (C) LShiftINode(result, intcon(0x2)));
5740  Node* adr = basic_plus_adr(top(), base, ConvI2X(offset));
5741  result = make_load(control(), adr, TypeInt::INT, T_INT, MemNode::unordered);
5742
5743  crc = _gvn.transform(new (C) URShiftINode(crc, intcon(8)));
5744  result = _gvn.transform(new (C) XorINode(crc, result));
5745  result = _gvn.transform(new (C) XorINode(result, M1));
5746  set_result(result);
5747  return true;
5748}
5749
5750/**
5751 * Calculate CRC32 for byte[] array.
5752 * int java.util.zip.CRC32.updateBytes(int crc, byte[] buf, int off, int len)
5753 */
5754bool LibraryCallKit::inline_updateBytesCRC32() {
5755  assert(UseCRC32Intrinsics, "need AVX and LCMUL instructions support");
5756  assert(callee()->signature()->size() == 4, "updateBytes has 4 parameters");
5757  // no receiver since it is static method
5758  Node* crc     = argument(0); // type: int
5759  Node* src     = argument(1); // type: oop
5760  Node* offset  = argument(2); // type: int
5761  Node* length  = argument(3); // type: int
5762
5763  const Type* src_type = src->Value(&_gvn);
5764  const TypeAryPtr* top_src = src_type->isa_aryptr();
5765  if (top_src  == NULL || top_src->klass()  == NULL) {
5766    // failed array check
5767    return false;
5768  }
5769
5770  // Figure out the size and type of the elements we will be copying.
5771  BasicType src_elem = src_type->isa_aryptr()->klass()->as_array_klass()->element_type()->basic_type();
5772  if (src_elem != T_BYTE) {
5773    return false;
5774  }
5775
5776  // 'src_start' points to src array + scaled offset
5777  Node* src_start = array_element_address(src, offset, src_elem);
5778
5779  // We assume that range check is done by caller.
5780  // TODO: generate range check (offset+length < src.length) in debug VM.
5781
5782  // Call the stub.
5783  address stubAddr = StubRoutines::updateBytesCRC32();
5784  const char *stubName = "updateBytesCRC32";
5785
5786  Node* call = make_runtime_call(RC_LEAF|RC_NO_FP, OptoRuntime::updateBytesCRC32_Type(),
5787                                 stubAddr, stubName, TypePtr::BOTTOM,
5788                                 crc, src_start, length);
5789  Node* result = _gvn.transform(new (C) ProjNode(call, TypeFunc::Parms));
5790  set_result(result);
5791  return true;
5792}
5793
5794/**
5795 * Calculate CRC32 for ByteBuffer.
5796 * int java.util.zip.CRC32.updateByteBuffer(int crc, long buf, int off, int len)
5797 */
5798bool LibraryCallKit::inline_updateByteBufferCRC32() {
5799  assert(UseCRC32Intrinsics, "need AVX and LCMUL instructions support");
5800  assert(callee()->signature()->size() == 5, "updateByteBuffer has 4 parameters and one is long");
5801  // no receiver since it is static method
5802  Node* crc     = argument(0); // type: int
5803  Node* src     = argument(1); // type: long
5804  Node* offset  = argument(3); // type: int
5805  Node* length  = argument(4); // type: int
5806
5807  src = ConvL2X(src);  // adjust Java long to machine word
5808  Node* base = _gvn.transform(new (C) CastX2PNode(src));
5809  offset = ConvI2X(offset);
5810
5811  // 'src_start' points to src array + scaled offset
5812  Node* src_start = basic_plus_adr(top(), base, offset);
5813
5814  // Call the stub.
5815  address stubAddr = StubRoutines::updateBytesCRC32();
5816  const char *stubName = "updateBytesCRC32";
5817
5818  Node* call = make_runtime_call(RC_LEAF|RC_NO_FP, OptoRuntime::updateBytesCRC32_Type(),
5819                                 stubAddr, stubName, TypePtr::BOTTOM,
5820                                 crc, src_start, length);
5821  Node* result = _gvn.transform(new (C) ProjNode(call, TypeFunc::Parms));
5822  set_result(result);
5823  return true;
5824}
5825
5826//----------------------------inline_reference_get----------------------------
5827// public T java.lang.ref.Reference.get();
5828bool LibraryCallKit::inline_reference_get() {
5829  const int referent_offset = java_lang_ref_Reference::referent_offset;
5830  guarantee(referent_offset > 0, "should have already been set");
5831
5832  // Get the argument:
5833  Node* reference_obj = null_check_receiver();
5834  if (stopped()) return true;
5835
5836  Node* adr = basic_plus_adr(reference_obj, reference_obj, referent_offset);
5837
5838  ciInstanceKlass* klass = env()->Object_klass();
5839  const TypeOopPtr* object_type = TypeOopPtr::make_from_klass(klass);
5840
5841  Node* no_ctrl = NULL;
5842  Node* result = make_load(no_ctrl, adr, object_type, T_OBJECT, MemNode::unordered);
5843
5844  // Use the pre-barrier to record the value in the referent field
5845  pre_barrier(false /* do_load */,
5846              control(),
5847              NULL /* obj */, NULL /* adr */, max_juint /* alias_idx */, NULL /* val */, NULL /* val_type */,
5848              result /* pre_val */,
5849              T_OBJECT);
5850
5851  // Add memory barrier to prevent commoning reads from this field
5852  // across safepoint since GC can change its value.
5853  insert_mem_bar(Op_MemBarCPUOrder);
5854
5855  set_result(result);
5856  return true;
5857}
5858
5859
5860Node * LibraryCallKit::load_field_from_object(Node * fromObj, const char * fieldName, const char * fieldTypeString,
5861                                              bool is_exact=true, bool is_static=false) {
5862
5863  const TypeInstPtr* tinst = _gvn.type(fromObj)->isa_instptr();
5864  assert(tinst != NULL, "obj is null");
5865  assert(tinst->klass()->is_loaded(), "obj is not loaded");
5866  assert(!is_exact || tinst->klass_is_exact(), "klass not exact");
5867
5868  ciField* field = tinst->klass()->as_instance_klass()->get_field_by_name(ciSymbol::make(fieldName),
5869                                                                          ciSymbol::make(fieldTypeString),
5870                                                                          is_static);
5871  if (field == NULL) return (Node *) NULL;
5872  assert (field != NULL, "undefined field");
5873
5874  // Next code  copied from Parse::do_get_xxx():
5875
5876  // Compute address and memory type.
5877  int offset  = field->offset_in_bytes();
5878  bool is_vol = field->is_volatile();
5879  ciType* field_klass = field->type();
5880  assert(field_klass->is_loaded(), "should be loaded");
5881  const TypePtr* adr_type = C->alias_type(field)->adr_type();
5882  Node *adr = basic_plus_adr(fromObj, fromObj, offset);
5883  BasicType bt = field->layout_type();
5884
5885  // Build the resultant type of the load
5886  const Type *type = TypeOopPtr::make_from_klass(field_klass->as_klass());
5887
5888  // Build the load.
5889  Node* loadedField = make_load(NULL, adr, type, bt, adr_type, MemNode::unordered, is_vol);
5890  return loadedField;
5891}
5892
5893
5894//------------------------------inline_aescrypt_Block-----------------------
5895bool LibraryCallKit::inline_aescrypt_Block(vmIntrinsics::ID id) {
5896  address stubAddr;
5897  const char *stubName;
5898  assert(UseAES, "need AES instruction support");
5899
5900  switch(id) {
5901  case vmIntrinsics::_aescrypt_encryptBlock:
5902    stubAddr = StubRoutines::aescrypt_encryptBlock();
5903    stubName = "aescrypt_encryptBlock";
5904    break;
5905  case vmIntrinsics::_aescrypt_decryptBlock:
5906    stubAddr = StubRoutines::aescrypt_decryptBlock();
5907    stubName = "aescrypt_decryptBlock";
5908    break;
5909  }
5910  if (stubAddr == NULL) return false;
5911
5912  Node* aescrypt_object = argument(0);
5913  Node* src             = argument(1);
5914  Node* src_offset      = argument(2);
5915  Node* dest            = argument(3);
5916  Node* dest_offset     = argument(4);
5917
5918  // (1) src and dest are arrays.
5919  const Type* src_type = src->Value(&_gvn);
5920  const Type* dest_type = dest->Value(&_gvn);
5921  const TypeAryPtr* top_src = src_type->isa_aryptr();
5922  const TypeAryPtr* top_dest = dest_type->isa_aryptr();
5923  assert (top_src  != NULL && top_src->klass()  != NULL &&  top_dest != NULL && top_dest->klass() != NULL, "args are strange");
5924
5925  // for the quick and dirty code we will skip all the checks.
5926  // we are just trying to get the call to be generated.
5927  Node* src_start  = src;
5928  Node* dest_start = dest;
5929  if (src_offset != NULL || dest_offset != NULL) {
5930    assert(src_offset != NULL && dest_offset != NULL, "");
5931    src_start  = array_element_address(src,  src_offset,  T_BYTE);
5932    dest_start = array_element_address(dest, dest_offset, T_BYTE);
5933  }
5934
5935  // now need to get the start of its expanded key array
5936  // this requires a newer class file that has this array as littleEndian ints, otherwise we revert to java
5937  Node* k_start = get_key_start_from_aescrypt_object(aescrypt_object);
5938  if (k_start == NULL) return false;
5939
5940  // Call the stub.
5941  make_runtime_call(RC_LEAF|RC_NO_FP, OptoRuntime::aescrypt_block_Type(),
5942                    stubAddr, stubName, TypePtr::BOTTOM,
5943                    src_start, dest_start, k_start);
5944
5945  return true;
5946}
5947
5948//------------------------------inline_cipherBlockChaining_AESCrypt-----------------------
5949bool LibraryCallKit::inline_cipherBlockChaining_AESCrypt(vmIntrinsics::ID id) {
5950  address stubAddr;
5951  const char *stubName;
5952
5953  assert(UseAES, "need AES instruction support");
5954
5955  switch(id) {
5956  case vmIntrinsics::_cipherBlockChaining_encryptAESCrypt:
5957    stubAddr = StubRoutines::cipherBlockChaining_encryptAESCrypt();
5958    stubName = "cipherBlockChaining_encryptAESCrypt";
5959    break;
5960  case vmIntrinsics::_cipherBlockChaining_decryptAESCrypt:
5961    stubAddr = StubRoutines::cipherBlockChaining_decryptAESCrypt();
5962    stubName = "cipherBlockChaining_decryptAESCrypt";
5963    break;
5964  }
5965  if (stubAddr == NULL) return false;
5966
5967  Node* cipherBlockChaining_object = argument(0);
5968  Node* src                        = argument(1);
5969  Node* src_offset                 = argument(2);
5970  Node* len                        = argument(3);
5971  Node* dest                       = argument(4);
5972  Node* dest_offset                = argument(5);
5973
5974  // (1) src and dest are arrays.
5975  const Type* src_type = src->Value(&_gvn);
5976  const Type* dest_type = dest->Value(&_gvn);
5977  const TypeAryPtr* top_src = src_type->isa_aryptr();
5978  const TypeAryPtr* top_dest = dest_type->isa_aryptr();
5979  assert (top_src  != NULL && top_src->klass()  != NULL
5980          &&  top_dest != NULL && top_dest->klass() != NULL, "args are strange");
5981
5982  // checks are the responsibility of the caller
5983  Node* src_start  = src;
5984  Node* dest_start = dest;
5985  if (src_offset != NULL || dest_offset != NULL) {
5986    assert(src_offset != NULL && dest_offset != NULL, "");
5987    src_start  = array_element_address(src,  src_offset,  T_BYTE);
5988    dest_start = array_element_address(dest, dest_offset, T_BYTE);
5989  }
5990
5991  // if we are in this set of code, we "know" the embeddedCipher is an AESCrypt object
5992  // (because of the predicated logic executed earlier).
5993  // so we cast it here safely.
5994  // this requires a newer class file that has this array as littleEndian ints, otherwise we revert to java
5995
5996  Node* embeddedCipherObj = load_field_from_object(cipherBlockChaining_object, "embeddedCipher", "Lcom/sun/crypto/provider/SymmetricCipher;", /*is_exact*/ false);
5997  if (embeddedCipherObj == NULL) return false;
5998
5999  // cast it to what we know it will be at runtime
6000  const TypeInstPtr* tinst = _gvn.type(cipherBlockChaining_object)->isa_instptr();
6001  assert(tinst != NULL, "CBC obj is null");
6002  assert(tinst->klass()->is_loaded(), "CBC obj is not loaded");
6003  ciKlass* klass_AESCrypt = tinst->klass()->as_instance_klass()->find_klass(ciSymbol::make("com/sun/crypto/provider/AESCrypt"));
6004  if (!klass_AESCrypt->is_loaded()) return false;
6005
6006  ciInstanceKlass* instklass_AESCrypt = klass_AESCrypt->as_instance_klass();
6007  const TypeKlassPtr* aklass = TypeKlassPtr::make(instklass_AESCrypt);
6008  const TypeOopPtr* xtype = aklass->as_instance_type();
6009  Node* aescrypt_object = new(C) CheckCastPPNode(control(), embeddedCipherObj, xtype);
6010  aescrypt_object = _gvn.transform(aescrypt_object);
6011
6012  // we need to get the start of the aescrypt_object's expanded key array
6013  Node* k_start = get_key_start_from_aescrypt_object(aescrypt_object);
6014  if (k_start == NULL) return false;
6015
6016  // similarly, get the start address of the r vector
6017  Node* objRvec = load_field_from_object(cipherBlockChaining_object, "r", "[B", /*is_exact*/ false);
6018  if (objRvec == NULL) return false;
6019  Node* r_start = array_element_address(objRvec, intcon(0), T_BYTE);
6020
6021  // Call the stub, passing src_start, dest_start, k_start, r_start and src_len
6022  make_runtime_call(RC_LEAF|RC_NO_FP,
6023                    OptoRuntime::cipherBlockChaining_aescrypt_Type(),
6024                    stubAddr, stubName, TypePtr::BOTTOM,
6025                    src_start, dest_start, k_start, r_start, len);
6026
6027  // return is void so no result needs to be pushed
6028
6029  return true;
6030}
6031
6032//------------------------------get_key_start_from_aescrypt_object-----------------------
6033Node * LibraryCallKit::get_key_start_from_aescrypt_object(Node *aescrypt_object) {
6034  Node* objAESCryptKey = load_field_from_object(aescrypt_object, "K", "[I", /*is_exact*/ false);
6035  assert (objAESCryptKey != NULL, "wrong version of com.sun.crypto.provider.AESCrypt");
6036  if (objAESCryptKey == NULL) return (Node *) NULL;
6037
6038  // now have the array, need to get the start address of the K array
6039  Node* k_start = array_element_address(objAESCryptKey, intcon(0), T_INT);
6040  return k_start;
6041}
6042
6043//----------------------------inline_cipherBlockChaining_AESCrypt_predicate----------------------------
6044// Return node representing slow path of predicate check.
6045// the pseudo code we want to emulate with this predicate is:
6046// for encryption:
6047//    if (embeddedCipherObj instanceof AESCrypt) do_intrinsic, else do_javapath
6048// for decryption:
6049//    if ((embeddedCipherObj instanceof AESCrypt) && (cipher!=plain)) do_intrinsic, else do_javapath
6050//    note cipher==plain is more conservative than the original java code but that's OK
6051//
6052Node* LibraryCallKit::inline_cipherBlockChaining_AESCrypt_predicate(bool decrypting) {
6053  // First, check receiver for NULL since it is virtual method.
6054  Node* objCBC = argument(0);
6055  objCBC = null_check(objCBC);
6056
6057  if (stopped()) return NULL; // Always NULL
6058
6059  // Load embeddedCipher field of CipherBlockChaining object.
6060  Node* embeddedCipherObj = load_field_from_object(objCBC, "embeddedCipher", "Lcom/sun/crypto/provider/SymmetricCipher;", /*is_exact*/ false);
6061
6062  // get AESCrypt klass for instanceOf check
6063  // AESCrypt might not be loaded yet if some other SymmetricCipher got us to this compile point
6064  // will have same classloader as CipherBlockChaining object
6065  const TypeInstPtr* tinst = _gvn.type(objCBC)->isa_instptr();
6066  assert(tinst != NULL, "CBCobj is null");
6067  assert(tinst->klass()->is_loaded(), "CBCobj is not loaded");
6068
6069  // we want to do an instanceof comparison against the AESCrypt class
6070  ciKlass* klass_AESCrypt = tinst->klass()->as_instance_klass()->find_klass(ciSymbol::make("com/sun/crypto/provider/AESCrypt"));
6071  if (!klass_AESCrypt->is_loaded()) {
6072    // if AESCrypt is not even loaded, we never take the intrinsic fast path
6073    Node* ctrl = control();
6074    set_control(top()); // no regular fast path
6075    return ctrl;
6076  }
6077  ciInstanceKlass* instklass_AESCrypt = klass_AESCrypt->as_instance_klass();
6078
6079  Node* instof = gen_instanceof(embeddedCipherObj, makecon(TypeKlassPtr::make(instklass_AESCrypt)));
6080  Node* cmp_instof  = _gvn.transform(new (C) CmpINode(instof, intcon(1)));
6081  Node* bool_instof  = _gvn.transform(new (C) BoolNode(cmp_instof, BoolTest::ne));
6082
6083  Node* instof_false = generate_guard(bool_instof, NULL, PROB_MIN);
6084
6085  // for encryption, we are done
6086  if (!decrypting)
6087    return instof_false;  // even if it is NULL
6088
6089  // for decryption, we need to add a further check to avoid
6090  // taking the intrinsic path when cipher and plain are the same
6091  // see the original java code for why.
6092  RegionNode* region = new(C) RegionNode(3);
6093  region->init_req(1, instof_false);
6094  Node* src = argument(1);
6095  Node* dest = argument(4);
6096  Node* cmp_src_dest = _gvn.transform(new (C) CmpPNode(src, dest));
6097  Node* bool_src_dest = _gvn.transform(new (C) BoolNode(cmp_src_dest, BoolTest::eq));
6098  Node* src_dest_conjoint = generate_guard(bool_src_dest, NULL, PROB_MIN);
6099  region->init_req(2, src_dest_conjoint);
6100
6101  record_for_igvn(region);
6102  return _gvn.transform(region);
6103}
6104