1/* Calculate branch probabilities, and basic block execution counts.
2   Copyright (C) 1990-2020 Free Software Foundation, Inc.
3   Contributed by James E. Wilson, UC Berkeley/Cygnus Support;
4   based on some ideas from Dain Samples of UC Berkeley.
5   Further mangling by Bob Manson, Cygnus Support.
6   Converted to use trees by Dale Johannesen, Apple Computer.
7
8This file is part of GCC.
9
10GCC is free software; you can redistribute it and/or modify it under
11the terms of the GNU General Public License as published by the Free
12Software Foundation; either version 3, or (at your option) any later
13version.
14
15GCC is distributed in the hope that it will be useful, but WITHOUT ANY
16WARRANTY; without even the implied warranty of MERCHANTABILITY or
17FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
18for more details.
19
20You should have received a copy of the GNU General Public License
21along with GCC; see the file COPYING3.  If not see
22<http://www.gnu.org/licenses/>.  */
23
24/* Generate basic block profile instrumentation and auxiliary files.
25   Tree-based version.  See profile.c for overview.  */
26
27#include "config.h"
28#include "system.h"
29#include "coretypes.h"
30#include "memmodel.h"
31#include "backend.h"
32#include "target.h"
33#include "tree.h"
34#include "gimple.h"
35#include "cfghooks.h"
36#include "tree-pass.h"
37#include "ssa.h"
38#include "cgraph.h"
39#include "coverage.h"
40#include "diagnostic-core.h"
41#include "fold-const.h"
42#include "varasm.h"
43#include "tree-nested.h"
44#include "gimplify.h"
45#include "gimple-iterator.h"
46#include "gimplify-me.h"
47#include "tree-cfg.h"
48#include "tree-into-ssa.h"
49#include "value-prof.h"
50#include "profile.h"
51#include "tree-cfgcleanup.h"
52#include "stringpool.h"
53#include "attribs.h"
54#include "tree-pretty-print.h"
55#include "langhooks.h"
56#include "stor-layout.h"
57#include "xregex.h"
58
59static GTY(()) tree gcov_type_node;
60static GTY(()) tree tree_interval_profiler_fn;
61static GTY(()) tree tree_pow2_profiler_fn;
62static GTY(()) tree tree_topn_values_profiler_fn;
63static GTY(()) tree tree_indirect_call_profiler_fn;
64static GTY(()) tree tree_average_profiler_fn;
65static GTY(()) tree tree_ior_profiler_fn;
66static GTY(()) tree tree_time_profiler_counter;
67
68
69static GTY(()) tree ic_tuple_var;
70static GTY(()) tree ic_tuple_counters_field;
71static GTY(()) tree ic_tuple_callee_field;
72
73/* Do initialization work for the edge profiler.  */
74
75/* Add code:
76   __thread gcov*	__gcov_indirect_call.counters; // pointer to actual counter
77   __thread void*	__gcov_indirect_call.callee; // actual callee address
78   __thread int __gcov_function_counter; // time profiler function counter
79*/
80static void
81init_ic_make_global_vars (void)
82{
83  tree gcov_type_ptr;
84
85  gcov_type_ptr = build_pointer_type (get_gcov_type ());
86
87  tree tuple_type = lang_hooks.types.make_type (RECORD_TYPE);
88
89  /* callee */
90  ic_tuple_callee_field = build_decl (BUILTINS_LOCATION, FIELD_DECL, NULL_TREE,
91				      ptr_type_node);
92
93  /* counters */
94  ic_tuple_counters_field = build_decl (BUILTINS_LOCATION, FIELD_DECL,
95					NULL_TREE, gcov_type_ptr);
96  DECL_CHAIN (ic_tuple_counters_field) = ic_tuple_callee_field;
97
98  finish_builtin_struct (tuple_type, "indirect_call_tuple",
99			 ic_tuple_counters_field, NULL_TREE);
100
101  ic_tuple_var
102    = build_decl (UNKNOWN_LOCATION, VAR_DECL,
103		  get_identifier ("__gcov_indirect_call"), tuple_type);
104  TREE_PUBLIC (ic_tuple_var) = 1;
105  DECL_ARTIFICIAL (ic_tuple_var) = 1;
106  DECL_INITIAL (ic_tuple_var) = NULL;
107  DECL_EXTERNAL (ic_tuple_var) = 1;
108  if (targetm.have_tls)
109    set_decl_tls_model (ic_tuple_var, decl_default_tls_model (ic_tuple_var));
110}
111
112/* Create the type and function decls for the interface with gcov.  */
113
114void
115gimple_init_gcov_profiler (void)
116{
117  tree interval_profiler_fn_type;
118  tree pow2_profiler_fn_type;
119  tree topn_values_profiler_fn_type;
120  tree gcov_type_ptr;
121  tree ic_profiler_fn_type;
122  tree average_profiler_fn_type;
123  const char *fn_name;
124
125  if (!gcov_type_node)
126    {
127      const char *fn_suffix
128	= flag_profile_update == PROFILE_UPDATE_ATOMIC ? "_atomic" : "";
129
130      gcov_type_node = get_gcov_type ();
131      gcov_type_ptr = build_pointer_type (gcov_type_node);
132
133      /* void (*) (gcov_type *, gcov_type, int, unsigned)  */
134      interval_profiler_fn_type
135	      = build_function_type_list (void_type_node,
136					  gcov_type_ptr, gcov_type_node,
137					  integer_type_node,
138					  unsigned_type_node, NULL_TREE);
139      fn_name = concat ("__gcov_interval_profiler", fn_suffix, NULL);
140      tree_interval_profiler_fn = build_fn_decl (fn_name,
141						 interval_profiler_fn_type);
142      free (CONST_CAST (char *, fn_name));
143      TREE_NOTHROW (tree_interval_profiler_fn) = 1;
144      DECL_ATTRIBUTES (tree_interval_profiler_fn)
145	= tree_cons (get_identifier ("leaf"), NULL,
146		     DECL_ATTRIBUTES (tree_interval_profiler_fn));
147
148      /* void (*) (gcov_type *, gcov_type)  */
149      pow2_profiler_fn_type
150	      = build_function_type_list (void_type_node,
151					  gcov_type_ptr, gcov_type_node,
152					  NULL_TREE);
153      fn_name = concat ("__gcov_pow2_profiler", fn_suffix, NULL);
154      tree_pow2_profiler_fn = build_fn_decl (fn_name, pow2_profiler_fn_type);
155      free (CONST_CAST (char *, fn_name));
156      TREE_NOTHROW (tree_pow2_profiler_fn) = 1;
157      DECL_ATTRIBUTES (tree_pow2_profiler_fn)
158	= tree_cons (get_identifier ("leaf"), NULL,
159		     DECL_ATTRIBUTES (tree_pow2_profiler_fn));
160
161      /* void (*) (gcov_type *, gcov_type)  */
162      topn_values_profiler_fn_type
163	      = build_function_type_list (void_type_node,
164					  gcov_type_ptr, gcov_type_node,
165					  NULL_TREE);
166      fn_name = concat ("__gcov_topn_values_profiler", fn_suffix, NULL);
167      tree_topn_values_profiler_fn
168	= build_fn_decl (fn_name, topn_values_profiler_fn_type);
169      free (CONST_CAST (char *, fn_name));
170
171      TREE_NOTHROW (tree_topn_values_profiler_fn) = 1;
172      DECL_ATTRIBUTES (tree_topn_values_profiler_fn)
173	= tree_cons (get_identifier ("leaf"), NULL,
174		     DECL_ATTRIBUTES (tree_topn_values_profiler_fn));
175
176      init_ic_make_global_vars ();
177
178      /* void (*) (gcov_type, void *)  */
179      ic_profiler_fn_type
180	       = build_function_type_list (void_type_node,
181					  gcov_type_node,
182					  ptr_type_node,
183					  NULL_TREE);
184      fn_name = concat ("__gcov_indirect_call_profiler_v4", fn_suffix, NULL);
185      tree_indirect_call_profiler_fn
186	= build_fn_decl (fn_name, ic_profiler_fn_type);
187      free (CONST_CAST (char *, fn_name));
188
189      TREE_NOTHROW (tree_indirect_call_profiler_fn) = 1;
190      DECL_ATTRIBUTES (tree_indirect_call_profiler_fn)
191	= tree_cons (get_identifier ("leaf"), NULL,
192		     DECL_ATTRIBUTES (tree_indirect_call_profiler_fn));
193
194      tree_time_profiler_counter
195	= build_decl (UNKNOWN_LOCATION, VAR_DECL,
196		      get_identifier ("__gcov_time_profiler_counter"),
197		      get_gcov_type ());
198      TREE_PUBLIC (tree_time_profiler_counter) = 1;
199      DECL_EXTERNAL (tree_time_profiler_counter) = 1;
200      TREE_STATIC (tree_time_profiler_counter) = 1;
201      DECL_ARTIFICIAL (tree_time_profiler_counter) = 1;
202      DECL_INITIAL (tree_time_profiler_counter) = NULL;
203
204      /* void (*) (gcov_type *, gcov_type)  */
205      average_profiler_fn_type
206	      = build_function_type_list (void_type_node,
207					  gcov_type_ptr, gcov_type_node, NULL_TREE);
208      fn_name = concat ("__gcov_average_profiler", fn_suffix, NULL);
209      tree_average_profiler_fn = build_fn_decl (fn_name,
210						average_profiler_fn_type);
211      free (CONST_CAST (char *, fn_name));
212      TREE_NOTHROW (tree_average_profiler_fn) = 1;
213      DECL_ATTRIBUTES (tree_average_profiler_fn)
214	= tree_cons (get_identifier ("leaf"), NULL,
215		     DECL_ATTRIBUTES (tree_average_profiler_fn));
216      fn_name = concat ("__gcov_ior_profiler", fn_suffix, NULL);
217      tree_ior_profiler_fn = build_fn_decl (fn_name, average_profiler_fn_type);
218      free (CONST_CAST (char *, fn_name));
219      TREE_NOTHROW (tree_ior_profiler_fn) = 1;
220      DECL_ATTRIBUTES (tree_ior_profiler_fn)
221	= tree_cons (get_identifier ("leaf"), NULL,
222		     DECL_ATTRIBUTES (tree_ior_profiler_fn));
223
224      /* LTO streamer needs assembler names.  Because we create these decls
225         late, we need to initialize them by hand.  */
226      DECL_ASSEMBLER_NAME (tree_interval_profiler_fn);
227      DECL_ASSEMBLER_NAME (tree_pow2_profiler_fn);
228      DECL_ASSEMBLER_NAME (tree_topn_values_profiler_fn);
229      DECL_ASSEMBLER_NAME (tree_indirect_call_profiler_fn);
230      DECL_ASSEMBLER_NAME (tree_average_profiler_fn);
231      DECL_ASSEMBLER_NAME (tree_ior_profiler_fn);
232    }
233}
234
235/* Output instructions as GIMPLE trees to increment the edge
236   execution count, and insert them on E.  We rely on
237   gsi_insert_on_edge to preserve the order.  */
238
239void
240gimple_gen_edge_profiler (int edgeno, edge e)
241{
242  tree one;
243
244  one = build_int_cst (gcov_type_node, 1);
245
246  if (flag_profile_update == PROFILE_UPDATE_ATOMIC)
247    {
248      /* __atomic_fetch_add (&counter, 1, MEMMODEL_RELAXED); */
249      tree addr = tree_coverage_counter_addr (GCOV_COUNTER_ARCS, edgeno);
250      tree f = builtin_decl_explicit (LONG_LONG_TYPE_SIZE > 32
251				      ? BUILT_IN_ATOMIC_FETCH_ADD_8:
252				      BUILT_IN_ATOMIC_FETCH_ADD_4);
253      gcall *stmt = gimple_build_call (f, 3, addr, one,
254				       build_int_cst (integer_type_node,
255						      MEMMODEL_RELAXED));
256      gsi_insert_on_edge (e, stmt);
257    }
258  else
259    {
260      tree ref = tree_coverage_counter_ref (GCOV_COUNTER_ARCS, edgeno);
261      tree gcov_type_tmp_var = make_temp_ssa_name (gcov_type_node,
262						   NULL, "PROF_edge_counter");
263      gassign *stmt1 = gimple_build_assign (gcov_type_tmp_var, ref);
264      gcov_type_tmp_var = make_temp_ssa_name (gcov_type_node,
265					      NULL, "PROF_edge_counter");
266      gassign *stmt2 = gimple_build_assign (gcov_type_tmp_var, PLUS_EXPR,
267					    gimple_assign_lhs (stmt1), one);
268      gassign *stmt3 = gimple_build_assign (unshare_expr (ref),
269					    gimple_assign_lhs (stmt2));
270      gsi_insert_on_edge (e, stmt1);
271      gsi_insert_on_edge (e, stmt2);
272      gsi_insert_on_edge (e, stmt3);
273    }
274}
275
276/* Emits code to get VALUE to instrument at GSI, and returns the
277   variable containing the value.  */
278
279static tree
280prepare_instrumented_value (gimple_stmt_iterator *gsi, histogram_value value)
281{
282  tree val = value->hvalue.value;
283  if (POINTER_TYPE_P (TREE_TYPE (val)))
284    val = fold_convert (build_nonstandard_integer_type
285			  (TYPE_PRECISION (TREE_TYPE (val)), 1), val);
286  return force_gimple_operand_gsi (gsi, fold_convert (gcov_type_node, val),
287				   true, NULL_TREE, true, GSI_SAME_STMT);
288}
289
290/* Output instructions as GIMPLE trees to increment the interval histogram
291   counter.  VALUE is the expression whose value is profiled.  TAG is the
292   tag of the section for counters, BASE is offset of the counter position.  */
293
294void
295gimple_gen_interval_profiler (histogram_value value, unsigned tag)
296{
297  gimple *stmt = value->hvalue.stmt;
298  gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
299  tree ref = tree_coverage_counter_ref (tag, 0), ref_ptr;
300  gcall *call;
301  tree val;
302  tree start = build_int_cst_type (integer_type_node,
303				   value->hdata.intvl.int_start);
304  tree steps = build_int_cst_type (unsigned_type_node,
305				   value->hdata.intvl.steps);
306
307  ref_ptr = force_gimple_operand_gsi (&gsi,
308				      build_addr (ref),
309				      true, NULL_TREE, true, GSI_SAME_STMT);
310  val = prepare_instrumented_value (&gsi, value);
311  call = gimple_build_call (tree_interval_profiler_fn, 4,
312			    ref_ptr, val, start, steps);
313  gsi_insert_before (&gsi, call, GSI_NEW_STMT);
314}
315
316/* Output instructions as GIMPLE trees to increment the power of two histogram
317   counter.  VALUE is the expression whose value is profiled.  TAG is the tag
318   of the section for counters.  */
319
320void
321gimple_gen_pow2_profiler (histogram_value value, unsigned tag)
322{
323  gimple *stmt = value->hvalue.stmt;
324  gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
325  tree ref_ptr = tree_coverage_counter_addr (tag, 0);
326  gcall *call;
327  tree val;
328
329  ref_ptr = force_gimple_operand_gsi (&gsi, ref_ptr,
330				      true, NULL_TREE, true, GSI_SAME_STMT);
331  val = prepare_instrumented_value (&gsi, value);
332  call = gimple_build_call (tree_pow2_profiler_fn, 2, ref_ptr, val);
333  gsi_insert_before (&gsi, call, GSI_NEW_STMT);
334}
335
336/* Output instructions as GIMPLE trees for code to find the most N common
337   values.  VALUE is the expression whose value is profiled.  TAG is the tag
338   of the section for counters.  */
339
340void
341gimple_gen_topn_values_profiler (histogram_value value, unsigned tag)
342{
343  gimple *stmt = value->hvalue.stmt;
344  gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
345  tree ref_ptr = tree_coverage_counter_addr (tag, 0);
346  gcall *call;
347  tree val;
348
349  ref_ptr = force_gimple_operand_gsi (&gsi, ref_ptr,
350				      true, NULL_TREE, true, GSI_SAME_STMT);
351  val = prepare_instrumented_value (&gsi, value);
352  call = gimple_build_call (tree_topn_values_profiler_fn, 2, ref_ptr, val);
353  gsi_insert_before (&gsi, call, GSI_NEW_STMT);
354}
355
356
357/* Output instructions as GIMPLE trees for code to find the most
358   common called function in indirect call.
359   VALUE is the call expression whose indirect callee is profiled.
360   TAG is the tag of the section for counters.  */
361
362void
363gimple_gen_ic_profiler (histogram_value value, unsigned tag)
364{
365  tree tmp1;
366  gassign *stmt1, *stmt2, *stmt3;
367  gimple *stmt = value->hvalue.stmt;
368  gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
369  tree ref_ptr = tree_coverage_counter_addr (tag, 0);
370
371  ref_ptr = force_gimple_operand_gsi (&gsi, ref_ptr,
372				      true, NULL_TREE, true, GSI_SAME_STMT);
373
374  /* Insert code:
375
376    stmt1: __gcov_indirect_call.counters = get_relevant_counter_ptr ();
377    stmt2: tmp1 = (void *) (indirect call argument value)
378    stmt3: __gcov_indirect_call.callee = tmp1;
379
380    Example:
381      f_1 = foo;
382      __gcov_indirect_call.counters = &__gcov4.main[0];
383      PROF_9 = f_1;
384      __gcov_indirect_call.callee = PROF_9;
385      _4 = f_1 ();
386   */
387
388  tree gcov_type_ptr = build_pointer_type (get_gcov_type ());
389
390  tree counter_ref = build3 (COMPONENT_REF, gcov_type_ptr,
391			     ic_tuple_var, ic_tuple_counters_field, NULL_TREE);
392
393  stmt1 = gimple_build_assign (counter_ref, ref_ptr);
394  tmp1 = make_temp_ssa_name (ptr_type_node, NULL, "PROF");
395  stmt2 = gimple_build_assign (tmp1, unshare_expr (value->hvalue.value));
396  tree callee_ref = build3 (COMPONENT_REF, ptr_type_node,
397			     ic_tuple_var, ic_tuple_callee_field, NULL_TREE);
398  stmt3 = gimple_build_assign (callee_ref, tmp1);
399
400  gsi_insert_before (&gsi, stmt1, GSI_SAME_STMT);
401  gsi_insert_before (&gsi, stmt2, GSI_SAME_STMT);
402  gsi_insert_before (&gsi, stmt3, GSI_SAME_STMT);
403}
404
405
406/* Output instructions as GIMPLE trees for code to find the most
407   common called function in indirect call. Insert instructions at the
408   beginning of every possible called function.
409  */
410
411void
412gimple_gen_ic_func_profiler (void)
413{
414  struct cgraph_node * c_node = cgraph_node::get (current_function_decl);
415  gcall *stmt1;
416  tree tree_uid, cur_func, void0;
417
418  if (c_node->only_called_directly_p ())
419    return;
420
421  gimple_init_gcov_profiler ();
422
423  basic_block entry = ENTRY_BLOCK_PTR_FOR_FN (cfun);
424  basic_block cond_bb = split_edge (single_succ_edge (entry));
425  basic_block update_bb = split_edge (single_succ_edge (cond_bb));
426
427  /* We need to do an extra split in order to not create an input
428     for a possible PHI node.  */
429  split_edge (single_succ_edge (update_bb));
430
431  edge true_edge = single_succ_edge (cond_bb);
432  true_edge->flags = EDGE_TRUE_VALUE;
433
434  profile_probability probability;
435  if (DECL_VIRTUAL_P (current_function_decl))
436    probability = profile_probability::very_likely ();
437  else
438    probability = profile_probability::unlikely ();
439
440  true_edge->probability = probability;
441  edge e = make_edge (cond_bb, single_succ_edge (update_bb)->dest,
442		      EDGE_FALSE_VALUE);
443  e->probability = true_edge->probability.invert ();
444
445  /* Insert code:
446
447     if (__gcov_indirect_call.callee != NULL)
448       __gcov_indirect_call_profiler_v3 (profile_id, &current_function_decl);
449
450     The function __gcov_indirect_call_profiler_v3 is responsible for
451     resetting __gcov_indirect_call.callee to NULL.  */
452
453  gimple_stmt_iterator gsi = gsi_start_bb (cond_bb);
454  void0 = build_int_cst (ptr_type_node, 0);
455
456  tree callee_ref = build3 (COMPONENT_REF, ptr_type_node,
457			    ic_tuple_var, ic_tuple_callee_field, NULL_TREE);
458
459  tree ref = force_gimple_operand_gsi (&gsi, callee_ref, true, NULL_TREE,
460				       true, GSI_SAME_STMT);
461
462  gcond *cond = gimple_build_cond (NE_EXPR, ref,
463				   void0, NULL, NULL);
464  gsi_insert_before (&gsi, cond, GSI_NEW_STMT);
465
466  gsi = gsi_after_labels (update_bb);
467
468  cur_func = force_gimple_operand_gsi (&gsi,
469				       build_addr (current_function_decl),
470				       true, NULL_TREE,
471				       true, GSI_SAME_STMT);
472  tree_uid = build_int_cst
473	      (gcov_type_node,
474	       cgraph_node::get (current_function_decl)->profile_id);
475  stmt1 = gimple_build_call (tree_indirect_call_profiler_fn, 2,
476			     tree_uid, cur_func);
477  gsi_insert_before (&gsi, stmt1, GSI_SAME_STMT);
478}
479
480/* Output instructions as GIMPLE tree at the beginning for each function.
481   TAG is the tag of the section for counters, BASE is offset of the
482   counter position and GSI is the iterator we place the counter.  */
483
484void
485gimple_gen_time_profiler (unsigned tag)
486{
487  tree type = get_gcov_type ();
488  basic_block entry = ENTRY_BLOCK_PTR_FOR_FN (cfun);
489  basic_block cond_bb = split_edge (single_succ_edge (entry));
490  basic_block update_bb = split_edge (single_succ_edge (cond_bb));
491
492  /* We need to do an extra split in order to not create an input
493     for a possible PHI node.  */
494  split_edge (single_succ_edge (update_bb));
495
496  edge true_edge = single_succ_edge (cond_bb);
497  true_edge->flags = EDGE_TRUE_VALUE;
498  true_edge->probability = profile_probability::unlikely ();
499  edge e
500    = make_edge (cond_bb, single_succ_edge (update_bb)->dest, EDGE_FALSE_VALUE);
501  e->probability = true_edge->probability.invert ();
502
503  gimple_stmt_iterator gsi = gsi_start_bb (cond_bb);
504  tree original_ref = tree_coverage_counter_ref (tag, 0);
505  tree ref = force_gimple_operand_gsi (&gsi, original_ref, true, NULL_TREE,
506				       true, GSI_SAME_STMT);
507  tree one = build_int_cst (type, 1);
508
509  /* Emit: if (counters[0] != 0).  */
510  gcond *cond = gimple_build_cond (EQ_EXPR, ref, build_int_cst (type, 0),
511				   NULL, NULL);
512  gsi_insert_before (&gsi, cond, GSI_NEW_STMT);
513
514  gsi = gsi_start_bb (update_bb);
515
516  /* Emit: counters[0] = ++__gcov_time_profiler_counter.  */
517  if (flag_profile_update == PROFILE_UPDATE_ATOMIC)
518    {
519      tree ptr = make_temp_ssa_name (build_pointer_type (type), NULL,
520				     "time_profiler_counter_ptr");
521      tree addr = build1 (ADDR_EXPR, TREE_TYPE (ptr),
522			  tree_time_profiler_counter);
523      gassign *assign = gimple_build_assign (ptr, NOP_EXPR, addr);
524      gsi_insert_before (&gsi, assign, GSI_NEW_STMT);
525      tree f = builtin_decl_explicit (LONG_LONG_TYPE_SIZE > 32
526				      ? BUILT_IN_ATOMIC_ADD_FETCH_8:
527				      BUILT_IN_ATOMIC_ADD_FETCH_4);
528      gcall *stmt = gimple_build_call (f, 3, ptr, one,
529				       build_int_cst (integer_type_node,
530						      MEMMODEL_RELAXED));
531      tree result_type = TREE_TYPE (TREE_TYPE (f));
532      tree tmp = make_temp_ssa_name (result_type, NULL, "time_profile");
533      gimple_set_lhs (stmt, tmp);
534      gsi_insert_after (&gsi, stmt, GSI_NEW_STMT);
535      tmp = make_temp_ssa_name (type, NULL, "time_profile");
536      assign = gimple_build_assign (tmp, NOP_EXPR,
537				    gimple_call_lhs (stmt));
538      gsi_insert_after (&gsi, assign, GSI_NEW_STMT);
539      assign = gimple_build_assign (original_ref, tmp);
540      gsi_insert_after (&gsi, assign, GSI_NEW_STMT);
541    }
542  else
543    {
544      tree tmp = make_temp_ssa_name (type, NULL, "time_profile");
545      gassign *assign = gimple_build_assign (tmp, tree_time_profiler_counter);
546      gsi_insert_before (&gsi, assign, GSI_NEW_STMT);
547
548      tmp = make_temp_ssa_name (type, NULL, "time_profile");
549      assign = gimple_build_assign (tmp, PLUS_EXPR, gimple_assign_lhs (assign),
550				    one);
551      gsi_insert_after (&gsi, assign, GSI_NEW_STMT);
552      assign = gimple_build_assign (original_ref, tmp);
553      gsi_insert_after (&gsi, assign, GSI_NEW_STMT);
554      assign = gimple_build_assign (tree_time_profiler_counter, tmp);
555      gsi_insert_after (&gsi, assign, GSI_NEW_STMT);
556    }
557}
558
559/* Output instructions as GIMPLE trees to increment the average histogram
560   counter.  VALUE is the expression whose value is profiled.  TAG is the
561   tag of the section for counters, BASE is offset of the counter position.  */
562
563void
564gimple_gen_average_profiler (histogram_value value, unsigned tag)
565{
566  gimple *stmt = value->hvalue.stmt;
567  gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
568  tree ref_ptr = tree_coverage_counter_addr (tag, 0);
569  gcall *call;
570  tree val;
571
572  ref_ptr = force_gimple_operand_gsi (&gsi, ref_ptr,
573				      true, NULL_TREE,
574				      true, GSI_SAME_STMT);
575  val = prepare_instrumented_value (&gsi, value);
576  call = gimple_build_call (tree_average_profiler_fn, 2, ref_ptr, val);
577  gsi_insert_before (&gsi, call, GSI_NEW_STMT);
578}
579
580/* Output instructions as GIMPLE trees to increment the ior histogram
581   counter.  VALUE is the expression whose value is profiled.  TAG is the
582   tag of the section for counters, BASE is offset of the counter position.  */
583
584void
585gimple_gen_ior_profiler (histogram_value value, unsigned tag)
586{
587  gimple *stmt = value->hvalue.stmt;
588  gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
589  tree ref_ptr = tree_coverage_counter_addr (tag, 0);
590  gcall *call;
591  tree val;
592
593  ref_ptr = force_gimple_operand_gsi (&gsi, ref_ptr,
594				      true, NULL_TREE, true, GSI_SAME_STMT);
595  val = prepare_instrumented_value (&gsi, value);
596  call = gimple_build_call (tree_ior_profiler_fn, 2, ref_ptr, val);
597  gsi_insert_before (&gsi, call, GSI_NEW_STMT);
598}
599
600static vec<regex_t> profile_filter_files;
601static vec<regex_t> profile_exclude_files;
602
603/* Parse list of provided REGEX (separated with semi-collon) and
604   create expressions (of type regex_t) and save them into V vector.
605   If there is a regular expression parsing error, error message is
606   printed for FLAG_NAME.  */
607
608static void
609parse_profile_filter (const char *regex, vec<regex_t> *v,
610		      const char *flag_name)
611{
612  v->create (4);
613  if (regex != NULL)
614    {
615      char *str = xstrdup (regex);
616      for (char *p = strtok (str, ";"); p != NULL; p = strtok (NULL, ";"))
617	{
618	  regex_t r;
619	  if (regcomp (&r, p, REG_EXTENDED | REG_NOSUB) != 0)
620	    {
621	      error ("invalid regular expression %qs in %qs",
622		     p, flag_name);
623	      return;
624	    }
625
626	  v->safe_push (r);
627	}
628    }
629}
630
631/* Parse values of -fprofile-filter-files and -fprofile-exclude-files
632   options.  */
633
634static void
635parse_profile_file_filtering ()
636{
637  parse_profile_filter (flag_profile_filter_files, &profile_filter_files,
638			"-fprofile-filter-files");
639  parse_profile_filter (flag_profile_exclude_files, &profile_exclude_files,
640			"-fprofile-exclude-files");
641}
642
643/* Parse vectors of regular expressions.  */
644
645static void
646release_profile_file_filtering ()
647{
648  profile_filter_files.release ();
649  profile_exclude_files.release ();
650}
651
652/* Return true when FILENAME should be instrumented based on
653   -fprofile-filter-files and -fprofile-exclude-files options.  */
654
655static bool
656include_source_file_for_profile (const char *filename)
657{
658  /* First check whether file is included in flag_profile_exclude_files.  */
659  for (unsigned i = 0; i < profile_exclude_files.length (); i++)
660    if (regexec (&profile_exclude_files[i],
661		 filename, 0, NULL, 0) == REG_NOERROR)
662      return false;
663
664  /* For non-empty flag_profile_filter_files include only files matching a
665     regex in the flag.  */
666  if (profile_filter_files.is_empty ())
667    return true;
668
669  for (unsigned i = 0; i < profile_filter_files.length (); i++)
670    if (regexec (&profile_filter_files[i], filename, 0, NULL, 0) == REG_NOERROR)
671      return true;
672
673  return false;
674}
675
676#ifndef HAVE_sync_compare_and_swapsi
677#define HAVE_sync_compare_and_swapsi 0
678#endif
679#ifndef HAVE_atomic_compare_and_swapsi
680#define HAVE_atomic_compare_and_swapsi 0
681#endif
682
683#ifndef HAVE_sync_compare_and_swapdi
684#define HAVE_sync_compare_and_swapdi 0
685#endif
686#ifndef HAVE_atomic_compare_and_swapdi
687#define HAVE_atomic_compare_and_swapdi 0
688#endif
689
690/* Profile all functions in the callgraph.  */
691
692static unsigned int
693tree_profiling (void)
694{
695  struct cgraph_node *node;
696
697  /* Verify whether we can utilize atomic update operations.  */
698  bool can_support_atomic = false;
699  unsigned HOST_WIDE_INT gcov_type_size
700    = tree_to_uhwi (TYPE_SIZE_UNIT (get_gcov_type ()));
701  if (gcov_type_size == 4)
702    can_support_atomic
703      = HAVE_sync_compare_and_swapsi || HAVE_atomic_compare_and_swapsi;
704  else if (gcov_type_size == 8)
705    can_support_atomic
706      = HAVE_sync_compare_and_swapdi || HAVE_atomic_compare_and_swapdi;
707
708  if (flag_profile_update == PROFILE_UPDATE_ATOMIC
709      && !can_support_atomic)
710    {
711      warning (0, "target does not support atomic profile update, "
712	       "single mode is selected");
713      flag_profile_update = PROFILE_UPDATE_SINGLE;
714    }
715  else if (flag_profile_update == PROFILE_UPDATE_PREFER_ATOMIC)
716    flag_profile_update = can_support_atomic
717      ? PROFILE_UPDATE_ATOMIC : PROFILE_UPDATE_SINGLE;
718
719  /* This is a small-ipa pass that gets called only once, from
720     cgraphunit.c:ipa_passes().  */
721  gcc_assert (symtab->state == IPA_SSA);
722
723  init_node_map (true);
724  parse_profile_file_filtering ();
725
726  FOR_EACH_DEFINED_FUNCTION (node)
727    {
728      bool thunk = false;
729      if (!gimple_has_body_p (node->decl) && !node->thunk.thunk_p)
730	continue;
731
732      /* Don't profile functions produced for builtin stuff.  */
733      if (DECL_SOURCE_LOCATION (node->decl) == BUILTINS_LOCATION)
734	continue;
735
736      if (lookup_attribute ("no_profile_instrument_function",
737			    DECL_ATTRIBUTES (node->decl)))
738	continue;
739      /* Do not instrument extern inline functions when testing coverage.
740	 While this is not perfectly consistent (early inlined extern inlines
741	 will get acocunted), testsuite expects that.  */
742      if (DECL_EXTERNAL (node->decl)
743	  && flag_test_coverage)
744	continue;
745
746      const char *file = LOCATION_FILE (DECL_SOURCE_LOCATION (node->decl));
747      if (!include_source_file_for_profile (file))
748	continue;
749
750      if (node->thunk.thunk_p)
751	{
752	  /* We cannot expand variadic thunks to Gimple.  */
753	  if (stdarg_p (TREE_TYPE (node->decl)))
754	    continue;
755	  thunk = true;
756	  /* When generate profile, expand thunk to gimple so it can be
757	     instrumented same way as other functions.  */
758	  if (profile_arc_flag)
759	    node->expand_thunk (false, true);
760	  /* Read cgraph profile but keep function as thunk at profile-use
761	     time.  */
762	  else
763	    {
764	      read_thunk_profile (node);
765	      continue;
766	    }
767	}
768
769      push_cfun (DECL_STRUCT_FUNCTION (node->decl));
770
771      if (dump_file)
772	dump_function_header (dump_file, cfun->decl, dump_flags);
773
774      /* Local pure-const may imply need to fixup the cfg.  */
775      if (gimple_has_body_p (node->decl)
776	  && (execute_fixup_cfg () & TODO_cleanup_cfg))
777	cleanup_tree_cfg ();
778
779      branch_prob (thunk);
780
781      if (! flag_branch_probabilities
782	  && flag_profile_values)
783	gimple_gen_ic_func_profiler ();
784
785      if (flag_branch_probabilities
786	  && !thunk
787	  && flag_profile_values
788	  && flag_value_profile_transformations
789	  && profile_status_for_fn (cfun) == PROFILE_READ)
790	gimple_value_profile_transformations ();
791
792      /* The above could hose dominator info.  Currently there is
793	 none coming in, this is a safety valve.  It should be
794	 easy to adjust it, if and when there is some.  */
795      free_dominance_info (CDI_DOMINATORS);
796      free_dominance_info (CDI_POST_DOMINATORS);
797      pop_cfun ();
798    }
799
800  release_profile_file_filtering ();
801
802  /* Drop pure/const flags from instrumented functions.  */
803  if (profile_arc_flag || flag_test_coverage)
804    FOR_EACH_DEFINED_FUNCTION (node)
805      {
806	if (!gimple_has_body_p (node->decl)
807	    || !(!node->clone_of
808	    || node->decl != node->clone_of->decl))
809	  continue;
810
811	/* Don't profile functions produced for builtin stuff.  */
812	if (DECL_SOURCE_LOCATION (node->decl) == BUILTINS_LOCATION)
813	  continue;
814
815	node->set_const_flag (false, false);
816	node->set_pure_flag (false, false);
817      }
818
819  /* Update call statements and rebuild the cgraph.  */
820  FOR_EACH_DEFINED_FUNCTION (node)
821    {
822      basic_block bb;
823
824      if (!gimple_has_body_p (node->decl)
825	  || !(!node->clone_of
826	  || node->decl != node->clone_of->decl))
827	continue;
828
829      /* Don't profile functions produced for builtin stuff.  */
830      if (DECL_SOURCE_LOCATION (node->decl) == BUILTINS_LOCATION)
831	continue;
832
833      push_cfun (DECL_STRUCT_FUNCTION (node->decl));
834
835      FOR_EACH_BB_FN (bb, cfun)
836	{
837	  gimple_stmt_iterator gsi;
838	  for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi))
839	    {
840	      gimple *stmt = gsi_stmt (gsi);
841	      if (is_gimple_call (stmt))
842		update_stmt (stmt);
843	    }
844	}
845
846      /* re-merge split blocks.  */
847      cleanup_tree_cfg ();
848      update_ssa (TODO_update_ssa);
849
850      cgraph_edge::rebuild_edges ();
851
852      pop_cfun ();
853    }
854
855  handle_missing_profiles ();
856
857  del_node_map ();
858  return 0;
859}
860
861namespace {
862
863const pass_data pass_data_ipa_tree_profile =
864{
865  SIMPLE_IPA_PASS, /* type */
866  "profile", /* name */
867  OPTGROUP_NONE, /* optinfo_flags */
868  TV_IPA_PROFILE, /* tv_id */
869  0, /* properties_required */
870  0, /* properties_provided */
871  0, /* properties_destroyed */
872  0, /* todo_flags_start */
873  TODO_dump_symtab, /* todo_flags_finish */
874};
875
876class pass_ipa_tree_profile : public simple_ipa_opt_pass
877{
878public:
879  pass_ipa_tree_profile (gcc::context *ctxt)
880    : simple_ipa_opt_pass (pass_data_ipa_tree_profile, ctxt)
881  {}
882
883  /* opt_pass methods: */
884  virtual bool gate (function *);
885  virtual unsigned int execute (function *) { return tree_profiling (); }
886
887}; // class pass_ipa_tree_profile
888
889bool
890pass_ipa_tree_profile::gate (function *)
891{
892  /* When profile instrumentation, use or test coverage shall be performed.
893     But for AutoFDO, this there is no instrumentation, thus this pass is
894     disabled.  */
895  return (!in_lto_p && !flag_auto_profile
896	  && (flag_branch_probabilities || flag_test_coverage
897	      || profile_arc_flag));
898}
899
900} // anon namespace
901
902simple_ipa_opt_pass *
903make_pass_ipa_tree_profile (gcc::context *ctxt)
904{
905  return new pass_ipa_tree_profile (ctxt);
906}
907
908#include "gt-tree-profile.h"
909