1/* Copyright (C) 2005-2020 Free Software Foundation, Inc.
2   Contributed by Richard Henderson <rth@redhat.com>.
3
4   This file is part of the GNU Offloading and Multi Processing Library
5   (libgomp).
6
7   Libgomp is free software; you can redistribute it and/or modify it
8   under the terms of the GNU General Public License as published by
9   the Free Software Foundation; either version 3, or (at your option)
10   any later version.
11
12   Libgomp is distributed in the hope that it will be useful, but WITHOUT ANY
13   WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
14   FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
15   more details.
16
17   Under Section 7 of GPL version 3, you are granted additional
18   permissions described in the GCC Runtime Library Exception, version
19   3.1, as published by the Free Software Foundation.
20
21   You should have received a copy of the GNU General Public License and
22   a copy of the GCC Runtime Library Exception along with this program;
23   see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
24   <http://www.gnu.org/licenses/>.  */
25
26/* This is a Linux specific implementation of a barrier synchronization
27   mechanism for libgomp.  This type is private to the library.  This
28   implementation uses atomic instructions and the futex syscall.  */
29
30#include <limits.h>
31#include "wait.h"
32
33
34void
35gomp_barrier_wait_end (gomp_barrier_t *bar, gomp_barrier_state_t state)
36{
37  if (__builtin_expect (state & BAR_WAS_LAST, 0))
38    {
39      /* Next time we'll be awaiting TOTAL threads again.  */
40      bar->awaited = bar->total;
41      __atomic_store_n (&bar->generation, bar->generation + BAR_INCR,
42			MEMMODEL_RELEASE);
43      futex_wake ((int *) &bar->generation, INT_MAX);
44    }
45  else
46    {
47      do
48	do_wait ((int *) &bar->generation, state);
49      while (__atomic_load_n (&bar->generation, MEMMODEL_ACQUIRE) == state);
50    }
51}
52
53void
54gomp_barrier_wait (gomp_barrier_t *bar)
55{
56  gomp_barrier_wait_end (bar, gomp_barrier_wait_start (bar));
57}
58
59/* Like gomp_barrier_wait, except that if the encountering thread
60   is not the last one to hit the barrier, it returns immediately.
61   The intended usage is that a thread which intends to gomp_barrier_destroy
62   this barrier calls gomp_barrier_wait, while all other threads
63   call gomp_barrier_wait_last.  When gomp_barrier_wait returns,
64   the barrier can be safely destroyed.  */
65
66void
67gomp_barrier_wait_last (gomp_barrier_t *bar)
68{
69  gomp_barrier_state_t state = gomp_barrier_wait_start (bar);
70  if (state & BAR_WAS_LAST)
71    gomp_barrier_wait_end (bar, state);
72}
73
74void
75gomp_team_barrier_wake (gomp_barrier_t *bar, int count)
76{
77  futex_wake ((int *) &bar->generation, count == 0 ? INT_MAX : count);
78}
79
80void
81gomp_team_barrier_wait_end (gomp_barrier_t *bar, gomp_barrier_state_t state)
82{
83  unsigned int generation, gen;
84
85  if (__builtin_expect (state & BAR_WAS_LAST, 0))
86    {
87      /* Next time we'll be awaiting TOTAL threads again.  */
88      struct gomp_thread *thr = gomp_thread ();
89      struct gomp_team *team = thr->ts.team;
90
91      bar->awaited = bar->total;
92      team->work_share_cancelled = 0;
93      if (__builtin_expect (team->task_count, 0))
94	{
95	  gomp_barrier_handle_tasks (state);
96	  state &= ~BAR_WAS_LAST;
97	}
98      else
99	{
100	  state &= ~BAR_CANCELLED;
101	  state += BAR_INCR - BAR_WAS_LAST;
102	  __atomic_store_n (&bar->generation, state, MEMMODEL_RELEASE);
103	  futex_wake ((int *) &bar->generation, INT_MAX);
104	  return;
105	}
106    }
107
108  generation = state;
109  state &= ~BAR_CANCELLED;
110  do
111    {
112      do_wait ((int *) &bar->generation, generation);
113      gen = __atomic_load_n (&bar->generation, MEMMODEL_ACQUIRE);
114      if (__builtin_expect (gen & BAR_TASK_PENDING, 0))
115	{
116	  gomp_barrier_handle_tasks (state);
117	  gen = __atomic_load_n (&bar->generation, MEMMODEL_ACQUIRE);
118	}
119      generation |= gen & BAR_WAITING_FOR_TASK;
120    }
121  while (gen != state + BAR_INCR);
122}
123
124void
125gomp_team_barrier_wait (gomp_barrier_t *bar)
126{
127  gomp_team_barrier_wait_end (bar, gomp_barrier_wait_start (bar));
128}
129
130void
131gomp_team_barrier_wait_final (gomp_barrier_t *bar)
132{
133  gomp_barrier_state_t state = gomp_barrier_wait_final_start (bar);
134  if (__builtin_expect (state & BAR_WAS_LAST, 0))
135    bar->awaited_final = bar->total;
136  gomp_team_barrier_wait_end (bar, state);
137}
138
139bool
140gomp_team_barrier_wait_cancel_end (gomp_barrier_t *bar,
141				   gomp_barrier_state_t state)
142{
143  unsigned int generation, gen;
144
145  if (__builtin_expect (state & BAR_WAS_LAST, 0))
146    {
147      /* Next time we'll be awaiting TOTAL threads again.  */
148      /* BAR_CANCELLED should never be set in state here, because
149	 cancellation means that at least one of the threads has been
150	 cancelled, thus on a cancellable barrier we should never see
151	 all threads to arrive.  */
152      struct gomp_thread *thr = gomp_thread ();
153      struct gomp_team *team = thr->ts.team;
154
155      bar->awaited = bar->total;
156      team->work_share_cancelled = 0;
157      if (__builtin_expect (team->task_count, 0))
158	{
159	  gomp_barrier_handle_tasks (state);
160	  state &= ~BAR_WAS_LAST;
161	}
162      else
163	{
164	  state += BAR_INCR - BAR_WAS_LAST;
165	  __atomic_store_n (&bar->generation, state, MEMMODEL_RELEASE);
166	  futex_wake ((int *) &bar->generation, INT_MAX);
167	  return false;
168	}
169    }
170
171  if (__builtin_expect (state & BAR_CANCELLED, 0))
172    return true;
173
174  generation = state;
175  do
176    {
177      do_wait ((int *) &bar->generation, generation);
178      gen = __atomic_load_n (&bar->generation, MEMMODEL_ACQUIRE);
179      if (__builtin_expect (gen & BAR_CANCELLED, 0))
180	return true;
181      if (__builtin_expect (gen & BAR_TASK_PENDING, 0))
182	{
183	  gomp_barrier_handle_tasks (state);
184	  gen = __atomic_load_n (&bar->generation, MEMMODEL_ACQUIRE);
185	}
186      generation |= gen & BAR_WAITING_FOR_TASK;
187    }
188  while (gen != state + BAR_INCR);
189
190  return false;
191}
192
193bool
194gomp_team_barrier_wait_cancel (gomp_barrier_t *bar)
195{
196  return gomp_team_barrier_wait_cancel_end (bar, gomp_barrier_wait_start (bar));
197}
198
199void
200gomp_team_barrier_cancel (struct gomp_team *team)
201{
202  gomp_mutex_lock (&team->task_lock);
203  if (team->barrier.generation & BAR_CANCELLED)
204    {
205      gomp_mutex_unlock (&team->task_lock);
206      return;
207    }
208  team->barrier.generation |= BAR_CANCELLED;
209  gomp_mutex_unlock (&team->task_lock);
210  futex_wake ((int *) &team->barrier.generation, INT_MAX);
211}
212