bar.c revision 1.10
12224Sctornqvi/* Copyright (C) 2005-2019 Free Software Foundation, Inc. 22553Siignatyev Contributed by Richard Henderson <rth@redhat.com>. 32224Sctornqvi 42224Sctornqvi This file is part of the GNU Offloading and Multi Processing Library 52224Sctornqvi (libgomp). 62224Sctornqvi 72224Sctornqvi Libgomp is free software; you can redistribute it and/or modify it 82224Sctornqvi under the terms of the GNU General Public License as published by 92224Sctornqvi the Free Software Foundation; either version 3, or (at your option) 102224Sctornqvi any later version. 112224Sctornqvi 122224Sctornqvi Libgomp is distributed in the hope that it will be useful, but WITHOUT ANY 132224Sctornqvi WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 142224Sctornqvi FOR A PARTICULAR PURPOSE. See the GNU General Public License for 152224Sctornqvi more details. 162224Sctornqvi 172224Sctornqvi Under Section 7 of GPL version 3, you are granted additional 182224Sctornqvi permissions described in the GCC Runtime Library Exception, version 192224Sctornqvi 3.1, as published by the Free Software Foundation. 202224Sctornqvi 212224Sctornqvi You should have received a copy of the GNU General Public License and 222224Sctornqvi a copy of the GCC Runtime Library Exception along with this program; 232224Sctornqvi see the files COPYING3 and COPYING.RUNTIME respectively. If not, see 242224Sctornqvi <http://www.gnu.org/licenses/>. */ 252224Sctornqvi 262224Sctornqvi/* This is a Linux specific implementation of a barrier synchronization 272224Sctornqvi mechanism for libgomp. This type is private to the library. This 282224Sctornqvi implementation uses atomic instructions and the futex syscall. */ 292224Sctornqvi 302224Sctornqvi#include <limits.h> 312553Siignatyev#include "wait.h" 322224Sctornqvi 332224Sctornqvi 342224Sctornqvivoid 352224Sctornqvigomp_barrier_wait_end (gomp_barrier_t *bar, gomp_barrier_state_t state) 362224Sctornqvi{ 372224Sctornqvi if (__builtin_expect (state & BAR_WAS_LAST, 0)) 382224Sctornqvi { 392224Sctornqvi /* Next time we'll be awaiting TOTAL threads again. */ 402224Sctornqvi bar->awaited = bar->total; 412224Sctornqvi __atomic_store_n (&bar->generation, bar->generation + BAR_INCR, 422224Sctornqvi MEMMODEL_RELEASE); 432224Sctornqvi futex_wake ((int *) &bar->generation, INT_MAX); 442224Sctornqvi } 452224Sctornqvi else 462224Sctornqvi { 472224Sctornqvi do 482224Sctornqvi do_wait ((int *) &bar->generation, state); 492224Sctornqvi while (__atomic_load_n (&bar->generation, MEMMODEL_ACQUIRE) == state); 502224Sctornqvi } 512224Sctornqvi} 522224Sctornqvi 532224Sctornqvivoid 542224Sctornqvigomp_barrier_wait (gomp_barrier_t *bar) 552224Sctornqvi{ 562224Sctornqvi gomp_barrier_wait_end (bar, gomp_barrier_wait_start (bar)); 572224Sctornqvi} 582224Sctornqvi 592224Sctornqvi/* Like gomp_barrier_wait, except that if the encountering thread 602224Sctornqvi is not the last one to hit the barrier, it returns immediately. 612224Sctornqvi The intended usage is that a thread which intends to gomp_barrier_destroy 622224Sctornqvi this barrier calls gomp_barrier_wait, while all other threads 632224Sctornqvi call gomp_barrier_wait_last. When gomp_barrier_wait returns, 642224Sctornqvi the barrier can be safely destroyed. */ 652224Sctornqvi 662224Sctornqvivoid 672224Sctornqvigomp_barrier_wait_last (gomp_barrier_t *bar) 682224Sctornqvi{ 692224Sctornqvi gomp_barrier_state_t state = gomp_barrier_wait_start (bar); 702224Sctornqvi if (state & BAR_WAS_LAST) 712224Sctornqvi gomp_barrier_wait_end (bar, state); 722224Sctornqvi} 732224Sctornqvi 742224Sctornqvivoid 752224Sctornqvigomp_team_barrier_wake (gomp_barrier_t *bar, int count) 762224Sctornqvi{ 772224Sctornqvi futex_wake ((int *) &bar->generation, count == 0 ? INT_MAX : count); 782224Sctornqvi} 792224Sctornqvi 802224Sctornqvivoid 812224Sctornqvigomp_team_barrier_wait_end (gomp_barrier_t *bar, gomp_barrier_state_t state) 822224Sctornqvi{ 832224Sctornqvi unsigned int generation, gen; 842224Sctornqvi 852224Sctornqvi if (__builtin_expect (state & BAR_WAS_LAST, 0)) 862224Sctornqvi { 872224Sctornqvi /* Next time we'll be awaiting TOTAL threads again. */ 882224Sctornqvi struct gomp_thread *thr = gomp_thread (); 892224Sctornqvi struct gomp_team *team = thr->ts.team; 902224Sctornqvi 912224Sctornqvi bar->awaited = bar->total; 922224Sctornqvi team->work_share_cancelled = 0; 932224Sctornqvi if (__builtin_expect (team->task_count, 0)) 942224Sctornqvi { 952224Sctornqvi gomp_barrier_handle_tasks (state); 962224Sctornqvi state &= ~BAR_WAS_LAST; 972224Sctornqvi } 982224Sctornqvi else 992224Sctornqvi { 1002224Sctornqvi state &= ~BAR_CANCELLED; 1012224Sctornqvi state += BAR_INCR - BAR_WAS_LAST; 1022224Sctornqvi __atomic_store_n (&bar->generation, state, MEMMODEL_RELEASE); 1032224Sctornqvi futex_wake ((int *) &bar->generation, INT_MAX); 1042224Sctornqvi return; 1052224Sctornqvi } 1062224Sctornqvi } 1072224Sctornqvi 1082224Sctornqvi generation = state; 1092553Siignatyev state &= ~BAR_CANCELLED; 1102224Sctornqvi do 1112224Sctornqvi { 1122224Sctornqvi do_wait ((int *) &bar->generation, generation); 1132224Sctornqvi gen = __atomic_load_n (&bar->generation, MEMMODEL_ACQUIRE); 1142224Sctornqvi if (__builtin_expect (gen & BAR_TASK_PENDING, 0)) 1152224Sctornqvi { 1162224Sctornqvi gomp_barrier_handle_tasks (state); 1172224Sctornqvi gen = __atomic_load_n (&bar->generation, MEMMODEL_ACQUIRE); 1182224Sctornqvi } 1192224Sctornqvi generation |= gen & BAR_WAITING_FOR_TASK; 1202224Sctornqvi } 1212224Sctornqvi while (gen != state + BAR_INCR); 1222224Sctornqvi} 1232224Sctornqvi 1242224Sctornqvivoid 1252224Sctornqvigomp_team_barrier_wait (gomp_barrier_t *bar) 1262224Sctornqvi{ 1272224Sctornqvi gomp_team_barrier_wait_end (bar, gomp_barrier_wait_start (bar)); 1282224Sctornqvi} 1292224Sctornqvi 1302224Sctornqvivoid 1312224Sctornqvigomp_team_barrier_wait_final (gomp_barrier_t *bar) 1322224Sctornqvi{ 1332224Sctornqvi gomp_barrier_state_t state = gomp_barrier_wait_final_start (bar); 1342224Sctornqvi if (__builtin_expect (state & BAR_WAS_LAST, 0)) 1352224Sctornqvi bar->awaited_final = bar->total; 1362224Sctornqvi gomp_team_barrier_wait_end (bar, state); 1372224Sctornqvi} 1382224Sctornqvi 1392224Sctornqvibool 1402224Sctornqvigomp_team_barrier_wait_cancel_end (gomp_barrier_t *bar, 1412224Sctornqvi gomp_barrier_state_t state) 1422224Sctornqvi{ 1432224Sctornqvi unsigned int generation, gen; 1442224Sctornqvi 1452224Sctornqvi if (__builtin_expect (state & BAR_WAS_LAST, 0)) 1462224Sctornqvi { 1472224Sctornqvi /* Next time we'll be awaiting TOTAL threads again. */ 1482224Sctornqvi /* BAR_CANCELLED should never be set in state here, because 1492224Sctornqvi cancellation means that at least one of the threads has been 1502224Sctornqvi cancelled, thus on a cancellable barrier we should never see 1512224Sctornqvi all threads to arrive. */ 1522224Sctornqvi struct gomp_thread *thr = gomp_thread (); 1532224Sctornqvi struct gomp_team *team = thr->ts.team; 1542224Sctornqvi 1552224Sctornqvi bar->awaited = bar->total; 1562224Sctornqvi team->work_share_cancelled = 0; 1572224Sctornqvi if (__builtin_expect (team->task_count, 0)) 1582224Sctornqvi { 1592224Sctornqvi gomp_barrier_handle_tasks (state); 1602224Sctornqvi state &= ~BAR_WAS_LAST; 1612224Sctornqvi } 1622224Sctornqvi else 1632224Sctornqvi { 1642224Sctornqvi state += BAR_INCR - BAR_WAS_LAST; 1652224Sctornqvi __atomic_store_n (&bar->generation, state, MEMMODEL_RELEASE); 1662224Sctornqvi futex_wake ((int *) &bar->generation, INT_MAX); 1672224Sctornqvi return false; 1682224Sctornqvi } 1692224Sctornqvi } 1702224Sctornqvi 1712224Sctornqvi if (__builtin_expect (state & BAR_CANCELLED, 0)) 1722224Sctornqvi return true; 1732224Sctornqvi 1742224Sctornqvi generation = state; 1752224Sctornqvi do 1762224Sctornqvi { 1772224Sctornqvi do_wait ((int *) &bar->generation, generation); 1782224Sctornqvi gen = __atomic_load_n (&bar->generation, MEMMODEL_ACQUIRE); 1792224Sctornqvi if (__builtin_expect (gen & BAR_CANCELLED, 0)) 1802224Sctornqvi return true; 1812224Sctornqvi if (__builtin_expect (gen & BAR_TASK_PENDING, 0)) 1822224Sctornqvi { 1832224Sctornqvi gomp_barrier_handle_tasks (state); 1842224Sctornqvi gen = __atomic_load_n (&bar->generation, MEMMODEL_ACQUIRE); 1852224Sctornqvi } 1862224Sctornqvi generation |= gen & BAR_WAITING_FOR_TASK; 1872224Sctornqvi } 1882224Sctornqvi while (gen != state + BAR_INCR); 1892224Sctornqvi 1902224Sctornqvi return false; 1912224Sctornqvi} 1922224Sctornqvi 1932224Sctornqvibool 1942224Sctornqvigomp_team_barrier_wait_cancel (gomp_barrier_t *bar) 1952224Sctornqvi{ 1962224Sctornqvi return gomp_team_barrier_wait_cancel_end (bar, gomp_barrier_wait_start (bar)); 1972224Sctornqvi} 1982224Sctornqvi 1992224Sctornqvivoid 2002224Sctornqvigomp_team_barrier_cancel (struct gomp_team *team) 2012224Sctornqvi{ 2022224Sctornqvi gomp_mutex_lock (&team->task_lock); 2032508Sjcm if (team->barrier.generation & BAR_CANCELLED) 2042508Sjcm { 2052508Sjcm gomp_mutex_unlock (&team->task_lock); 2062508Sjcm return; 2072224Sctornqvi } 2082224Sctornqvi team->barrier.generation |= BAR_CANCELLED; 2092224Sctornqvi gomp_mutex_unlock (&team->task_lock); 2102224Sctornqvi futex_wake ((int *) &team->barrier.generation, INT_MAX); 2112224Sctornqvi} 2122224Sctornqvi