Lines Matching refs:thr_bar
51 kmp_bstate_t *thr_bar = &this_thr->th.th_bar[bt].bb;
74 team->t.t_id, 0, &thr_bar->b_arrived, thr_bar->b_arrived,
75 thr_bar->b_arrived + KMP_BARRIER_STATE_BUMP));
81 kmp_flag_64 flag(&thr_bar->b_arrived, other_threads[0]);
157 kmp_bstate_t *thr_bar = &this_thr->th.th_bar[bt].bb;
213 gtid, &thr_bar->b_go, KMP_BARRIER_STATE_BUMP));
214 kmp_flag_64 flag(&thr_bar->b_go, KMP_BARRIER_STATE_BUMP);
251 TCW_4(thr_bar->b_go, KMP_INIT_BARRIER_STATE);
254 gtid, team->t.t_id, tid, &thr_bar->b_go, KMP_INIT_BARRIER_STATE));
299 kmp_bstate_t *thr_bar = &this_thr->th.th_bar[bt].bb;
379 team->t.t_id, parent_tid, &thr_bar->b_arrived, thr_bar->b_arrived,
380 thr_bar->b_arrived + KMP_BARRIER_STATE_BUMP));
387 kmp_flag_64 flag(&thr_bar->b_arrived, other_threads[parent_tid]);
410 kmp_bstate_t *thr_bar = &this_thr->th.th_bar[bt].bb;
421 &thr_bar->b_go, KMP_BARRIER_STATE_BUMP));
423 kmp_flag_64 flag(&thr_bar->b_go, KMP_BARRIER_STATE_BUMP);
452 TCW_4(thr_bar->b_go, KMP_INIT_BARRIER_STATE);
455 team->t.t_id, tid, &thr_bar->b_go, KMP_INIT_BARRIER_STATE));
519 kmp_bstate_t *thr_bar = &this_thr->th.th_bar[bt].bb;
543 kmp_flag_64 p_flag(&thr_bar->b_arrived);
556 team->t.t_id, parent_tid, &thr_bar->b_arrived,
557 thr_bar->b_arrived,
558 thr_bar->b_arrived + KMP_BARRIER_STATE_BUMP));
641 kmp_bstate_t *thr_bar = &this_thr->th.th_bar[bt].bb;
663 copy_icvs(&thr_bar->th_fixed_icvs,
669 &thr_bar->b_go, KMP_BARRIER_STATE_BUMP));
671 kmp_flag_64 flag(&thr_bar->b_go, KMP_BARRIER_STATE_BUMP);
699 TCW_4(thr_bar->b_go, KMP_INIT_BARRIER_STATE);
702 gtid, team->t.t_id, tid, &thr_bar->b_go, KMP_INIT_BARRIER_STATE));
761 copy_icvs(&child_bar->th_fixed_icvs, &thr_bar->th_fixed_icvs);
784 &thr_bar->th_fixed_icvs);
804 kmp_bstate_t *thr_bar,
808 bool uninitialized = thr_bar->team == NULL;
809 bool team_changed = team != thr_bar->team;
810 bool team_sz_changed = nproc != thr_bar->nproc;
811 bool tid_changed = tid != thr_bar->old_tid;
815 __kmp_get_hierarchy(nproc, thr_bar);
819 thr_bar->my_level = thr_bar->depth - 1; // default for master
820 thr_bar->parent_tid = -1; // default for master
824 while (d < thr_bar->depth) { // find parent based on level of thread in
827 if (d == thr_bar->depth - 2) { // reached level right below the master
828 thr_bar->parent_tid = 0;
829 thr_bar->my_level = d;
831 } else if ((rem = tid % thr_bar->skip_per_level[d + 1]) !=
834 thr_bar->parent_tid = tid - rem;
835 thr_bar->my_level = d;
841 thr_bar->offset = 7 - (tid - thr_bar->parent_tid - 1);
842 thr_bar->old_tid = tid;
843 thr_bar->wait_flag = KMP_BARRIER_NOT_WAITING;
844 thr_bar->team = team;
845 thr_bar->parent_bar =
846 &team->t.t_threads[thr_bar->parent_tid]->th.th_bar[bt].bb;
849 thr_bar->team = team;
850 thr_bar->parent_bar =
851 &team->t.t_threads[thr_bar->parent_tid]->th.th_bar[bt].bb;
855 thr_bar->nproc = nproc;
856 thr_bar->leaf_kids = thr_bar->base_leaf_kids;
857 if (thr_bar->my_level == 0)
858 thr_bar->leaf_kids = 0;
859 if (thr_bar->leaf_kids && (kmp_uint32)tid + thr_bar->leaf_kids + 1 > nproc)
860 thr_bar->leaf_kids = nproc - tid - 1;
861 thr_bar->leaf_state = 0;
862 for (int i = 0; i < thr_bar->leaf_kids; ++i)
863 ((char *)&(thr_bar->leaf_state))[7 - i] = 1;
873 kmp_bstate_t *thr_bar = &this_thr->th.th_bar[bt].bb;
884 thr_bar->use_oncore_barrier = 1;
886 thr_bar->use_oncore_barrier = 0; // Do not use oncore barrier when nested
900 (void)__kmp_init_hierarchical_barrier_thread(bt, thr_bar, nproc, gtid, tid,
903 if (thr_bar->my_level) { // not a leaf (my_level==0 means leaf)
908 thr_bar->use_oncore_barrier) {
909 if (thr_bar->leaf_kids) {
913 ? thr_bar->b_arrived | thr_bar->leaf_state
914 : team->t.t_bar[bt].b_arrived | thr_bar->leaf_state;
918 kmp_flag_64 flag(&thr_bar->b_arrived, leaf_state);
924 for (child_tid = tid + 1; child_tid <= tid + thr_bar->leaf_kids;
940 KMP_TEST_THEN_AND64(&thr_bar->b_arrived, ~(thr_bar->leaf_state));
943 for (kmp_uint32 d = 1; d < thr_bar->my_level;
945 kmp_uint32 last = tid + thr_bar->skip_per_level[d + 1],
946 skip = thr_bar->skip_per_level[d];
976 for (kmp_uint32 d = 0; d < thr_bar->my_level;
978 kmp_uint32 last = tid + thr_bar->skip_per_level[d + 1],
979 skip = thr_bar->skip_per_level[d];
1016 __kmp_gtid_from_tid(thr_bar->parent_tid, team), team->t.t_id,
1017 thr_bar->parent_tid, &thr_bar->b_arrived, thr_bar->b_arrived,
1018 thr_bar->b_arrived + KMP_BARRIER_STATE_BUMP));
1022 if (thr_bar->my_level || __kmp_dflt_blocktime != KMP_MAX_BLOCKTIME ||
1023 !thr_bar->use_oncore_barrier) { // Parent is waiting on my b_arrived
1026 kmp_flag_64 flag(&thr_bar->b_arrived, other_threads[thr_bar->parent_tid]);
1030 thr_bar->b_arrived = team->t.t_bar[bt].b_arrived + KMP_BARRIER_STATE_BUMP;
1031 kmp_flag_oncore flag(&thr_bar->parent_bar->b_arrived, thr_bar->offset);
1032 flag.set_waiter(other_threads[thr_bar->parent_tid]);
1053 kmp_bstate_t *thr_bar = &this_thr->th.th_bar[bt].bb;
1065 if (!thr_bar->use_oncore_barrier ||
1066 __kmp_dflt_blocktime != KMP_MAX_BLOCKTIME || thr_bar->my_level != 0 ||
1067 thr_bar->team == NULL) {
1069 thr_bar->wait_flag = KMP_BARRIER_OWN_FLAG;
1070 kmp_flag_64 flag(&thr_bar->b_go, KMP_BARRIER_STATE_BUMP);
1073 TCW_8(thr_bar->b_go,
1078 thr_bar->wait_flag = KMP_BARRIER_PARENT_FLAG;
1079 kmp_flag_oncore flag(&thr_bar->parent_bar->b_go, KMP_BARRIER_STATE_BUMP,
1080 thr_bar->offset, bt,
1083 if (thr_bar->wait_flag ==
1085 TCW_8(thr_bar->b_go,
1089 &(thr_bar->parent_bar->b_go)))[thr_bar->offset] = 0;
1092 thr_bar->wait_flag = KMP_BARRIER_NOT_WAITING;
1104 gtid, team->t.t_id, tid, &thr_bar->b_go, KMP_INIT_BARRIER_STATE));
1119 thr_bar->use_oncore_barrier = 1;
1121 thr_bar->use_oncore_barrier = 0; // Do not use oncore barrier when nested
1125 unsigned short int old_leaf_kids = thr_bar->leaf_kids;
1126 kmp_uint64 old_leaf_state = thr_bar->leaf_state;
1127 team_change = __kmp_init_hierarchical_barrier_thread(bt, thr_bar, nproc, gtid,
1139 copy_icvs(&thr_bar->th_fixed_icvs,
1142 thr_bar->use_oncore_barrier) { // optimization for inf blocktime
1143 if (!thr_bar->my_level) // I'm a leaf in the hierarchy (my_level==0)
1147 &thr_bar->parent_bar->th_fixed_icvs);
1150 if (thr_bar->my_level) // not a leaf; copy ICVs to my fixed ICVs child can
1152 copy_icvs(&thr_bar->th_fixed_icvs, &thr_bar->parent_bar->th_fixed_icvs);
1155 &thr_bar->parent_bar->th_fixed_icvs);
1161 if (thr_bar->my_level) { // not a leaf
1165 thr_bar->use_oncore_barrier) {
1169 thr_bar->b_go = KMP_BARRIER_STATE_BUMP;
1172 ngo_load(&thr_bar->th_fixed_icvs);
1175 for (child_tid = thr_bar->skip_per_level[1]; child_tid < (int)nproc;
1176 child_tid += thr_bar->skip_per_level[1]) {
1188 ngo_store_go(&child_bar->th_fixed_icvs, &thr_bar->th_fixed_icvs);
1192 TCW_8(thr_bar->b_go,
1195 if (thr_bar->leaf_kids) { // if there are any
1198 old_leaf_kids < thr_bar->leaf_kids) { // some old, some new
1200 thr_bar->b_go |= old_leaf_state;
1203 last = tid + thr_bar->skip_per_level[1];
1224 thr_bar->b_go |= thr_bar->leaf_state;
1228 for (int d = thr_bar->my_level - 1; d >= 0;
1230 last = tid + thr_bar->skip_per_level[d + 1];
1231 kmp_uint32 skip = thr_bar->skip_per_level[d];
1254 &thr_bar->th_fixed_icvs);