Lines Matching defs:thr_bar
51 kmp_bstate_t *thr_bar = &this_thr->th.th_bar[bt].bb;
74 team->t.t_id, 0, &thr_bar->b_arrived, thr_bar->b_arrived,
75 thr_bar->b_arrived + KMP_BARRIER_STATE_BUMP));
81 kmp_flag_64 flag(&thr_bar->b_arrived, other_threads[0]);
157 kmp_bstate_t *thr_bar = &this_thr->th.th_bar[bt].bb;
213 gtid, &thr_bar->b_go, KMP_BARRIER_STATE_BUMP));
214 kmp_flag_64 flag(&thr_bar->b_go, KMP_BARRIER_STATE_BUMP);
251 TCW_4(thr_bar->b_go, KMP_INIT_BARRIER_STATE);
254 gtid, team->t.t_id, tid, &thr_bar->b_go, KMP_INIT_BARRIER_STATE));
299 kmp_bstate_t *thr_bar = &this_thr->th.th_bar[bt].bb;
379 team->t.t_id, parent_tid, &thr_bar->b_arrived, thr_bar->b_arrived,
380 thr_bar->b_arrived + KMP_BARRIER_STATE_BUMP));
387 kmp_flag_64 flag(&thr_bar->b_arrived, other_threads[parent_tid]);
410 kmp_bstate_t *thr_bar = &this_thr->th.th_bar[bt].bb;
421 &thr_bar->b_go, KMP_BARRIER_STATE_BUMP));
423 kmp_flag_64 flag(&thr_bar->b_go, KMP_BARRIER_STATE_BUMP);
452 TCW_4(thr_bar->b_go, KMP_INIT_BARRIER_STATE);
455 team->t.t_id, tid, &thr_bar->b_go, KMP_INIT_BARRIER_STATE));
519 kmp_bstate_t *thr_bar = &this_thr->th.th_bar[bt].bb;
543 kmp_flag_64 p_flag(&thr_bar->b_arrived);
557 team->t.t_id, parent_tid, &thr_bar->b_arrived,
558 thr_bar->b_arrived,
559 thr_bar->b_arrived + KMP_BARRIER_STATE_BUMP));
643 kmp_bstate_t *thr_bar = &this_thr->th.th_bar[bt].bb;
665 copy_icvs(&thr_bar->th_fixed_icvs,
671 &thr_bar->b_go, KMP_BARRIER_STATE_BUMP));
673 kmp_flag_64 flag(&thr_bar->b_go, KMP_BARRIER_STATE_BUMP);
701 TCW_4(thr_bar->b_go, KMP_INIT_BARRIER_STATE);
704 gtid, team->t.t_id, tid, &thr_bar->b_go, KMP_INIT_BARRIER_STATE));
763 copy_icvs(&child_bar->th_fixed_icvs, &thr_bar->th_fixed_icvs);
786 &thr_bar->th_fixed_icvs);
806 kmp_bstate_t *thr_bar,
810 bool uninitialized = thr_bar->team == NULL;
811 bool team_changed = team != thr_bar->team;
812 bool team_sz_changed = nproc != thr_bar->nproc;
813 bool tid_changed = tid != thr_bar->old_tid;
817 __kmp_get_hierarchy(nproc, thr_bar);
821 thr_bar->my_level = thr_bar->depth - 1; // default for master
822 thr_bar->parent_tid = -1; // default for master
826 while (d < thr_bar->depth) { // find parent based on level of thread in
829 if (d == thr_bar->depth - 2) { // reached level right below the master
830 thr_bar->parent_tid = 0;
831 thr_bar->my_level = d;
833 } else if ((rem = tid % thr_bar->skip_per_level[d + 1]) !=
836 thr_bar->parent_tid = tid - rem;
837 thr_bar->my_level = d;
843 thr_bar->offset = 7 - (tid - thr_bar->parent_tid - 1);
844 thr_bar->old_tid = tid;
845 thr_bar->wait_flag = KMP_BARRIER_NOT_WAITING;
846 thr_bar->team = team;
847 thr_bar->parent_bar =
848 &team->t.t_threads[thr_bar->parent_tid]->th.th_bar[bt].bb;
851 thr_bar->team = team;
852 thr_bar->parent_bar =
853 &team->t.t_threads[thr_bar->parent_tid]->th.th_bar[bt].bb;
857 thr_bar->nproc = nproc;
858 thr_bar->leaf_kids = thr_bar->base_leaf_kids;
859 if (thr_bar->my_level == 0)
860 thr_bar->leaf_kids = 0;
861 if (thr_bar->leaf_kids && (kmp_uint32)tid + thr_bar->leaf_kids + 1 > nproc)
862 thr_bar->leaf_kids = nproc - tid - 1;
863 thr_bar->leaf_state = 0;
864 for (int i = 0; i < thr_bar->leaf_kids; ++i)
865 ((char *)&(thr_bar->leaf_state))[7 - i] = 1;
875 kmp_bstate_t *thr_bar = &this_thr->th.th_bar[bt].bb;
886 thr_bar->use_oncore_barrier = 1;
888 thr_bar->use_oncore_barrier = 0; // Do not use oncore barrier when nested
902 (void)__kmp_init_hierarchical_barrier_thread(bt, thr_bar, nproc, gtid, tid,
905 if (thr_bar->my_level) { // not a leaf (my_level==0 means leaf)
910 thr_bar->use_oncore_barrier) {
911 if (thr_bar->leaf_kids) {
915 ? thr_bar->b_arrived | thr_bar->leaf_state
916 : team->t.t_bar[bt].b_arrived | thr_bar->leaf_state;
920 kmp_flag_64 flag(&thr_bar->b_arrived, leaf_state);
926 for (child_tid = tid + 1; child_tid <= tid + thr_bar->leaf_kids;
942 KMP_TEST_THEN_AND64(&thr_bar->b_arrived, ~(thr_bar->leaf_state));
945 for (kmp_uint32 d = 1; d < thr_bar->my_level;
947 kmp_uint32 last = tid + thr_bar->skip_per_level[d + 1],
948 skip = thr_bar->skip_per_level[d];
978 for (kmp_uint32 d = 0; d < thr_bar->my_level;
980 kmp_uint32 last = tid + thr_bar->skip_per_level[d + 1],
981 skip = thr_bar->skip_per_level[d];
1018 __kmp_gtid_from_tid(thr_bar->parent_tid, team), team->t.t_id,
1019 thr_bar->parent_tid, &thr_bar->b_arrived, thr_bar->b_arrived,
1020 thr_bar->b_arrived + KMP_BARRIER_STATE_BUMP));
1024 if (thr_bar->my_level || __kmp_dflt_blocktime != KMP_MAX_BLOCKTIME ||
1025 !thr_bar->use_oncore_barrier) { // Parent is waiting on my b_arrived
1028 kmp_flag_64 flag(&thr_bar->b_arrived, other_threads[thr_bar->parent_tid]);
1032 thr_bar->b_arrived = team->t.t_bar[bt].b_arrived + KMP_BARRIER_STATE_BUMP;
1033 kmp_flag_oncore flag(&thr_bar->parent_bar->b_arrived, thr_bar->offset);
1034 flag.set_waiter(other_threads[thr_bar->parent_tid]);
1055 kmp_bstate_t *thr_bar = &this_thr->th.th_bar[bt].bb;
1067 if (!thr_bar->use_oncore_barrier ||
1068 __kmp_dflt_blocktime != KMP_MAX_BLOCKTIME || thr_bar->my_level != 0 ||
1069 thr_bar->team == NULL) {
1071 thr_bar->wait_flag = KMP_BARRIER_OWN_FLAG;
1072 kmp_flag_64 flag(&thr_bar->b_go, KMP_BARRIER_STATE_BUMP);
1075 TCW_8(thr_bar->b_go,
1080 thr_bar->wait_flag = KMP_BARRIER_PARENT_FLAG;
1081 kmp_flag_oncore flag(&thr_bar->parent_bar->b_go, KMP_BARRIER_STATE_BUMP,
1082 thr_bar->offset, bt,
1085 if (thr_bar->wait_flag ==
1087 TCW_8(thr_bar->b_go,
1091 &(thr_bar->parent_bar->b_go)))[thr_bar->offset] = 0;
1094 thr_bar->wait_flag = KMP_BARRIER_NOT_WAITING;
1106 gtid, team->t.t_id, tid, &thr_bar->b_go, KMP_INIT_BARRIER_STATE));
1121 thr_bar->use_oncore_barrier = 1;
1123 thr_bar->use_oncore_barrier = 0; // Do not use oncore barrier when nested
1127 unsigned short int old_leaf_kids = thr_bar->leaf_kids;
1128 kmp_uint64 old_leaf_state = thr_bar->leaf_state;
1129 team_change = __kmp_init_hierarchical_barrier_thread(bt, thr_bar, nproc, gtid,
1141 copy_icvs(&thr_bar->th_fixed_icvs,
1144 thr_bar->use_oncore_barrier) { // optimization for inf blocktime
1145 if (!thr_bar->my_level) // I'm a leaf in the hierarchy (my_level==0)
1149 &thr_bar->parent_bar->th_fixed_icvs);
1152 if (thr_bar->my_level) // not a leaf; copy ICVs to my fixed ICVs child can
1154 copy_icvs(&thr_bar->th_fixed_icvs, &thr_bar->parent_bar->th_fixed_icvs);
1157 &thr_bar->parent_bar->th_fixed_icvs);
1163 if (thr_bar->my_level) { // not a leaf
1167 thr_bar->use_oncore_barrier) {
1171 thr_bar->b_go = KMP_BARRIER_STATE_BUMP;
1174 ngo_load(&thr_bar->th_fixed_icvs);
1177 for (child_tid = thr_bar->skip_per_level[1]; child_tid < (int)nproc;
1178 child_tid += thr_bar->skip_per_level[1]) {
1190 ngo_store_go(&child_bar->th_fixed_icvs, &thr_bar->th_fixed_icvs);
1194 TCW_8(thr_bar->b_go,
1197 if (thr_bar->leaf_kids) { // if there are any
1200 old_leaf_kids < thr_bar->leaf_kids) { // some old, some new
1202 thr_bar->b_go |= old_leaf_state;
1205 last = tid + thr_bar->skip_per_level[1];
1226 thr_bar->b_go |= thr_bar->leaf_state;
1230 for (int d = thr_bar->my_level - 1; d >= 0;
1232 last = tid + thr_bar->skip_per_level[d + 1];
1233 kmp_uint32 skip = thr_bar->skip_per_level[d];
1256 &thr_bar->th_fixed_icvs);