common/os/lgrp.c

69 #include <sys/lgrp.h>
111 extern struct lgrp_stats lgrp_stats[];	/* table of per-lgrp stats */
138  * During system bootstrap cp_default does not contain the list of lgrp load
147  * The lpl_bootstrap_list is maintained by the code in lgrp.c. Every other
152  * lgrp will use lpl_bootstrap as a default lpl. This is necessary because
164  * an lgrp. lgrp_mem_choose() uses this macro to detect the case where
165  * a thread is trying to allocate memory close to a CPU that has no lgrp.
422  * true when lgrp initialization has been completed.
427  * True when lgrp topology is constructed.
474 		 * Notify the PG subsystem that the CPU's lgrp
515  * Finish lgrp initialization after all CPUS are brought on-line.
539 	lgrp_t		*lgrp;
543 		lgrp = lgrp_table[i];
545 		if (!LGRP_EXISTS(lgrp))
549 		    lgrp->lgrp_latency == oldtime) ||
550 		    (hand != LGRP_NULL_HANDLE && lgrp->lgrp_plathand == hand))
551 			lgrp->lgrp_latency = (int)newtime;
576 		 * Initialize the new CPU's lgrp related next/prev
704  * Called to add lgrp info into cpu structure from cpu_add_unit;
710  * for the lgrp structs and stats.
751 		 * Create new lgrp and add it to lgroup topology
801 			lgrp_t		*lgrp;
803 			lgrp = lgrp_table[i];
804 			if (!LGRP_EXISTS(lgrp) ||
805 			    !lgrp_rsets_member(lgrp->lgrp_set, lgrpid))
808 			klgrpset_add(lgrp->lgrp_set[LGRP_RSRC_CPU], lgrpid);
824 	 * link the CPU into the lgrp's CPU list
915 lgrp_destroy(lgrp_t *lgrp)
928 	if (!LGRP_EXISTS(lgrp))
935 	if (lgrp_alloc_hint == -1 || lgrp->lgrp_id < lgrp_alloc_hint)
936 		lgrp_alloc_hint = lgrp->lgrp_id;
942 	lgrp->lgrp_id = LGRP_NONE;
943 	lgrp->lgrp_latency = 0;
944 	lgrp->lgrp_plathand = LGRP_NULL_HANDLE;
945 	lgrp->lgrp_parent = NULL;
946 	lgrp->lgrp_childcnt = 0;
948 	klgrpset_clear(lgrp->lgrp_children);
949 	klgrpset_clear(lgrp->lgrp_leaves);
951 		klgrpset_clear(lgrp->lgrp_set[i]);
953 	lgrp->lgrp_mnodes = (mnodeset_t)0;
954 	lgrp->lgrp_nmnodes = 0;
956 	lgrp->lgrp_cpu = NULL;
957 	lgrp->lgrp_cpucnt = 0;
963  * Initialize kstat data. Called from lgrp intialization code.
978  * initialize an lgrp's kstats if needed
981  * memory leaving the lgrp yet...
999 	lgrp_kstat = kstat_create("lgrp", lgrpid, NULL, "misc",
1080 			lgrp_t		*lgrp;
1082 			lgrp = lgrp_table[i];
1083 			if (!LGRP_EXISTS(lgrp) ||
1084 			    !klgrpset_ismember(lgrp->lgrp_set[LGRP_RSRC_CPU],
1088 			klgrpset_del(lgrp->lgrp_set[LGRP_RSRC_CPU], lgrpid);
1107 	lgrp_t	*lgrp;
1124 		lgrp = lgrp_table[i];
1125 		if (!klgrpset_ismember(target, i) || !LGRP_EXISTS(lgrp)) {
1134 		if (lgrp->lgrp_childcnt && lgrp != lgrp_root) {
1135 			lgrp->lgrp_mnodes = (mnodeset_t)0;
1136 			lgrp->lgrp_nmnodes = 0;
1152 			    !klgrpset_ismember(lgrp->lgrp_set[LGRP_RSRC_MEM],
1165 				    !(lgrp->lgrp_mnodes & mnode_mask)) {
1166 					lgrp->lgrp_mnodes |= mnode_mask;
1167 					lgrp->lgrp_nmnodes++;
1172 				klgrpset_add(*changed, lgrp->lgrp_id);
1207  * lgrp with non-empty lgrp_mnodes. To deal with the special case above,
1226  * Called to indicate that the lgrp with platform handle "hand" now
1291 		/* new lgrp */
1334 			lgrp_t		*lgrp;
1336 			lgrp = lgrp_table[i];
1337 			if (!LGRP_EXISTS(lgrp) ||
1338 			    !lgrp_rsets_member(lgrp->lgrp_set, lgrpid))
1341 			klgrpset_add(lgrp->lgrp_set[LGRP_RSRC_MEM], lgrpid);
1342 			klgrpset_add(changed, lgrp->lgrp_id);
1413 	 * The lgrp *must* be pre-existing
1422 		lgrp_t *lgrp = lgrp_table[i];
1427 		if (!LGRP_EXISTS(lgrp) ||
1428 		    !(lgrp->lgrp_mnodes & mnodes_mask))
1436 		    (lgrp == lgrp_root) && (lgrp->lgrp_mnodes == mnodes_mask))
1442 		lgrp->lgrp_mnodes &= ~mnodes_mask;
1443 		lgrp->lgrp_nmnodes--;
1444 		ASSERT(lgrp->lgrp_nmnodes >= 0);
1492 			lgrp_t		*lgrp;
1494 			lgrp = lgrp_table[i];
1495 			if (!LGRP_EXISTS(lgrp) ||
1496 			    !klgrpset_ismember(lgrp->lgrp_set[LGRP_RSRC_MEM],
1500 			klgrpset_del(lgrp->lgrp_set[LGRP_RSRC_MEM], lgrpid);
1514 	lgrp_t	*lgrp;
1520 		lgrp = lgrp_table[i];
1521 		if (LGRP_EXISTS(lgrp) && lgrp->lgrp_plathand == hand)
1522 			return (lgrp);
1541 	lgrp_t	*lgrp;
1550 	lgrp = lgrp_table[lpl->lpl_lgrpid];
1554 	return (lgrp);
1565 	lgrp_id_t	lgrp;
1581 	lgrp = lpl->lpl_lgrpid;
1585 	return (lgrp);
1596 	lgrp_t		*lgrp;
1601 			lgrp = lgrp_table[i];
1602 			if (LGRP_EXISTS(lgrp) && lgrp->lgrp_plathand == hand)
1603 				return (lgrp);
1616 	lgrp_t		*lgrp;
1623 			lgrp = lgrp_table[i];
1624 			if (LGRP_EXISTS(lgrp) && lgrp->lgrp_plathand == hand)
1625 				return (lgrp);
1644  * Return the sum of the partition loads in an lgrp divided by
1645  * the number of CPUs in the lgrp.  This is our best approximation
1649 lgrp_sum_loadavgs(lgrp_t *lgrp)
1657 	cpu = lgrp->lgrp_cpu;
1658 	ncpu = lgrp->lgrp_cpucnt;
1668 	} while (cpu != lgrp->lgrp_cpu);
1706  * Reset all kstats for lgrp specified by its lgrpid.
1722  * Collect all per-lgrp statistics for the lgrp associated with this
1726  * lgrp by writing to any of the lgrp's stats.
1733 	lgrp_t			*lgrp;
1736 	lgrp = (lgrp_t *)ksp->ks_private;
1741 	lgrpid = lgrp->lgrp_id;
1745 		 * Return all zeroes as stats for freed lgrp.
1766 		ksd[stat + LGRP_NUM_CPUS].value.i64 = lgrp->lgrp_cpucnt;
1773 		ksd[stat + LGRP_LOADAVG].value.i64 = lgrp_sum_loadavgs(lgrp);
1886  * The lgrp topology is used as the reference since it is fully
1945 		 * Update the lgrp id <=> rset mapping
1995 		 * Don't adjust if the lgrp isn't there, if we're the leaf lpl
1996 		 * for the cpu in question, or if the current lgrp and leaf
2019  * Initialize lpl with given resources and specified lgrp
2022 lpl_init(lpl_t *lpl, lpl_t *lpl_leaf, lgrp_t *lgrp)
2024 	lpl->lpl_lgrpid = lgrp->lgrp_id;
2033 	lpl->lpl_lgrp = lgrp;
2076 	lgrp_t		*lgrp;
2093 		lgrp = lgrp_table[i];
2105 		if (!LGRP_EXISTS(lgrp)) {
2115 		ASSERT(lgrp->lgrp_id == lpl->lpl_lgrpid);
2118 		if (!klgrpset_intersects(lgrp->lgrp_leaves,
2147 		ASSERT(lgrp == lpl->lpl_lgrp);
2148 		if (lgrp != lpl->lpl_lgrp) {
2153 		if (lgrp->lgrp_parent) {
2155 			ASSERT(lgrp->lgrp_parent->lgrp_id ==
2160 			} else if (lgrp->lgrp_parent->lgrp_id !=
2169 			/* verify that lgrp is also a leaf */
2170 			ASSERT((lgrp->lgrp_childcnt == 0) &&
2171 			    (klgrpset_ismember(lgrp->lgrp_leaves,
2174 			if ((lgrp->lgrp_childcnt > 0) ||
2175 			    (!klgrpset_ismember(lgrp->lgrp_leaves,
2180 			ASSERT((lgrp->lgrp_cpucnt >= lpl->lpl_ncpu) &&
2182 			if ((lgrp->lgrp_cpucnt < lpl->lpl_ncpu) ||
2240 				    lgrp->lgrp_id)) {
2288 		/* make sure lpl rset matches lgrp rset */
2289 		klgrpset_diff(rset, lgrp->lgrp_set[LGRP_RSRC_CPU]);
2364 				 * whose parent had their lgrp deleted.  (And
2417 	klgrpset_t	rset_intersect; /* resources in cpupart and lgrp */
2423 		 * Don't insert if the lgrp isn't there, if the leaf isn't
2424 		 * contained within the current lgrp, or if the current lgrp has
2437 			/* if lgrp has a parent, assign it properly */
2471 				lgrp_t	*lgrp_cand;	/* candidate lgrp */
2531 		 * based upon lgrp membership but not necessarily cpu-part
2557  * add a cpu to a partition in terms of lgrp load avg bookeeping
2571  * 2. The lpl to contain the resources in this cpu-partition for this lgrp does
2575  * routine, as is the cpu-partition state for the lgrp membership.  This routine
2593 	/* don't add non-existent lgrp */
2633  * remove a cpu from a partition in terms of lgrp load avg bookeeping
2650  * cpu partition in question as no longer containing resources from the lgrp of
2706  * Recompute load average for the specified partition/lgrp fragment.
2860 		 * lgrp id <=> rset mapping storage,
2914 	 * dispatcher prior to their lgrp/lpl initialization.
3115 			 * has threads on this lgrp, so this is a preferred
3125 			 * The process doesn't have any threads on this lgrp,
3126 			 * but we're willing to consider this lgrp if the load
3179  * Decide if lpl1 is a better candidate than lpl2 for lgrp homing.
3212  * process that uses text replication changed home lgrp. This info is used by
3251  *	cpu_lock, kpreempt_disable(), or thread_lock -- to assure t's new lgrp
3621  * should be mindful of the possibility that the lgrp returned
3622  * may be deleted. If this happens, dereferences of the lgrp
3623  * pointer will still be safe, but the resources in the lgrp will
3630 	lgrp_t			*lgrp;
3641 	 * Just return null if the lgrp framework hasn't finished
3666 					lgrp = lgrp_table[id];
3667 					if (!LGRP_EXISTS(lgrp)) {
3672 						return (lgrp);
3683 	lgrp = lgrp_home_lgrp();
3686 	 * When homing threads on root lgrp, override default memory
3689 	if (lgrp == lgrp_root)
3700 		 * If the CPU isn't currently in an lgrp, then opt to
3704 		 * the current CPU from going away before lgrp is found.
3707 			lgrp = lgrp_root;
3710 			lgrp = lgrp_cpu_to_lgrp(CPU);
3726 		if (lgrp != lgrp_root) {
3728 				lgrp_stat_add(lgrp->lgrp_id, LGRP_NUM_NEXT, 1);
3730 				lgrp_stat_add(lgrp->lgrp_id,
3749 		 * configured. In this case just return the root lgrp.
3752 			lgrp = lgrp_root;
3770 				lgrp = lgrp_table[i];
3771 				lgrp_stat_add(lgrp->lgrp_id, LGRP_NUM_RANDOM,
3815 			lgrp = lgrp_root;
3833 				lgrp = lgrp_table[i];
3834 				lgrp_stat_add(lgrp->lgrp_id, LGRP_NUM_RANDOM,
3853 		i = lgrp->lgrp_id;
3858 			lgrp = lgrp_table[i];
3862 		lgrp_stat_add(lgrp->lgrp_id, LGRP_NUM_ROUNDROBIN, 1);
3867 	ASSERT(lgrp != NULL);
3868 	return (lgrp);
3880 	lgrp_t *lgrp;
3882 	lgrp = lgrp_table[lgrpid];
3883 	if (!LGRP_EXISTS(lgrp) ||
3884 	    klgrpset_isempty(lgrp->lgrp_set[LGRP_RSRC_MEM]) ||
3885 	    !klgrpset_ismember(lgrp->lgrp_set[LGRP_RSRC_MEM], lgrpid))
3888 	return (lgrp_plat_mem_size(lgrp->lgrp_plathand, query));
4385  *	- traverse is 0, and all the memnodes in "lgrp" have been returned.