linux-master/block/blk-iocost.c

474 	 * `weight`.  `inuse` is the surplus adjusted active weight.
475 	 * `active` and `inuse` are used to calculate `hweight_active` and
478 	 * `last_inuse` remembers `inuse` while an iocg is idle to persist
481 	 * `inuse` may be adjusted dynamically during period. `saved_*` are used
487 	u32				inuse;
1080  * Update @iocg's `active` and `inuse` to @active and @inuse, update level
1082  * is saved to be used as reference for later inuse in-period adjustments.
1084 static void __propagate_weights(struct ioc_gq *iocg, u32 active, u32 inuse,
1093 	 * For an active leaf node, its inuse shouldn't be zero or exceed
1094 	 * @active. An active internal node's inuse is solely determined by the
1095 	 * inuse to active ratio of its children regardless of @inuse.
1098 		inuse = DIV64_U64_ROUND_UP(active * iocg->child_inuse_sum,
1101 		inuse = clamp_t(u32, inuse, 1, active);
1104 	iocg->last_inuse = iocg->inuse;
1108 	if (active == iocg->active && inuse == iocg->inuse)
1118 		parent->child_inuse_sum += (s32)(inuse - child->inuse);
1121 		child->inuse = inuse;
1124 		 * The delta between inuse and active sums indicates that
1125 		 * much of weight is being given away.  Parent's inuse
1137 		    parent_inuse == parent->inuse)
1141 		inuse = parent_inuse;
1159 static void propagate_weights(struct ioc_gq *iocg, u32 active, u32 inuse,
1162 	__propagate_weights(iocg, active, inuse, save, now);
1197 		u32 inuse = READ_ONCE(child->inuse);
1206 		inuse_sum = max_t(u64, inuse, inuse_sum);
1207 		hwi = div64_u64((u64)hwi * inuse, inuse_sum);
1221  * Calculate the hweight_inuse @iocg would get with max @inuse assuming all the
1227 	u32 inuse = iocg->active;
1237 		child_inuse_sum = parent->child_inuse_sum + inuse - child->inuse;
1238 		hwm = div64_u64((u64)hwm * inuse, child_inuse_sum);
1239 		inuse = DIV64_U64_ROUND_UP(parent->active * child_inuse_sum,
1257 		propagate_weights(iocg, weight, iocg->inuse, true, now);
1421 	 * Once in debt, debt handling owns inuse. @iocg stays at the minimum
1422 	 * inuse donating all of it share to others until its debt is paid off.
1447 	WARN_ON_ONCE(iocg->inuse > 1);
1451 	/* if debt is paid in full, restore inuse */
1750 	/* debt handling owns inuse for debtors */
1795  * bumping up vrate accordingly 2. lowering the donating iocg's inuse weight.
1798  * global hweight_inuse updates when idle iocg's get activated or inuse weights
1810  * inuse should be adjusted to achieve the target donation amounts. This is done
1824  * achieved by scaling inuse weights of donating leaves and propagating the
1829  * Combining the above two, we can determine how each leaf iocg's inuse should
1983 	 * Calculate adjusted hwi, child_adjusted_sum and inuse for the inner
1988 		u32 inuse, wpt, wptp;
2007 		inuse = DIV64_U64_ROUND_UP(
2020 			(u64)inuse * iocg->hweight_after_donation,
2032 		u32 inuse;
2036 		 * the minimum target hweight_inuse. Configuring inuse
2038 		 * @iocg->inuse stay at the minimum and we don't wanna
2042 			WARN_ON_ONCE(iocg->inuse > 1);
2047 		inuse = DIV64_U64_ROUND_UP(
2052 				iocg->inuse, inuse,
2056 		__propagate_weights(iocg, iocg->active, inuse, true, now);
2355 				 * its inuse to active.
2357 				 * Don't reset debtors as their inuse's are
2364 						iocg->inuse, iocg->active,
2465 	u32 __maybe_unused old_inuse = iocg->inuse, __maybe_unused old_hwi;
2476 	/* debt handling owns inuse for debtors */
2481 	 * We only increase inuse during period and do so if the margin has
2485 	    iocg->inuse == iocg->active)
2490 	/* we own inuse only when @iocg is in the normal active state */
2497 	 * Bump up inuse till @abs_cost fits in the existing budget.
2503 	new_inuse = iocg->inuse;
2511 		 iocg->inuse != iocg->active);
2516 			old_inuse, iocg->inuse, old_hwi, hwi);
2688 	/* guarantee that iocgs w/ waiters have maximum inuse */
2689 	if (!iocg->abs_vdebt && iocg->inuse != iocg->active) {