linux-master/lib/dynamic_queue_limits.c

18 static void dql_check_stall(struct dql *dql, unsigned short stall_thrs)
27 	if (time_after_eq(now, dql->last_reap + stall_thrs)) {
35 		hist_head = READ_ONCE(dql->history_head);
45 		if (time_before(start, dql->last_reap + 1))
46 			start = dql->last_reap + 1;
60 				     dql->history))
68 		if (hist_head != READ_ONCE(dql->history_head))
71 		dql->stall_cnt++;
72 		dql->stall_max = max_t(unsigned short, dql->stall_max, now - t);
74 		trace_dql_stall_detected(dql->stall_thrs, now - t,
75 					 dql->last_reap, dql->history_head,
76 					 now, dql->history);
79 	dql->last_reap = now;
83 void dql_completed(struct dql *dql, unsigned int count)
90 	num_queued = READ_ONCE(dql->num_queued);
96 	stall_thrs = READ_ONCE(dql->stall_thrs);
99 	BUG_ON(count > num_queued - dql->num_completed);
101 	completed = dql->num_completed + count;
102 	limit = dql->limit;
103 	ovlimit = POSDIFF(num_queued - dql->num_completed, limit);
105 	prev_inprogress = dql->prev_num_queued - dql->num_completed;
106 	all_prev_completed = AFTER_EQ(completed, dql->prev_num_queued);
109 	    (dql->prev_ovlimit && all_prev_completed)) {
125 		limit += POSDIFF(completed, dql->prev_num_queued) +
126 		     dql->prev_ovlimit;
127 		dql->slack_start_time = jiffies;
128 		dql->lowest_slack = UINT_MAX;
154 		slack = POSDIFF(limit + dql->prev_ovlimit,
155 		    2 * (completed - dql->num_completed));
156 		slack_last_objs = dql->prev_ovlimit ?
157 		    POSDIFF(dql->prev_last_obj_cnt, dql->prev_ovlimit) : 0;
161 		if (slack < dql->lowest_slack)
162 			dql->lowest_slack = slack;
165 			       dql->slack_start_time + dql->slack_hold_time)) {
166 			limit = POSDIFF(limit, dql->lowest_slack);
167 			dql->slack_start_time = jiffies;
168 			dql->lowest_slack = UINT_MAX;
173 	limit = clamp(limit, dql->min_limit, dql->max_limit);
175 	if (limit != dql->limit) {
176 		dql->limit = limit;
180 	dql->adj_limit = limit + completed;
181 	dql->prev_ovlimit = ovlimit;
182 	dql->prev_last_obj_cnt = dql->last_obj_cnt;
183 	dql->num_completed = completed;
184 	dql->prev_num_queued = num_queued;
186 	dql_check_stall(dql, stall_thrs);
190 void dql_reset(struct dql *dql)
193 	dql->limit = 0;
194 	dql->num_queued = 0;
195 	dql->num_completed = 0;
196 	dql->last_obj_cnt = 0;
197 	dql->prev_num_queued = 0;
198 	dql->prev_last_obj_cnt = 0;
199 	dql->prev_ovlimit = 0;
200 	dql->lowest_slack = UINT_MAX;
201 	dql->slack_start_time = jiffies;
203 	dql->last_reap = jiffies;
204 	dql->history_head = jiffies / BITS_PER_LONG;
205 	memset(dql->history, 0, sizeof(dql->history));
209 void dql_init(struct dql *dql, unsigned int hold_time)
211 	dql->max_limit = DQL_MAX_LIMIT;
212 	dql->min_limit = 0;
213 	dql->slack_hold_time = hold_time;
214 	dql->stall_thrs = 0;
215 	dql_reset(dql);