1145132Sanholt// SPDX-License-Identifier: GPL-2.0 2145132Sanholt 3145132Sanholt#include "blk-rq-qos.h" 4145132Sanholt 5145132Sanholt/* 6145132Sanholt * Increment 'v', if 'v' is below 'below'. Returns true if we succeeded, 7145132Sanholt * false if 'v' + 1 would be bigger than 'below'. 8145132Sanholt */ 9145132Sanholtstatic bool atomic_inc_below(atomic_t *v, unsigned int below) 10145132Sanholt{ 11145132Sanholt unsigned int cur = atomic_read(v); 12145132Sanholt 13145132Sanholt do { 14145132Sanholt if (cur >= below) 15145132Sanholt return false; 16145132Sanholt } while (!atomic_try_cmpxchg(v, &cur, cur + 1)); 17145132Sanholt 18145132Sanholt return true; 19145132Sanholt} 20145132Sanholt 21145132Sanholtbool rq_wait_inc_below(struct rq_wait *rq_wait, unsigned int limit) 22145132Sanholt{ 23145132Sanholt return atomic_inc_below(&rq_wait->inflight, limit); 24145132Sanholt} 25145132Sanholt 26145132Sanholtvoid __rq_qos_cleanup(struct rq_qos *rqos, struct bio *bio) 27145132Sanholt{ 28145132Sanholt do { 29145132Sanholt if (rqos->ops->cleanup) 30145132Sanholt rqos->ops->cleanup(rqos, bio); 31152909Sanholt rqos = rqos->next; 32152909Sanholt } while (rqos); 33152909Sanholt} 34182080Srnoland 35182080Srnolandvoid __rq_qos_done(struct rq_qos *rqos, struct request *rq) 36182080Srnoland{ 37182080Srnoland do { 38182080Srnoland if (rqos->ops->done) 39145132Sanholt rqos->ops->done(rqos, rq); 40145132Sanholt rqos = rqos->next; 41182080Srnoland } while (rqos); 42182080Srnoland} 43182080Srnoland 44182080Srnolandvoid __rq_qos_issue(struct rq_qos *rqos, struct request *rq) 45182080Srnoland{ 46182080Srnoland do { 47182080Srnoland if (rqos->ops->issue) 48182080Srnoland rqos->ops->issue(rqos, rq); 49182080Srnoland rqos = rqos->next; 50145132Sanholt } while (rqos); 51182080Srnoland} 52182080Srnoland 53182080Srnolandvoid __rq_qos_requeue(struct rq_qos *rqos, struct request *rq) 54182080Srnoland{ 55182080Srnoland do { 56182080Srnoland if (rqos->ops->requeue) 57145132Sanholt rqos->ops->requeue(rqos, rq); 58182080Srnoland rqos = rqos->next; 59182080Srnoland } while (rqos); 60145132Sanholt} 61182080Srnoland 62182080Srnolandvoid __rq_qos_throttle(struct rq_qos *rqos, struct bio *bio) 63182080Srnoland{ 64182080Srnoland do { 65182080Srnoland if (rqos->ops->throttle) 66182080Srnoland rqos->ops->throttle(rqos, bio); 67182080Srnoland rqos = rqos->next; 68182080Srnoland } while (rqos); 69182080Srnoland} 70182080Srnoland 71182080Srnolandvoid __rq_qos_track(struct rq_qos *rqos, struct request *rq, struct bio *bio) 72182080Srnoland{ 73182080Srnoland do { 74183573Srnoland if (rqos->ops->track) 75182080Srnoland rqos->ops->track(rqos, rq, bio); 76182080Srnoland rqos = rqos->next; 77183834Srnoland } while (rqos); 78183834Srnoland} 79182080Srnoland 80182080Srnolandvoid __rq_qos_merge(struct rq_qos *rqos, struct request *rq, struct bio *bio) 81182080Srnoland{ 82182080Srnoland do { 83182080Srnoland if (rqos->ops->merge) 84182080Srnoland rqos->ops->merge(rqos, rq, bio); 85182080Srnoland rqos = rqos->next; 86182080Srnoland } while (rqos); 87182080Srnoland} 88182080Srnoland 89182080Srnolandvoid __rq_qos_done_bio(struct rq_qos *rqos, struct bio *bio) 90145132Sanholt{ 91145132Sanholt do { 92145132Sanholt if (rqos->ops->done_bio) 93182080Srnoland rqos->ops->done_bio(rqos, bio); 94145132Sanholt rqos = rqos->next; 95183573Srnoland } while (rqos); 96182080Srnoland} 97182080Srnoland 98182080Srnolandvoid __rq_qos_queue_depth_changed(struct rq_qos *rqos) 99182080Srnoland{ 100182080Srnoland do { 101182080Srnoland if (rqos->ops->queue_depth_changed) 102182080Srnoland rqos->ops->queue_depth_changed(rqos); 103182080Srnoland rqos = rqos->next; 104182080Srnoland } while (rqos); 105183834Srnoland} 106183834Srnoland 107182080Srnoland/* 108182080Srnoland * Return true, if we can't increase the depth further by scaling 109182080Srnoland */ 110182080Srnolandbool rq_depth_calc_max_depth(struct rq_depth *rqd) 111182080Srnoland{ 112145132Sanholt unsigned int depth; 113182080Srnoland bool ret = false; 114182080Srnoland 115182080Srnoland /* 116182080Srnoland * For QD=1 devices, this is a special case. It's important for those 117182080Srnoland * to have one request ready when one completes, so force a depth of 118182080Srnoland * 2 for those devices. On the backend, it'll be a depth of 1 anyway, 119182080Srnoland * since the device can't have more than that in flight. If we're 120182080Srnoland * scaling down, then keep a setting of 1/1/1. 121182080Srnoland */ 122182080Srnoland if (rqd->queue_depth == 1) { 123182080Srnoland if (rqd->scale_step > 0) 124182080Srnoland rqd->max_depth = 1; 125182080Srnoland else { 126182080Srnoland rqd->max_depth = 2; 127182080Srnoland ret = true; 128182080Srnoland } 129183834Srnoland } else { 130182080Srnoland /* 131182080Srnoland * scale_step == 0 is our default state. If we have suffered 132182080Srnoland * latency spikes, step will be > 0, and we shrink the 133182080Srnoland * allowed write depths. If step is < 0, we're only doing 134182080Srnoland * writes, and we allow a temporarily higher depth to 135182080Srnoland * increase performance. 136182080Srnoland */ 137182080Srnoland depth = min_t(unsigned int, rqd->default_depth, 138183834Srnoland rqd->queue_depth); 139183834Srnoland if (rqd->scale_step > 0) 140182080Srnoland depth = 1 + ((depth - 1) >> min(31, rqd->scale_step)); 141182080Srnoland else if (rqd->scale_step < 0) { 142182080Srnoland unsigned int maxd = 3 * rqd->queue_depth / 4; 143182080Srnoland 144182080Srnoland depth = 1 + ((depth - 1) << -rqd->scale_step); 145182080Srnoland if (depth > maxd) { 146182080Srnoland depth = maxd; 147182080Srnoland ret = true; 148182080Srnoland } 149182080Srnoland } 150182080Srnoland 151182080Srnoland rqd->max_depth = depth; 152182080Srnoland } 153182080Srnoland 154182080Srnoland return ret; 155182080Srnoland} 156182080Srnoland 157182080Srnoland/* Returns true on success and false if scaling up wasn't possible */ 158182080Srnolandbool rq_depth_scale_up(struct rq_depth *rqd) 159182080Srnoland{ 160182080Srnoland /* 161182080Srnoland * Hit max in previous round, stop here 162182080Srnoland */ 163182080Srnoland if (rqd->scaled_max) 164182080Srnoland return false; 165182080Srnoland 166182080Srnoland rqd->scale_step--; 167182080Srnoland 168183834Srnoland rqd->scaled_max = rq_depth_calc_max_depth(rqd); 169183834Srnoland return true; 170182080Srnoland} 171182080Srnoland 172182080Srnoland/* 173182080Srnoland * Scale rwb down. If 'hard_throttle' is set, do it quicker, since we 174 * had a latency violation. Returns true on success and returns false if 175 * scaling down wasn't possible. 176 */ 177bool rq_depth_scale_down(struct rq_depth *rqd, bool hard_throttle) 178{ 179 /* 180 * Stop scaling down when we've hit the limit. This also prevents 181 * ->scale_step from going to crazy values, if the device can't 182 * keep up. 183 */ 184 if (rqd->max_depth == 1) 185 return false; 186 187 if (rqd->scale_step < 0 && hard_throttle) 188 rqd->scale_step = 0; 189 else 190 rqd->scale_step++; 191 192 rqd->scaled_max = false; 193 rq_depth_calc_max_depth(rqd); 194 return true; 195} 196 197struct rq_qos_wait_data { 198 struct wait_queue_entry wq; 199 struct task_struct *task; 200 struct rq_wait *rqw; 201 acquire_inflight_cb_t *cb; 202 void *private_data; 203 bool got_token; 204}; 205 206static int rq_qos_wake_function(struct wait_queue_entry *curr, 207 unsigned int mode, int wake_flags, void *key) 208{ 209 struct rq_qos_wait_data *data = container_of(curr, 210 struct rq_qos_wait_data, 211 wq); 212 213 /* 214 * If we fail to get a budget, return -1 to interrupt the wake up loop 215 * in __wake_up_common. 216 */ 217 if (!data->cb(data->rqw, data->private_data)) 218 return -1; 219 220 data->got_token = true; 221 smp_wmb(); 222 list_del_init(&curr->entry); 223 wake_up_process(data->task); 224 return 1; 225} 226 227/** 228 * rq_qos_wait - throttle on a rqw if we need to 229 * @rqw: rqw to throttle on 230 * @private_data: caller provided specific data 231 * @acquire_inflight_cb: inc the rqw->inflight counter if we can 232 * @cleanup_cb: the callback to cleanup in case we race with a waker 233 * 234 * This provides a uniform place for the rq_qos users to do their throttling. 235 * Since you can end up with a lot of things sleeping at once, this manages the 236 * waking up based on the resources available. The acquire_inflight_cb should 237 * inc the rqw->inflight if we have the ability to do so, or return false if not 238 * and then we will sleep until the room becomes available. 239 * 240 * cleanup_cb is in case that we race with a waker and need to cleanup the 241 * inflight count accordingly. 242 */ 243void rq_qos_wait(struct rq_wait *rqw, void *private_data, 244 acquire_inflight_cb_t *acquire_inflight_cb, 245 cleanup_cb_t *cleanup_cb) 246{ 247 struct rq_qos_wait_data data = { 248 .wq = { 249 .func = rq_qos_wake_function, 250 .entry = LIST_HEAD_INIT(data.wq.entry), 251 }, 252 .task = current, 253 .rqw = rqw, 254 .cb = acquire_inflight_cb, 255 .private_data = private_data, 256 }; 257 bool has_sleeper; 258 259 has_sleeper = wq_has_sleeper(&rqw->wait); 260 if (!has_sleeper && acquire_inflight_cb(rqw, private_data)) 261 return; 262 263 has_sleeper = !prepare_to_wait_exclusive(&rqw->wait, &data.wq, 264 TASK_UNINTERRUPTIBLE); 265 do { 266 /* The memory barrier in set_task_state saves us here. */ 267 if (data.got_token) 268 break; 269 if (!has_sleeper && acquire_inflight_cb(rqw, private_data)) { 270 finish_wait(&rqw->wait, &data.wq); 271 272 /* 273 * We raced with rq_qos_wake_function() getting a token, 274 * which means we now have two. Put our local token 275 * and wake anyone else potentially waiting for one. 276 */ 277 smp_rmb(); 278 if (data.got_token) 279 cleanup_cb(rqw, private_data); 280 break; 281 } 282 io_schedule(); 283 has_sleeper = true; 284 set_current_state(TASK_UNINTERRUPTIBLE); 285 } while (1); 286 finish_wait(&rqw->wait, &data.wq); 287} 288 289void rq_qos_exit(struct request_queue *q) 290{ 291 mutex_lock(&q->rq_qos_mutex); 292 while (q->rq_qos) { 293 struct rq_qos *rqos = q->rq_qos; 294 q->rq_qos = rqos->next; 295 rqos->ops->exit(rqos); 296 } 297 mutex_unlock(&q->rq_qos_mutex); 298} 299 300int rq_qos_add(struct rq_qos *rqos, struct gendisk *disk, enum rq_qos_id id, 301 const struct rq_qos_ops *ops) 302{ 303 struct request_queue *q = disk->queue; 304 305 lockdep_assert_held(&q->rq_qos_mutex); 306 307 rqos->disk = disk; 308 rqos->id = id; 309 rqos->ops = ops; 310 311 /* 312 * No IO can be in-flight when adding rqos, so freeze queue, which 313 * is fine since we only support rq_qos for blk-mq queue. 314 */ 315 blk_mq_freeze_queue(q); 316 317 if (rq_qos_id(q, rqos->id)) 318 goto ebusy; 319 rqos->next = q->rq_qos; 320 q->rq_qos = rqos; 321 322 blk_mq_unfreeze_queue(q); 323 324 if (rqos->ops->debugfs_attrs) { 325 mutex_lock(&q->debugfs_mutex); 326 blk_mq_debugfs_register_rqos(rqos); 327 mutex_unlock(&q->debugfs_mutex); 328 } 329 330 return 0; 331ebusy: 332 blk_mq_unfreeze_queue(q); 333 return -EBUSY; 334} 335 336void rq_qos_del(struct rq_qos *rqos) 337{ 338 struct request_queue *q = rqos->disk->queue; 339 struct rq_qos **cur; 340 341 lockdep_assert_held(&q->rq_qos_mutex); 342 343 blk_mq_freeze_queue(q); 344 for (cur = &q->rq_qos; *cur; cur = &(*cur)->next) { 345 if (*cur == rqos) { 346 *cur = rqos->next; 347 break; 348 } 349 } 350 blk_mq_unfreeze_queue(q); 351 352 mutex_lock(&q->debugfs_mutex); 353 blk_mq_debugfs_unregister_rqos(rqos); 354 mutex_unlock(&q->debugfs_mutex); 355} 356