1258945Sroberto// SPDX-License-Identifier: GPL-2.0 2258945Sroberto/* 3258945Sroberto * Block rq-qos policy for assigning an I/O priority class to requests. 4258945Sroberto * 5258945Sroberto * Using an rq-qos policy for assigning I/O priority class has two advantages 6258945Sroberto * over using the ioprio_set() system call: 7258945Sroberto * 8258945Sroberto * - This policy is cgroup based so it has all the advantages of cgroups. 9258945Sroberto * - While ioprio_set() does not affect page cache writeback I/O, this rq-qos 10258945Sroberto * controller affects page cache writeback I/O for filesystems that support 11258945Sroberto * assiociating a cgroup with writeback I/O. See also 12258945Sroberto * Documentation/admin-guide/cgroup-v2.rst. 13258945Sroberto */ 14258945Sroberto 15258945Sroberto#include <linux/blk-mq.h> 16258945Sroberto#include <linux/blk_types.h> 17258945Sroberto#include <linux/kernel.h> 18258945Sroberto#include <linux/module.h> 19258945Sroberto#include "blk-cgroup.h" 20258945Sroberto#include "blk-ioprio.h" 21258945Sroberto#include "blk-rq-qos.h" 22258945Sroberto 23258945Sroberto/** 24258945Sroberto * enum prio_policy - I/O priority class policy. 25258945Sroberto * @POLICY_NO_CHANGE: (default) do not modify the I/O priority class. 26258945Sroberto * @POLICY_PROMOTE_TO_RT: modify no-IOPRIO_CLASS_RT to IOPRIO_CLASS_RT. 27258945Sroberto * @POLICY_RESTRICT_TO_BE: modify IOPRIO_CLASS_NONE and IOPRIO_CLASS_RT into 28258945Sroberto * IOPRIO_CLASS_BE. 29258945Sroberto * @POLICY_ALL_TO_IDLE: change the I/O priority class into IOPRIO_CLASS_IDLE. 30258945Sroberto * @POLICY_NONE_TO_RT: an alias for POLICY_PROMOTE_TO_RT. 31258945Sroberto * 32258945Sroberto * See also <linux/ioprio.h>. 33258945Sroberto */ 34258945Srobertoenum prio_policy { 35258945Sroberto POLICY_NO_CHANGE = 0, 36258945Sroberto POLICY_PROMOTE_TO_RT = 1, 37258945Sroberto POLICY_RESTRICT_TO_BE = 2, 38258945Sroberto POLICY_ALL_TO_IDLE = 3, 39258945Sroberto POLICY_NONE_TO_RT = 4, 40258945Sroberto}; 41258945Sroberto 42258945Srobertostatic const char *policy_name[] = { 43258945Sroberto [POLICY_NO_CHANGE] = "no-change", 44258945Sroberto [POLICY_PROMOTE_TO_RT] = "promote-to-rt", 45258945Sroberto [POLICY_RESTRICT_TO_BE] = "restrict-to-be", 46258945Sroberto [POLICY_ALL_TO_IDLE] = "idle", 47258945Sroberto [POLICY_NONE_TO_RT] = "none-to-rt", 48258945Sroberto}; 49258945Sroberto 50258945Srobertostatic struct blkcg_policy ioprio_policy; 51258945Sroberto 52258945Sroberto/** 53258945Sroberto * struct ioprio_blkg - Per (cgroup, request queue) data. 54258945Sroberto * @pd: blkg_policy_data structure. 55258945Sroberto */ 56258945Srobertostruct ioprio_blkg { 57258945Sroberto struct blkg_policy_data pd; 58258945Sroberto}; 59258945Sroberto 60258945Sroberto/** 61258945Sroberto * struct ioprio_blkcg - Per cgroup data. 62258945Sroberto * @cpd: blkcg_policy_data structure. 63258945Sroberto * @prio_policy: One of the IOPRIO_CLASS_* values. See also <linux/ioprio.h>. 64258945Sroberto */ 65258945Srobertostruct ioprio_blkcg { 66258945Sroberto struct blkcg_policy_data cpd; 67258945Sroberto enum prio_policy prio_policy; 68258945Sroberto}; 69258945Sroberto 70258945Srobertostatic inline struct ioprio_blkg *pd_to_ioprio(struct blkg_policy_data *pd) 71258945Sroberto{ 72258945Sroberto return pd ? container_of(pd, struct ioprio_blkg, pd) : NULL; 73258945Sroberto} 74258945Sroberto 75258945Srobertostatic struct ioprio_blkcg *blkcg_to_ioprio_blkcg(struct blkcg *blkcg) 76258945Sroberto{ 77 return container_of(blkcg_to_cpd(blkcg, &ioprio_policy), 78 struct ioprio_blkcg, cpd); 79} 80 81static struct ioprio_blkcg * 82ioprio_blkcg_from_css(struct cgroup_subsys_state *css) 83{ 84 return blkcg_to_ioprio_blkcg(css_to_blkcg(css)); 85} 86 87static struct ioprio_blkcg *ioprio_blkcg_from_bio(struct bio *bio) 88{ 89 struct blkg_policy_data *pd = blkg_to_pd(bio->bi_blkg, &ioprio_policy); 90 91 if (!pd) 92 return NULL; 93 94 return blkcg_to_ioprio_blkcg(pd->blkg->blkcg); 95} 96 97static int ioprio_show_prio_policy(struct seq_file *sf, void *v) 98{ 99 struct ioprio_blkcg *blkcg = ioprio_blkcg_from_css(seq_css(sf)); 100 101 seq_printf(sf, "%s\n", policy_name[blkcg->prio_policy]); 102 return 0; 103} 104 105static ssize_t ioprio_set_prio_policy(struct kernfs_open_file *of, char *buf, 106 size_t nbytes, loff_t off) 107{ 108 struct ioprio_blkcg *blkcg = ioprio_blkcg_from_css(of_css(of)); 109 int ret; 110 111 if (off != 0) 112 return -EIO; 113 /* kernfs_fop_write_iter() terminates 'buf' with '\0'. */ 114 ret = sysfs_match_string(policy_name, buf); 115 if (ret < 0) 116 return ret; 117 blkcg->prio_policy = ret; 118 return nbytes; 119} 120 121static struct blkg_policy_data * 122ioprio_alloc_pd(struct gendisk *disk, struct blkcg *blkcg, gfp_t gfp) 123{ 124 struct ioprio_blkg *ioprio_blkg; 125 126 ioprio_blkg = kzalloc(sizeof(*ioprio_blkg), gfp); 127 if (!ioprio_blkg) 128 return NULL; 129 130 return &ioprio_blkg->pd; 131} 132 133static void ioprio_free_pd(struct blkg_policy_data *pd) 134{ 135 struct ioprio_blkg *ioprio_blkg = pd_to_ioprio(pd); 136 137 kfree(ioprio_blkg); 138} 139 140static struct blkcg_policy_data *ioprio_alloc_cpd(gfp_t gfp) 141{ 142 struct ioprio_blkcg *blkcg; 143 144 blkcg = kzalloc(sizeof(*blkcg), gfp); 145 if (!blkcg) 146 return NULL; 147 blkcg->prio_policy = POLICY_NO_CHANGE; 148 return &blkcg->cpd; 149} 150 151static void ioprio_free_cpd(struct blkcg_policy_data *cpd) 152{ 153 struct ioprio_blkcg *blkcg = container_of(cpd, typeof(*blkcg), cpd); 154 155 kfree(blkcg); 156} 157 158#define IOPRIO_ATTRS \ 159 { \ 160 .name = "prio.class", \ 161 .seq_show = ioprio_show_prio_policy, \ 162 .write = ioprio_set_prio_policy, \ 163 }, \ 164 { } /* sentinel */ 165 166/* cgroup v2 attributes */ 167static struct cftype ioprio_files[] = { 168 IOPRIO_ATTRS 169}; 170 171/* cgroup v1 attributes */ 172static struct cftype ioprio_legacy_files[] = { 173 IOPRIO_ATTRS 174}; 175 176static struct blkcg_policy ioprio_policy = { 177 .dfl_cftypes = ioprio_files, 178 .legacy_cftypes = ioprio_legacy_files, 179 180 .cpd_alloc_fn = ioprio_alloc_cpd, 181 .cpd_free_fn = ioprio_free_cpd, 182 183 .pd_alloc_fn = ioprio_alloc_pd, 184 .pd_free_fn = ioprio_free_pd, 185}; 186 187void blkcg_set_ioprio(struct bio *bio) 188{ 189 struct ioprio_blkcg *blkcg = ioprio_blkcg_from_bio(bio); 190 u16 prio; 191 192 if (!blkcg || blkcg->prio_policy == POLICY_NO_CHANGE) 193 return; 194 195 if (blkcg->prio_policy == POLICY_PROMOTE_TO_RT || 196 blkcg->prio_policy == POLICY_NONE_TO_RT) { 197 /* 198 * For RT threads, the default priority level is 4 because 199 * task_nice is 0. By promoting non-RT io-priority to RT-class 200 * and default level 4, those requests that are already 201 * RT-class but need a higher io-priority can use ioprio_set() 202 * to achieve this. 203 */ 204 if (IOPRIO_PRIO_CLASS(bio->bi_ioprio) != IOPRIO_CLASS_RT) 205 bio->bi_ioprio = IOPRIO_PRIO_VALUE(IOPRIO_CLASS_RT, 4); 206 return; 207 } 208 209 /* 210 * Except for IOPRIO_CLASS_NONE, higher I/O priority numbers 211 * correspond to a lower priority. Hence, the max_t() below selects 212 * the lower priority of bi_ioprio and the cgroup I/O priority class. 213 * If the bio I/O priority equals IOPRIO_CLASS_NONE, the cgroup I/O 214 * priority is assigned to the bio. 215 */ 216 prio = max_t(u16, bio->bi_ioprio, 217 IOPRIO_PRIO_VALUE(blkcg->prio_policy, 0)); 218 if (prio > bio->bi_ioprio) 219 bio->bi_ioprio = prio; 220} 221 222void blk_ioprio_exit(struct gendisk *disk) 223{ 224 blkcg_deactivate_policy(disk, &ioprio_policy); 225} 226 227int blk_ioprio_init(struct gendisk *disk) 228{ 229 return blkcg_activate_policy(disk, &ioprio_policy); 230} 231 232static int __init ioprio_init(void) 233{ 234 return blkcg_policy_register(&ioprio_policy); 235} 236 237static void __exit ioprio_exit(void) 238{ 239 blkcg_policy_unregister(&ioprio_policy); 240} 241 242module_init(ioprio_init); 243module_exit(ioprio_exit); 244