1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Copyright (C) 2020 Oracle Corporation
4 *
5 * Module Author: Mike Christie
6 */
7#include "dm-path-selector.h"
8
9#include <linux/device-mapper.h>
10#include <linux/module.h>
11
12#define DM_MSG_PREFIX "multipath io-affinity"
13
14struct path_info {
15	struct dm_path *path;
16	cpumask_var_t cpumask;
17	refcount_t refcount;
18	bool failed;
19};
20
21struct selector {
22	struct path_info **path_map;
23	cpumask_var_t path_mask;
24	atomic_t map_misses;
25};
26
27static void ioa_free_path(struct selector *s, unsigned int cpu)
28{
29	struct path_info *pi = s->path_map[cpu];
30
31	if (!pi)
32		return;
33
34	if (refcount_dec_and_test(&pi->refcount)) {
35		cpumask_clear_cpu(cpu, s->path_mask);
36		free_cpumask_var(pi->cpumask);
37		kfree(pi);
38
39		s->path_map[cpu] = NULL;
40	}
41}
42
43static int ioa_add_path(struct path_selector *ps, struct dm_path *path,
44			int argc, char **argv, char **error)
45{
46	struct selector *s = ps->context;
47	struct path_info *pi = NULL;
48	unsigned int cpu;
49	int ret;
50
51	if (argc != 1) {
52		*error = "io-affinity ps: invalid number of arguments";
53		return -EINVAL;
54	}
55
56	pi = kzalloc(sizeof(*pi), GFP_KERNEL);
57	if (!pi) {
58		*error = "io-affinity ps: Error allocating path context";
59		return -ENOMEM;
60	}
61
62	pi->path = path;
63	path->pscontext = pi;
64	refcount_set(&pi->refcount, 1);
65
66	if (!zalloc_cpumask_var(&pi->cpumask, GFP_KERNEL)) {
67		*error = "io-affinity ps: Error allocating cpumask context";
68		ret = -ENOMEM;
69		goto free_pi;
70	}
71
72	ret = cpumask_parse(argv[0], pi->cpumask);
73	if (ret) {
74		*error = "io-affinity ps: invalid cpumask";
75		ret = -EINVAL;
76		goto free_mask;
77	}
78
79	for_each_cpu(cpu, pi->cpumask) {
80		if (cpu >= nr_cpu_ids) {
81			DMWARN_LIMIT("Ignoring mapping for CPU %u. Max CPU is %u",
82				     cpu, nr_cpu_ids);
83			break;
84		}
85
86		if (s->path_map[cpu]) {
87			DMWARN("CPU mapping for %u exists. Ignoring.", cpu);
88			continue;
89		}
90
91		cpumask_set_cpu(cpu, s->path_mask);
92		s->path_map[cpu] = pi;
93		refcount_inc(&pi->refcount);
94	}
95
96	if (refcount_dec_and_test(&pi->refcount)) {
97		*error = "io-affinity ps: No new/valid CPU mapping found";
98		ret = -EINVAL;
99		goto free_mask;
100	}
101
102	return 0;
103
104free_mask:
105	free_cpumask_var(pi->cpumask);
106free_pi:
107	kfree(pi);
108	return ret;
109}
110
111static int ioa_create(struct path_selector *ps, unsigned int argc, char **argv)
112{
113	struct selector *s;
114
115	s = kmalloc(sizeof(*s), GFP_KERNEL);
116	if (!s)
117		return -ENOMEM;
118
119	s->path_map = kzalloc(nr_cpu_ids * sizeof(struct path_info *),
120			      GFP_KERNEL);
121	if (!s->path_map)
122		goto free_selector;
123
124	if (!zalloc_cpumask_var(&s->path_mask, GFP_KERNEL))
125		goto free_map;
126
127	atomic_set(&s->map_misses, 0);
128	ps->context = s;
129	return 0;
130
131free_map:
132	kfree(s->path_map);
133free_selector:
134	kfree(s);
135	return -ENOMEM;
136}
137
138static void ioa_destroy(struct path_selector *ps)
139{
140	struct selector *s = ps->context;
141	unsigned int cpu;
142
143	for_each_cpu(cpu, s->path_mask)
144		ioa_free_path(s, cpu);
145
146	free_cpumask_var(s->path_mask);
147	kfree(s->path_map);
148	kfree(s);
149
150	ps->context = NULL;
151}
152
153static int ioa_status(struct path_selector *ps, struct dm_path *path,
154		      status_type_t type, char *result, unsigned int maxlen)
155{
156	struct selector *s = ps->context;
157	struct path_info *pi;
158	int sz = 0;
159
160	if (!path) {
161		DMEMIT("0 ");
162		return sz;
163	}
164
165	switch (type) {
166	case STATUSTYPE_INFO:
167		DMEMIT("%d ", atomic_read(&s->map_misses));
168		break;
169	case STATUSTYPE_TABLE:
170		pi = path->pscontext;
171		DMEMIT("%*pb ", cpumask_pr_args(pi->cpumask));
172		break;
173	case STATUSTYPE_IMA:
174		*result = '\0';
175		break;
176	}
177
178	return sz;
179}
180
181static void ioa_fail_path(struct path_selector *ps, struct dm_path *p)
182{
183	struct path_info *pi = p->pscontext;
184
185	pi->failed = true;
186}
187
188static int ioa_reinstate_path(struct path_selector *ps, struct dm_path *p)
189{
190	struct path_info *pi = p->pscontext;
191
192	pi->failed = false;
193	return 0;
194}
195
196static struct dm_path *ioa_select_path(struct path_selector *ps,
197				       size_t nr_bytes)
198{
199	unsigned int cpu, node;
200	struct selector *s = ps->context;
201	const struct cpumask *cpumask;
202	struct path_info *pi;
203	int i;
204
205	cpu = get_cpu();
206
207	pi = s->path_map[cpu];
208	if (pi && !pi->failed)
209		goto done;
210
211	/*
212	 * Perf is not optimal, but we at least try the local node then just
213	 * try not to fail.
214	 */
215	if (!pi)
216		atomic_inc(&s->map_misses);
217
218	node = cpu_to_node(cpu);
219	cpumask = cpumask_of_node(node);
220	for_each_cpu(i, cpumask) {
221		pi = s->path_map[i];
222		if (pi && !pi->failed)
223			goto done;
224	}
225
226	for_each_cpu(i, s->path_mask) {
227		pi = s->path_map[i];
228		if (pi && !pi->failed)
229			goto done;
230	}
231	pi = NULL;
232
233done:
234	put_cpu();
235	return pi ? pi->path : NULL;
236}
237
238static struct path_selector_type ioa_ps = {
239	.name		= "io-affinity",
240	.module		= THIS_MODULE,
241	.table_args	= 1,
242	.info_args	= 1,
243	.create		= ioa_create,
244	.destroy	= ioa_destroy,
245	.status		= ioa_status,
246	.add_path	= ioa_add_path,
247	.fail_path	= ioa_fail_path,
248	.reinstate_path	= ioa_reinstate_path,
249	.select_path	= ioa_select_path,
250};
251
252static int __init dm_ioa_init(void)
253{
254	int ret = dm_register_path_selector(&ioa_ps);
255
256	if (ret < 0)
257		DMERR("register failed %d", ret);
258	return ret;
259}
260
261static void __exit dm_ioa_exit(void)
262{
263	int ret = dm_unregister_path_selector(&ioa_ps);
264
265	if (ret < 0)
266		DMERR("unregister failed %d", ret);
267}
268
269module_init(dm_ioa_init);
270module_exit(dm_ioa_exit);
271
272MODULE_DESCRIPTION(DM_NAME " multipath path selector that selects paths based on the CPU IO is being executed on");
273MODULE_AUTHOR("Mike Christie <michael.christie@oracle.com>");
274MODULE_LICENSE("GPL");
275