1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * VIRTIO based driver for vDPA device
4 *
5 * Copyright (c) 2020, Red Hat. All rights reserved.
6 *     Author: Jason Wang <jasowang@redhat.com>
7 *
8 */
9
10#include <linux/init.h>
11#include <linux/module.h>
12#include <linux/device.h>
13#include <linux/kernel.h>
14#include <linux/slab.h>
15#include <linux/uuid.h>
16#include <linux/group_cpus.h>
17#include <linux/virtio.h>
18#include <linux/vdpa.h>
19#include <linux/virtio_config.h>
20#include <linux/virtio_ring.h>
21
22#define MOD_VERSION  "0.1"
23#define MOD_AUTHOR   "Jason Wang <jasowang@redhat.com>"
24#define MOD_DESC     "vDPA bus driver for virtio devices"
25#define MOD_LICENSE  "GPL v2"
26
27struct virtio_vdpa_device {
28	struct virtio_device vdev;
29	struct vdpa_device *vdpa;
30	u64 features;
31
32	/* The lock to protect virtqueue list */
33	spinlock_t lock;
34	/* List of virtio_vdpa_vq_info */
35	struct list_head virtqueues;
36};
37
38struct virtio_vdpa_vq_info {
39	/* the actual virtqueue */
40	struct virtqueue *vq;
41
42	/* the list node for the virtqueues list */
43	struct list_head node;
44};
45
46static inline struct virtio_vdpa_device *
47to_virtio_vdpa_device(struct virtio_device *dev)
48{
49	return container_of(dev, struct virtio_vdpa_device, vdev);
50}
51
52static struct vdpa_device *vd_get_vdpa(struct virtio_device *vdev)
53{
54	return to_virtio_vdpa_device(vdev)->vdpa;
55}
56
57static void virtio_vdpa_get(struct virtio_device *vdev, unsigned int offset,
58			    void *buf, unsigned int len)
59{
60	struct vdpa_device *vdpa = vd_get_vdpa(vdev);
61
62	vdpa_get_config(vdpa, offset, buf, len);
63}
64
65static void virtio_vdpa_set(struct virtio_device *vdev, unsigned int offset,
66			    const void *buf, unsigned int len)
67{
68	struct vdpa_device *vdpa = vd_get_vdpa(vdev);
69
70	vdpa_set_config(vdpa, offset, buf, len);
71}
72
73static u32 virtio_vdpa_generation(struct virtio_device *vdev)
74{
75	struct vdpa_device *vdpa = vd_get_vdpa(vdev);
76	const struct vdpa_config_ops *ops = vdpa->config;
77
78	if (ops->get_generation)
79		return ops->get_generation(vdpa);
80
81	return 0;
82}
83
84static u8 virtio_vdpa_get_status(struct virtio_device *vdev)
85{
86	struct vdpa_device *vdpa = vd_get_vdpa(vdev);
87	const struct vdpa_config_ops *ops = vdpa->config;
88
89	return ops->get_status(vdpa);
90}
91
92static void virtio_vdpa_set_status(struct virtio_device *vdev, u8 status)
93{
94	struct vdpa_device *vdpa = vd_get_vdpa(vdev);
95
96	return vdpa_set_status(vdpa, status);
97}
98
99static void virtio_vdpa_reset(struct virtio_device *vdev)
100{
101	struct vdpa_device *vdpa = vd_get_vdpa(vdev);
102
103	vdpa_reset(vdpa, 0);
104}
105
106static bool virtio_vdpa_notify(struct virtqueue *vq)
107{
108	struct vdpa_device *vdpa = vd_get_vdpa(vq->vdev);
109	const struct vdpa_config_ops *ops = vdpa->config;
110
111	ops->kick_vq(vdpa, vq->index);
112
113	return true;
114}
115
116static bool virtio_vdpa_notify_with_data(struct virtqueue *vq)
117{
118	struct vdpa_device *vdpa = vd_get_vdpa(vq->vdev);
119	const struct vdpa_config_ops *ops = vdpa->config;
120	u32 data = vring_notification_data(vq);
121
122	ops->kick_vq_with_data(vdpa, data);
123
124	return true;
125}
126
127static irqreturn_t virtio_vdpa_config_cb(void *private)
128{
129	struct virtio_vdpa_device *vd_dev = private;
130
131	virtio_config_changed(&vd_dev->vdev);
132
133	return IRQ_HANDLED;
134}
135
136static irqreturn_t virtio_vdpa_virtqueue_cb(void *private)
137{
138	struct virtio_vdpa_vq_info *info = private;
139
140	return vring_interrupt(0, info->vq);
141}
142
143static struct virtqueue *
144virtio_vdpa_setup_vq(struct virtio_device *vdev, unsigned int index,
145		     void (*callback)(struct virtqueue *vq),
146		     const char *name, bool ctx)
147{
148	struct virtio_vdpa_device *vd_dev = to_virtio_vdpa_device(vdev);
149	struct vdpa_device *vdpa = vd_get_vdpa(vdev);
150	struct device *dma_dev;
151	const struct vdpa_config_ops *ops = vdpa->config;
152	struct virtio_vdpa_vq_info *info;
153	bool (*notify)(struct virtqueue *vq) = virtio_vdpa_notify;
154	struct vdpa_callback cb;
155	struct virtqueue *vq;
156	u64 desc_addr, driver_addr, device_addr;
157	/* Assume split virtqueue, switch to packed if necessary */
158	struct vdpa_vq_state state = {0};
159	unsigned long flags;
160	u32 align, max_num, min_num = 1;
161	bool may_reduce_num = true;
162	int err;
163
164	if (!name)
165		return NULL;
166
167	if (index >= vdpa->nvqs)
168		return ERR_PTR(-ENOENT);
169
170	/* We cannot accept VIRTIO_F_NOTIFICATION_DATA without kick_vq_with_data */
171	if (__virtio_test_bit(vdev, VIRTIO_F_NOTIFICATION_DATA)) {
172		if (ops->kick_vq_with_data)
173			notify = virtio_vdpa_notify_with_data;
174		else
175			__virtio_clear_bit(vdev, VIRTIO_F_NOTIFICATION_DATA);
176	}
177
178	/* Queue shouldn't already be set up. */
179	if (ops->get_vq_ready(vdpa, index))
180		return ERR_PTR(-ENOENT);
181
182	/* Allocate and fill out our active queue description */
183	info = kmalloc(sizeof(*info), GFP_KERNEL);
184	if (!info)
185		return ERR_PTR(-ENOMEM);
186	if (ops->get_vq_size)
187		max_num = ops->get_vq_size(vdpa, index);
188	else
189		max_num = ops->get_vq_num_max(vdpa);
190
191	if (max_num == 0) {
192		err = -ENOENT;
193		goto error_new_virtqueue;
194	}
195
196	if (ops->get_vq_num_min)
197		min_num = ops->get_vq_num_min(vdpa);
198
199	may_reduce_num = (max_num == min_num) ? false : true;
200
201	/* Create the vring */
202	align = ops->get_vq_align(vdpa);
203
204	if (ops->get_vq_dma_dev)
205		dma_dev = ops->get_vq_dma_dev(vdpa, index);
206	else
207		dma_dev = vdpa_get_dma_dev(vdpa);
208	vq = vring_create_virtqueue_dma(index, max_num, align, vdev,
209					true, may_reduce_num, ctx,
210					notify, callback, name, dma_dev);
211	if (!vq) {
212		err = -ENOMEM;
213		goto error_new_virtqueue;
214	}
215
216	vq->num_max = max_num;
217
218	/* Setup virtqueue callback */
219	cb.callback = callback ? virtio_vdpa_virtqueue_cb : NULL;
220	cb.private = info;
221	cb.trigger = NULL;
222	ops->set_vq_cb(vdpa, index, &cb);
223	ops->set_vq_num(vdpa, index, virtqueue_get_vring_size(vq));
224
225	desc_addr = virtqueue_get_desc_addr(vq);
226	driver_addr = virtqueue_get_avail_addr(vq);
227	device_addr = virtqueue_get_used_addr(vq);
228
229	if (ops->set_vq_address(vdpa, index,
230				desc_addr, driver_addr,
231				device_addr)) {
232		err = -EINVAL;
233		goto err_vq;
234	}
235
236	/* reset virtqueue state index */
237	if (virtio_has_feature(vdev, VIRTIO_F_RING_PACKED)) {
238		struct vdpa_vq_state_packed *s = &state.packed;
239
240		s->last_avail_counter = 1;
241		s->last_avail_idx = 0;
242		s->last_used_counter = 1;
243		s->last_used_idx = 0;
244	}
245	err = ops->set_vq_state(vdpa, index, &state);
246	if (err)
247		goto err_vq;
248
249	ops->set_vq_ready(vdpa, index, 1);
250
251	vq->priv = info;
252	info->vq = vq;
253
254	spin_lock_irqsave(&vd_dev->lock, flags);
255	list_add(&info->node, &vd_dev->virtqueues);
256	spin_unlock_irqrestore(&vd_dev->lock, flags);
257
258	return vq;
259
260err_vq:
261	vring_del_virtqueue(vq);
262error_new_virtqueue:
263	ops->set_vq_ready(vdpa, index, 0);
264	/* VDPA driver should make sure vq is stopeed here */
265	WARN_ON(ops->get_vq_ready(vdpa, index));
266	kfree(info);
267	return ERR_PTR(err);
268}
269
270static void virtio_vdpa_del_vq(struct virtqueue *vq)
271{
272	struct virtio_vdpa_device *vd_dev = to_virtio_vdpa_device(vq->vdev);
273	struct vdpa_device *vdpa = vd_dev->vdpa;
274	const struct vdpa_config_ops *ops = vdpa->config;
275	struct virtio_vdpa_vq_info *info = vq->priv;
276	unsigned int index = vq->index;
277	unsigned long flags;
278
279	spin_lock_irqsave(&vd_dev->lock, flags);
280	list_del(&info->node);
281	spin_unlock_irqrestore(&vd_dev->lock, flags);
282
283	/* Select and deactivate the queue (best effort) */
284	ops->set_vq_ready(vdpa, index, 0);
285
286	vring_del_virtqueue(vq);
287
288	kfree(info);
289}
290
291static void virtio_vdpa_del_vqs(struct virtio_device *vdev)
292{
293	struct virtqueue *vq, *n;
294
295	list_for_each_entry_safe(vq, n, &vdev->vqs, list)
296		virtio_vdpa_del_vq(vq);
297}
298
299static void default_calc_sets(struct irq_affinity *affd, unsigned int affvecs)
300{
301	affd->nr_sets = 1;
302	affd->set_size[0] = affvecs;
303}
304
305static struct cpumask *
306create_affinity_masks(unsigned int nvecs, struct irq_affinity *affd)
307{
308	unsigned int affvecs = 0, curvec, usedvecs, i;
309	struct cpumask *masks = NULL;
310
311	if (nvecs > affd->pre_vectors + affd->post_vectors)
312		affvecs = nvecs - affd->pre_vectors - affd->post_vectors;
313
314	if (!affd->calc_sets)
315		affd->calc_sets = default_calc_sets;
316
317	affd->calc_sets(affd, affvecs);
318
319	if (!affvecs)
320		return NULL;
321
322	masks = kcalloc(nvecs, sizeof(*masks), GFP_KERNEL);
323	if (!masks)
324		return NULL;
325
326	/* Fill out vectors at the beginning that don't need affinity */
327	for (curvec = 0; curvec < affd->pre_vectors; curvec++)
328		cpumask_setall(&masks[curvec]);
329
330	for (i = 0, usedvecs = 0; i < affd->nr_sets; i++) {
331		unsigned int this_vecs = affd->set_size[i];
332		int j;
333		struct cpumask *result = group_cpus_evenly(this_vecs);
334
335		if (!result) {
336			kfree(masks);
337			return NULL;
338		}
339
340		for (j = 0; j < this_vecs; j++)
341			cpumask_copy(&masks[curvec + j], &result[j]);
342		kfree(result);
343
344		curvec += this_vecs;
345		usedvecs += this_vecs;
346	}
347
348	/* Fill out vectors at the end that don't need affinity */
349	if (usedvecs >= affvecs)
350		curvec = affd->pre_vectors + affvecs;
351	else
352		curvec = affd->pre_vectors + usedvecs;
353	for (; curvec < nvecs; curvec++)
354		cpumask_setall(&masks[curvec]);
355
356	return masks;
357}
358
359static int virtio_vdpa_find_vqs(struct virtio_device *vdev, unsigned int nvqs,
360				struct virtqueue *vqs[],
361				vq_callback_t *callbacks[],
362				const char * const names[],
363				const bool *ctx,
364				struct irq_affinity *desc)
365{
366	struct virtio_vdpa_device *vd_dev = to_virtio_vdpa_device(vdev);
367	struct vdpa_device *vdpa = vd_get_vdpa(vdev);
368	const struct vdpa_config_ops *ops = vdpa->config;
369	struct irq_affinity default_affd = { 0 };
370	struct cpumask *masks;
371	struct vdpa_callback cb;
372	bool has_affinity = desc && ops->set_vq_affinity;
373	int i, err, queue_idx = 0;
374
375	if (has_affinity) {
376		masks = create_affinity_masks(nvqs, desc ? desc : &default_affd);
377		if (!masks)
378			return -ENOMEM;
379	}
380
381	for (i = 0; i < nvqs; ++i) {
382		if (!names[i]) {
383			vqs[i] = NULL;
384			continue;
385		}
386
387		vqs[i] = virtio_vdpa_setup_vq(vdev, queue_idx++,
388					      callbacks[i], names[i], ctx ?
389					      ctx[i] : false);
390		if (IS_ERR(vqs[i])) {
391			err = PTR_ERR(vqs[i]);
392			goto err_setup_vq;
393		}
394
395		if (has_affinity)
396			ops->set_vq_affinity(vdpa, i, &masks[i]);
397	}
398
399	cb.callback = virtio_vdpa_config_cb;
400	cb.private = vd_dev;
401	ops->set_config_cb(vdpa, &cb);
402	if (has_affinity)
403		kfree(masks);
404
405	return 0;
406
407err_setup_vq:
408	virtio_vdpa_del_vqs(vdev);
409	if (has_affinity)
410		kfree(masks);
411	return err;
412}
413
414static u64 virtio_vdpa_get_features(struct virtio_device *vdev)
415{
416	struct vdpa_device *vdpa = vd_get_vdpa(vdev);
417	const struct vdpa_config_ops *ops = vdpa->config;
418
419	return ops->get_device_features(vdpa);
420}
421
422static int virtio_vdpa_finalize_features(struct virtio_device *vdev)
423{
424	struct vdpa_device *vdpa = vd_get_vdpa(vdev);
425
426	/* Give virtio_ring a chance to accept features. */
427	vring_transport_features(vdev);
428
429	return vdpa_set_features(vdpa, vdev->features);
430}
431
432static const char *virtio_vdpa_bus_name(struct virtio_device *vdev)
433{
434	struct virtio_vdpa_device *vd_dev = to_virtio_vdpa_device(vdev);
435	struct vdpa_device *vdpa = vd_dev->vdpa;
436
437	return dev_name(&vdpa->dev);
438}
439
440static int virtio_vdpa_set_vq_affinity(struct virtqueue *vq,
441				       const struct cpumask *cpu_mask)
442{
443	struct virtio_vdpa_device *vd_dev = to_virtio_vdpa_device(vq->vdev);
444	struct vdpa_device *vdpa = vd_dev->vdpa;
445	const struct vdpa_config_ops *ops = vdpa->config;
446	unsigned int index = vq->index;
447
448	if (ops->set_vq_affinity)
449		return ops->set_vq_affinity(vdpa, index, cpu_mask);
450
451	return 0;
452}
453
454static const struct cpumask *
455virtio_vdpa_get_vq_affinity(struct virtio_device *vdev, int index)
456{
457	struct vdpa_device *vdpa = vd_get_vdpa(vdev);
458	const struct vdpa_config_ops *ops = vdpa->config;
459
460	if (ops->get_vq_affinity)
461		return ops->get_vq_affinity(vdpa, index);
462
463	return NULL;
464}
465
466static const struct virtio_config_ops virtio_vdpa_config_ops = {
467	.get		= virtio_vdpa_get,
468	.set		= virtio_vdpa_set,
469	.generation	= virtio_vdpa_generation,
470	.get_status	= virtio_vdpa_get_status,
471	.set_status	= virtio_vdpa_set_status,
472	.reset		= virtio_vdpa_reset,
473	.find_vqs	= virtio_vdpa_find_vqs,
474	.del_vqs	= virtio_vdpa_del_vqs,
475	.get_features	= virtio_vdpa_get_features,
476	.finalize_features = virtio_vdpa_finalize_features,
477	.bus_name	= virtio_vdpa_bus_name,
478	.set_vq_affinity = virtio_vdpa_set_vq_affinity,
479	.get_vq_affinity = virtio_vdpa_get_vq_affinity,
480};
481
482static void virtio_vdpa_release_dev(struct device *_d)
483{
484	struct virtio_device *vdev =
485	       container_of(_d, struct virtio_device, dev);
486	struct virtio_vdpa_device *vd_dev =
487	       container_of(vdev, struct virtio_vdpa_device, vdev);
488
489	kfree(vd_dev);
490}
491
492static int virtio_vdpa_probe(struct vdpa_device *vdpa)
493{
494	const struct vdpa_config_ops *ops = vdpa->config;
495	struct virtio_vdpa_device *vd_dev, *reg_dev = NULL;
496	int ret = -EINVAL;
497
498	vd_dev = kzalloc(sizeof(*vd_dev), GFP_KERNEL);
499	if (!vd_dev)
500		return -ENOMEM;
501
502	vd_dev->vdev.dev.parent = vdpa_get_dma_dev(vdpa);
503	vd_dev->vdev.dev.release = virtio_vdpa_release_dev;
504	vd_dev->vdev.config = &virtio_vdpa_config_ops;
505	vd_dev->vdpa = vdpa;
506	INIT_LIST_HEAD(&vd_dev->virtqueues);
507	spin_lock_init(&vd_dev->lock);
508
509	vd_dev->vdev.id.device = ops->get_device_id(vdpa);
510	if (vd_dev->vdev.id.device == 0)
511		goto err;
512
513	vd_dev->vdev.id.vendor = ops->get_vendor_id(vdpa);
514	ret = register_virtio_device(&vd_dev->vdev);
515	reg_dev = vd_dev;
516	if (ret)
517		goto err;
518
519	vdpa_set_drvdata(vdpa, vd_dev);
520
521	return 0;
522
523err:
524	if (reg_dev)
525		put_device(&vd_dev->vdev.dev);
526	else
527		kfree(vd_dev);
528	return ret;
529}
530
531static void virtio_vdpa_remove(struct vdpa_device *vdpa)
532{
533	struct virtio_vdpa_device *vd_dev = vdpa_get_drvdata(vdpa);
534
535	unregister_virtio_device(&vd_dev->vdev);
536}
537
538static struct vdpa_driver virtio_vdpa_driver = {
539	.driver = {
540		.name	= "virtio_vdpa",
541	},
542	.probe	= virtio_vdpa_probe,
543	.remove = virtio_vdpa_remove,
544};
545
546module_vdpa_driver(virtio_vdpa_driver);
547
548MODULE_VERSION(MOD_VERSION);
549MODULE_LICENSE(MOD_LICENSE);
550MODULE_AUTHOR(MOD_AUTHOR);
551MODULE_DESCRIPTION(MOD_DESC);
552