1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Xen grant DMA-mapping layer - contains special DMA-mapping routines
4 * for providing grant references as DMA addresses to be used by frontends
5 * (e.g. virtio) in Xen guests
6 *
7 * Copyright (c) 2021, Juergen Gross <jgross@suse.com>
8 */
9
10#include <linux/module.h>
11#include <linux/dma-map-ops.h>
12#include <linux/of.h>
13#include <linux/pci.h>
14#include <linux/pfn.h>
15#include <linux/xarray.h>
16#include <linux/virtio_anchor.h>
17#include <linux/virtio.h>
18#include <xen/xen.h>
19#include <xen/xen-ops.h>
20#include <xen/grant_table.h>
21
22struct xen_grant_dma_data {
23	/* The ID of backend domain */
24	domid_t backend_domid;
25	/* Is device behaving sane? */
26	bool broken;
27};
28
29static DEFINE_XARRAY_FLAGS(xen_grant_dma_devices, XA_FLAGS_LOCK_IRQ);
30
31#define XEN_GRANT_DMA_ADDR_OFF	(1ULL << 63)
32
33static inline dma_addr_t grant_to_dma(grant_ref_t grant)
34{
35	return XEN_GRANT_DMA_ADDR_OFF | ((dma_addr_t)grant << XEN_PAGE_SHIFT);
36}
37
38static inline grant_ref_t dma_to_grant(dma_addr_t dma)
39{
40	return (grant_ref_t)((dma & ~XEN_GRANT_DMA_ADDR_OFF) >> XEN_PAGE_SHIFT);
41}
42
43static struct xen_grant_dma_data *find_xen_grant_dma_data(struct device *dev)
44{
45	struct xen_grant_dma_data *data;
46	unsigned long flags;
47
48	xa_lock_irqsave(&xen_grant_dma_devices, flags);
49	data = xa_load(&xen_grant_dma_devices, (unsigned long)dev);
50	xa_unlock_irqrestore(&xen_grant_dma_devices, flags);
51
52	return data;
53}
54
55static int store_xen_grant_dma_data(struct device *dev,
56				    struct xen_grant_dma_data *data)
57{
58	unsigned long flags;
59	int ret;
60
61	xa_lock_irqsave(&xen_grant_dma_devices, flags);
62	ret = xa_err(__xa_store(&xen_grant_dma_devices, (unsigned long)dev, data,
63			GFP_ATOMIC));
64	xa_unlock_irqrestore(&xen_grant_dma_devices, flags);
65
66	return ret;
67}
68
69/*
70 * DMA ops for Xen frontends (e.g. virtio).
71 *
72 * Used to act as a kind of software IOMMU for Xen guests by using grants as
73 * DMA addresses.
74 * Such a DMA address is formed by using the grant reference as a frame
75 * number and setting the highest address bit (this bit is for the backend
76 * to be able to distinguish it from e.g. a mmio address).
77 */
78static void *xen_grant_dma_alloc(struct device *dev, size_t size,
79				 dma_addr_t *dma_handle, gfp_t gfp,
80				 unsigned long attrs)
81{
82	struct xen_grant_dma_data *data;
83	unsigned int i, n_pages = XEN_PFN_UP(size);
84	unsigned long pfn;
85	grant_ref_t grant;
86	void *ret;
87
88	data = find_xen_grant_dma_data(dev);
89	if (!data)
90		return NULL;
91
92	if (unlikely(data->broken))
93		return NULL;
94
95	ret = alloc_pages_exact(n_pages * XEN_PAGE_SIZE, gfp);
96	if (!ret)
97		return NULL;
98
99	pfn = virt_to_pfn(ret);
100
101	if (gnttab_alloc_grant_reference_seq(n_pages, &grant)) {
102		free_pages_exact(ret, n_pages * XEN_PAGE_SIZE);
103		return NULL;
104	}
105
106	for (i = 0; i < n_pages; i++) {
107		gnttab_grant_foreign_access_ref(grant + i, data->backend_domid,
108				pfn_to_gfn(pfn + i), 0);
109	}
110
111	*dma_handle = grant_to_dma(grant);
112
113	return ret;
114}
115
116static void xen_grant_dma_free(struct device *dev, size_t size, void *vaddr,
117			       dma_addr_t dma_handle, unsigned long attrs)
118{
119	struct xen_grant_dma_data *data;
120	unsigned int i, n_pages = XEN_PFN_UP(size);
121	grant_ref_t grant;
122
123	data = find_xen_grant_dma_data(dev);
124	if (!data)
125		return;
126
127	if (unlikely(data->broken))
128		return;
129
130	grant = dma_to_grant(dma_handle);
131
132	for (i = 0; i < n_pages; i++) {
133		if (unlikely(!gnttab_end_foreign_access_ref(grant + i))) {
134			dev_alert(dev, "Grant still in use by backend domain, disabled for further use\n");
135			data->broken = true;
136			return;
137		}
138	}
139
140	gnttab_free_grant_reference_seq(grant, n_pages);
141
142	free_pages_exact(vaddr, n_pages * XEN_PAGE_SIZE);
143}
144
145static struct page *xen_grant_dma_alloc_pages(struct device *dev, size_t size,
146					      dma_addr_t *dma_handle,
147					      enum dma_data_direction dir,
148					      gfp_t gfp)
149{
150	void *vaddr;
151
152	vaddr = xen_grant_dma_alloc(dev, size, dma_handle, gfp, 0);
153	if (!vaddr)
154		return NULL;
155
156	return virt_to_page(vaddr);
157}
158
159static void xen_grant_dma_free_pages(struct device *dev, size_t size,
160				     struct page *vaddr, dma_addr_t dma_handle,
161				     enum dma_data_direction dir)
162{
163	xen_grant_dma_free(dev, size, page_to_virt(vaddr), dma_handle, 0);
164}
165
166static dma_addr_t xen_grant_dma_map_page(struct device *dev, struct page *page,
167					 unsigned long offset, size_t size,
168					 enum dma_data_direction dir,
169					 unsigned long attrs)
170{
171	struct xen_grant_dma_data *data;
172	unsigned long dma_offset = xen_offset_in_page(offset),
173			pfn_offset = XEN_PFN_DOWN(offset);
174	unsigned int i, n_pages = XEN_PFN_UP(dma_offset + size);
175	grant_ref_t grant;
176	dma_addr_t dma_handle;
177
178	if (WARN_ON(dir == DMA_NONE))
179		return DMA_MAPPING_ERROR;
180
181	data = find_xen_grant_dma_data(dev);
182	if (!data)
183		return DMA_MAPPING_ERROR;
184
185	if (unlikely(data->broken))
186		return DMA_MAPPING_ERROR;
187
188	if (gnttab_alloc_grant_reference_seq(n_pages, &grant))
189		return DMA_MAPPING_ERROR;
190
191	for (i = 0; i < n_pages; i++) {
192		gnttab_grant_foreign_access_ref(grant + i, data->backend_domid,
193				pfn_to_gfn(page_to_xen_pfn(page) + i + pfn_offset),
194				dir == DMA_TO_DEVICE);
195	}
196
197	dma_handle = grant_to_dma(grant) + dma_offset;
198
199	return dma_handle;
200}
201
202static void xen_grant_dma_unmap_page(struct device *dev, dma_addr_t dma_handle,
203				     size_t size, enum dma_data_direction dir,
204				     unsigned long attrs)
205{
206	struct xen_grant_dma_data *data;
207	unsigned long dma_offset = xen_offset_in_page(dma_handle);
208	unsigned int i, n_pages = XEN_PFN_UP(dma_offset + size);
209	grant_ref_t grant;
210
211	if (WARN_ON(dir == DMA_NONE))
212		return;
213
214	data = find_xen_grant_dma_data(dev);
215	if (!data)
216		return;
217
218	if (unlikely(data->broken))
219		return;
220
221	grant = dma_to_grant(dma_handle);
222
223	for (i = 0; i < n_pages; i++) {
224		if (unlikely(!gnttab_end_foreign_access_ref(grant + i))) {
225			dev_alert(dev, "Grant still in use by backend domain, disabled for further use\n");
226			data->broken = true;
227			return;
228		}
229	}
230
231	gnttab_free_grant_reference_seq(grant, n_pages);
232}
233
234static void xen_grant_dma_unmap_sg(struct device *dev, struct scatterlist *sg,
235				   int nents, enum dma_data_direction dir,
236				   unsigned long attrs)
237{
238	struct scatterlist *s;
239	unsigned int i;
240
241	if (WARN_ON(dir == DMA_NONE))
242		return;
243
244	for_each_sg(sg, s, nents, i)
245		xen_grant_dma_unmap_page(dev, s->dma_address, sg_dma_len(s), dir,
246				attrs);
247}
248
249static int xen_grant_dma_map_sg(struct device *dev, struct scatterlist *sg,
250				int nents, enum dma_data_direction dir,
251				unsigned long attrs)
252{
253	struct scatterlist *s;
254	unsigned int i;
255
256	if (WARN_ON(dir == DMA_NONE))
257		return -EINVAL;
258
259	for_each_sg(sg, s, nents, i) {
260		s->dma_address = xen_grant_dma_map_page(dev, sg_page(s), s->offset,
261				s->length, dir, attrs);
262		if (s->dma_address == DMA_MAPPING_ERROR)
263			goto out;
264
265		sg_dma_len(s) = s->length;
266	}
267
268	return nents;
269
270out:
271	xen_grant_dma_unmap_sg(dev, sg, i, dir, attrs | DMA_ATTR_SKIP_CPU_SYNC);
272	sg_dma_len(sg) = 0;
273
274	return -EIO;
275}
276
277static int xen_grant_dma_supported(struct device *dev, u64 mask)
278{
279	return mask == DMA_BIT_MASK(64);
280}
281
282static const struct dma_map_ops xen_grant_dma_ops = {
283	.alloc = xen_grant_dma_alloc,
284	.free = xen_grant_dma_free,
285	.alloc_pages = xen_grant_dma_alloc_pages,
286	.free_pages = xen_grant_dma_free_pages,
287	.mmap = dma_common_mmap,
288	.get_sgtable = dma_common_get_sgtable,
289	.map_page = xen_grant_dma_map_page,
290	.unmap_page = xen_grant_dma_unmap_page,
291	.map_sg = xen_grant_dma_map_sg,
292	.unmap_sg = xen_grant_dma_unmap_sg,
293	.dma_supported = xen_grant_dma_supported,
294};
295
296static struct device_node *xen_dt_get_node(struct device *dev)
297{
298	if (dev_is_pci(dev)) {
299		struct pci_dev *pdev = to_pci_dev(dev);
300		struct pci_bus *bus = pdev->bus;
301
302		/* Walk up to the root bus to look for PCI Host controller */
303		while (!pci_is_root_bus(bus))
304			bus = bus->parent;
305
306		if (!bus->bridge->parent)
307			return NULL;
308		return of_node_get(bus->bridge->parent->of_node);
309	}
310
311	return of_node_get(dev->of_node);
312}
313
314static int xen_dt_grant_init_backend_domid(struct device *dev,
315					   struct device_node *np,
316					   domid_t *backend_domid)
317{
318	struct of_phandle_args iommu_spec = { .args_count = 1 };
319
320	if (dev_is_pci(dev)) {
321		struct pci_dev *pdev = to_pci_dev(dev);
322		u32 rid = PCI_DEVID(pdev->bus->number, pdev->devfn);
323
324		if (of_map_id(np, rid, "iommu-map", "iommu-map-mask", &iommu_spec.np,
325				iommu_spec.args)) {
326			dev_dbg(dev, "Cannot translate ID\n");
327			return -ESRCH;
328		}
329	} else {
330		if (of_parse_phandle_with_args(np, "iommus", "#iommu-cells",
331				0, &iommu_spec)) {
332			dev_dbg(dev, "Cannot parse iommus property\n");
333			return -ESRCH;
334		}
335	}
336
337	if (!of_device_is_compatible(iommu_spec.np, "xen,grant-dma") ||
338			iommu_spec.args_count != 1) {
339		dev_dbg(dev, "Incompatible IOMMU node\n");
340		of_node_put(iommu_spec.np);
341		return -ESRCH;
342	}
343
344	of_node_put(iommu_spec.np);
345
346	/*
347	 * The endpoint ID here means the ID of the domain where the
348	 * corresponding backend is running
349	 */
350	*backend_domid = iommu_spec.args[0];
351
352	return 0;
353}
354
355static int xen_grant_init_backend_domid(struct device *dev,
356					domid_t *backend_domid)
357{
358	struct device_node *np;
359	int ret = -ENODEV;
360
361	np = xen_dt_get_node(dev);
362	if (np) {
363		ret = xen_dt_grant_init_backend_domid(dev, np, backend_domid);
364		of_node_put(np);
365	} else if (IS_ENABLED(CONFIG_XEN_VIRTIO_FORCE_GRANT) || xen_pv_domain()) {
366		dev_info(dev, "Using dom0 as backend\n");
367		*backend_domid = 0;
368		ret = 0;
369	}
370
371	return ret;
372}
373
374static void xen_grant_setup_dma_ops(struct device *dev, domid_t backend_domid)
375{
376	struct xen_grant_dma_data *data;
377
378	data = find_xen_grant_dma_data(dev);
379	if (data) {
380		dev_err(dev, "Xen grant DMA data is already created\n");
381		return;
382	}
383
384	data = devm_kzalloc(dev, sizeof(*data), GFP_KERNEL);
385	if (!data)
386		goto err;
387
388	data->backend_domid = backend_domid;
389
390	if (store_xen_grant_dma_data(dev, data)) {
391		dev_err(dev, "Cannot store Xen grant DMA data\n");
392		goto err;
393	}
394
395	dev->dma_ops = &xen_grant_dma_ops;
396
397	return;
398
399err:
400	devm_kfree(dev, data);
401	dev_err(dev, "Cannot set up Xen grant DMA ops, retain platform DMA ops\n");
402}
403
404bool xen_virtio_restricted_mem_acc(struct virtio_device *dev)
405{
406	domid_t backend_domid;
407
408	if (!xen_grant_init_backend_domid(dev->dev.parent, &backend_domid)) {
409		xen_grant_setup_dma_ops(dev->dev.parent, backend_domid);
410		return true;
411	}
412
413	return false;
414}
415
416MODULE_DESCRIPTION("Xen grant DMA-mapping layer");
417MODULE_AUTHOR("Juergen Gross <jgross@suse.com>");
418MODULE_LICENSE("GPL");
419