1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES
4 */
5#include <linux/interval_tree.h>
6#include <linux/iommufd.h>
7#include <linux/iommu.h>
8#include <uapi/linux/iommufd.h>
9
10#include "io_pagetable.h"
11
12void iommufd_ioas_destroy(struct iommufd_object *obj)
13{
14	struct iommufd_ioas *ioas = container_of(obj, struct iommufd_ioas, obj);
15	int rc;
16
17	rc = iopt_unmap_all(&ioas->iopt, NULL);
18	WARN_ON(rc && rc != -ENOENT);
19	iopt_destroy_table(&ioas->iopt);
20	mutex_destroy(&ioas->mutex);
21}
22
23struct iommufd_ioas *iommufd_ioas_alloc(struct iommufd_ctx *ictx)
24{
25	struct iommufd_ioas *ioas;
26
27	ioas = iommufd_object_alloc(ictx, ioas, IOMMUFD_OBJ_IOAS);
28	if (IS_ERR(ioas))
29		return ioas;
30
31	iopt_init_table(&ioas->iopt);
32	INIT_LIST_HEAD(&ioas->hwpt_list);
33	mutex_init(&ioas->mutex);
34	return ioas;
35}
36
37int iommufd_ioas_alloc_ioctl(struct iommufd_ucmd *ucmd)
38{
39	struct iommu_ioas_alloc *cmd = ucmd->cmd;
40	struct iommufd_ioas *ioas;
41	int rc;
42
43	if (cmd->flags)
44		return -EOPNOTSUPP;
45
46	ioas = iommufd_ioas_alloc(ucmd->ictx);
47	if (IS_ERR(ioas))
48		return PTR_ERR(ioas);
49
50	cmd->out_ioas_id = ioas->obj.id;
51	rc = iommufd_ucmd_respond(ucmd, sizeof(*cmd));
52	if (rc)
53		goto out_table;
54	iommufd_object_finalize(ucmd->ictx, &ioas->obj);
55	return 0;
56
57out_table:
58	iommufd_object_abort_and_destroy(ucmd->ictx, &ioas->obj);
59	return rc;
60}
61
62int iommufd_ioas_iova_ranges(struct iommufd_ucmd *ucmd)
63{
64	struct iommu_iova_range __user *ranges;
65	struct iommu_ioas_iova_ranges *cmd = ucmd->cmd;
66	struct iommufd_ioas *ioas;
67	struct interval_tree_span_iter span;
68	u32 max_iovas;
69	int rc;
70
71	if (cmd->__reserved)
72		return -EOPNOTSUPP;
73
74	ioas = iommufd_get_ioas(ucmd->ictx, cmd->ioas_id);
75	if (IS_ERR(ioas))
76		return PTR_ERR(ioas);
77
78	down_read(&ioas->iopt.iova_rwsem);
79	max_iovas = cmd->num_iovas;
80	ranges = u64_to_user_ptr(cmd->allowed_iovas);
81	cmd->num_iovas = 0;
82	cmd->out_iova_alignment = ioas->iopt.iova_alignment;
83	interval_tree_for_each_span(&span, &ioas->iopt.reserved_itree, 0,
84				    ULONG_MAX) {
85		if (!span.is_hole)
86			continue;
87		if (cmd->num_iovas < max_iovas) {
88			struct iommu_iova_range elm = {
89				.start = span.start_hole,
90				.last = span.last_hole,
91			};
92
93			if (copy_to_user(&ranges[cmd->num_iovas], &elm,
94					 sizeof(elm))) {
95				rc = -EFAULT;
96				goto out_put;
97			}
98		}
99		cmd->num_iovas++;
100	}
101	rc = iommufd_ucmd_respond(ucmd, sizeof(*cmd));
102	if (rc)
103		goto out_put;
104	if (cmd->num_iovas > max_iovas)
105		rc = -EMSGSIZE;
106out_put:
107	up_read(&ioas->iopt.iova_rwsem);
108	iommufd_put_object(ucmd->ictx, &ioas->obj);
109	return rc;
110}
111
112static int iommufd_ioas_load_iovas(struct rb_root_cached *itree,
113				   struct iommu_iova_range __user *ranges,
114				   u32 num)
115{
116	u32 i;
117
118	for (i = 0; i != num; i++) {
119		struct iommu_iova_range range;
120		struct iopt_allowed *allowed;
121
122		if (copy_from_user(&range, ranges + i, sizeof(range)))
123			return -EFAULT;
124
125		if (range.start >= range.last)
126			return -EINVAL;
127
128		if (interval_tree_iter_first(itree, range.start, range.last))
129			return -EINVAL;
130
131		allowed = kzalloc(sizeof(*allowed), GFP_KERNEL_ACCOUNT);
132		if (!allowed)
133			return -ENOMEM;
134		allowed->node.start = range.start;
135		allowed->node.last = range.last;
136
137		interval_tree_insert(&allowed->node, itree);
138	}
139	return 0;
140}
141
142int iommufd_ioas_allow_iovas(struct iommufd_ucmd *ucmd)
143{
144	struct iommu_ioas_allow_iovas *cmd = ucmd->cmd;
145	struct rb_root_cached allowed_iova = RB_ROOT_CACHED;
146	struct interval_tree_node *node;
147	struct iommufd_ioas *ioas;
148	struct io_pagetable *iopt;
149	int rc = 0;
150
151	if (cmd->__reserved)
152		return -EOPNOTSUPP;
153
154	ioas = iommufd_get_ioas(ucmd->ictx, cmd->ioas_id);
155	if (IS_ERR(ioas))
156		return PTR_ERR(ioas);
157	iopt = &ioas->iopt;
158
159	rc = iommufd_ioas_load_iovas(&allowed_iova,
160				     u64_to_user_ptr(cmd->allowed_iovas),
161				     cmd->num_iovas);
162	if (rc)
163		goto out_free;
164
165	/*
166	 * We want the allowed tree update to be atomic, so we have to keep the
167	 * original nodes around, and keep track of the new nodes as we allocate
168	 * memory for them. The simplest solution is to have a new/old tree and
169	 * then swap new for old. On success we free the old tree, on failure we
170	 * free the new tree.
171	 */
172	rc = iopt_set_allow_iova(iopt, &allowed_iova);
173out_free:
174	while ((node = interval_tree_iter_first(&allowed_iova, 0, ULONG_MAX))) {
175		interval_tree_remove(node, &allowed_iova);
176		kfree(container_of(node, struct iopt_allowed, node));
177	}
178	iommufd_put_object(ucmd->ictx, &ioas->obj);
179	return rc;
180}
181
182static int conv_iommu_prot(u32 map_flags)
183{
184	/*
185	 * We provide no manual cache coherency ioctls to userspace and most
186	 * architectures make the CPU ops for cache flushing privileged.
187	 * Therefore we require the underlying IOMMU to support CPU coherent
188	 * operation. Support for IOMMU_CACHE is enforced by the
189	 * IOMMU_CAP_CACHE_COHERENCY test during bind.
190	 */
191	int iommu_prot = IOMMU_CACHE;
192
193	if (map_flags & IOMMU_IOAS_MAP_WRITEABLE)
194		iommu_prot |= IOMMU_WRITE;
195	if (map_flags & IOMMU_IOAS_MAP_READABLE)
196		iommu_prot |= IOMMU_READ;
197	return iommu_prot;
198}
199
200int iommufd_ioas_map(struct iommufd_ucmd *ucmd)
201{
202	struct iommu_ioas_map *cmd = ucmd->cmd;
203	unsigned long iova = cmd->iova;
204	struct iommufd_ioas *ioas;
205	unsigned int flags = 0;
206	int rc;
207
208	if ((cmd->flags &
209	     ~(IOMMU_IOAS_MAP_FIXED_IOVA | IOMMU_IOAS_MAP_WRITEABLE |
210	       IOMMU_IOAS_MAP_READABLE)) ||
211	    cmd->__reserved)
212		return -EOPNOTSUPP;
213	if (cmd->iova >= ULONG_MAX || cmd->length >= ULONG_MAX)
214		return -EOVERFLOW;
215
216	ioas = iommufd_get_ioas(ucmd->ictx, cmd->ioas_id);
217	if (IS_ERR(ioas))
218		return PTR_ERR(ioas);
219
220	if (!(cmd->flags & IOMMU_IOAS_MAP_FIXED_IOVA))
221		flags = IOPT_ALLOC_IOVA;
222	rc = iopt_map_user_pages(ucmd->ictx, &ioas->iopt, &iova,
223				 u64_to_user_ptr(cmd->user_va), cmd->length,
224				 conv_iommu_prot(cmd->flags), flags);
225	if (rc)
226		goto out_put;
227
228	cmd->iova = iova;
229	rc = iommufd_ucmd_respond(ucmd, sizeof(*cmd));
230out_put:
231	iommufd_put_object(ucmd->ictx, &ioas->obj);
232	return rc;
233}
234
235int iommufd_ioas_copy(struct iommufd_ucmd *ucmd)
236{
237	struct iommu_ioas_copy *cmd = ucmd->cmd;
238	struct iommufd_ioas *src_ioas;
239	struct iommufd_ioas *dst_ioas;
240	unsigned int flags = 0;
241	LIST_HEAD(pages_list);
242	unsigned long iova;
243	int rc;
244
245	iommufd_test_syz_conv_iova_id(ucmd, cmd->src_ioas_id, &cmd->src_iova,
246				      &cmd->flags);
247
248	if ((cmd->flags &
249	     ~(IOMMU_IOAS_MAP_FIXED_IOVA | IOMMU_IOAS_MAP_WRITEABLE |
250	       IOMMU_IOAS_MAP_READABLE)))
251		return -EOPNOTSUPP;
252	if (cmd->length >= ULONG_MAX || cmd->src_iova >= ULONG_MAX ||
253	    cmd->dst_iova >= ULONG_MAX)
254		return -EOVERFLOW;
255
256	src_ioas = iommufd_get_ioas(ucmd->ictx, cmd->src_ioas_id);
257	if (IS_ERR(src_ioas))
258		return PTR_ERR(src_ioas);
259	rc = iopt_get_pages(&src_ioas->iopt, cmd->src_iova, cmd->length,
260			    &pages_list);
261	iommufd_put_object(ucmd->ictx, &src_ioas->obj);
262	if (rc)
263		return rc;
264
265	dst_ioas = iommufd_get_ioas(ucmd->ictx, cmd->dst_ioas_id);
266	if (IS_ERR(dst_ioas)) {
267		rc = PTR_ERR(dst_ioas);
268		goto out_pages;
269	}
270
271	if (!(cmd->flags & IOMMU_IOAS_MAP_FIXED_IOVA))
272		flags = IOPT_ALLOC_IOVA;
273	iova = cmd->dst_iova;
274	rc = iopt_map_pages(&dst_ioas->iopt, &pages_list, cmd->length, &iova,
275			    conv_iommu_prot(cmd->flags), flags);
276	if (rc)
277		goto out_put_dst;
278
279	cmd->dst_iova = iova;
280	rc = iommufd_ucmd_respond(ucmd, sizeof(*cmd));
281out_put_dst:
282	iommufd_put_object(ucmd->ictx, &dst_ioas->obj);
283out_pages:
284	iopt_free_pages_list(&pages_list);
285	return rc;
286}
287
288int iommufd_ioas_unmap(struct iommufd_ucmd *ucmd)
289{
290	struct iommu_ioas_unmap *cmd = ucmd->cmd;
291	struct iommufd_ioas *ioas;
292	unsigned long unmapped = 0;
293	int rc;
294
295	ioas = iommufd_get_ioas(ucmd->ictx, cmd->ioas_id);
296	if (IS_ERR(ioas))
297		return PTR_ERR(ioas);
298
299	if (cmd->iova == 0 && cmd->length == U64_MAX) {
300		rc = iopt_unmap_all(&ioas->iopt, &unmapped);
301		if (rc)
302			goto out_put;
303	} else {
304		if (cmd->iova >= ULONG_MAX || cmd->length >= ULONG_MAX) {
305			rc = -EOVERFLOW;
306			goto out_put;
307		}
308		rc = iopt_unmap_iova(&ioas->iopt, cmd->iova, cmd->length,
309				     &unmapped);
310		if (rc)
311			goto out_put;
312	}
313
314	cmd->length = unmapped;
315	rc = iommufd_ucmd_respond(ucmd, sizeof(*cmd));
316
317out_put:
318	iommufd_put_object(ucmd->ictx, &ioas->obj);
319	return rc;
320}
321
322int iommufd_option_rlimit_mode(struct iommu_option *cmd,
323			       struct iommufd_ctx *ictx)
324{
325	if (cmd->object_id)
326		return -EOPNOTSUPP;
327
328	if (cmd->op == IOMMU_OPTION_OP_GET) {
329		cmd->val64 = ictx->account_mode == IOPT_PAGES_ACCOUNT_MM;
330		return 0;
331	}
332	if (cmd->op == IOMMU_OPTION_OP_SET) {
333		int rc = 0;
334
335		if (!capable(CAP_SYS_RESOURCE))
336			return -EPERM;
337
338		xa_lock(&ictx->objects);
339		if (!xa_empty(&ictx->objects)) {
340			rc = -EBUSY;
341		} else {
342			if (cmd->val64 == 0)
343				ictx->account_mode = IOPT_PAGES_ACCOUNT_USER;
344			else if (cmd->val64 == 1)
345				ictx->account_mode = IOPT_PAGES_ACCOUNT_MM;
346			else
347				rc = -EINVAL;
348		}
349		xa_unlock(&ictx->objects);
350
351		return rc;
352	}
353	return -EOPNOTSUPP;
354}
355
356static int iommufd_ioas_option_huge_pages(struct iommu_option *cmd,
357					  struct iommufd_ioas *ioas)
358{
359	if (cmd->op == IOMMU_OPTION_OP_GET) {
360		cmd->val64 = !ioas->iopt.disable_large_pages;
361		return 0;
362	}
363	if (cmd->op == IOMMU_OPTION_OP_SET) {
364		if (cmd->val64 == 0)
365			return iopt_disable_large_pages(&ioas->iopt);
366		if (cmd->val64 == 1) {
367			iopt_enable_large_pages(&ioas->iopt);
368			return 0;
369		}
370		return -EINVAL;
371	}
372	return -EOPNOTSUPP;
373}
374
375int iommufd_ioas_option(struct iommufd_ucmd *ucmd)
376{
377	struct iommu_option *cmd = ucmd->cmd;
378	struct iommufd_ioas *ioas;
379	int rc = 0;
380
381	if (cmd->__reserved)
382		return -EOPNOTSUPP;
383
384	ioas = iommufd_get_ioas(ucmd->ictx, cmd->object_id);
385	if (IS_ERR(ioas))
386		return PTR_ERR(ioas);
387
388	switch (cmd->option_id) {
389	case IOMMU_OPTION_HUGE_PAGES:
390		rc = iommufd_ioas_option_huge_pages(cmd, ioas);
391		break;
392	default:
393		rc = -EOPNOTSUPP;
394	}
395
396	iommufd_put_object(ucmd->ictx, &ioas->obj);
397	return rc;
398}
399