1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES
4 */
5#include <linux/iommu.h>
6#include <uapi/linux/iommufd.h>
7
8#include "../iommu-priv.h"
9#include "iommufd_private.h"
10
11void iommufd_hwpt_paging_destroy(struct iommufd_object *obj)
12{
13	struct iommufd_hwpt_paging *hwpt_paging =
14		container_of(obj, struct iommufd_hwpt_paging, common.obj);
15
16	if (!list_empty(&hwpt_paging->hwpt_item)) {
17		mutex_lock(&hwpt_paging->ioas->mutex);
18		list_del(&hwpt_paging->hwpt_item);
19		mutex_unlock(&hwpt_paging->ioas->mutex);
20
21		iopt_table_remove_domain(&hwpt_paging->ioas->iopt,
22					 hwpt_paging->common.domain);
23	}
24
25	if (hwpt_paging->common.domain)
26		iommu_domain_free(hwpt_paging->common.domain);
27
28	refcount_dec(&hwpt_paging->ioas->obj.users);
29}
30
31void iommufd_hwpt_paging_abort(struct iommufd_object *obj)
32{
33	struct iommufd_hwpt_paging *hwpt_paging =
34		container_of(obj, struct iommufd_hwpt_paging, common.obj);
35
36	/* The ioas->mutex must be held until finalize is called. */
37	lockdep_assert_held(&hwpt_paging->ioas->mutex);
38
39	if (!list_empty(&hwpt_paging->hwpt_item)) {
40		list_del_init(&hwpt_paging->hwpt_item);
41		iopt_table_remove_domain(&hwpt_paging->ioas->iopt,
42					 hwpt_paging->common.domain);
43	}
44	iommufd_hwpt_paging_destroy(obj);
45}
46
47void iommufd_hwpt_nested_destroy(struct iommufd_object *obj)
48{
49	struct iommufd_hwpt_nested *hwpt_nested =
50		container_of(obj, struct iommufd_hwpt_nested, common.obj);
51
52	if (hwpt_nested->common.domain)
53		iommu_domain_free(hwpt_nested->common.domain);
54
55	refcount_dec(&hwpt_nested->parent->common.obj.users);
56}
57
58void iommufd_hwpt_nested_abort(struct iommufd_object *obj)
59{
60	iommufd_hwpt_nested_destroy(obj);
61}
62
63static int
64iommufd_hwpt_paging_enforce_cc(struct iommufd_hwpt_paging *hwpt_paging)
65{
66	struct iommu_domain *paging_domain = hwpt_paging->common.domain;
67
68	if (hwpt_paging->enforce_cache_coherency)
69		return 0;
70
71	if (paging_domain->ops->enforce_cache_coherency)
72		hwpt_paging->enforce_cache_coherency =
73			paging_domain->ops->enforce_cache_coherency(
74				paging_domain);
75	if (!hwpt_paging->enforce_cache_coherency)
76		return -EINVAL;
77	return 0;
78}
79
80/**
81 * iommufd_hwpt_paging_alloc() - Get a PAGING iommu_domain for a device
82 * @ictx: iommufd context
83 * @ioas: IOAS to associate the domain with
84 * @idev: Device to get an iommu_domain for
85 * @flags: Flags from userspace
86 * @immediate_attach: True if idev should be attached to the hwpt
87 * @user_data: The user provided driver specific data describing the domain to
88 *             create
89 *
90 * Allocate a new iommu_domain and return it as a hw_pagetable. The HWPT
91 * will be linked to the given ioas and upon return the underlying iommu_domain
92 * is fully popoulated.
93 *
94 * The caller must hold the ioas->mutex until after
95 * iommufd_object_abort_and_destroy() or iommufd_object_finalize() is called on
96 * the returned hwpt.
97 */
98struct iommufd_hwpt_paging *
99iommufd_hwpt_paging_alloc(struct iommufd_ctx *ictx, struct iommufd_ioas *ioas,
100			  struct iommufd_device *idev, u32 flags,
101			  bool immediate_attach,
102			  const struct iommu_user_data *user_data)
103{
104	const u32 valid_flags = IOMMU_HWPT_ALLOC_NEST_PARENT |
105				IOMMU_HWPT_ALLOC_DIRTY_TRACKING;
106	const struct iommu_ops *ops = dev_iommu_ops(idev->dev);
107	struct iommufd_hwpt_paging *hwpt_paging;
108	struct iommufd_hw_pagetable *hwpt;
109	int rc;
110
111	lockdep_assert_held(&ioas->mutex);
112
113	if ((flags || user_data) && !ops->domain_alloc_user)
114		return ERR_PTR(-EOPNOTSUPP);
115	if (flags & ~valid_flags)
116		return ERR_PTR(-EOPNOTSUPP);
117
118	hwpt_paging = __iommufd_object_alloc(
119		ictx, hwpt_paging, IOMMUFD_OBJ_HWPT_PAGING, common.obj);
120	if (IS_ERR(hwpt_paging))
121		return ERR_CAST(hwpt_paging);
122	hwpt = &hwpt_paging->common;
123
124	INIT_LIST_HEAD(&hwpt_paging->hwpt_item);
125	/* Pairs with iommufd_hw_pagetable_destroy() */
126	refcount_inc(&ioas->obj.users);
127	hwpt_paging->ioas = ioas;
128	hwpt_paging->nest_parent = flags & IOMMU_HWPT_ALLOC_NEST_PARENT;
129
130	if (ops->domain_alloc_user) {
131		hwpt->domain = ops->domain_alloc_user(idev->dev, flags, NULL,
132						      user_data);
133		if (IS_ERR(hwpt->domain)) {
134			rc = PTR_ERR(hwpt->domain);
135			hwpt->domain = NULL;
136			goto out_abort;
137		}
138		hwpt->domain->owner = ops;
139	} else {
140		hwpt->domain = iommu_domain_alloc(idev->dev->bus);
141		if (!hwpt->domain) {
142			rc = -ENOMEM;
143			goto out_abort;
144		}
145	}
146
147	/*
148	 * Set the coherency mode before we do iopt_table_add_domain() as some
149	 * iommus have a per-PTE bit that controls it and need to decide before
150	 * doing any maps. It is an iommu driver bug to report
151	 * IOMMU_CAP_ENFORCE_CACHE_COHERENCY but fail enforce_cache_coherency on
152	 * a new domain.
153	 *
154	 * The cache coherency mode must be configured here and unchanged later.
155	 * Note that a HWPT (non-CC) created for a device (non-CC) can be later
156	 * reused by another device (either non-CC or CC). However, A HWPT (CC)
157	 * created for a device (CC) cannot be reused by another device (non-CC)
158	 * but only devices (CC). Instead user space in this case would need to
159	 * allocate a separate HWPT (non-CC).
160	 */
161	if (idev->enforce_cache_coherency) {
162		rc = iommufd_hwpt_paging_enforce_cc(hwpt_paging);
163		if (WARN_ON(rc))
164			goto out_abort;
165	}
166
167	/*
168	 * immediate_attach exists only to accommodate iommu drivers that cannot
169	 * directly allocate a domain. These drivers do not finish creating the
170	 * domain until attach is completed. Thus we must have this call
171	 * sequence. Once those drivers are fixed this should be removed.
172	 */
173	if (immediate_attach) {
174		rc = iommufd_hw_pagetable_attach(hwpt, idev);
175		if (rc)
176			goto out_abort;
177	}
178
179	rc = iopt_table_add_domain(&ioas->iopt, hwpt->domain);
180	if (rc)
181		goto out_detach;
182	list_add_tail(&hwpt_paging->hwpt_item, &ioas->hwpt_list);
183	return hwpt_paging;
184
185out_detach:
186	if (immediate_attach)
187		iommufd_hw_pagetable_detach(idev);
188out_abort:
189	iommufd_object_abort_and_destroy(ictx, &hwpt->obj);
190	return ERR_PTR(rc);
191}
192
193/**
194 * iommufd_hwpt_nested_alloc() - Get a NESTED iommu_domain for a device
195 * @ictx: iommufd context
196 * @parent: Parent PAGING-type hwpt to associate the domain with
197 * @idev: Device to get an iommu_domain for
198 * @flags: Flags from userspace
199 * @user_data: user_data pointer. Must be valid
200 *
201 * Allocate a new iommu_domain (must be IOMMU_DOMAIN_NESTED) and return it as
202 * a NESTED hw_pagetable. The given parent PAGING-type hwpt must be capable of
203 * being a parent.
204 */
205static struct iommufd_hwpt_nested *
206iommufd_hwpt_nested_alloc(struct iommufd_ctx *ictx,
207			  struct iommufd_hwpt_paging *parent,
208			  struct iommufd_device *idev, u32 flags,
209			  const struct iommu_user_data *user_data)
210{
211	const struct iommu_ops *ops = dev_iommu_ops(idev->dev);
212	struct iommufd_hwpt_nested *hwpt_nested;
213	struct iommufd_hw_pagetable *hwpt;
214	int rc;
215
216	if (flags || !user_data->len || !ops->domain_alloc_user)
217		return ERR_PTR(-EOPNOTSUPP);
218	if (parent->auto_domain || !parent->nest_parent)
219		return ERR_PTR(-EINVAL);
220
221	hwpt_nested = __iommufd_object_alloc(
222		ictx, hwpt_nested, IOMMUFD_OBJ_HWPT_NESTED, common.obj);
223	if (IS_ERR(hwpt_nested))
224		return ERR_CAST(hwpt_nested);
225	hwpt = &hwpt_nested->common;
226
227	refcount_inc(&parent->common.obj.users);
228	hwpt_nested->parent = parent;
229
230	hwpt->domain = ops->domain_alloc_user(idev->dev, flags,
231					      parent->common.domain, user_data);
232	if (IS_ERR(hwpt->domain)) {
233		rc = PTR_ERR(hwpt->domain);
234		hwpt->domain = NULL;
235		goto out_abort;
236	}
237	hwpt->domain->owner = ops;
238
239	if (WARN_ON_ONCE(hwpt->domain->type != IOMMU_DOMAIN_NESTED)) {
240		rc = -EINVAL;
241		goto out_abort;
242	}
243	return hwpt_nested;
244
245out_abort:
246	iommufd_object_abort_and_destroy(ictx, &hwpt->obj);
247	return ERR_PTR(rc);
248}
249
250int iommufd_hwpt_alloc(struct iommufd_ucmd *ucmd)
251{
252	struct iommu_hwpt_alloc *cmd = ucmd->cmd;
253	const struct iommu_user_data user_data = {
254		.type = cmd->data_type,
255		.uptr = u64_to_user_ptr(cmd->data_uptr),
256		.len = cmd->data_len,
257	};
258	struct iommufd_hw_pagetable *hwpt;
259	struct iommufd_ioas *ioas = NULL;
260	struct iommufd_object *pt_obj;
261	struct iommufd_device *idev;
262	int rc;
263
264	if (cmd->__reserved)
265		return -EOPNOTSUPP;
266	if ((cmd->data_type == IOMMU_HWPT_DATA_NONE && cmd->data_len) ||
267	    (cmd->data_type != IOMMU_HWPT_DATA_NONE && !cmd->data_len))
268		return -EINVAL;
269
270	idev = iommufd_get_device(ucmd, cmd->dev_id);
271	if (IS_ERR(idev))
272		return PTR_ERR(idev);
273
274	pt_obj = iommufd_get_object(ucmd->ictx, cmd->pt_id, IOMMUFD_OBJ_ANY);
275	if (IS_ERR(pt_obj)) {
276		rc = -EINVAL;
277		goto out_put_idev;
278	}
279
280	if (pt_obj->type == IOMMUFD_OBJ_IOAS) {
281		struct iommufd_hwpt_paging *hwpt_paging;
282
283		ioas = container_of(pt_obj, struct iommufd_ioas, obj);
284		mutex_lock(&ioas->mutex);
285		hwpt_paging = iommufd_hwpt_paging_alloc(
286			ucmd->ictx, ioas, idev, cmd->flags, false,
287			user_data.len ? &user_data : NULL);
288		if (IS_ERR(hwpt_paging)) {
289			rc = PTR_ERR(hwpt_paging);
290			goto out_unlock;
291		}
292		hwpt = &hwpt_paging->common;
293	} else if (pt_obj->type == IOMMUFD_OBJ_HWPT_PAGING) {
294		struct iommufd_hwpt_nested *hwpt_nested;
295
296		hwpt_nested = iommufd_hwpt_nested_alloc(
297			ucmd->ictx,
298			container_of(pt_obj, struct iommufd_hwpt_paging,
299				     common.obj),
300			idev, cmd->flags, &user_data);
301		if (IS_ERR(hwpt_nested)) {
302			rc = PTR_ERR(hwpt_nested);
303			goto out_unlock;
304		}
305		hwpt = &hwpt_nested->common;
306	} else {
307		rc = -EINVAL;
308		goto out_put_pt;
309	}
310
311	cmd->out_hwpt_id = hwpt->obj.id;
312	rc = iommufd_ucmd_respond(ucmd, sizeof(*cmd));
313	if (rc)
314		goto out_hwpt;
315	iommufd_object_finalize(ucmd->ictx, &hwpt->obj);
316	goto out_unlock;
317
318out_hwpt:
319	iommufd_object_abort_and_destroy(ucmd->ictx, &hwpt->obj);
320out_unlock:
321	if (ioas)
322		mutex_unlock(&ioas->mutex);
323out_put_pt:
324	iommufd_put_object(ucmd->ictx, pt_obj);
325out_put_idev:
326	iommufd_put_object(ucmd->ictx, &idev->obj);
327	return rc;
328}
329
330int iommufd_hwpt_set_dirty_tracking(struct iommufd_ucmd *ucmd)
331{
332	struct iommu_hwpt_set_dirty_tracking *cmd = ucmd->cmd;
333	struct iommufd_hwpt_paging *hwpt_paging;
334	struct iommufd_ioas *ioas;
335	int rc = -EOPNOTSUPP;
336	bool enable;
337
338	if (cmd->flags & ~IOMMU_HWPT_DIRTY_TRACKING_ENABLE)
339		return rc;
340
341	hwpt_paging = iommufd_get_hwpt_paging(ucmd, cmd->hwpt_id);
342	if (IS_ERR(hwpt_paging))
343		return PTR_ERR(hwpt_paging);
344
345	ioas = hwpt_paging->ioas;
346	enable = cmd->flags & IOMMU_HWPT_DIRTY_TRACKING_ENABLE;
347
348	rc = iopt_set_dirty_tracking(&ioas->iopt, hwpt_paging->common.domain,
349				     enable);
350
351	iommufd_put_object(ucmd->ictx, &hwpt_paging->common.obj);
352	return rc;
353}
354
355int iommufd_hwpt_get_dirty_bitmap(struct iommufd_ucmd *ucmd)
356{
357	struct iommu_hwpt_get_dirty_bitmap *cmd = ucmd->cmd;
358	struct iommufd_hwpt_paging *hwpt_paging;
359	struct iommufd_ioas *ioas;
360	int rc = -EOPNOTSUPP;
361
362	if ((cmd->flags & ~(IOMMU_HWPT_GET_DIRTY_BITMAP_NO_CLEAR)) ||
363	    cmd->__reserved)
364		return -EOPNOTSUPP;
365
366	hwpt_paging = iommufd_get_hwpt_paging(ucmd, cmd->hwpt_id);
367	if (IS_ERR(hwpt_paging))
368		return PTR_ERR(hwpt_paging);
369
370	ioas = hwpt_paging->ioas;
371	rc = iopt_read_and_clear_dirty_data(
372		&ioas->iopt, hwpt_paging->common.domain, cmd->flags, cmd);
373
374	iommufd_put_object(ucmd->ictx, &hwpt_paging->common.obj);
375	return rc;
376}
377
378int iommufd_hwpt_invalidate(struct iommufd_ucmd *ucmd)
379{
380	struct iommu_hwpt_invalidate *cmd = ucmd->cmd;
381	struct iommu_user_data_array data_array = {
382		.type = cmd->data_type,
383		.uptr = u64_to_user_ptr(cmd->data_uptr),
384		.entry_len = cmd->entry_len,
385		.entry_num = cmd->entry_num,
386	};
387	struct iommufd_hw_pagetable *hwpt;
388	u32 done_num = 0;
389	int rc;
390
391	if (cmd->__reserved) {
392		rc = -EOPNOTSUPP;
393		goto out;
394	}
395
396	if (cmd->entry_num && (!cmd->data_uptr || !cmd->entry_len)) {
397		rc = -EINVAL;
398		goto out;
399	}
400
401	hwpt = iommufd_get_hwpt_nested(ucmd, cmd->hwpt_id);
402	if (IS_ERR(hwpt)) {
403		rc = PTR_ERR(hwpt);
404		goto out;
405	}
406
407	rc = hwpt->domain->ops->cache_invalidate_user(hwpt->domain,
408						      &data_array);
409	done_num = data_array.entry_num;
410
411	iommufd_put_object(ucmd->ictx, &hwpt->obj);
412out:
413	cmd->entry_num = done_num;
414	if (iommufd_ucmd_respond(ucmd, sizeof(*cmd)))
415		return -EFAULT;
416	return rc;
417}
418