1// SPDX-License-Identifier: GPL-2.0 or BSD-3-Clause
2/*
3 * Copyright(c) 2023 - Cornelis Networks, Inc.
4 */
5
6#include <linux/types.h>
7
8#include "hfi.h"
9#include "common.h"
10#include "device.h"
11#include "pinning.h"
12#include "mmu_rb.h"
13#include "user_sdma.h"
14#include "trace.h"
15
16struct sdma_mmu_node {
17	struct mmu_rb_node rb;
18	struct hfi1_user_sdma_pkt_q *pq;
19	struct page **pages;
20	unsigned int npages;
21};
22
23static bool sdma_rb_filter(struct mmu_rb_node *node, unsigned long addr,
24			   unsigned long len);
25static int sdma_rb_evict(void *arg, struct mmu_rb_node *mnode, void *arg2,
26			 bool *stop);
27static void sdma_rb_remove(void *arg, struct mmu_rb_node *mnode);
28
29static struct mmu_rb_ops sdma_rb_ops = {
30	.filter = sdma_rb_filter,
31	.evict = sdma_rb_evict,
32	.remove = sdma_rb_remove,
33};
34
35int hfi1_init_system_pinning(struct hfi1_user_sdma_pkt_q *pq)
36{
37	struct hfi1_devdata *dd = pq->dd;
38	int ret;
39
40	ret = hfi1_mmu_rb_register(pq, &sdma_rb_ops, dd->pport->hfi1_wq,
41				   &pq->handler);
42	if (ret)
43		dd_dev_err(dd,
44			   "[%u:%u] Failed to register system memory DMA support with MMU: %d\n",
45			   pq->ctxt, pq->subctxt, ret);
46	return ret;
47}
48
49void hfi1_free_system_pinning(struct hfi1_user_sdma_pkt_q *pq)
50{
51	if (pq->handler)
52		hfi1_mmu_rb_unregister(pq->handler);
53}
54
55static u32 sdma_cache_evict(struct hfi1_user_sdma_pkt_q *pq, u32 npages)
56{
57	struct evict_data evict_data;
58
59	evict_data.cleared = 0;
60	evict_data.target = npages;
61	hfi1_mmu_rb_evict(pq->handler, &evict_data);
62	return evict_data.cleared;
63}
64
65static void unpin_vector_pages(struct mm_struct *mm, struct page **pages,
66			       unsigned int start, unsigned int npages)
67{
68	hfi1_release_user_pages(mm, pages + start, npages, false);
69	kfree(pages);
70}
71
72static inline struct mm_struct *mm_from_sdma_node(struct sdma_mmu_node *node)
73{
74	return node->rb.handler->mn.mm;
75}
76
77static void free_system_node(struct sdma_mmu_node *node)
78{
79	if (node->npages) {
80		unpin_vector_pages(mm_from_sdma_node(node), node->pages, 0,
81				   node->npages);
82		atomic_sub(node->npages, &node->pq->n_locked);
83	}
84	kfree(node);
85}
86
87/*
88 * kref_get()'s an additional kref on the returned rb_node to prevent rb_node
89 * from being released until after rb_node is assigned to an SDMA descriptor
90 * (struct sdma_desc) under add_system_iovec_to_sdma_packet(), even if the
91 * virtual address range for rb_node is invalidated between now and then.
92 */
93static struct sdma_mmu_node *find_system_node(struct mmu_rb_handler *handler,
94					      unsigned long start,
95					      unsigned long end)
96{
97	struct mmu_rb_node *rb_node;
98	unsigned long flags;
99
100	spin_lock_irqsave(&handler->lock, flags);
101	rb_node = hfi1_mmu_rb_get_first(handler, start, (end - start));
102	if (!rb_node) {
103		spin_unlock_irqrestore(&handler->lock, flags);
104		return NULL;
105	}
106
107	/* "safety" kref to prevent release before add_system_iovec_to_sdma_packet() */
108	kref_get(&rb_node->refcount);
109	spin_unlock_irqrestore(&handler->lock, flags);
110
111	return container_of(rb_node, struct sdma_mmu_node, rb);
112}
113
114static int pin_system_pages(struct user_sdma_request *req,
115			    uintptr_t start_address, size_t length,
116			    struct sdma_mmu_node *node, int npages)
117{
118	struct hfi1_user_sdma_pkt_q *pq = req->pq;
119	int pinned, cleared;
120	struct page **pages;
121
122	pages = kcalloc(npages, sizeof(*pages), GFP_KERNEL);
123	if (!pages)
124		return -ENOMEM;
125
126retry:
127	if (!hfi1_can_pin_pages(pq->dd, current->mm, atomic_read(&pq->n_locked),
128				npages)) {
129		SDMA_DBG(req, "Evicting: nlocked %u npages %u",
130			 atomic_read(&pq->n_locked), npages);
131		cleared = sdma_cache_evict(pq, npages);
132		if (cleared >= npages)
133			goto retry;
134	}
135
136	SDMA_DBG(req, "Acquire user pages start_address %lx node->npages %u npages %u",
137		 start_address, node->npages, npages);
138	pinned = hfi1_acquire_user_pages(current->mm, start_address, npages, 0,
139					 pages);
140
141	if (pinned < 0) {
142		kfree(pages);
143		SDMA_DBG(req, "pinned %d", pinned);
144		return pinned;
145	}
146	if (pinned != npages) {
147		unpin_vector_pages(current->mm, pages, node->npages, pinned);
148		SDMA_DBG(req, "npages %u pinned %d", npages, pinned);
149		return -EFAULT;
150	}
151	node->rb.addr = start_address;
152	node->rb.len = length;
153	node->pages = pages;
154	node->npages = npages;
155	atomic_add(pinned, &pq->n_locked);
156	SDMA_DBG(req, "done. pinned %d", pinned);
157	return 0;
158}
159
160/*
161 * kref refcount on *node_p will be 2 on successful addition: one kref from
162 * kref_init() for mmu_rb_handler and one kref to prevent *node_p from being
163 * released until after *node_p is assigned to an SDMA descriptor (struct
164 * sdma_desc) under add_system_iovec_to_sdma_packet(), even if the virtual
165 * address range for *node_p is invalidated between now and then.
166 */
167static int add_system_pinning(struct user_sdma_request *req,
168			      struct sdma_mmu_node **node_p,
169			      unsigned long start, unsigned long len)
170
171{
172	struct hfi1_user_sdma_pkt_q *pq = req->pq;
173	struct sdma_mmu_node *node;
174	int ret;
175
176	node = kzalloc(sizeof(*node), GFP_KERNEL);
177	if (!node)
178		return -ENOMEM;
179
180	/* First kref "moves" to mmu_rb_handler */
181	kref_init(&node->rb.refcount);
182
183	/* "safety" kref to prevent release before add_system_iovec_to_sdma_packet() */
184	kref_get(&node->rb.refcount);
185
186	node->pq = pq;
187	ret = pin_system_pages(req, start, len, node, PFN_DOWN(len));
188	if (ret == 0) {
189		ret = hfi1_mmu_rb_insert(pq->handler, &node->rb);
190		if (ret)
191			free_system_node(node);
192		else
193			*node_p = node;
194
195		return ret;
196	}
197
198	kfree(node);
199	return ret;
200}
201
202static int get_system_cache_entry(struct user_sdma_request *req,
203				  struct sdma_mmu_node **node_p,
204				  size_t req_start, size_t req_len)
205{
206	struct hfi1_user_sdma_pkt_q *pq = req->pq;
207	u64 start = ALIGN_DOWN(req_start, PAGE_SIZE);
208	u64 end = PFN_ALIGN(req_start + req_len);
209	int ret;
210
211	if ((end - start) == 0) {
212		SDMA_DBG(req,
213			 "Request for empty cache entry req_start %lx req_len %lx start %llx end %llx",
214			 req_start, req_len, start, end);
215		return -EINVAL;
216	}
217
218	SDMA_DBG(req, "req_start %lx req_len %lu", req_start, req_len);
219
220	while (1) {
221		struct sdma_mmu_node *node =
222			find_system_node(pq->handler, start, end);
223		u64 prepend_len = 0;
224
225		SDMA_DBG(req, "node %p start %llx end %llu", node, start, end);
226		if (!node) {
227			ret = add_system_pinning(req, node_p, start,
228						 end - start);
229			if (ret == -EEXIST) {
230				/*
231				 * Another execution context has inserted a
232				 * conficting entry first.
233				 */
234				continue;
235			}
236			return ret;
237		}
238
239		if (node->rb.addr <= start) {
240			/*
241			 * This entry covers at least part of the region. If it doesn't extend
242			 * to the end, then this will be called again for the next segment.
243			 */
244			*node_p = node;
245			return 0;
246		}
247
248		SDMA_DBG(req, "prepend: node->rb.addr %lx, node->rb.refcount %d",
249			 node->rb.addr, kref_read(&node->rb.refcount));
250		prepend_len = node->rb.addr - start;
251
252		/*
253		 * This node will not be returned, instead a new node
254		 * will be. So release the reference.
255		 */
256		kref_put(&node->rb.refcount, hfi1_mmu_rb_release);
257
258		/* Prepend a node to cover the beginning of the allocation */
259		ret = add_system_pinning(req, node_p, start, prepend_len);
260		if (ret == -EEXIST) {
261			/* Another execution context has inserted a conficting entry first. */
262			continue;
263		}
264		return ret;
265	}
266}
267
268static void sdma_mmu_rb_node_get(void *ctx)
269{
270	struct mmu_rb_node *node = ctx;
271
272	kref_get(&node->refcount);
273}
274
275static void sdma_mmu_rb_node_put(void *ctx)
276{
277	struct sdma_mmu_node *node = ctx;
278
279	kref_put(&node->rb.refcount, hfi1_mmu_rb_release);
280}
281
282static int add_mapping_to_sdma_packet(struct user_sdma_request *req,
283				      struct user_sdma_txreq *tx,
284				      struct sdma_mmu_node *cache_entry,
285				      size_t start,
286				      size_t from_this_cache_entry)
287{
288	struct hfi1_user_sdma_pkt_q *pq = req->pq;
289	unsigned int page_offset;
290	unsigned int from_this_page;
291	size_t page_index;
292	void *ctx;
293	int ret;
294
295	/*
296	 * Because the cache may be more fragmented than the memory that is being accessed,
297	 * it's not strictly necessary to have a descriptor per cache entry.
298	 */
299
300	while (from_this_cache_entry) {
301		page_index = PFN_DOWN(start - cache_entry->rb.addr);
302
303		if (page_index >= cache_entry->npages) {
304			SDMA_DBG(req,
305				 "Request for page_index %zu >= cache_entry->npages %u",
306				 page_index, cache_entry->npages);
307			return -EINVAL;
308		}
309
310		page_offset = start - ALIGN_DOWN(start, PAGE_SIZE);
311		from_this_page = PAGE_SIZE - page_offset;
312
313		if (from_this_page < from_this_cache_entry) {
314			ctx = NULL;
315		} else {
316			/*
317			 * In the case they are equal the next line has no practical effect,
318			 * but it's better to do a register to register copy than a conditional
319			 * branch.
320			 */
321			from_this_page = from_this_cache_entry;
322			ctx = cache_entry;
323		}
324
325		ret = sdma_txadd_page(pq->dd, &tx->txreq,
326				      cache_entry->pages[page_index],
327				      page_offset, from_this_page,
328				      ctx,
329				      sdma_mmu_rb_node_get,
330				      sdma_mmu_rb_node_put);
331		if (ret) {
332			/*
333			 * When there's a failure, the entire request is freed by
334			 * user_sdma_send_pkts().
335			 */
336			SDMA_DBG(req,
337				 "sdma_txadd_page failed %d page_index %lu page_offset %u from_this_page %u",
338				 ret, page_index, page_offset, from_this_page);
339			return ret;
340		}
341		start += from_this_page;
342		from_this_cache_entry -= from_this_page;
343	}
344	return 0;
345}
346
347static int add_system_iovec_to_sdma_packet(struct user_sdma_request *req,
348					   struct user_sdma_txreq *tx,
349					   struct user_sdma_iovec *iovec,
350					   size_t from_this_iovec)
351{
352	while (from_this_iovec > 0) {
353		struct sdma_mmu_node *cache_entry;
354		size_t from_this_cache_entry;
355		size_t start;
356		int ret;
357
358		start = (uintptr_t)iovec->iov.iov_base + iovec->offset;
359		ret = get_system_cache_entry(req, &cache_entry, start,
360					     from_this_iovec);
361		if (ret) {
362			SDMA_DBG(req, "pin system segment failed %d", ret);
363			return ret;
364		}
365
366		from_this_cache_entry = cache_entry->rb.len - (start - cache_entry->rb.addr);
367		if (from_this_cache_entry > from_this_iovec)
368			from_this_cache_entry = from_this_iovec;
369
370		ret = add_mapping_to_sdma_packet(req, tx, cache_entry, start,
371						 from_this_cache_entry);
372
373		/*
374		 * Done adding cache_entry to zero or more sdma_desc. Can
375		 * kref_put() the "safety" kref taken under
376		 * get_system_cache_entry().
377		 */
378		kref_put(&cache_entry->rb.refcount, hfi1_mmu_rb_release);
379
380		if (ret) {
381			SDMA_DBG(req, "add system segment failed %d", ret);
382			return ret;
383		}
384
385		iovec->offset += from_this_cache_entry;
386		from_this_iovec -= from_this_cache_entry;
387	}
388
389	return 0;
390}
391
392/*
393 * Add up to pkt_data_remaining bytes to the txreq, starting at the current
394 * offset in the given iovec entry and continuing until all data has been added
395 * to the iovec or the iovec entry type changes.
396 *
397 * On success, prior to returning, adjust pkt_data_remaining, req->iov_idx, and
398 * the offset value in req->iov[req->iov_idx] to reflect the data that has been
399 * consumed.
400 */
401int hfi1_add_pages_to_sdma_packet(struct user_sdma_request *req,
402				  struct user_sdma_txreq *tx,
403				  struct user_sdma_iovec *iovec,
404				  u32 *pkt_data_remaining)
405{
406	size_t remaining_to_add = *pkt_data_remaining;
407	/*
408	 * Walk through iovec entries, ensure the associated pages
409	 * are pinned and mapped, add data to the packet until no more
410	 * data remains to be added or the iovec entry type changes.
411	 */
412	while (remaining_to_add > 0) {
413		struct user_sdma_iovec *cur_iovec;
414		size_t from_this_iovec;
415		int ret;
416
417		cur_iovec = iovec;
418		from_this_iovec = iovec->iov.iov_len - iovec->offset;
419
420		if (from_this_iovec > remaining_to_add) {
421			from_this_iovec = remaining_to_add;
422		} else {
423			/* The current iovec entry will be consumed by this pass. */
424			req->iov_idx++;
425			iovec++;
426		}
427
428		ret = add_system_iovec_to_sdma_packet(req, tx, cur_iovec,
429						      from_this_iovec);
430		if (ret)
431			return ret;
432
433		remaining_to_add -= from_this_iovec;
434	}
435	*pkt_data_remaining = remaining_to_add;
436
437	return 0;
438}
439
440static bool sdma_rb_filter(struct mmu_rb_node *node, unsigned long addr,
441			   unsigned long len)
442{
443	return (bool)(node->addr == addr);
444}
445
446/*
447 * Return 1 to remove the node from the rb tree and call the remove op.
448 *
449 * Called with the rb tree lock held.
450 */
451static int sdma_rb_evict(void *arg, struct mmu_rb_node *mnode,
452			 void *evict_arg, bool *stop)
453{
454	struct sdma_mmu_node *node =
455		container_of(mnode, struct sdma_mmu_node, rb);
456	struct evict_data *evict_data = evict_arg;
457
458	/* this node will be evicted, add its pages to our count */
459	evict_data->cleared += node->npages;
460
461	/* have enough pages been cleared? */
462	if (evict_data->cleared >= evict_data->target)
463		*stop = true;
464
465	return 1; /* remove this node */
466}
467
468static void sdma_rb_remove(void *arg, struct mmu_rb_node *mnode)
469{
470	struct sdma_mmu_node *node =
471		container_of(mnode, struct sdma_mmu_node, rb);
472
473	free_system_node(node);
474}
475