1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Copyright (C) 2018 Intel Corporation.
4 * Copyright 2018 Google LLC.
5 *
6 * Author: Tuukka Toivonen <tuukka.toivonen@intel.com>
7 * Author: Sakari Ailus <sakari.ailus@linux.intel.com>
8 * Author: Samu Onkalo <samu.onkalo@intel.com>
9 * Author: Tomasz Figa <tfiga@chromium.org>
10 *
11 */
12
13#include <linux/dma-mapping.h>
14#include <linux/iopoll.h>
15#include <linux/pm_runtime.h>
16#include <linux/slab.h>
17#include <linux/vmalloc.h>
18
19#include <asm/set_memory.h>
20
21#include "ipu3-mmu.h"
22
23#define IPU3_PT_BITS		10
24#define IPU3_PT_PTES		(1UL << IPU3_PT_BITS)
25#define IPU3_PT_SIZE		(IPU3_PT_PTES << 2)
26#define IPU3_PT_ORDER		(IPU3_PT_SIZE >> PAGE_SHIFT)
27
28#define IPU3_ADDR2PTE(addr)	((addr) >> IPU3_PAGE_SHIFT)
29#define IPU3_PTE2ADDR(pte)	((phys_addr_t)(pte) << IPU3_PAGE_SHIFT)
30
31#define IPU3_L2PT_SHIFT		IPU3_PT_BITS
32#define IPU3_L2PT_MASK		((1UL << IPU3_L2PT_SHIFT) - 1)
33
34#define IPU3_L1PT_SHIFT		IPU3_PT_BITS
35#define IPU3_L1PT_MASK		((1UL << IPU3_L1PT_SHIFT) - 1)
36
37#define IPU3_MMU_ADDRESS_BITS	(IPU3_PAGE_SHIFT + \
38				 IPU3_L2PT_SHIFT + \
39				 IPU3_L1PT_SHIFT)
40
41#define IMGU_REG_BASE		0x4000
42#define REG_TLB_INVALIDATE	(IMGU_REG_BASE + 0x300)
43#define TLB_INVALIDATE		1
44#define REG_L1_PHYS		(IMGU_REG_BASE + 0x304) /* 27-bit pfn */
45#define REG_GP_HALT		(IMGU_REG_BASE + 0x5dc)
46#define REG_GP_HALTED		(IMGU_REG_BASE + 0x5e0)
47
48struct imgu_mmu {
49	struct device *dev;
50	void __iomem *base;
51	/* protect access to l2pts, l1pt */
52	spinlock_t lock;
53
54	void *dummy_page;
55	u32 dummy_page_pteval;
56
57	u32 *dummy_l2pt;
58	u32 dummy_l2pt_pteval;
59
60	u32 **l2pts;
61	u32 *l1pt;
62
63	struct imgu_mmu_info geometry;
64};
65
66static inline struct imgu_mmu *to_imgu_mmu(struct imgu_mmu_info *info)
67{
68	return container_of(info, struct imgu_mmu, geometry);
69}
70
71/**
72 * imgu_mmu_tlb_invalidate - invalidate translation look-aside buffer
73 * @mmu: MMU to perform the invalidate operation on
74 *
75 * This function invalidates the whole TLB. Must be called when the hardware
76 * is powered on.
77 */
78static void imgu_mmu_tlb_invalidate(struct imgu_mmu *mmu)
79{
80	writel(TLB_INVALIDATE, mmu->base + REG_TLB_INVALIDATE);
81}
82
83static void call_if_imgu_is_powered(struct imgu_mmu *mmu,
84				    void (*func)(struct imgu_mmu *mmu))
85{
86	if (!pm_runtime_get_if_in_use(mmu->dev))
87		return;
88
89	func(mmu);
90	pm_runtime_put(mmu->dev);
91}
92
93/**
94 * imgu_mmu_set_halt - set CIO gate halt bit
95 * @mmu: MMU to set the CIO gate bit in.
96 * @halt: Desired state of the gate bit.
97 *
98 * This function sets the CIO gate bit that controls whether external memory
99 * accesses are allowed. Must be called when the hardware is powered on.
100 */
101static void imgu_mmu_set_halt(struct imgu_mmu *mmu, bool halt)
102{
103	int ret;
104	u32 val;
105
106	writel(halt, mmu->base + REG_GP_HALT);
107	ret = readl_poll_timeout(mmu->base + REG_GP_HALTED,
108				 val, (val & 1) == halt, 1000, 100000);
109
110	if (ret)
111		dev_err(mmu->dev, "failed to %s CIO gate halt\n",
112			halt ? "set" : "clear");
113}
114
115/**
116 * imgu_mmu_alloc_page_table - allocate a pre-filled page table
117 * @pteval: Value to initialize for page table entries with.
118 *
119 * Return: Pointer to allocated page table or NULL on failure.
120 */
121static u32 *imgu_mmu_alloc_page_table(u32 pteval)
122{
123	u32 *pt;
124	int pte;
125
126	pt = (u32 *)__get_free_page(GFP_KERNEL);
127	if (!pt)
128		return NULL;
129
130	for (pte = 0; pte < IPU3_PT_PTES; pte++)
131		pt[pte] = pteval;
132
133	set_memory_uc((unsigned long)pt, IPU3_PT_ORDER);
134
135	return pt;
136}
137
138/**
139 * imgu_mmu_free_page_table - free page table
140 * @pt: Page table to free.
141 */
142static void imgu_mmu_free_page_table(u32 *pt)
143{
144	set_memory_wb((unsigned long)pt, IPU3_PT_ORDER);
145	free_page((unsigned long)pt);
146}
147
148/**
149 * address_to_pte_idx - split IOVA into L1 and L2 page table indices
150 * @iova: IOVA to split.
151 * @l1pt_idx: Output for the L1 page table index.
152 * @l2pt_idx: Output for the L2 page index.
153 */
154static inline void address_to_pte_idx(unsigned long iova, u32 *l1pt_idx,
155				      u32 *l2pt_idx)
156{
157	iova >>= IPU3_PAGE_SHIFT;
158
159	if (l2pt_idx)
160		*l2pt_idx = iova & IPU3_L2PT_MASK;
161
162	iova >>= IPU3_L2PT_SHIFT;
163
164	if (l1pt_idx)
165		*l1pt_idx = iova & IPU3_L1PT_MASK;
166}
167
168static u32 *imgu_mmu_get_l2pt(struct imgu_mmu *mmu, u32 l1pt_idx)
169{
170	unsigned long flags;
171	u32 *l2pt, *new_l2pt;
172	u32 pteval;
173
174	spin_lock_irqsave(&mmu->lock, flags);
175
176	l2pt = mmu->l2pts[l1pt_idx];
177	if (l2pt) {
178		spin_unlock_irqrestore(&mmu->lock, flags);
179		return l2pt;
180	}
181
182	spin_unlock_irqrestore(&mmu->lock, flags);
183
184	new_l2pt = imgu_mmu_alloc_page_table(mmu->dummy_page_pteval);
185	if (!new_l2pt)
186		return NULL;
187
188	spin_lock_irqsave(&mmu->lock, flags);
189
190	dev_dbg(mmu->dev, "allocated page table %p for l1pt_idx %u\n",
191		new_l2pt, l1pt_idx);
192
193	l2pt = mmu->l2pts[l1pt_idx];
194	if (l2pt) {
195		spin_unlock_irqrestore(&mmu->lock, flags);
196		imgu_mmu_free_page_table(new_l2pt);
197		return l2pt;
198	}
199
200	l2pt = new_l2pt;
201	mmu->l2pts[l1pt_idx] = new_l2pt;
202
203	pteval = IPU3_ADDR2PTE(virt_to_phys(new_l2pt));
204	mmu->l1pt[l1pt_idx] = pteval;
205
206	spin_unlock_irqrestore(&mmu->lock, flags);
207	return l2pt;
208}
209
210static int __imgu_mmu_map(struct imgu_mmu *mmu, unsigned long iova,
211			  phys_addr_t paddr)
212{
213	u32 l1pt_idx, l2pt_idx;
214	unsigned long flags;
215	u32 *l2pt;
216
217	if (!mmu)
218		return -ENODEV;
219
220	address_to_pte_idx(iova, &l1pt_idx, &l2pt_idx);
221
222	l2pt = imgu_mmu_get_l2pt(mmu, l1pt_idx);
223	if (!l2pt)
224		return -ENOMEM;
225
226	spin_lock_irqsave(&mmu->lock, flags);
227
228	if (l2pt[l2pt_idx] != mmu->dummy_page_pteval) {
229		spin_unlock_irqrestore(&mmu->lock, flags);
230		return -EBUSY;
231	}
232
233	l2pt[l2pt_idx] = IPU3_ADDR2PTE(paddr);
234
235	spin_unlock_irqrestore(&mmu->lock, flags);
236
237	return 0;
238}
239
240/**
241 * imgu_mmu_map - map a buffer to a physical address
242 *
243 * @info: MMU mappable range
244 * @iova: the virtual address
245 * @paddr: the physical address
246 * @size: length of the mappable area
247 *
248 * The function has been adapted from iommu_map() in
249 * drivers/iommu/iommu.c .
250 */
251int imgu_mmu_map(struct imgu_mmu_info *info, unsigned long iova,
252		 phys_addr_t paddr, size_t size)
253{
254	struct imgu_mmu *mmu = to_imgu_mmu(info);
255	int ret = 0;
256
257	/*
258	 * both the virtual address and the physical one, as well as
259	 * the size of the mapping, must be aligned (at least) to the
260	 * size of the smallest page supported by the hardware
261	 */
262	if (!IS_ALIGNED(iova | paddr | size, IPU3_PAGE_SIZE)) {
263		dev_err(mmu->dev, "unaligned: iova 0x%lx pa %pa size 0x%zx\n",
264			iova, &paddr, size);
265		return -EINVAL;
266	}
267
268	dev_dbg(mmu->dev, "map: iova 0x%lx pa %pa size 0x%zx\n",
269		iova, &paddr, size);
270
271	while (size) {
272		dev_dbg(mmu->dev, "mapping: iova 0x%lx pa %pa\n", iova, &paddr);
273
274		ret = __imgu_mmu_map(mmu, iova, paddr);
275		if (ret)
276			break;
277
278		iova += IPU3_PAGE_SIZE;
279		paddr += IPU3_PAGE_SIZE;
280		size -= IPU3_PAGE_SIZE;
281	}
282
283	call_if_imgu_is_powered(mmu, imgu_mmu_tlb_invalidate);
284
285	return ret;
286}
287
288/**
289 * imgu_mmu_map_sg - Map a scatterlist
290 *
291 * @info: MMU mappable range
292 * @iova: the virtual address
293 * @sg: the scatterlist to map
294 * @nents: number of entries in the scatterlist
295 *
296 * The function has been adapted from default_iommu_map_sg() in
297 * drivers/iommu/iommu.c .
298 */
299size_t imgu_mmu_map_sg(struct imgu_mmu_info *info, unsigned long iova,
300		       struct scatterlist *sg, unsigned int nents)
301{
302	struct imgu_mmu *mmu = to_imgu_mmu(info);
303	struct scatterlist *s;
304	size_t s_length, mapped = 0;
305	unsigned int i;
306	int ret;
307
308	for_each_sg(sg, s, nents, i) {
309		phys_addr_t phys = page_to_phys(sg_page(s)) + s->offset;
310
311		s_length = s->length;
312
313		if (!IS_ALIGNED(s->offset, IPU3_PAGE_SIZE))
314			goto out_err;
315
316		/* must be IPU3_PAGE_SIZE aligned to be mapped singlely */
317		if (i == nents - 1 && !IS_ALIGNED(s->length, IPU3_PAGE_SIZE))
318			s_length = PAGE_ALIGN(s->length);
319
320		ret = imgu_mmu_map(info, iova + mapped, phys, s_length);
321		if (ret)
322			goto out_err;
323
324		mapped += s_length;
325	}
326
327	call_if_imgu_is_powered(mmu, imgu_mmu_tlb_invalidate);
328
329	return mapped;
330
331out_err:
332	/* undo mappings already done */
333	imgu_mmu_unmap(info, iova, mapped);
334
335	return 0;
336}
337
338static size_t __imgu_mmu_unmap(struct imgu_mmu *mmu,
339			       unsigned long iova, size_t size)
340{
341	u32 l1pt_idx, l2pt_idx;
342	unsigned long flags;
343	size_t unmap = size;
344	u32 *l2pt;
345
346	if (!mmu)
347		return 0;
348
349	address_to_pte_idx(iova, &l1pt_idx, &l2pt_idx);
350
351	spin_lock_irqsave(&mmu->lock, flags);
352
353	l2pt = mmu->l2pts[l1pt_idx];
354	if (!l2pt) {
355		spin_unlock_irqrestore(&mmu->lock, flags);
356		return 0;
357	}
358
359	if (l2pt[l2pt_idx] == mmu->dummy_page_pteval)
360		unmap = 0;
361
362	l2pt[l2pt_idx] = mmu->dummy_page_pteval;
363
364	spin_unlock_irqrestore(&mmu->lock, flags);
365
366	return unmap;
367}
368
369/**
370 * imgu_mmu_unmap - Unmap a buffer
371 *
372 * @info: MMU mappable range
373 * @iova: the virtual address
374 * @size: the length of the buffer
375 *
376 * The function has been adapted from iommu_unmap() in
377 * drivers/iommu/iommu.c .
378 */
379size_t imgu_mmu_unmap(struct imgu_mmu_info *info, unsigned long iova,
380		      size_t size)
381{
382	struct imgu_mmu *mmu = to_imgu_mmu(info);
383	size_t unmapped_page, unmapped = 0;
384
385	/*
386	 * The virtual address, as well as the size of the mapping, must be
387	 * aligned (at least) to the size of the smallest page supported
388	 * by the hardware
389	 */
390	if (!IS_ALIGNED(iova | size, IPU3_PAGE_SIZE)) {
391		dev_err(mmu->dev, "unaligned: iova 0x%lx size 0x%zx\n",
392			iova, size);
393		return -EINVAL;
394	}
395
396	dev_dbg(mmu->dev, "unmap this: iova 0x%lx size 0x%zx\n", iova, size);
397
398	/*
399	 * Keep iterating until we either unmap 'size' bytes (or more)
400	 * or we hit an area that isn't mapped.
401	 */
402	while (unmapped < size) {
403		unmapped_page = __imgu_mmu_unmap(mmu, iova, IPU3_PAGE_SIZE);
404		if (!unmapped_page)
405			break;
406
407		dev_dbg(mmu->dev, "unmapped: iova 0x%lx size 0x%zx\n",
408			iova, unmapped_page);
409
410		iova += unmapped_page;
411		unmapped += unmapped_page;
412	}
413
414	call_if_imgu_is_powered(mmu, imgu_mmu_tlb_invalidate);
415
416	return unmapped;
417}
418
419/**
420 * imgu_mmu_init() - initialize IPU3 MMU block
421 *
422 * @parent:	struct device parent
423 * @base:	IOMEM base of hardware registers.
424 *
425 * Return: Pointer to IPU3 MMU private data pointer or ERR_PTR() on error.
426 */
427struct imgu_mmu_info *imgu_mmu_init(struct device *parent, void __iomem *base)
428{
429	struct imgu_mmu *mmu;
430	u32 pteval;
431
432	mmu = kzalloc(sizeof(*mmu), GFP_KERNEL);
433	if (!mmu)
434		return ERR_PTR(-ENOMEM);
435
436	mmu->dev = parent;
437	mmu->base = base;
438	spin_lock_init(&mmu->lock);
439
440	/* Disallow external memory access when having no valid page tables. */
441	imgu_mmu_set_halt(mmu, true);
442
443	/*
444	 * The MMU does not have a "valid" bit, so we have to use a dummy
445	 * page for invalid entries.
446	 */
447	mmu->dummy_page = (void *)__get_free_page(GFP_KERNEL);
448	if (!mmu->dummy_page)
449		goto fail_group;
450	pteval = IPU3_ADDR2PTE(virt_to_phys(mmu->dummy_page));
451	mmu->dummy_page_pteval = pteval;
452
453	/*
454	 * Allocate a dummy L2 page table with all entries pointing to
455	 * the dummy page.
456	 */
457	mmu->dummy_l2pt = imgu_mmu_alloc_page_table(pteval);
458	if (!mmu->dummy_l2pt)
459		goto fail_dummy_page;
460	pteval = IPU3_ADDR2PTE(virt_to_phys(mmu->dummy_l2pt));
461	mmu->dummy_l2pt_pteval = pteval;
462
463	/*
464	 * Allocate the array of L2PT CPU pointers, initialized to zero,
465	 * which means the dummy L2PT allocated above.
466	 */
467	mmu->l2pts = vzalloc(IPU3_PT_PTES * sizeof(*mmu->l2pts));
468	if (!mmu->l2pts)
469		goto fail_l2pt;
470
471	/* Allocate the L1 page table. */
472	mmu->l1pt = imgu_mmu_alloc_page_table(mmu->dummy_l2pt_pteval);
473	if (!mmu->l1pt)
474		goto fail_l2pts;
475
476	pteval = IPU3_ADDR2PTE(virt_to_phys(mmu->l1pt));
477	writel(pteval, mmu->base + REG_L1_PHYS);
478	imgu_mmu_tlb_invalidate(mmu);
479	imgu_mmu_set_halt(mmu, false);
480
481	mmu->geometry.aperture_start = 0;
482	mmu->geometry.aperture_end = DMA_BIT_MASK(IPU3_MMU_ADDRESS_BITS);
483
484	return &mmu->geometry;
485
486fail_l2pts:
487	vfree(mmu->l2pts);
488fail_l2pt:
489	imgu_mmu_free_page_table(mmu->dummy_l2pt);
490fail_dummy_page:
491	free_page((unsigned long)mmu->dummy_page);
492fail_group:
493	kfree(mmu);
494
495	return ERR_PTR(-ENOMEM);
496}
497
498/**
499 * imgu_mmu_exit() - clean up IPU3 MMU block
500 *
501 * @info: MMU mappable range
502 */
503void imgu_mmu_exit(struct imgu_mmu_info *info)
504{
505	struct imgu_mmu *mmu = to_imgu_mmu(info);
506
507	/* We are going to free our page tables, no more memory access. */
508	imgu_mmu_set_halt(mmu, true);
509	imgu_mmu_tlb_invalidate(mmu);
510
511	imgu_mmu_free_page_table(mmu->l1pt);
512	vfree(mmu->l2pts);
513	imgu_mmu_free_page_table(mmu->dummy_l2pt);
514	free_page((unsigned long)mmu->dummy_page);
515	kfree(mmu);
516}
517
518void imgu_mmu_suspend(struct imgu_mmu_info *info)
519{
520	struct imgu_mmu *mmu = to_imgu_mmu(info);
521
522	imgu_mmu_set_halt(mmu, true);
523}
524
525void imgu_mmu_resume(struct imgu_mmu_info *info)
526{
527	struct imgu_mmu *mmu = to_imgu_mmu(info);
528	u32 pteval;
529
530	imgu_mmu_set_halt(mmu, true);
531
532	pteval = IPU3_ADDR2PTE(virt_to_phys(mmu->l1pt));
533	writel(pteval, mmu->base + REG_L1_PHYS);
534
535	imgu_mmu_tlb_invalidate(mmu);
536	imgu_mmu_set_halt(mmu, false);
537}
538