1// SPDX-License-Identifier: GPL-2.0 OR MIT
2/* Copyright 2017-2019 Qiang Yu <yuq825@gmail.com> */
3
4#include <linux/interrupt.h>
5#include <linux/iopoll.h>
6#include <linux/device.h>
7#include <linux/slab.h>
8
9#include <drm/lima_drm.h>
10
11#include "lima_device.h"
12#include "lima_gp.h"
13#include "lima_regs.h"
14#include "lima_gem.h"
15#include "lima_vm.h"
16
17#define gp_write(reg, data) writel(data, ip->iomem + reg)
18#define gp_read(reg) readl(ip->iomem + reg)
19
20static irqreturn_t lima_gp_irq_handler(int irq, void *data)
21{
22	struct lima_ip *ip = data;
23	struct lima_device *dev = ip->dev;
24	struct lima_sched_pipe *pipe = dev->pipe + lima_pipe_gp;
25	struct lima_sched_task *task = pipe->current_task;
26	u32 state = gp_read(LIMA_GP_INT_STAT);
27	u32 status = gp_read(LIMA_GP_STATUS);
28	bool done = false;
29
30	/* for shared irq case */
31	if (!state)
32		return IRQ_NONE;
33
34	if (state & LIMA_GP_IRQ_MASK_ERROR) {
35		if ((state & LIMA_GP_IRQ_MASK_ERROR) ==
36		    LIMA_GP_IRQ_PLBU_OUT_OF_MEM) {
37			dev_dbg(dev->dev, "%s out of heap irq status=%x\n",
38				lima_ip_name(ip), status);
39		} else {
40			dev_err(dev->dev, "%s error irq state=%x status=%x\n",
41				lima_ip_name(ip), state, status);
42			if (task)
43				task->recoverable = false;
44		}
45
46		/* mask all interrupts before hard reset */
47		gp_write(LIMA_GP_INT_MASK, 0);
48
49		pipe->error = true;
50		done = true;
51	} else {
52		bool valid = state & (LIMA_GP_IRQ_VS_END_CMD_LST |
53				      LIMA_GP_IRQ_PLBU_END_CMD_LST);
54		bool active = status & (LIMA_GP_STATUS_VS_ACTIVE |
55					LIMA_GP_STATUS_PLBU_ACTIVE);
56		done = valid && !active;
57		pipe->error = false;
58	}
59
60	gp_write(LIMA_GP_INT_CLEAR, state);
61
62	if (done)
63		lima_sched_pipe_task_done(pipe);
64
65	return IRQ_HANDLED;
66}
67
68static void lima_gp_soft_reset_async(struct lima_ip *ip)
69{
70	if (ip->data.async_reset)
71		return;
72
73	gp_write(LIMA_GP_INT_MASK, 0);
74	gp_write(LIMA_GP_INT_CLEAR, LIMA_GP_IRQ_RESET_COMPLETED);
75	gp_write(LIMA_GP_CMD, LIMA_GP_CMD_SOFT_RESET);
76	ip->data.async_reset = true;
77}
78
79static int lima_gp_soft_reset_async_wait(struct lima_ip *ip)
80{
81	struct lima_device *dev = ip->dev;
82	int err;
83	u32 v;
84
85	if (!ip->data.async_reset)
86		return 0;
87
88	err = readl_poll_timeout(ip->iomem + LIMA_GP_INT_RAWSTAT, v,
89				 v & LIMA_GP_IRQ_RESET_COMPLETED,
90				 0, 100);
91	if (err) {
92		dev_err(dev->dev, "%s soft reset time out\n",
93			lima_ip_name(ip));
94		return err;
95	}
96
97	gp_write(LIMA_GP_INT_CLEAR, LIMA_GP_IRQ_MASK_ALL);
98	gp_write(LIMA_GP_INT_MASK, LIMA_GP_IRQ_MASK_USED);
99
100	ip->data.async_reset = false;
101	return 0;
102}
103
104static int lima_gp_task_validate(struct lima_sched_pipe *pipe,
105				 struct lima_sched_task *task)
106{
107	struct drm_lima_gp_frame *frame = task->frame;
108	u32 *f = frame->frame;
109	(void)pipe;
110
111	if (f[LIMA_GP_VSCL_START_ADDR >> 2] >
112	    f[LIMA_GP_VSCL_END_ADDR >> 2] ||
113	    f[LIMA_GP_PLBUCL_START_ADDR >> 2] >
114	    f[LIMA_GP_PLBUCL_END_ADDR >> 2] ||
115	    f[LIMA_GP_PLBU_ALLOC_START_ADDR >> 2] >
116	    f[LIMA_GP_PLBU_ALLOC_END_ADDR >> 2])
117		return -EINVAL;
118
119	if (f[LIMA_GP_VSCL_START_ADDR >> 2] ==
120	    f[LIMA_GP_VSCL_END_ADDR >> 2] &&
121	    f[LIMA_GP_PLBUCL_START_ADDR >> 2] ==
122	    f[LIMA_GP_PLBUCL_END_ADDR >> 2])
123		return -EINVAL;
124
125	return 0;
126}
127
128static void lima_gp_task_run(struct lima_sched_pipe *pipe,
129			     struct lima_sched_task *task)
130{
131	struct lima_ip *ip = pipe->processor[0];
132	struct drm_lima_gp_frame *frame = task->frame;
133	u32 *f = frame->frame;
134	u32 cmd = 0;
135	int i;
136
137	/* update real heap buffer size for GP */
138	for (i = 0; i < task->num_bos; i++) {
139		struct lima_bo *bo = task->bos[i];
140
141		if (bo->heap_size &&
142		    lima_vm_get_va(task->vm, bo) ==
143		    f[LIMA_GP_PLBU_ALLOC_START_ADDR >> 2]) {
144			f[LIMA_GP_PLBU_ALLOC_END_ADDR >> 2] =
145				f[LIMA_GP_PLBU_ALLOC_START_ADDR >> 2] +
146				bo->heap_size;
147			task->recoverable = true;
148			task->heap = bo;
149			break;
150		}
151	}
152
153	if (f[LIMA_GP_VSCL_START_ADDR >> 2] !=
154	    f[LIMA_GP_VSCL_END_ADDR >> 2])
155		cmd |= LIMA_GP_CMD_START_VS;
156	if (f[LIMA_GP_PLBUCL_START_ADDR >> 2] !=
157	    f[LIMA_GP_PLBUCL_END_ADDR >> 2])
158		cmd |= LIMA_GP_CMD_START_PLBU;
159
160	/* before any hw ops, wait last success task async soft reset */
161	lima_gp_soft_reset_async_wait(ip);
162
163	for (i = 0; i < LIMA_GP_FRAME_REG_NUM; i++)
164		writel(f[i], ip->iomem + LIMA_GP_VSCL_START_ADDR + i * 4);
165
166	gp_write(LIMA_GP_CMD, LIMA_GP_CMD_UPDATE_PLBU_ALLOC);
167	gp_write(LIMA_GP_CMD, cmd);
168}
169
170static int lima_gp_bus_stop_poll(struct lima_ip *ip)
171{
172	return !!(gp_read(LIMA_GP_STATUS) & LIMA_GP_STATUS_BUS_STOPPED);
173}
174
175static int lima_gp_hard_reset_poll(struct lima_ip *ip)
176{
177	gp_write(LIMA_GP_PERF_CNT_0_LIMIT, 0xC01A0000);
178	return gp_read(LIMA_GP_PERF_CNT_0_LIMIT) == 0xC01A0000;
179}
180
181static int lima_gp_hard_reset(struct lima_ip *ip)
182{
183	struct lima_device *dev = ip->dev;
184	int ret;
185
186	gp_write(LIMA_GP_PERF_CNT_0_LIMIT, 0xC0FFE000);
187	gp_write(LIMA_GP_INT_MASK, 0);
188
189	gp_write(LIMA_GP_CMD, LIMA_GP_CMD_STOP_BUS);
190	ret = lima_poll_timeout(ip, lima_gp_bus_stop_poll, 10, 100);
191	if (ret) {
192		dev_err(dev->dev, "%s bus stop timeout\n", lima_ip_name(ip));
193		return ret;
194	}
195	gp_write(LIMA_GP_CMD, LIMA_GP_CMD_RESET);
196	ret = lima_poll_timeout(ip, lima_gp_hard_reset_poll, 10, 100);
197	if (ret) {
198		dev_err(dev->dev, "%s hard reset timeout\n", lima_ip_name(ip));
199		return ret;
200	}
201
202	gp_write(LIMA_GP_PERF_CNT_0_LIMIT, 0);
203	gp_write(LIMA_GP_INT_CLEAR, LIMA_GP_IRQ_MASK_ALL);
204	gp_write(LIMA_GP_INT_MASK, LIMA_GP_IRQ_MASK_USED);
205
206	/*
207	 * if there was an async soft reset queued,
208	 * don't wait for it in the next job
209	 */
210	ip->data.async_reset = false;
211
212	return 0;
213}
214
215static void lima_gp_task_fini(struct lima_sched_pipe *pipe)
216{
217	lima_gp_soft_reset_async(pipe->processor[0]);
218}
219
220static void lima_gp_task_error(struct lima_sched_pipe *pipe)
221{
222	struct lima_ip *ip = pipe->processor[0];
223
224	dev_err(ip->dev->dev, "%s task error int_state=%x status=%x\n",
225		lima_ip_name(ip), gp_read(LIMA_GP_INT_STAT),
226		gp_read(LIMA_GP_STATUS));
227
228	lima_gp_hard_reset(ip);
229}
230
231static void lima_gp_task_mmu_error(struct lima_sched_pipe *pipe)
232{
233	lima_sched_pipe_task_done(pipe);
234}
235
236static int lima_gp_task_recover(struct lima_sched_pipe *pipe)
237{
238	struct lima_ip *ip = pipe->processor[0];
239	struct lima_sched_task *task = pipe->current_task;
240	struct drm_lima_gp_frame *frame = task->frame;
241	u32 *f = frame->frame;
242	size_t fail_size =
243		f[LIMA_GP_PLBU_ALLOC_END_ADDR >> 2] -
244		f[LIMA_GP_PLBU_ALLOC_START_ADDR >> 2];
245
246	if (fail_size == task->heap->heap_size) {
247		int ret;
248
249		ret = lima_heap_alloc(task->heap, task->vm);
250		if (ret < 0)
251			return ret;
252	}
253
254	gp_write(LIMA_GP_INT_MASK, LIMA_GP_IRQ_MASK_USED);
255	/* Resume from where we stopped, i.e. new start is old end */
256	gp_write(LIMA_GP_PLBU_ALLOC_START_ADDR,
257		 f[LIMA_GP_PLBU_ALLOC_END_ADDR >> 2]);
258	f[LIMA_GP_PLBU_ALLOC_END_ADDR >> 2] =
259		f[LIMA_GP_PLBU_ALLOC_START_ADDR >> 2] + task->heap->heap_size;
260	gp_write(LIMA_GP_PLBU_ALLOC_END_ADDR,
261		 f[LIMA_GP_PLBU_ALLOC_END_ADDR >> 2]);
262	gp_write(LIMA_GP_CMD, LIMA_GP_CMD_UPDATE_PLBU_ALLOC);
263	return 0;
264}
265
266static void lima_gp_print_version(struct lima_ip *ip)
267{
268	u32 version, major, minor;
269	char *name;
270
271	version = gp_read(LIMA_GP_VERSION);
272	major = (version >> 8) & 0xFF;
273	minor = version & 0xFF;
274	switch (version >> 16) {
275	case 0xA07:
276	    name = "mali200";
277		break;
278	case 0xC07:
279		name = "mali300";
280		break;
281	case 0xB07:
282		name = "mali400";
283		break;
284	case 0xD07:
285		name = "mali450";
286		break;
287	default:
288		name = "unknown";
289		break;
290	}
291	dev_info(ip->dev->dev, "%s - %s version major %d minor %d\n",
292		 lima_ip_name(ip), name, major, minor);
293}
294
295static struct kmem_cache *lima_gp_task_slab;
296static int lima_gp_task_slab_refcnt;
297
298static int lima_gp_hw_init(struct lima_ip *ip)
299{
300	ip->data.async_reset = false;
301	lima_gp_soft_reset_async(ip);
302	return lima_gp_soft_reset_async_wait(ip);
303}
304
305int lima_gp_resume(struct lima_ip *ip)
306{
307	return lima_gp_hw_init(ip);
308}
309
310void lima_gp_suspend(struct lima_ip *ip)
311{
312
313}
314
315int lima_gp_init(struct lima_ip *ip)
316{
317	struct lima_device *dev = ip->dev;
318	int err;
319
320	lima_gp_print_version(ip);
321
322	err = lima_gp_hw_init(ip);
323	if (err)
324		return err;
325
326	err = devm_request_irq(dev->dev, ip->irq, lima_gp_irq_handler,
327			       IRQF_SHARED, lima_ip_name(ip), ip);
328	if (err) {
329		dev_err(dev->dev, "%s fail to request irq\n",
330			lima_ip_name(ip));
331		return err;
332	}
333
334	dev->gp_version = gp_read(LIMA_GP_VERSION);
335
336	return 0;
337}
338
339void lima_gp_fini(struct lima_ip *ip)
340{
341
342}
343
344int lima_gp_pipe_init(struct lima_device *dev)
345{
346	int frame_size = sizeof(struct drm_lima_gp_frame);
347	struct lima_sched_pipe *pipe = dev->pipe + lima_pipe_gp;
348
349	if (!lima_gp_task_slab) {
350		lima_gp_task_slab = kmem_cache_create_usercopy(
351			"lima_gp_task", sizeof(struct lima_sched_task) + frame_size,
352			0, SLAB_HWCACHE_ALIGN, sizeof(struct lima_sched_task),
353			frame_size, NULL);
354		if (!lima_gp_task_slab)
355			return -ENOMEM;
356	}
357	lima_gp_task_slab_refcnt++;
358
359	pipe->frame_size = frame_size;
360	pipe->task_slab = lima_gp_task_slab;
361
362	pipe->task_validate = lima_gp_task_validate;
363	pipe->task_run = lima_gp_task_run;
364	pipe->task_fini = lima_gp_task_fini;
365	pipe->task_error = lima_gp_task_error;
366	pipe->task_mmu_error = lima_gp_task_mmu_error;
367	pipe->task_recover = lima_gp_task_recover;
368
369	return 0;
370}
371
372void lima_gp_pipe_fini(struct lima_device *dev)
373{
374	if (!--lima_gp_task_slab_refcnt) {
375		kmem_cache_destroy(lima_gp_task_slab);
376		lima_gp_task_slab = NULL;
377	}
378}
379