1// SPDX-License-Identifier: MIT
2/*
3 * Copyright �� 2022 Intel Corporation
4 */
5
6#include "xe_reg_sr.h"
7
8#include <kunit/visibility.h>
9#include <linux/align.h>
10#include <linux/string_helpers.h>
11#include <linux/xarray.h>
12
13#include <drm/drm_managed.h>
14#include <drm/drm_print.h>
15
16#include "regs/xe_engine_regs.h"
17#include "regs/xe_gt_regs.h"
18#include "xe_device_types.h"
19#include "xe_force_wake.h"
20#include "xe_gt.h"
21#include "xe_gt_mcr.h"
22#include "xe_gt_printk.h"
23#include "xe_hw_engine_types.h"
24#include "xe_macros.h"
25#include "xe_mmio.h"
26#include "xe_reg_whitelist.h"
27#include "xe_rtp_types.h"
28
29#define XE_REG_SR_GROW_STEP_DEFAULT	16
30
31static void reg_sr_fini(struct drm_device *drm, void *arg)
32{
33	struct xe_reg_sr *sr = arg;
34
35	xa_destroy(&sr->xa);
36	kfree(sr->pool.arr);
37	memset(&sr->pool, 0, sizeof(sr->pool));
38}
39
40int xe_reg_sr_init(struct xe_reg_sr *sr, const char *name, struct xe_device *xe)
41{
42	xa_init(&sr->xa);
43	memset(&sr->pool, 0, sizeof(sr->pool));
44	sr->pool.grow_step = XE_REG_SR_GROW_STEP_DEFAULT;
45	sr->name = name;
46
47	return drmm_add_action_or_reset(&xe->drm, reg_sr_fini, sr);
48}
49EXPORT_SYMBOL_IF_KUNIT(xe_reg_sr_init);
50
51static struct xe_reg_sr_entry *alloc_entry(struct xe_reg_sr *sr)
52{
53	if (sr->pool.used == sr->pool.allocated) {
54		struct xe_reg_sr_entry *arr;
55
56		arr = krealloc_array(sr->pool.arr,
57				     ALIGN(sr->pool.allocated + 1, sr->pool.grow_step),
58				     sizeof(*arr), GFP_KERNEL);
59		if (!arr)
60			return NULL;
61
62		sr->pool.arr = arr;
63		sr->pool.allocated += sr->pool.grow_step;
64	}
65
66	return &sr->pool.arr[sr->pool.used++];
67}
68
69static bool compatible_entries(const struct xe_reg_sr_entry *e1,
70			       const struct xe_reg_sr_entry *e2)
71{
72	/*
73	 * Don't allow overwriting values: clr_bits/set_bits should be disjoint
74	 * when operating in the same register
75	 */
76	if (e1->clr_bits & e2->clr_bits || e1->set_bits & e2->set_bits ||
77	    e1->clr_bits & e2->set_bits || e1->set_bits & e2->clr_bits)
78		return false;
79
80	if (e1->reg.raw != e2->reg.raw)
81		return false;
82
83	return true;
84}
85
86static void reg_sr_inc_error(struct xe_reg_sr *sr)
87{
88#if IS_ENABLED(CONFIG_DRM_XE_KUNIT_TEST)
89	sr->errors++;
90#endif
91}
92
93int xe_reg_sr_add(struct xe_reg_sr *sr,
94		  const struct xe_reg_sr_entry *e,
95		  struct xe_gt *gt)
96{
97	unsigned long idx = e->reg.addr;
98	struct xe_reg_sr_entry *pentry = xa_load(&sr->xa, idx);
99	int ret;
100
101	if (pentry) {
102		if (!compatible_entries(pentry, e)) {
103			ret = -EINVAL;
104			goto fail;
105		}
106
107		pentry->clr_bits |= e->clr_bits;
108		pentry->set_bits |= e->set_bits;
109		pentry->read_mask |= e->read_mask;
110
111		return 0;
112	}
113
114	pentry = alloc_entry(sr);
115	if (!pentry) {
116		ret = -ENOMEM;
117		goto fail;
118	}
119
120	*pentry = *e;
121	ret = xa_err(xa_store(&sr->xa, idx, pentry, GFP_KERNEL));
122	if (ret)
123		goto fail;
124
125	return 0;
126
127fail:
128	xe_gt_err(gt,
129		  "discarding save-restore reg %04lx (clear: %08x, set: %08x, masked: %s, mcr: %s): ret=%d\n",
130		  idx, e->clr_bits, e->set_bits,
131		  str_yes_no(e->reg.masked),
132		  str_yes_no(e->reg.mcr),
133		  ret);
134	reg_sr_inc_error(sr);
135
136	return ret;
137}
138
139/*
140 * Convert back from encoded value to type-safe, only to be used when reg.mcr
141 * is true
142 */
143static struct xe_reg_mcr to_xe_reg_mcr(const struct xe_reg reg)
144{
145	return (const struct xe_reg_mcr){.__reg.raw = reg.raw };
146}
147
148static void apply_one_mmio(struct xe_gt *gt, struct xe_reg_sr_entry *entry)
149{
150	struct xe_reg reg = entry->reg;
151	struct xe_reg_mcr reg_mcr = to_xe_reg_mcr(reg);
152	u32 val;
153
154	/*
155	 * If this is a masked register, need to set the upper 16 bits.
156	 * Set them to clr_bits since that is always a superset of the bits
157	 * being modified.
158	 *
159	 * When it's not masked, we have to read it from hardware, unless we are
160	 * supposed to set all bits.
161	 */
162	if (reg.masked)
163		val = entry->clr_bits << 16;
164	else if (entry->clr_bits + 1)
165		val = (reg.mcr ?
166		       xe_gt_mcr_unicast_read_any(gt, reg_mcr) :
167		       xe_mmio_read32(gt, reg)) & (~entry->clr_bits);
168	else
169		val = 0;
170
171	/*
172	 * TODO: add selftest to validate all tables, regardless of platform:
173	 *   - Masked registers can't have set_bits with upper bits set
174	 *   - set_bits must be contained in clr_bits
175	 */
176	val |= entry->set_bits;
177
178	xe_gt_dbg(gt, "REG[0x%x] = 0x%08x", reg.addr, val);
179
180	if (entry->reg.mcr)
181		xe_gt_mcr_multicast_write(gt, reg_mcr, val);
182	else
183		xe_mmio_write32(gt, reg, val);
184}
185
186void xe_reg_sr_apply_mmio(struct xe_reg_sr *sr, struct xe_gt *gt)
187{
188	struct xe_reg_sr_entry *entry;
189	unsigned long reg;
190	int err;
191
192	if (xa_empty(&sr->xa))
193		return;
194
195	xe_gt_dbg(gt, "Applying %s save-restore MMIOs\n", sr->name);
196
197	err = xe_force_wake_get(&gt->mmio.fw, XE_FORCEWAKE_ALL);
198	if (err)
199		goto err_force_wake;
200
201	xa_for_each(&sr->xa, reg, entry)
202		apply_one_mmio(gt, entry);
203
204	err = xe_force_wake_put(&gt->mmio.fw, XE_FORCEWAKE_ALL);
205	XE_WARN_ON(err);
206
207	return;
208
209err_force_wake:
210	xe_gt_err(gt, "Failed to apply, err=%d\n", err);
211}
212
213void xe_reg_sr_apply_whitelist(struct xe_hw_engine *hwe)
214{
215	struct xe_reg_sr *sr = &hwe->reg_whitelist;
216	struct xe_gt *gt = hwe->gt;
217	struct xe_device *xe = gt_to_xe(gt);
218	struct xe_reg_sr_entry *entry;
219	struct drm_printer p;
220	u32 mmio_base = hwe->mmio_base;
221	unsigned long reg;
222	unsigned int slot = 0;
223	int err;
224
225	if (xa_empty(&sr->xa))
226		return;
227
228	drm_dbg(&xe->drm, "Whitelisting %s registers\n", sr->name);
229
230	err = xe_force_wake_get(&gt->mmio.fw, XE_FORCEWAKE_ALL);
231	if (err)
232		goto err_force_wake;
233
234	p = drm_dbg_printer(&xe->drm, DRM_UT_DRIVER, NULL);
235	xa_for_each(&sr->xa, reg, entry) {
236		if (slot == RING_MAX_NONPRIV_SLOTS) {
237			xe_gt_err(gt,
238				  "hwe %s: maximum register whitelist slots (%d) reached, refusing to add more\n",
239				  hwe->name, RING_MAX_NONPRIV_SLOTS);
240			break;
241		}
242
243		xe_reg_whitelist_print_entry(&p, 0, reg, entry);
244		xe_mmio_write32(gt, RING_FORCE_TO_NONPRIV(mmio_base, slot),
245				reg | entry->set_bits);
246		slot++;
247	}
248
249	/* And clear the rest just in case of garbage */
250	for (; slot < RING_MAX_NONPRIV_SLOTS; slot++) {
251		u32 addr = RING_NOPID(mmio_base).addr;
252
253		xe_mmio_write32(gt, RING_FORCE_TO_NONPRIV(mmio_base, slot), addr);
254	}
255
256	err = xe_force_wake_put(&gt->mmio.fw, XE_FORCEWAKE_ALL);
257	XE_WARN_ON(err);
258
259	return;
260
261err_force_wake:
262	drm_err(&xe->drm, "Failed to apply, err=%d\n", err);
263}
264
265/**
266 * xe_reg_sr_dump - print all save/restore entries
267 * @sr: Save/restore entries
268 * @p: DRM printer
269 */
270void xe_reg_sr_dump(struct xe_reg_sr *sr, struct drm_printer *p)
271{
272	struct xe_reg_sr_entry *entry;
273	unsigned long reg;
274
275	if (!sr->name || xa_empty(&sr->xa))
276		return;
277
278	drm_printf(p, "%s\n", sr->name);
279	xa_for_each(&sr->xa, reg, entry)
280		drm_printf(p, "\tREG[0x%lx] clr=0x%08x set=0x%08x masked=%s mcr=%s\n",
281			   reg, entry->clr_bits, entry->set_bits,
282			   str_yes_no(entry->reg.masked),
283			   str_yes_no(entry->reg.mcr));
284}
285