1/* SPDX-License-Identifier: GPL-2.0-or-later */
2/*
3 * Copyright 2016-17 IBM Corp.
4 */
5
6#ifndef _VAS_H
7#define _VAS_H
8#include <linux/atomic.h>
9#include <linux/idr.h>
10#include <asm/vas.h>
11#include <linux/io.h>
12#include <linux/dcache.h>
13#include <linux/mutex.h>
14#include <linux/stringify.h>
15
16/*
17 * Overview of Virtual Accelerator Switchboard (VAS).
18 *
19 * VAS is a hardware "switchboard" that allows senders and receivers to
20 * exchange messages with _minimal_ kernel involvment. The receivers are
21 * typically NX coprocessor engines that perform compression or encryption
22 * in hardware, but receivers can also be other software threads.
23 *
24 * Senders are user/kernel threads that submit compression/encryption or
25 * other requests to the receivers. Senders must format their messages as
26 * Coprocessor Request Blocks (CRB)s and submit them using the "copy" and
27 * "paste" instructions which were introduced in Power9.
28 *
29 * A Power node can have (upto?) 8 Power chips. There is one instance of
30 * VAS in each Power9 chip. Each instance of VAS has 64K windows or ports,
31 * Senders and receivers must each connect to a separate window before they
32 * can exchange messages through the switchboard.
33 *
34 * Each window is described by two types of window contexts:
35 *
36 *	Hypervisor Window Context (HVWC) of size VAS_HVWC_SIZE bytes
37 *
38 *	OS/User Window Context (UWC) of size VAS_UWC_SIZE bytes.
39 *
40 * A window context can be viewed as a set of 64-bit registers. The settings
41 * in these registers configure/control/determine the behavior of the VAS
42 * hardware when messages are sent/received through the window. The registers
43 * in the HVWC are configured by the kernel while the registers in the UWC can
44 * be configured by the kernel or by the user space application that is using
45 * the window.
46 *
47 * The HVWCs for all windows on a specific instance of VAS are in a contiguous
48 * range of hardware addresses or Base address region (BAR) referred to as the
49 * HVWC BAR for the instance. Similarly the UWCs for all windows on an instance
50 * are referred to as the UWC BAR for the instance.
51 *
52 * The two BARs for each instance are defined Power9 MMIO Ranges spreadsheet
53 * and available to the kernel in the VAS node's "reg" property in the device
54 * tree:
55 *
56 *	/proc/device-tree/vasm@.../reg
57 *
58 * (see vas_probe() for details on the reg property).
59 *
60 * The kernel maps the HVWC and UWC BAR regions into the kernel address
61 * space (hvwc_map and uwc_map). The kernel can then access the window
62 * contexts of a specific window using:
63 *
64 *	 hvwc = hvwc_map + winid * VAS_HVWC_SIZE.
65 *	 uwc = uwc_map + winid * VAS_UWC_SIZE.
66 *
67 * where winid is the window index (0..64K).
68 *
69 * As mentioned, a window context is used to "configure" a window. Besides
70 * this configuration address, each _send_ window also has a unique hardware
71 * "paste" address that is used to submit requests/CRBs (see vas_paste_crb()).
72 *
73 * The hardware paste address for a window is computed using the "paste
74 * base address" and "paste win id shift" reg properties in the VAS device
75 * tree node using:
76 *
77 *	paste_addr = paste_base + ((winid << paste_win_id_shift))
78 *
79 * (again, see vas_probe() for ->paste_base_addr and ->paste_win_id_shift).
80 *
81 * The kernel maps this hardware address into the sender's address space
82 * after which they can use the 'paste' instruction (new in Power9) to
83 * send a message (submit a request aka CRB) to the coprocessor.
84 *
85 * NOTE: In the initial version, senders can only in-kernel drivers/threads.
86 *	 Support for user space threads will be added in follow-on patches.
87 *
88 * TODO: Do we need to map the UWC into user address space so they can return
89 *	 credits? Its NA for NX but may be needed for other receive windows.
90 *
91 */
92
93#define VAS_WINDOWS_PER_CHIP		(64 << 10)
94
95/*
96 * Hypervisor and OS/USer Window Context sizes
97 */
98#define VAS_HVWC_SIZE			512
99#define VAS_UWC_SIZE			PAGE_SIZE
100
101/*
102 * Initial per-process credits.
103 * Max send window credits:    4K-1 (12-bits in VAS_TX_WCRED)
104 *
105 * TODO: Needs tuning for per-process credits
106 */
107#define VAS_TX_WCREDS_MAX		((4 << 10) - 1)
108#define VAS_WCREDS_DEFAULT		(1 << 10)
109
110/*
111 * VAS Window Context Register Offsets and bitmasks.
112 * See Section 3.1.4 of VAS Work book
113 */
114#define VAS_LPID_OFFSET			0x010
115#define VAS_LPID			PPC_BITMASK(0, 11)
116
117#define VAS_PID_OFFSET			0x018
118#define VAS_PID_ID			PPC_BITMASK(0, 19)
119
120#define VAS_XLATE_MSR_OFFSET		0x020
121#define VAS_XLATE_MSR_DR		PPC_BIT(0)
122#define VAS_XLATE_MSR_TA		PPC_BIT(1)
123#define VAS_XLATE_MSR_PR		PPC_BIT(2)
124#define VAS_XLATE_MSR_US		PPC_BIT(3)
125#define VAS_XLATE_MSR_HV		PPC_BIT(4)
126#define VAS_XLATE_MSR_SF		PPC_BIT(5)
127
128#define VAS_XLATE_LPCR_OFFSET		0x028
129#define VAS_XLATE_LPCR_PAGE_SIZE	PPC_BITMASK(0, 2)
130#define VAS_XLATE_LPCR_ISL		PPC_BIT(3)
131#define VAS_XLATE_LPCR_TC		PPC_BIT(4)
132#define VAS_XLATE_LPCR_SC		PPC_BIT(5)
133
134#define VAS_XLATE_CTL_OFFSET		0x030
135#define VAS_XLATE_MODE			PPC_BITMASK(0, 1)
136
137#define VAS_AMR_OFFSET			0x040
138#define VAS_AMR				PPC_BITMASK(0, 63)
139
140#define VAS_SEIDR_OFFSET		0x048
141#define VAS_SEIDR			PPC_BITMASK(0, 63)
142
143#define VAS_FAULT_TX_WIN_OFFSET		0x050
144#define VAS_FAULT_TX_WIN		PPC_BITMASK(48, 63)
145
146#define VAS_OSU_INTR_SRC_RA_OFFSET	0x060
147#define VAS_OSU_INTR_SRC_RA		PPC_BITMASK(8, 63)
148
149#define VAS_HV_INTR_SRC_RA_OFFSET	0x070
150#define VAS_HV_INTR_SRC_RA		PPC_BITMASK(8, 63)
151
152#define VAS_PSWID_OFFSET		0x078
153#define VAS_PSWID_EA_HANDLE		PPC_BITMASK(0, 31)
154
155#define VAS_SPARE1_OFFSET		0x080
156#define VAS_SPARE2_OFFSET		0x088
157#define VAS_SPARE3_OFFSET		0x090
158#define VAS_SPARE4_OFFSET		0x130
159#define VAS_SPARE5_OFFSET		0x160
160#define VAS_SPARE6_OFFSET		0x188
161
162#define VAS_LFIFO_BAR_OFFSET		0x0A0
163#define VAS_LFIFO_BAR			PPC_BITMASK(8, 53)
164#define VAS_PAGE_MIGRATION_SELECT	PPC_BITMASK(54, 56)
165
166#define VAS_LDATA_STAMP_CTL_OFFSET	0x0A8
167#define VAS_LDATA_STAMP			PPC_BITMASK(0, 1)
168#define VAS_XTRA_WRITE			PPC_BIT(2)
169
170#define VAS_LDMA_CACHE_CTL_OFFSET	0x0B0
171#define VAS_LDMA_TYPE			PPC_BITMASK(0, 1)
172#define VAS_LDMA_FIFO_DISABLE		PPC_BIT(2)
173
174#define VAS_LRFIFO_PUSH_OFFSET		0x0B8
175#define VAS_LRFIFO_PUSH			PPC_BITMASK(0, 15)
176
177#define VAS_CURR_MSG_COUNT_OFFSET	0x0C0
178#define VAS_CURR_MSG_COUNT		PPC_BITMASK(0, 7)
179
180#define VAS_LNOTIFY_AFTER_COUNT_OFFSET	0x0C8
181#define VAS_LNOTIFY_AFTER_COUNT		PPC_BITMASK(0, 7)
182
183#define VAS_LRX_WCRED_OFFSET		0x0E0
184#define VAS_LRX_WCRED			PPC_BITMASK(0, 15)
185
186#define VAS_LRX_WCRED_ADDER_OFFSET	0x190
187#define VAS_LRX_WCRED_ADDER		PPC_BITMASK(0, 15)
188
189#define VAS_TX_WCRED_OFFSET		0x0F0
190#define VAS_TX_WCRED			PPC_BITMASK(4, 15)
191
192#define VAS_TX_WCRED_ADDER_OFFSET	0x1A0
193#define VAS_TX_WCRED_ADDER		PPC_BITMASK(4, 15)
194
195#define VAS_LFIFO_SIZE_OFFSET		0x100
196#define VAS_LFIFO_SIZE			PPC_BITMASK(0, 3)
197
198#define VAS_WINCTL_OFFSET		0x108
199#define VAS_WINCTL_OPEN			PPC_BIT(0)
200#define VAS_WINCTL_REJ_NO_CREDIT	PPC_BIT(1)
201#define VAS_WINCTL_PIN			PPC_BIT(2)
202#define VAS_WINCTL_TX_WCRED_MODE	PPC_BIT(3)
203#define VAS_WINCTL_RX_WCRED_MODE	PPC_BIT(4)
204#define VAS_WINCTL_TX_WORD_MODE		PPC_BIT(5)
205#define VAS_WINCTL_RX_WORD_MODE		PPC_BIT(6)
206#define VAS_WINCTL_RSVD_TXBUF		PPC_BIT(7)
207#define VAS_WINCTL_THRESH_CTL		PPC_BITMASK(8, 9)
208#define VAS_WINCTL_FAULT_WIN		PPC_BIT(10)
209#define VAS_WINCTL_NX_WIN		PPC_BIT(11)
210
211#define VAS_WIN_STATUS_OFFSET		0x110
212#define VAS_WIN_BUSY			PPC_BIT(1)
213
214#define VAS_WIN_CTX_CACHING_CTL_OFFSET	0x118
215#define VAS_CASTOUT_REQ			PPC_BIT(0)
216#define VAS_PUSH_TO_MEM			PPC_BIT(1)
217#define VAS_WIN_CACHE_STATUS		PPC_BIT(4)
218
219#define VAS_TX_RSVD_BUF_COUNT_OFFSET	0x120
220#define VAS_RXVD_BUF_COUNT		PPC_BITMASK(58, 63)
221
222#define VAS_LRFIFO_WIN_PTR_OFFSET	0x128
223#define VAS_LRX_WIN_ID			PPC_BITMASK(0, 15)
224
225/*
226 * Local Notification Control Register controls what happens in _response_
227 * to a paste command and hence applies only to receive windows.
228 */
229#define VAS_LNOTIFY_CTL_OFFSET		0x138
230#define VAS_NOTIFY_DISABLE		PPC_BIT(0)
231#define VAS_INTR_DISABLE		PPC_BIT(1)
232#define VAS_NOTIFY_EARLY		PPC_BIT(2)
233#define VAS_NOTIFY_OSU_INTR		PPC_BIT(3)
234
235#define VAS_LNOTIFY_PID_OFFSET		0x140
236#define VAS_LNOTIFY_PID			PPC_BITMASK(0, 19)
237
238#define VAS_LNOTIFY_LPID_OFFSET		0x148
239#define VAS_LNOTIFY_LPID		PPC_BITMASK(0, 11)
240
241#define VAS_LNOTIFY_TID_OFFSET		0x150
242#define VAS_LNOTIFY_TID			PPC_BITMASK(0, 15)
243
244#define VAS_LNOTIFY_SCOPE_OFFSET	0x158
245#define VAS_LNOTIFY_MIN_SCOPE		PPC_BITMASK(0, 1)
246#define VAS_LNOTIFY_MAX_SCOPE		PPC_BITMASK(2, 3)
247
248#define VAS_NX_UTIL_OFFSET		0x1B0
249#define VAS_NX_UTIL			PPC_BITMASK(0, 63)
250
251/* SE: Side effects */
252#define VAS_NX_UTIL_SE_OFFSET		0x1B8
253#define VAS_NX_UTIL_SE			PPC_BITMASK(0, 63)
254
255#define VAS_NX_UTIL_ADDER_OFFSET	0x180
256#define VAS_NX_UTIL_ADDER		PPC_BITMASK(32, 63)
257
258/*
259 * VREG(x):
260 * Expand a register's short name (eg: LPID) into two parameters:
261 *	- the register's short name in string form ("LPID"), and
262 *	- the name of the macro (eg: VAS_LPID_OFFSET), defining the
263 *	  register's offset in the window context
264 */
265#define VREG_SFX(n, s)	__stringify(n), VAS_##n##s
266#define VREG(r)		VREG_SFX(r, _OFFSET)
267
268/*
269 * Local Notify Scope Control Register. (Receive windows only).
270 */
271enum vas_notify_scope {
272	VAS_SCOPE_LOCAL,
273	VAS_SCOPE_GROUP,
274	VAS_SCOPE_VECTORED_GROUP,
275	VAS_SCOPE_UNUSED,
276};
277
278/*
279 * Local DMA Cache Control Register (Receive windows only).
280 */
281enum vas_dma_type {
282	VAS_DMA_TYPE_INJECT,
283	VAS_DMA_TYPE_WRITE,
284};
285
286/*
287 * Local Notify Scope Control Register. (Receive windows only).
288 * Not applicable to NX receive windows.
289 */
290enum vas_notify_after_count {
291	VAS_NOTIFY_AFTER_256 = 0,
292	VAS_NOTIFY_NONE,
293	VAS_NOTIFY_AFTER_2
294};
295
296/*
297 * NX can generate an interrupt for multiple faults and expects kernel
298 * to process all of them. So read all valid CRB entries until find the
299 * invalid one. So use pswid which is pasted by NX and ccw[0] (reserved
300 * bit in BE) to check valid CRB. CCW[0] will not be touched by user
301 * space. Application gets CRB formt error if it updates this bit.
302 *
303 * Invalidate FIFO during allocation and process all entries from last
304 * successful read until finds invalid pswid and ccw[0] values.
305 * After reading each CRB entry from fault FIFO, the kernel invalidate
306 * it by updating pswid with FIFO_INVALID_ENTRY and CCW[0] with
307 * CCW0_INVALID.
308 */
309#define FIFO_INVALID_ENTRY	0xffffffff
310#define CCW0_INVALID		1
311
312/*
313 * One per instance of VAS. Each instance will have a separate set of
314 * receive windows, one per coprocessor type.
315 *
316 * See also function header of set_vinst_win() for details on ->windows[]
317 * and ->rxwin[] tables.
318 */
319struct vas_instance {
320	int vas_id;
321	struct ida ida;
322	struct list_head node;
323	struct platform_device *pdev;
324
325	u64 hvwc_bar_start;
326	u64 uwc_bar_start;
327	u64 paste_base_addr;
328	u64 paste_win_id_shift;
329
330	u64 irq_port;
331	int virq;
332	int fault_crbs;
333	int fault_fifo_size;
334	int fifo_in_progress;	/* To wake up thread or return IRQ_HANDLED */
335	spinlock_t fault_lock;	/* Protects fifo_in_progress update */
336	void *fault_fifo;
337	struct pnv_vas_window *fault_win; /* Fault window */
338
339	struct mutex mutex;
340	struct pnv_vas_window *rxwin[VAS_COP_TYPE_MAX];
341	struct pnv_vas_window *windows[VAS_WINDOWS_PER_CHIP];
342
343	char *name;
344	char *dbgname;
345	struct dentry *dbgdir;
346};
347
348/*
349 * In-kernel state a VAS window on PowerNV. One per window.
350 */
351struct pnv_vas_window {
352	struct vas_window vas_win;
353	/* Fields common to send and receive windows */
354	struct vas_instance *vinst;
355	bool tx_win;		/* True if send window */
356	bool nx_win;		/* True if NX window */
357	bool user_win;		/* True if user space window */
358	void *hvwc_map;		/* HV window context */
359	void *uwc_map;		/* OS/User window context */
360
361	/* Fields applicable only to send windows */
362	void *paste_kaddr;
363	char *paste_addr_name;
364	struct pnv_vas_window *rxwin;
365
366	/* Fields applicable only to receive windows */
367	atomic_t num_txwins;
368};
369
370/*
371 * Container for the hardware state of a window. One per-window.
372 *
373 * A VAS Window context is a 512-byte area in the hardware that contains
374 * a set of 64-bit registers. Individual bit-fields in these registers
375 * determine the configuration/operation of the hardware. struct vas_winctx
376 * is a container for the register fields in the window context.
377 */
378struct vas_winctx {
379	u64 rx_fifo;
380	int rx_fifo_size;
381	int wcreds_max;
382	int rsvd_txbuf_count;
383
384	bool user_win;
385	bool nx_win;
386	bool fault_win;
387	bool rsvd_txbuf_enable;
388	bool pin_win;
389	bool rej_no_credit;
390	bool tx_wcred_mode;
391	bool rx_wcred_mode;
392	bool tx_word_mode;
393	bool rx_word_mode;
394	bool data_stamp;
395	bool xtra_write;
396	bool notify_disable;
397	bool intr_disable;
398	bool fifo_disable;
399	bool notify_early;
400	bool notify_os_intr_reg;
401
402	int lpid;
403	int pidr;		/* value from SPRN_PID, not linux pid */
404	int lnotify_lpid;
405	int lnotify_pid;
406	int lnotify_tid;
407	u32 pswid;
408	int rx_win_id;
409	int fault_win_id;
410	int tc_mode;
411
412	u64 irq_port;
413
414	enum vas_dma_type dma_type;
415	enum vas_notify_scope min_scope;
416	enum vas_notify_scope max_scope;
417	enum vas_notify_after_count notify_after_count;
418};
419
420extern struct mutex vas_mutex;
421
422extern struct vas_instance *find_vas_instance(int vasid);
423extern void vas_init_dbgdir(void);
424extern void vas_instance_init_dbgdir(struct vas_instance *vinst);
425extern void vas_window_init_dbgdir(struct pnv_vas_window *win);
426extern void vas_window_free_dbgdir(struct pnv_vas_window *win);
427extern int vas_setup_fault_window(struct vas_instance *vinst);
428extern irqreturn_t vas_fault_thread_fn(int irq, void *data);
429extern irqreturn_t vas_fault_handler(int irq, void *dev_id);
430extern void vas_return_credit(struct pnv_vas_window *window, bool tx);
431extern struct pnv_vas_window *vas_pswid_to_window(struct vas_instance *vinst,
432						uint32_t pswid);
433extern void vas_win_paste_addr(struct pnv_vas_window *window, u64 *addr,
434				int *len);
435
436static inline int vas_window_pid(struct vas_window *window)
437{
438	return pid_vnr(window->task_ref.pid);
439}
440
441static inline void vas_log_write(struct pnv_vas_window *win, char *name,
442			void *regptr, u64 val)
443{
444	if (val)
445		pr_debug("%swin #%d: %s reg %p, val 0x%016llx\n",
446				win->tx_win ? "Tx" : "Rx", win->vas_win.winid,
447				name, regptr, val);
448}
449
450static inline void write_uwc_reg(struct pnv_vas_window *win, char *name,
451			s32 reg, u64 val)
452{
453	void *regptr;
454
455	regptr = win->uwc_map + reg;
456	vas_log_write(win, name, regptr, val);
457
458	out_be64(regptr, val);
459}
460
461static inline void write_hvwc_reg(struct pnv_vas_window *win, char *name,
462			s32 reg, u64 val)
463{
464	void *regptr;
465
466	regptr = win->hvwc_map + reg;
467	vas_log_write(win, name, regptr, val);
468
469	out_be64(regptr, val);
470}
471
472static inline u64 read_hvwc_reg(struct pnv_vas_window *win,
473			char *name __maybe_unused, s32 reg)
474{
475	return in_be64(win->hvwc_map+reg);
476}
477
478/*
479 * Encode/decode the Partition Send Window ID (PSWID) for a window in
480 * a way that we can uniquely identify any window in the system. i.e.
481 * we should be able to locate the 'struct vas_window' given the PSWID.
482 *
483 *	Bits	Usage
484 *	0:7	VAS id (8 bits)
485 *	8:15	Unused, 0 (3 bits)
486 *	16:31	Window id (16 bits)
487 */
488static inline u32 encode_pswid(int vasid, int winid)
489{
490	return ((u32)winid | (vasid << (31 - 7)));
491}
492
493static inline void decode_pswid(u32 pswid, int *vasid, int *winid)
494{
495	if (vasid)
496		*vasid = pswid >> (31 - 7) & 0xFF;
497
498	if (winid)
499		*winid = pswid & 0xFFFF;
500}
501#endif /* _VAS_H */
502