1// SPDX-License-Identifier: GPL-2.0-or-later
2
3/*
4 * Copyright 2020 IBM Corp.
5 *
6 * Author: Bulent Abali <abali@us.ibm.com>
7 *
8 */
9#include <stdio.h>
10#include <stdlib.h>
11#include <string.h>
12#include <unistd.h>
13#include <stdint.h>
14#include <sys/types.h>
15#include <sys/stat.h>
16#include <sys/time.h>
17#include <sys/fcntl.h>
18#include <sys/mman.h>
19#include <endian.h>
20#include <bits/endian.h>
21#include <sys/ioctl.h>
22#include <assert.h>
23#include <errno.h>
24#include <signal.h>
25#include "vas-api.h"
26#include "nx.h"
27#include "copy-paste.h"
28#include "nxu.h"
29#include "nx_dbg.h"
30#include <sys/platform/ppc.h>
31
32#define barrier()
33#define hwsync()    ({ asm volatile("sync" ::: "memory"); })
34
35#ifndef NX_NO_CPU_PRI
36#define cpu_pri_default()  ({ asm volatile ("or 2, 2, 2"); })
37#define cpu_pri_low()      ({ asm volatile ("or 31, 31, 31"); })
38#else
39#define cpu_pri_default()
40#define cpu_pri_low()
41#endif
42
43void *nx_fault_storage_address;
44
45struct nx_handle {
46	int fd;
47	int function;
48	void *paste_addr;
49};
50
51static int open_device_nodes(char *devname, int pri, struct nx_handle *handle)
52{
53	int rc, fd;
54	void *addr;
55	struct vas_tx_win_open_attr txattr;
56
57	fd = open(devname, O_RDWR);
58	if (fd < 0) {
59		fprintf(stderr, " open device name %s\n", devname);
60		return -errno;
61	}
62
63	memset(&txattr, 0, sizeof(txattr));
64	txattr.version = 1;
65	txattr.vas_id = pri;
66	rc = ioctl(fd, VAS_TX_WIN_OPEN, (unsigned long)&txattr);
67	if (rc < 0) {
68		fprintf(stderr, "ioctl() n %d, error %d\n", rc, errno);
69		rc = -errno;
70		goto out;
71	}
72
73	addr = mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_SHARED, fd, 0ULL);
74	if (addr == MAP_FAILED) {
75		fprintf(stderr, "mmap() failed, errno %d\n", errno);
76		rc = -errno;
77		goto out;
78	}
79	handle->fd = fd;
80	handle->paste_addr = (void *)((char *)addr + 0x400);
81
82	rc = 0;
83out:
84	close(fd);
85	return rc;
86}
87
88void *nx_function_begin(int function, int pri)
89{
90	int rc;
91	char *devname = "/dev/crypto/nx-gzip";
92	struct nx_handle *nxhandle;
93
94	if (function != NX_FUNC_COMP_GZIP) {
95		errno = EINVAL;
96		fprintf(stderr, " NX_FUNC_COMP_GZIP not found\n");
97		return NULL;
98	}
99
100
101	nxhandle = malloc(sizeof(*nxhandle));
102	if (!nxhandle) {
103		errno = ENOMEM;
104		fprintf(stderr, " No memory\n");
105		return NULL;
106	}
107
108	nxhandle->function = function;
109	rc = open_device_nodes(devname, pri, nxhandle);
110	if (rc < 0) {
111		errno = -rc;
112		fprintf(stderr, " open_device_nodes failed\n");
113		return NULL;
114	}
115
116	return nxhandle;
117}
118
119int nx_function_end(void *handle)
120{
121	int rc = 0;
122	struct nx_handle *nxhandle = handle;
123
124	rc = munmap(nxhandle->paste_addr - 0x400, 4096);
125	if (rc < 0) {
126		fprintf(stderr, "munmap() failed, errno %d\n", errno);
127		return rc;
128	}
129	close(nxhandle->fd);
130	free(nxhandle);
131
132	return rc;
133}
134
135static int nx_wait_for_csb(struct nx_gzip_crb_cpb_t *cmdp)
136{
137	long poll = 0;
138	uint64_t t;
139
140	/* Save power and let other threads use the h/w. top may show
141	 * 100% but only because OS doesn't know we slowed the this
142	 * h/w thread while polling. We're letting other threads have
143	 * higher throughput on the core.
144	 */
145	cpu_pri_low();
146
147#define CSB_MAX_POLL 200000000UL
148#define USLEEP_TH     300000UL
149
150	t = __ppc_get_timebase();
151
152	while (getnn(cmdp->crb.csb, csb_v) == 0) {
153		++poll;
154		hwsync();
155
156		cpu_pri_low();
157
158		/* usleep(0) takes around 29000 ticks ~60 us.
159		 * 300000 is spinning for about 600 us then
160		 * start sleeping.
161		 */
162		if ((__ppc_get_timebase() - t) > USLEEP_TH) {
163			cpu_pri_default();
164			usleep(1);
165		}
166
167		if (poll > CSB_MAX_POLL)
168			break;
169
170		/* Fault address from signal handler */
171		if (nx_fault_storage_address) {
172			cpu_pri_default();
173			return -EAGAIN;
174		}
175
176	}
177
178	cpu_pri_default();
179
180	/* hw has updated csb and output buffer */
181	hwsync();
182
183	/* Check CSB flags. */
184	if (getnn(cmdp->crb.csb, csb_v) == 0) {
185		fprintf(stderr, "CSB still not valid after %d polls.\n",
186			(int) poll);
187		prt_err("CSB still not valid after %d polls, giving up.\n",
188			(int) poll);
189		return -ETIMEDOUT;
190	}
191
192	return 0;
193}
194
195static int nxu_run_job(struct nx_gzip_crb_cpb_t *cmdp, void *handle)
196{
197	int i, ret, retries;
198	struct nx_handle *nxhandle = handle;
199
200	assert(handle != NULL);
201	i = 0;
202	retries = 5000;
203	while (i++ < retries) {
204		hwsync();
205		vas_copy(&cmdp->crb, 0);
206		ret = vas_paste(nxhandle->paste_addr, 0);
207		hwsync();
208
209		NXPRT(fprintf(stderr, "Paste attempt %d/%d returns 0x%x\n",
210				i, retries, ret));
211
212		if ((ret == 2) || (ret == 3)) {
213
214			ret = nx_wait_for_csb(cmdp);
215			if (!ret) {
216				goto out;
217			} else if (ret == -EAGAIN) {
218				long x;
219
220				prt_err("Touching address %p, 0x%lx\n",
221					 nx_fault_storage_address,
222					 *(long *) nx_fault_storage_address);
223				x = *(long *) nx_fault_storage_address;
224				*(long *) nx_fault_storage_address = x;
225				nx_fault_storage_address = 0;
226				continue;
227			} else {
228				prt_err("wait_for_csb() returns %d\n", ret);
229				break;
230			}
231		} else {
232			if (i < 10) {
233				/* spin for few ticks */
234#define SPIN_TH 500UL
235				uint64_t fail_spin;
236
237				fail_spin = __ppc_get_timebase();
238				while ((__ppc_get_timebase() - fail_spin) <
239					 SPIN_TH)
240					;
241			} else {
242				/* sleep */
243				unsigned int pr = 0;
244
245				if (pr++ % 100 == 0) {
246					prt_err("Paste attempt %d/", i);
247					prt_err("%d, failed pid= %d\n", retries,
248						getpid());
249				}
250				usleep(1);
251			}
252			continue;
253		}
254	}
255
256out:
257	cpu_pri_default();
258
259	return ret;
260}
261
262int nxu_submit_job(struct nx_gzip_crb_cpb_t *cmdp, void *handle)
263{
264	int cc;
265
266	cc = nxu_run_job(cmdp, handle);
267
268	if (!cc)
269		cc = getnn(cmdp->crb.csb, csb_cc);      /* CC Table 6-8 */
270
271	return cc;
272}
273
274
275void nxu_sigsegv_handler(int sig, siginfo_t *info, void *ctx)
276{
277	fprintf(stderr, "%d: Got signal %d si_code %d, si_addr %p\n", getpid(),
278		sig, info->si_code, info->si_addr);
279
280	nx_fault_storage_address = info->si_addr;
281}
282
283/*
284 * Fault in pages prior to NX job submission.  wr=1 may be required to
285 * touch writeable pages.  System zero pages do not fault-in the page as
286 * intended.  Typically set wr=1 for NX target pages and set wr=0 for NX
287 * source pages.
288 */
289int nxu_touch_pages(void *buf, long buf_len, long page_len, int wr)
290{
291	char *begin = buf;
292	char *end = (char *) buf + buf_len - 1;
293	volatile char t;
294
295	assert(buf_len >= 0 && !!buf);
296
297	NXPRT(fprintf(stderr, "touch %p %p len 0x%lx wr=%d\n", buf,
298			(buf + buf_len), buf_len, wr));
299
300	if (buf_len <= 0 || buf == NULL)
301		return -1;
302
303	do {
304		t = *begin;
305		if (wr)
306			*begin = t;
307		begin = begin + page_len;
308	} while (begin < end);
309
310	/* When buf_sz is small or buf tail is in another page */
311	t = *end;
312	if (wr)
313		*end = t;
314
315	return 0;
316}
317