1/*-
2 * Copyright (c) 2003-2012 Broadcom Corporation
3 * All Rights Reserved
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 *
9 * 1. Redistributions of source code must retain the above copyright
10 *    notice, this list of conditions and the following disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 *    notice, this list of conditions and the following disclaimer in
13 *    the documentation and/or other materials provided with the
14 *    distribution.
15 *
16 * THIS SOFTWARE IS PROVIDED BY BROADCOM ``AS IS'' AND ANY EXPRESS OR
17 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
18 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19 * ARE DISCLAIMED. IN NO EVENT SHALL BROADCOM OR CONTRIBUTORS BE LIABLE
20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
21 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
22 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
23 * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
24 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
25 * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
26 * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27 */
28
29#include <sys/cdefs.h>
30__FBSDID("$FreeBSD$");
31
32#include <sys/cdefs.h>
33#include <sys/param.h>
34#include <sys/systm.h>
35#include <sys/proc.h>
36#include <sys/errno.h>
37#include <sys/endian.h>
38#include <sys/malloc.h>
39#include <sys/kernel.h>
40#include <sys/module.h>
41#include <sys/mbuf.h>
42#include <sys/lock.h>
43#include <sys/mutex.h>
44#include <sys/sysctl.h>
45#include <sys/bus.h>
46#include <sys/random.h>
47#include <sys/rman.h>
48#include <sys/uio.h>
49#include <sys/kobj.h>
50
51#include <dev/pci/pcivar.h>
52
53#include <opencrypto/cryptodev.h>
54
55#include "cryptodev_if.h"
56
57#include <vm/vm.h>
58#include <vm/pmap.h>
59
60#include <mips/nlm/hal/haldefs.h>
61#include <mips/nlm/hal/iomap.h>
62#include <mips/nlm/xlp.h>
63#include <mips/nlm/hal/sys.h>
64#include <mips/nlm/hal/fmn.h>
65#include <mips/nlm/hal/nlmsaelib.h>
66#include <mips/nlm/dev/sec/rsa_ucode.h>
67#include <mips/nlm/hal/cop2.h>
68#include <mips/nlm/hal/mips-extns.h>
69#include <mips/nlm/msgring.h>
70#include <mips/nlm/dev/sec/nlmrsalib.h>
71
72#ifdef NLM_RSA_DEBUG
73static	void print_krp_params(struct cryptkop *krp);
74#endif
75
76static	int xlp_rsa_init(struct xlp_rsa_softc *sc, int node);
77static	int xlp_rsa_newsession(device_t , uint32_t *, struct cryptoini *);
78static	int xlp_rsa_freesession(device_t , uint64_t);
79static	int xlp_rsa_kprocess(device_t , struct cryptkop *, int);
80static	int xlp_get_rsa_opsize(struct xlp_rsa_command *cmd, unsigned int bits);
81static	void xlp_free_cmd_params(struct xlp_rsa_command *cmd);
82static	int xlp_rsa_inp2hwformat(uint8_t *src, uint8_t *dst,
83    uint32_t paramsize, uint8_t result);
84
85static	int xlp_rsa_probe(device_t);
86static	int xlp_rsa_attach(device_t);
87static	int xlp_rsa_detach(device_t);
88
89static device_method_t xlp_rsa_methods[] = {
90	/* device interface */
91	DEVMETHOD(device_probe, xlp_rsa_probe),
92	DEVMETHOD(device_attach, xlp_rsa_attach),
93	DEVMETHOD(device_detach, xlp_rsa_detach),
94
95	/* bus interface */
96	DEVMETHOD(bus_print_child, bus_generic_print_child),
97	DEVMETHOD(bus_driver_added, bus_generic_driver_added),
98
99	/* crypto device methods */
100	DEVMETHOD(cryptodev_newsession, xlp_rsa_newsession),
101	DEVMETHOD(cryptodev_freesession, xlp_rsa_freesession),
102	DEVMETHOD(cryptodev_kprocess,   xlp_rsa_kprocess),
103
104	DEVMETHOD_END
105};
106
107static driver_t xlp_rsa_driver = {
108	"nlmrsa",
109	xlp_rsa_methods,
110	sizeof(struct xlp_rsa_softc)
111};
112static devclass_t xlp_rsa_devclass;
113
114DRIVER_MODULE(nlmrsa, pci, xlp_rsa_driver, xlp_rsa_devclass, 0, 0);
115MODULE_DEPEND(nlmrsa, crypto, 1, 1, 1);
116
117#ifdef NLM_RSA_DEBUG
118static void
119print_krp_params(struct cryptkop *krp)
120{
121	int i;
122
123	printf("krp->krp_op	:%d\n", krp->krp_op);
124	printf("krp->krp_status	:%d\n", krp->krp_status);
125	printf("krp->krp_iparams:%d\n", krp->krp_iparams);
126	printf("krp->krp_oparams:%d\n", krp->krp_oparams);
127	for (i = 0; i < krp->krp_iparams + krp->krp_oparams; i++) {
128		printf("krp->krp_param[%d].crp_p	:0x%llx\n", i,
129		    (unsigned long long)krp->krp_param[i].crp_p);
130		printf("krp->krp_param[%d].crp_nbits	:%d\n", i,
131		    krp->krp_param[i].crp_nbits);
132		printf("krp->krp_param[%d].crp_nbytes	:%d\n", i,
133		    howmany(krp->krp_param[i].crp_nbits, 8));
134	}
135}
136#endif
137
138static int
139xlp_rsa_init(struct xlp_rsa_softc *sc, int node)
140{
141	struct xlp_rsa_command *cmd = NULL;
142	uint32_t fbvc, dstvc, endsel, regval;
143	struct nlm_fmn_msg m;
144	int err, ret, i;
145	uint64_t base;
146
147	/* Register interrupt handler for the RSA/ECC CMS messages */
148	if (register_msgring_handler(sc->rsaecc_vc_start,
149	    sc->rsaecc_vc_end, nlm_xlprsaecc_msgring_handler, sc) != 0) {
150		err = -1;
151		printf("Couldn't register rsa/ecc msgring handler\n");
152		goto errout;
153	}
154	fbvc = nlm_cpuid() * 4 + XLPGE_FB_VC;
155	/* Do the CMS credit initialization */
156	/* Currently it is configured by default to 50 when kernel comes up */
157
158#if BYTE_ORDER == LITTLE_ENDIAN
159	for (i = 0; i < nitems(nlm_rsa_ucode_data); i++)
160		nlm_rsa_ucode_data[i] = htobe64(nlm_rsa_ucode_data[i]);
161#endif
162	for (dstvc = sc->rsaecc_vc_start; dstvc <= sc->rsaecc_vc_end; dstvc++) {
163		cmd = malloc(sizeof(struct xlp_rsa_command), M_DEVBUF,
164		    M_NOWAIT | M_ZERO);
165		KASSERT(cmd != NULL, ("%s:cmd is NULL\n", __func__));
166		cmd->rsasrc = contigmalloc(sizeof(nlm_rsa_ucode_data),
167		    M_DEVBUF,
168		    (M_WAITOK | M_ZERO),
169		    0UL /* low address */, -1UL /* high address */,
170		    XLP_L2L3_CACHELINE_SIZE /* alignment */,
171		    0UL /* boundary */);
172		KASSERT(cmd->rsasrc != NULL,
173		    ("%s:cmd->rsasrc is NULL\n", __func__));
174		memcpy(cmd->rsasrc, nlm_rsa_ucode_data,
175		    sizeof(nlm_rsa_ucode_data));
176		m.msg[0] = nlm_crypto_form_rsa_ecc_fmn_entry0(1, 0x70, 0,
177		    vtophys(cmd->rsasrc));
178		m.msg[1] = nlm_crypto_form_rsa_ecc_fmn_entry1(0, 1, fbvc,
179		    vtophys(cmd->rsasrc));
180		/* Software scratch pad */
181		m.msg[2] = (uintptr_t)cmd;
182		m.msg[3] = 0;
183
184		ret = nlm_fmn_msgsend(dstvc, 3, FMN_SWCODE_RSA, &m);
185		if (ret != 0) {
186			err = -1;
187			printf("%s: msgsnd failed (%x)\n", __func__, ret);
188			goto errout;
189		}
190	}
191	/* Configure so that all VCs send request to all RSA pipes */
192	base = nlm_get_rsa_regbase(node);
193	if (nlm_is_xlp3xx()) {
194		endsel = 1;
195		regval = 0xFFFF;
196	} else {
197		endsel = 3;
198		regval = 0x07FFFFFF;
199	}
200	for (i = 0; i < endsel; i++)
201		nlm_write_rsa_reg(base, RSA_ENG_SEL_0 + i, regval);
202	return (0);
203errout:
204	xlp_free_cmd_params(cmd);
205	return (err);
206}
207
208/* This function is called from an interrupt handler */
209void
210nlm_xlprsaecc_msgring_handler(int vc, int size, int code, int src_id,
211    struct nlm_fmn_msg *msg, void *data)
212{
213	struct xlp_rsa_command *cmd;
214	struct xlp_rsa_softc *sc;
215	struct crparam *outparam;
216	int ostart;
217
218	KASSERT(code == FMN_SWCODE_RSA,
219	    ("%s: bad code = %d, expected code = %d\n", __func__, code,
220	    FMN_SWCODE_RSA));
221
222	sc = data;
223	KASSERT(src_id >= sc->rsaecc_vc_start && src_id <= sc->rsaecc_vc_end,
224	    ("%s: bad src_id = %d, expect %d - %d\n", __func__,
225	    src_id, sc->rsaecc_vc_start, sc->rsaecc_vc_end));
226
227	cmd = (struct xlp_rsa_command *)(uintptr_t)msg->msg[1];
228	KASSERT(cmd != NULL, ("%s:cmd not received properly\n", __func__));
229
230	if (RSA_ERROR(msg->msg[0]) != 0) {
231		printf("%s: Message rcv msg0 %llx msg1 %llx err %x \n",
232		    __func__, (unsigned long long)msg->msg[0],
233		    (unsigned long long)msg->msg[1],
234		    (int)RSA_ERROR(msg->msg[0]));
235		cmd->krp->krp_status = EBADMSG;
236	}
237
238	if (cmd->krp != NULL) {
239		ostart = cmd->krp->krp_iparams;
240		outparam = &cmd->krp->krp_param[ostart];
241		xlp_rsa_inp2hwformat(cmd->rsasrc + cmd->rsaopsize * ostart,
242		    outparam->crp_p,
243		    howmany(outparam->crp_nbits, 8),
244		    1);
245		crypto_kdone(cmd->krp);
246	}
247
248	xlp_free_cmd_params(cmd);
249}
250
251static int
252xlp_rsa_probe(device_t dev)
253{
254	struct xlp_rsa_softc *sc;
255
256	if (pci_get_vendor(dev) == PCI_VENDOR_NETLOGIC &&
257	    pci_get_device(dev) == PCI_DEVICE_ID_NLM_RSA) {
258		sc = device_get_softc(dev);
259		return (BUS_PROBE_DEFAULT);
260	}
261	return (ENXIO);
262}
263
264/*
265 * Attach an interface that successfully probed.
266 */
267static int
268xlp_rsa_attach(device_t dev)
269{
270	struct xlp_rsa_softc *sc = device_get_softc(dev);
271	uint64_t base;
272	int qstart, qnum;
273	int freq, node;
274
275	sc->sc_dev = dev;
276
277	node = nlm_get_device_node(pci_get_slot(dev));
278	freq = nlm_set_device_frequency(node, DFS_DEVICE_RSA, 250);
279	if (bootverbose)
280		device_printf(dev, "RSA Freq: %dMHz\n", freq);
281	if (pci_get_device(dev) == PCI_DEVICE_ID_NLM_RSA) {
282		device_set_desc(dev, "XLP RSA/ECC Accelerator");
283		if ((sc->sc_cid = crypto_get_driverid(dev,
284		    CRYPTOCAP_F_HARDWARE)) < 0) {
285			printf("xlp_rsaecc-err:couldn't get the driver id\n");
286			goto error_exit;
287		}
288		if (crypto_kregister(sc->sc_cid, CRK_MOD_EXP, 0) != 0)
289			goto error_exit;
290
291		base = nlm_get_rsa_pcibase(node);
292		qstart = nlm_qidstart(base);
293		qnum = nlm_qnum(base);
294		sc->rsaecc_vc_start = qstart;
295		sc->rsaecc_vc_end = qstart + qnum - 1;
296	}
297	if (xlp_rsa_init(sc, node) != 0)
298		goto error_exit;
299	device_printf(dev, "RSA Initialization complete!\n");
300	return (0);
301
302error_exit:
303	return (ENXIO);
304}
305
306/*
307 * Detach an interface that successfully probed.
308 */
309static int
310xlp_rsa_detach(device_t dev)
311{
312	return (0);
313}
314
315/*
316 * Allocate a new 'session' and return an encoded session id.  'sidp'
317 * contains our registration id, and should contain an encoded session
318 * id on successful allocation.
319 */
320static int
321xlp_rsa_newsession(device_t dev, u_int32_t *sidp, struct cryptoini *cri)
322{
323	struct xlp_rsa_softc *sc = device_get_softc(dev);
324	struct xlp_rsa_session *ses = NULL;
325	int sesn;
326
327	if (sidp == NULL || cri == NULL || sc == NULL)
328		return (EINVAL);
329
330	if (sc->sc_sessions == NULL) {
331		ses = sc->sc_sessions = malloc(sizeof(struct xlp_rsa_session),
332		    M_DEVBUF, M_NOWAIT);
333		if (ses == NULL)
334			return (ENOMEM);
335		sesn = 0;
336		sc->sc_nsessions = 1;
337	} else {
338		for (sesn = 0; sesn < sc->sc_nsessions; sesn++) {
339			if (!sc->sc_sessions[sesn].hs_used) {
340				ses = &sc->sc_sessions[sesn];
341				break;
342			}
343		}
344
345		if (ses == NULL) {
346			sesn = sc->sc_nsessions;
347			ses = malloc((sesn + 1) * sizeof(*ses),
348			    M_DEVBUF, M_NOWAIT);
349			if (ses == NULL)
350				return (ENOMEM);
351			bcopy(sc->sc_sessions, ses, sesn * sizeof(*ses));
352			bzero(sc->sc_sessions, sesn * sizeof(*ses));
353			free(sc->sc_sessions, M_DEVBUF);
354			sc->sc_sessions = ses;
355			ses = &sc->sc_sessions[sesn];
356			sc->sc_nsessions++;
357		}
358	}
359	bzero(ses, sizeof(*ses));
360	ses->sessionid = sesn;
361	ses->hs_used = 1;
362
363	*sidp = XLP_RSA_SID(device_get_unit(sc->sc_dev), sesn);
364	return (0);
365}
366
367/*
368 * Deallocate a session.
369 * XXX this routine should run a zero'd mac/encrypt key into context ram.
370 * XXX to blow away any keys already stored there.
371 */
372static int
373xlp_rsa_freesession(device_t dev, u_int64_t tid)
374{
375	struct xlp_rsa_softc *sc = device_get_softc(dev);
376	int session;
377	u_int32_t sid = CRYPTO_SESID2LID(tid);
378
379	if (sc == NULL)
380		return (EINVAL);
381
382	session = XLP_RSA_SESSION(sid);
383	if (session >= sc->sc_nsessions)
384		return (EINVAL);
385
386	sc->sc_sessions[session].hs_used = 0;
387	return (0);
388}
389
390static void
391xlp_free_cmd_params(struct xlp_rsa_command *cmd)
392{
393
394	if (cmd == NULL)
395		return;
396	if (cmd->rsasrc != NULL) {
397		if (cmd->krp == NULL) /* Micro code load */
398			contigfree(cmd->rsasrc, sizeof(nlm_rsa_ucode_data),
399			    M_DEVBUF);
400		else
401			free(cmd->rsasrc, M_DEVBUF);
402	}
403	free(cmd, M_DEVBUF);
404}
405
406static int
407xlp_get_rsa_opsize(struct xlp_rsa_command *cmd, unsigned int bits)
408{
409
410	if (bits == 0 || bits > 8192)
411		return (-1);
412	/* XLP hardware expects always a fixed size with unused bytes
413	 * zeroed out in the input data */
414	if (bits <= 512) {
415		cmd->rsatype = 0x40;
416		cmd->rsaopsize = 64;
417	} else if (bits <= 1024) {
418		cmd->rsatype = 0x41;
419		cmd->rsaopsize = 128;
420	} else if (bits <= 2048) {
421		cmd->rsatype = 0x42;
422		cmd->rsaopsize = 256;
423	} else if (bits <= 4096) {
424		cmd->rsatype = 0x43;
425		cmd->rsaopsize = 512;
426	} else if (bits <= 8192) {
427		cmd->rsatype = 0x44;
428		cmd->rsaopsize = 1024;
429	}
430	return (0);
431}
432
433static int
434xlp_rsa_inp2hwformat(uint8_t *src, uint8_t *dst, uint32_t paramsize,
435    uint8_t result)
436{
437	uint32_t pdwords, pbytes;
438	int i, j, k;
439
440	pdwords = paramsize / 8;
441	pbytes = paramsize % 8;
442
443	for (i = 0, k = 0; i < pdwords; i++) {
444		/* copy dwords of inp/hw to hw/out format */
445		for (j = 7; j >= 0; j--, k++)
446			dst[i * 8 + j] = src[k];
447	}
448	if (pbytes) {
449		if (result == 0) {
450			/* copy rem bytes of input data to hw format */
451			for (j = 7; k < paramsize; j--, k++)
452				dst[i * 8 + j] = src[k];
453		} else {
454			/* copy rem bytes of hw data to exp output format */
455			for (j = 7; k < paramsize; j--, k++)
456				dst[k] = src[i * 8 + j];
457		}
458	}
459
460	return (0);
461}
462
463static int
464nlm_crypto_complete_rsa_request(struct xlp_rsa_softc *sc,
465    struct xlp_rsa_command *cmd)
466{
467	unsigned int fbvc;
468	struct nlm_fmn_msg m;
469	int ret;
470
471	fbvc = nlm_cpuid() * 4 + XLPGE_FB_VC;
472
473	m.msg[0] = nlm_crypto_form_rsa_ecc_fmn_entry0(1, cmd->rsatype,
474	    cmd->rsafn, vtophys(cmd->rsasrc));
475	m.msg[1] = nlm_crypto_form_rsa_ecc_fmn_entry1(0, 1, fbvc,
476	    vtophys(cmd->rsasrc + cmd->rsaopsize * cmd->krp->krp_iparams));
477	/* Software scratch pad */
478	m.msg[2] = (uintptr_t)cmd;
479	m.msg[3] = 0;
480
481	/* Send the message to rsa engine vc */
482	ret = nlm_fmn_msgsend(sc->rsaecc_vc_start, 3, FMN_SWCODE_RSA, &m);
483        if (ret != 0) {
484#ifdef NLM_SEC_DEBUG
485                printf("%s: msgsnd failed (%x)\n", __func__, ret);
486#endif
487		return (ERESTART);
488        }
489	return (0);
490}
491
492static int
493xlp_rsa_kprocess(device_t dev, struct cryptkop *krp, int hint)
494{
495	struct xlp_rsa_softc *sc = device_get_softc(dev);
496	struct xlp_rsa_command *cmd;
497	struct crparam *kp;
498	int err, i;
499
500	if (krp == NULL || krp->krp_callback == NULL)
501		return (EINVAL);
502
503	cmd = malloc(sizeof(struct xlp_rsa_command), M_DEVBUF,
504	    M_NOWAIT | M_ZERO);
505	KASSERT(cmd != NULL, ("%s:cmd is NULL\n", __func__));
506	cmd->krp = krp;
507
508#ifdef NLM_RSA_DEBUG
509	print_krp_params(krp);
510#endif
511	err = EOPNOTSUPP;
512	switch (krp->krp_op) {
513	case CRK_MOD_EXP:
514		if (krp->krp_iparams == 3 && krp->krp_oparams == 1)
515			break;
516		goto errout;
517	default:
518		device_printf(dev, "Op:%d not yet supported\n", krp->krp_op);
519		goto errout;
520	}
521
522	err = xlp_get_rsa_opsize(cmd,
523	    krp->krp_param[krp->krp_iparams - 1].crp_nbits);
524	if (err != 0) {
525		err = EINVAL;
526		goto errout;
527	}
528	cmd->rsafn = 0; /* Mod Exp */
529	cmd->rsasrc = malloc(
530	    cmd->rsaopsize * (krp->krp_iparams + krp->krp_oparams),
531	    M_DEVBUF,
532	    M_NOWAIT | M_ZERO);
533	if (cmd->rsasrc == NULL) {
534		err = ENOMEM;
535		goto errout;
536	}
537
538	for (i = 0, kp = krp->krp_param; i < krp->krp_iparams; i++, kp++) {
539		KASSERT(kp->crp_nbits != 0,
540		    ("%s: parameter[%d]'s length is zero\n", __func__, i));
541		xlp_rsa_inp2hwformat(kp->crp_p,
542		    cmd->rsasrc + i * cmd->rsaopsize,
543		    howmany(kp->crp_nbits, 8), 0);
544	}
545	err = nlm_crypto_complete_rsa_request(sc, cmd);
546	if (err != 0)
547		goto errout;
548
549	return (0);
550errout:
551	xlp_free_cmd_params(cmd);
552	krp->krp_status = err;
553	crypto_kdone(krp);
554	return (err);
555}
556