1/*
2 * Octeon Crypto for OCF
3 *
4 * Written by David McCullough <david_mccullough@securecomputing.com>
5 * Copyright (C) 2009 David McCullough
6 *
7 * LICENSE TERMS
8 *
9 * The free distribution and use of this software in both source and binary
10 * form is allowed (with or without changes) provided that:
11 *
12 *   1. distributions of this source code include the above copyright
13 *      notice, this list of conditions and the following disclaimer;
14 *
15 *   2. distributions in binary form include the above copyright
16 *      notice, this list of conditions and the following disclaimer
17 *      in the documentation and/or other associated materials;
18 *
19 *   3. the copyright holder's name is not used to endorse products
20 *      built using this software without specific written permission.
21 *
22 * DISCLAIMER
23 *
24 * This software is provided 'as is' with no explicit or implied warranties
25 * in respect of its properties, including, but not limited to, correctness
26 * and/or fitness for purpose.
27 * ---------------------------------------------------------------------------
28 */
29
30#include <sys/cdefs.h>
31__FBSDID("$FreeBSD$");
32
33#include <sys/param.h>
34#include <sys/systm.h>
35#include <sys/bus.h>
36#include <sys/kernel.h>
37#include <sys/module.h>
38#include <sys/malloc.h>
39#include <sys/mbuf.h>
40#include <sys/uio.h>
41
42#include <opencrypto/cryptodev.h>
43
44#include <contrib/octeon-sdk/cvmx.h>
45
46#include <mips/cavium/cryptocteon/cryptocteonvar.h>
47
48#include "cryptodev_if.h"
49
50struct cryptocteon_softc {
51	int32_t			sc_cid;		/* opencrypto id */
52	struct octo_sess	**sc_sessions;
53	uint32_t		sc_sesnum;
54};
55
56int cryptocteon_debug = 0;
57TUNABLE_INT("hw.cryptocteon.debug", &cryptocteon_debug);
58
59static void cryptocteon_identify(driver_t *, device_t);
60static int cryptocteon_probe(device_t);
61static int cryptocteon_attach(device_t);
62
63static int cryptocteon_process(device_t, struct cryptop *, int);
64static int cryptocteon_newsession(device_t, u_int32_t *, struct cryptoini *);
65static int cryptocteon_freesession(device_t, u_int64_t);
66
67static void
68cryptocteon_identify(driver_t *drv, device_t parent)
69{
70	if (octeon_has_feature(OCTEON_FEATURE_CRYPTO))
71		BUS_ADD_CHILD(parent, 0, "cryptocteon", 0);
72}
73
74static int
75cryptocteon_probe(device_t dev)
76{
77	device_set_desc(dev, "Octeon Secure Coprocessor");
78	return (0);
79}
80
81static int
82cryptocteon_attach(device_t dev)
83{
84	struct cryptocteon_softc *sc;
85
86	sc = device_get_softc(dev);
87
88	sc->sc_cid = crypto_get_driverid(dev, CRYPTOCAP_F_HARDWARE | CRYPTOCAP_F_SYNC);
89	if (sc->sc_cid < 0) {
90		device_printf(dev, "crypto_get_driverid ret %d\n", sc->sc_cid);
91		return (ENXIO);
92	}
93
94	crypto_register(sc->sc_cid, CRYPTO_MD5_HMAC, 0, 0);
95	crypto_register(sc->sc_cid, CRYPTO_SHA1_HMAC, 0, 0);
96	crypto_register(sc->sc_cid, CRYPTO_DES_CBC, 0, 0);
97	crypto_register(sc->sc_cid, CRYPTO_3DES_CBC, 0, 0);
98	crypto_register(sc->sc_cid, CRYPTO_AES_CBC, 0, 0);
99
100	return (0);
101}
102
103/*
104 * Generate a new octo session.  We artifically limit it to a single
105 * hash/cipher or hash-cipher combo just to make it easier, most callers
106 * do not expect more than this anyway.
107 */
108static int
109cryptocteon_newsession(device_t dev, u_int32_t *sid, struct cryptoini *cri)
110{
111	struct cryptoini *c, *encini = NULL, *macini = NULL;
112	struct cryptocteon_softc *sc;
113	struct octo_sess **ocd;
114	int i;
115
116	sc = device_get_softc(dev);
117
118	if (sid == NULL || cri == NULL || sc == NULL)
119		return (EINVAL);
120
121	/*
122	 * To keep it simple, we only handle hash, cipher or hash/cipher in a
123	 * session,  you cannot currently do multiple ciphers/hashes in one
124	 * session even though it would be possibel to code this driver to
125	 * handle it.
126	 */
127	for (i = 0, c = cri; c && i < 2; i++) {
128		if (c->cri_alg == CRYPTO_MD5_HMAC ||
129		    c->cri_alg == CRYPTO_SHA1_HMAC ||
130		    c->cri_alg == CRYPTO_NULL_HMAC) {
131			if (macini) {
132				break;
133			}
134			macini = c;
135		}
136		if (c->cri_alg == CRYPTO_DES_CBC ||
137		    c->cri_alg == CRYPTO_3DES_CBC ||
138		    c->cri_alg == CRYPTO_AES_CBC ||
139		    c->cri_alg == CRYPTO_NULL_CBC) {
140			if (encini) {
141				break;
142			}
143			encini = c;
144		}
145		c = c->cri_next;
146	}
147	if (!macini && !encini) {
148		dprintf("%s,%d - EINVAL bad cipher/hash or combination\n",
149				__FILE__, __LINE__);
150		return EINVAL;
151	}
152	if (c) {
153		dprintf("%s,%d - EINVAL cannot handle chained cipher/hash combos\n",
154				__FILE__, __LINE__);
155		return EINVAL;
156	}
157
158	/*
159	 * So we have something we can do, lets setup the session
160	 */
161
162	if (sc->sc_sessions) {
163		for (i = 1; i < sc->sc_sesnum; i++)
164			if (sc->sc_sessions[i] == NULL)
165				break;
166	} else
167		i = 1;		/* NB: to silence compiler warning */
168
169	if (sc->sc_sessions == NULL || i == sc->sc_sesnum) {
170		if (sc->sc_sessions == NULL) {
171			i = 1; /* We leave sc->sc_sessions[0] empty */
172			sc->sc_sesnum = CRYPTO_SW_SESSIONS;
173		} else
174			sc->sc_sesnum *= 2;
175
176		ocd = malloc(sc->sc_sesnum * sizeof(struct octo_sess *),
177			     M_DEVBUF, M_NOWAIT | M_ZERO);
178		if (ocd == NULL) {
179			/* Reset session number */
180			if (sc->sc_sesnum == CRYPTO_SW_SESSIONS)
181				sc->sc_sesnum = 0;
182			else
183				sc->sc_sesnum /= 2;
184			dprintf("%s,%d: ENOBUFS\n", __FILE__, __LINE__);
185			return ENOBUFS;
186		}
187
188		/* Copy existing sessions */
189		if (sc->sc_sessions) {
190			memcpy(ocd, sc->sc_sessions,
191			    (sc->sc_sesnum / 2) * sizeof(struct octo_sess *));
192			free(sc->sc_sessions, M_DEVBUF);
193		}
194
195		sc->sc_sessions = ocd;
196	}
197
198	ocd = &sc->sc_sessions[i];
199	*sid = i;
200
201	*ocd = malloc(sizeof(struct octo_sess), M_DEVBUF, M_NOWAIT | M_ZERO);
202	if (*ocd == NULL) {
203		cryptocteon_freesession(NULL, i);
204		dprintf("%s,%d: ENOBUFS\n", __FILE__, __LINE__);
205		return ENOBUFS;
206	}
207
208	if (encini && encini->cri_key) {
209		(*ocd)->octo_encklen = (encini->cri_klen + 7) / 8;
210		memcpy((*ocd)->octo_enckey, encini->cri_key, (*ocd)->octo_encklen);
211	}
212
213	if (macini && macini->cri_key) {
214		(*ocd)->octo_macklen = (macini->cri_klen + 7) / 8;
215		memcpy((*ocd)->octo_mackey, macini->cri_key, (*ocd)->octo_macklen);
216	}
217
218	(*ocd)->octo_mlen = 0;
219	if (encini && encini->cri_mlen)
220		(*ocd)->octo_mlen = encini->cri_mlen;
221	else if (macini && macini->cri_mlen)
222		(*ocd)->octo_mlen = macini->cri_mlen;
223	else
224		(*ocd)->octo_mlen = 12;
225
226	/*
227	 * point c at the enc if it exists, otherwise the mac
228	 */
229	c = encini ? encini : macini;
230
231	switch (c->cri_alg) {
232	case CRYPTO_DES_CBC:
233	case CRYPTO_3DES_CBC:
234		(*ocd)->octo_ivsize  = 8;
235		switch (macini ? macini->cri_alg : -1) {
236		case CRYPTO_MD5_HMAC:
237			(*ocd)->octo_encrypt = octo_des_cbc_md5_encrypt;
238			(*ocd)->octo_decrypt = octo_des_cbc_md5_decrypt;
239			octo_calc_hash(0, macini->cri_key, (*ocd)->octo_hminner,
240					(*ocd)->octo_hmouter);
241			break;
242		case CRYPTO_SHA1_HMAC:
243			(*ocd)->octo_encrypt = octo_des_cbc_sha1_encrypt;
244			(*ocd)->octo_decrypt = octo_des_cbc_sha1_encrypt;
245			octo_calc_hash(1, macini->cri_key, (*ocd)->octo_hminner,
246					(*ocd)->octo_hmouter);
247			break;
248		case -1:
249			(*ocd)->octo_encrypt = octo_des_cbc_encrypt;
250			(*ocd)->octo_decrypt = octo_des_cbc_decrypt;
251			break;
252		default:
253			cryptocteon_freesession(NULL, i);
254			dprintf("%s,%d: EINVALn", __FILE__, __LINE__);
255			return EINVAL;
256		}
257		break;
258	case CRYPTO_AES_CBC:
259		(*ocd)->octo_ivsize  = 16;
260		switch (macini ? macini->cri_alg : -1) {
261		case CRYPTO_MD5_HMAC:
262			(*ocd)->octo_encrypt = octo_aes_cbc_md5_encrypt;
263			(*ocd)->octo_decrypt = octo_aes_cbc_md5_decrypt;
264			octo_calc_hash(0, macini->cri_key, (*ocd)->octo_hminner,
265					(*ocd)->octo_hmouter);
266			break;
267		case CRYPTO_SHA1_HMAC:
268			(*ocd)->octo_encrypt = octo_aes_cbc_sha1_encrypt;
269			(*ocd)->octo_decrypt = octo_aes_cbc_sha1_decrypt;
270			octo_calc_hash(1, macini->cri_key, (*ocd)->octo_hminner,
271					(*ocd)->octo_hmouter);
272			break;
273		case -1:
274			(*ocd)->octo_encrypt = octo_aes_cbc_encrypt;
275			(*ocd)->octo_decrypt = octo_aes_cbc_decrypt;
276			break;
277		default:
278			cryptocteon_freesession(NULL, i);
279			dprintf("%s,%d: EINVALn", __FILE__, __LINE__);
280			return EINVAL;
281		}
282		break;
283	case CRYPTO_MD5_HMAC:
284		(*ocd)->octo_encrypt = octo_null_md5_encrypt;
285		(*ocd)->octo_decrypt = octo_null_md5_encrypt;
286		octo_calc_hash(0, macini->cri_key, (*ocd)->octo_hminner,
287				(*ocd)->octo_hmouter);
288		break;
289	case CRYPTO_SHA1_HMAC:
290		(*ocd)->octo_encrypt = octo_null_sha1_encrypt;
291		(*ocd)->octo_decrypt = octo_null_sha1_encrypt;
292		octo_calc_hash(1, macini->cri_key, (*ocd)->octo_hminner,
293				(*ocd)->octo_hmouter);
294		break;
295	default:
296		cryptocteon_freesession(NULL, i);
297		dprintf("%s,%d: EINVALn", __FILE__, __LINE__);
298		return EINVAL;
299	}
300
301	(*ocd)->octo_encalg = encini ? encini->cri_alg : -1;
302	(*ocd)->octo_macalg = macini ? macini->cri_alg : -1;
303
304	return 0;
305}
306
307/*
308 * Free a session.
309 */
310static int
311cryptocteon_freesession(device_t dev, u_int64_t tid)
312{
313	struct cryptocteon_softc *sc;
314	u_int32_t sid = CRYPTO_SESID2LID(tid);
315
316	sc = device_get_softc(dev);
317
318	if (sc == NULL)
319		return (EINVAL);
320
321	if (sid > sc->sc_sesnum || sc->sc_sessions == NULL ||
322	    sc->sc_sessions[sid] == NULL)
323		return (EINVAL);
324
325	/* Silently accept and return */
326	if (sid == 0)
327		return(0);
328
329	if (sc->sc_sessions[sid])
330		free(sc->sc_sessions[sid], M_DEVBUF);
331	sc->sc_sessions[sid] = NULL;
332	return 0;
333}
334
335/*
336 * Process a request.
337 */
338static int
339cryptocteon_process(device_t dev, struct cryptop *crp, int hint)
340{
341	struct cryptodesc *crd;
342	struct octo_sess *od;
343	u_int32_t lid;
344	size_t iovcnt, iovlen;
345	struct mbuf *m = NULL;
346	struct uio *uiop = NULL;
347	struct cryptodesc *enccrd = NULL, *maccrd = NULL;
348	unsigned char *ivp = NULL;
349	unsigned char iv_data[HASH_MAX_LEN];
350	int auth_off = 0, auth_len = 0, crypt_off = 0, crypt_len = 0, icv_off = 0;
351	struct cryptocteon_softc *sc;
352
353	sc = device_get_softc(dev);
354
355	if (sc == NULL || crp == NULL)
356		return EINVAL;
357
358	crp->crp_etype = 0;
359
360	if (crp->crp_desc == NULL || crp->crp_buf == NULL) {
361		dprintf("%s,%d: EINVAL\n", __FILE__, __LINE__);
362		crp->crp_etype = EINVAL;
363		goto done;
364	}
365
366	lid = crp->crp_sid & 0xffffffff;
367	if (lid >= sc->sc_sesnum || lid == 0 || sc->sc_sessions == NULL ||
368	    sc->sc_sessions[lid] == NULL) {
369		crp->crp_etype = ENOENT;
370		dprintf("%s,%d: ENOENT\n", __FILE__, __LINE__);
371		goto done;
372	}
373	od = sc->sc_sessions[lid];
374
375	/*
376	 * do some error checking outside of the loop for m and IOV processing
377	 * this leaves us with valid m or uiop pointers for later
378	 */
379	if (crp->crp_flags & CRYPTO_F_IMBUF) {
380		unsigned frags;
381
382		m = (struct mbuf *) crp->crp_buf;
383		for (frags = 0; m != NULL; frags++)
384			m = m->m_next;
385
386		if (frags >= UIO_MAXIOV) {
387			printf("%s,%d: %d frags > UIO_MAXIOV", __FILE__, __LINE__, frags);
388			goto done;
389		}
390
391		m = (struct mbuf *) crp->crp_buf;
392	} else if (crp->crp_flags & CRYPTO_F_IOV) {
393		uiop = (struct uio *) crp->crp_buf;
394		if (uiop->uio_iovcnt > UIO_MAXIOV) {
395			printf("%s,%d: %d uio_iovcnt > UIO_MAXIOV", __FILE__, __LINE__,
396			       uiop->uio_iovcnt);
397			goto done;
398		}
399	}
400
401	/* point our enccrd and maccrd appropriately */
402	crd = crp->crp_desc;
403	if (crd->crd_alg == od->octo_encalg) enccrd = crd;
404	if (crd->crd_alg == od->octo_macalg) maccrd = crd;
405	crd = crd->crd_next;
406	if (crd) {
407		if (crd->crd_alg == od->octo_encalg) enccrd = crd;
408		if (crd->crd_alg == od->octo_macalg) maccrd = crd;
409		crd = crd->crd_next;
410	}
411	if (crd) {
412		crp->crp_etype = EINVAL;
413		dprintf("%s,%d: ENOENT - descriptors do not match session\n",
414				__FILE__, __LINE__);
415		goto done;
416	}
417
418	if (enccrd) {
419		if (enccrd->crd_flags & CRD_F_IV_EXPLICIT) {
420			ivp = enccrd->crd_iv;
421		} else {
422			ivp = iv_data;
423			crypto_copydata(crp->crp_flags, crp->crp_buf,
424					enccrd->crd_inject, od->octo_ivsize, (caddr_t) ivp);
425		}
426
427		if (maccrd) {
428			auth_off = maccrd->crd_skip;
429			auth_len = maccrd->crd_len;
430			icv_off  = maccrd->crd_inject;
431		}
432
433		crypt_off = enccrd->crd_skip;
434		crypt_len = enccrd->crd_len;
435	} else { /* if (maccrd) */
436		auth_off = maccrd->crd_skip;
437		auth_len = maccrd->crd_len;
438		icv_off  = maccrd->crd_inject;
439	}
440
441	/*
442	 * setup the I/O vector to cover the buffer
443	 */
444	if (crp->crp_flags & CRYPTO_F_IMBUF) {
445		iovcnt = 0;
446		iovlen = 0;
447
448		while (m != NULL) {
449			od->octo_iov[iovcnt].iov_base = mtod(m, void *);
450			od->octo_iov[iovcnt].iov_len = m->m_len;
451
452			m = m->m_next;
453			iovlen += od->octo_iov[iovcnt++].iov_len;
454		}
455	} else if (crp->crp_flags & CRYPTO_F_IOV) {
456		iovlen = 0;
457		for (iovcnt = 0; iovcnt < uiop->uio_iovcnt; iovcnt++) {
458			od->octo_iov[iovcnt].iov_base = uiop->uio_iov[iovcnt].iov_base;
459			od->octo_iov[iovcnt].iov_len = uiop->uio_iov[iovcnt].iov_len;
460
461			iovlen += od->octo_iov[iovcnt].iov_len;
462		}
463	} else {
464		iovlen = crp->crp_ilen;
465		od->octo_iov[0].iov_base = crp->crp_buf;
466		od->octo_iov[0].iov_len = crp->crp_ilen;
467		iovcnt = 1;
468	}
469
470
471	/*
472	 * setup a new explicit key
473	 */
474	if (enccrd) {
475		if (enccrd->crd_flags & CRD_F_KEY_EXPLICIT) {
476			od->octo_encklen = (enccrd->crd_klen + 7) / 8;
477			memcpy(od->octo_enckey, enccrd->crd_key, od->octo_encklen);
478		}
479	}
480	if (maccrd) {
481		if (maccrd->crd_flags & CRD_F_KEY_EXPLICIT) {
482			od->octo_macklen = (maccrd->crd_klen + 7) / 8;
483			memcpy(od->octo_mackey, maccrd->crd_key, od->octo_macklen);
484			od->octo_mackey_set = 0;
485		}
486		if (!od->octo_mackey_set) {
487			octo_calc_hash(maccrd->crd_alg == CRYPTO_MD5_HMAC ? 0 : 1,
488				maccrd->crd_key, od->octo_hminner, od->octo_hmouter);
489			od->octo_mackey_set = 1;
490		}
491	}
492
493
494	if (!enccrd || (enccrd->crd_flags & CRD_F_ENCRYPT))
495		(*od->octo_encrypt)(od, od->octo_iov, iovcnt, iovlen,
496				auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp);
497	else
498		(*od->octo_decrypt)(od, od->octo_iov, iovcnt, iovlen,
499				auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp);
500
501done:
502	crypto_done(crp);
503	return 0;
504}
505
506static device_method_t cryptocteon_methods[] = {
507	/* device methods */
508	DEVMETHOD(device_identify,	cryptocteon_identify),
509	DEVMETHOD(device_probe,		cryptocteon_probe),
510	DEVMETHOD(device_attach,	cryptocteon_attach),
511
512	/* crypto device methods */
513	DEVMETHOD(cryptodev_newsession,	cryptocteon_newsession),
514	DEVMETHOD(cryptodev_freesession,cryptocteon_freesession),
515	DEVMETHOD(cryptodev_process,	cryptocteon_process),
516
517	{ 0, 0 }
518};
519
520static driver_t cryptocteon_driver = {
521	"cryptocteon",
522	cryptocteon_methods,
523	sizeof (struct cryptocteon_softc),
524};
525static devclass_t cryptocteon_devclass;
526DRIVER_MODULE(cryptocteon, nexus, cryptocteon_driver, cryptocteon_devclass, 0, 0);
527