crypto/ccp/ccp_hardware.c

123 ccp_queue_write_tail(struct ccp_queue *qp)
125 	ccp_write_queue_4(qp->cq_softc, qp->cq_qindex, CMD_Q_TAIL_LO_BASE,
126 	    ((uint32_t)qp->desc_ring_bus_addr) + (Q_DESC_SIZE * qp->cq_tail));
134 ccp_queue_lsb_entry(struct ccp_queue *qp, unsigned lsb_entry)
136 	return ((qp->private_lsb * LSB_REGION_LENGTH + lsb_entry));
144 ccp_queue_lsb_address(struct ccp_queue *qp, unsigned lsb_entry)
146 	return (ccp_queue_lsb_entry(qp, lsb_entry) * LSB_ENTRY_SIZE);
213 	struct ccp_queue *qp;
220 	qp = &sc->queues[queue];
232 	if (qp->lsb_mask == 0) {
255 	    ringsz, 0, NULL, NULL, &qp->ring_desc_tag);
259 	error = bus_dmamem_alloc(qp->ring_desc_tag, &desc,
260 	    BUS_DMA_ZERO | BUS_DMA_WAITOK, &qp->ring_desc_map);
264 	error = bus_dmamap_load(qp->ring_desc_tag, qp->ring_desc_map, desc,
265 	    ringsz, ccp_dmamap_cb, &qp->desc_ring_bus_addr, BUS_DMA_WAITOK);
269 	qp->desc_ring = desc;
270 	qp->completions_ring = malloc(num_descriptors *
271 	    sizeof(*qp->completions_ring), M_CCP, M_ZERO | M_WAITOK);
274 	qp->qcontrol = 0;
275 	ccp_write_queue_4(sc, queue, CMD_Q_CONTROL_BASE, qp->qcontrol);
282 	qp->qcontrol |= (sc->ring_size_order - 1) << CMD_Q_SIZE_SHIFT;
285 	    (uint32_t)qp->desc_ring_bus_addr);
287 	    (uint32_t)qp->desc_ring_bus_addr);
297 	qp->qcontrol |= (qp->desc_ring_bus_addr >> 32) << CMD_Q_PTR_HI_SHIFT;
298 	qp->qcontrol |= CMD_Q_RUN;
299 	ccp_write_queue_4(sc, queue, CMD_Q_CONTROL_BASE, qp->qcontrol);
303 		if (qp->desc_ring != NULL)
304 			bus_dmamap_unload(qp->ring_desc_tag,
305 			    qp->ring_desc_map);
307 			bus_dmamem_free(qp->ring_desc_tag, desc,
308 			    qp->ring_desc_map);
309 		if (qp->ring_desc_tag != NULL)
310 			bus_dma_tag_destroy(qp->ring_desc_tag);
319 	struct ccp_queue *qp;
322 	qp = &sc->queues[queue];
331 	free(qp->completions_ring, M_CCP);
332 	bus_dmamap_unload(qp->ring_desc_tag, qp->ring_desc_map);
333 	bus_dmamem_free(qp->ring_desc_tag, qp->desc_ring, qp->ring_desc_map);
334 	bus_dma_tag_destroy(qp->ring_desc_tag);
455 ccp_intr_handle_error(struct ccp_queue *qp, const struct ccp_desc *desc)
464 	sc = qp->cq_softc;
465 	q = qp->cq_qindex;
490 	idx = desc - qp->desc_ring;
501 		memset(&qp->desc_ring[idx], 0, sizeof(qp->desc_ring[idx]));
503 		cctx = &qp->completions_ring[idx];
521 			cctx->callback_fn(qp, cctx->session, cctx->callback_arg, errno);
527 	qp->cq_head = idx;
528 	qp->cq_waiting = false;
529 	wakeup(&qp->cq_tail);
530 	DPRINTF(sc->dev, "%s: wrote sw head:%u\n", __func__, qp->cq_head);
532 	    (uint32_t)qp->desc_ring_bus_addr + (idx * Q_DESC_SIZE));
533 	ccp_write_queue_4(sc, q, CMD_Q_CONTROL_BASE, qp->qcontrol);
538 ccp_intr_run_completions(struct ccp_queue *qp, uint32_t ints)
546 	sc = qp->cq_softc;
547 	q = qp->cq_qindex;
549 	mtx_lock(&qp->cq_lock);
557 	idx = (headlo - (uint32_t)qp->desc_ring_bus_addr) / Q_DESC_SIZE;
560 	    qp->cq_head);
562 	while (qp->cq_head != idx) {
563 		DPRINTF(sc->dev, "%s: completing:%u\n", __func__, qp->cq_head);
565 		cctx = &qp->completions_ring[qp->cq_head];
567 			cctx->callback_fn(qp, cctx->session,
573 		memset(&qp->desc_ring[qp->cq_head], 0,
574 		    sizeof(qp->desc_ring[qp->cq_head]));
576 		qp->cq_head = (qp->cq_head + 1) % (1 << sc->ring_size_order);
580 		qp->cq_waiting = false;
581 		wakeup(&qp->cq_tail);
584 	DPRINTF(sc->dev, "%s: wrote sw head:%u\n", __func__, qp->cq_head);
591 	desc = &qp->desc_ring[idx];
593 		ccp_intr_handle_error(qp, desc);
595 	mtx_unlock(&qp->cq_lock);
845 ccp_passthrough(struct ccp_queue *qp, bus_addr_t dst,
853 	if (ccp_queue_get_ring_space(qp) == 0)
856 	desc = &qp->desc_ring[qp->cq_tail];
875 		desc->lsb_ctx_id = ccp_queue_lsb_entry(qp, LSB_ENTRY_KEY);
878 		memcpy(&qp->completions_ring[qp->cq_tail], cctx, sizeof(*cctx));
880 	qp->cq_tail = (qp->cq_tail + 1) % (1 << qp->cq_softc->ring_size_order);
885 ccp_passthrough_sgl(struct ccp_queue *qp, bus_addr_t lsb_addr, bool tolsb,
900 			error = ccp_passthrough(qp, lsb_addr, CCP_MEMTYPE_SB,
906 			error = ccp_passthrough(qp, seg->ss_paddr,
1038 ccp_sha_single_desc(struct ccp_queue *qp, const struct SHA_Defn *defn,
1043 	if (ccp_queue_get_ring_space(qp) == 0)
1046 	desc = &qp->desc_ring[qp->cq_tail];
1065 	desc->lsb_ctx_id = ccp_queue_lsb_entry(qp, LSB_ENTRY_SHA);
1067 	qp->cq_tail = (qp->cq_tail + 1) % (1 << qp->cq_softc->ring_size_order);
1072 ccp_sha(struct ccp_queue *qp, enum sha_version version, struct sglist *sgl_src,
1092 	error = ccp_passthrough(qp, ccp_queue_lsb_address(qp, LSB_ENTRY_SHA),
1105 		error = ccp_sha_single_desc(qp, defn, seg->ss_paddr,
1114 	lsbaddr = ccp_queue_lsb_address(qp, LSB_ENTRY_SHA);
1120 		error = ccp_passthrough(qp, seg->ss_paddr, CCP_MEMTYPE_SYSTEM,
1202 ccp_do_hmac_done(struct ccp_queue *qp, struct ccp_session *s,
1247 ccp_hmac_done(struct ccp_queue *qp, struct ccp_session *s, void *vcrp,
1253 	ccp_do_hmac_done(qp, s, crp, error);
1257 ccp_do_hmac(struct ccp_queue *qp, struct ccp_session *s, struct cryptop *crp,
1264 	dev = qp->cq_softc->dev;
1272 	sglist_reset(qp->cq_sg_ulptx);
1273 	error = sglist_append(qp->cq_sg_ulptx, s->hmac.ipad, axf->blocksize);
1277 		error = sglist_append_sglist(qp->cq_sg_ulptx, qp->cq_sg_crp,
1282 	error = sglist_append_sglist(qp->cq_sg_ulptx, qp->cq_sg_crp,
1289 	sglist_reset(qp->cq_sg_dst);
1290 	error = sglist_append(qp->cq_sg_dst, s->hmac.res,
1295 	error = ccp_sha(qp, s->hmac.auth_mode, qp->cq_sg_ulptx, qp->cq_sg_dst,
1305 ccp_hmac(struct ccp_queue *qp, struct ccp_session *s, struct cryptop *crp)
1313 	return (ccp_do_hmac(qp, s, crp, &ctx));
1331 ccp_blkcipher_done(struct ccp_queue *qp, struct ccp_session *s, void *vcrp,
1345 	DPRINTF(qp->cq_softc->dev, "%s: qp=%p crp=%p\n", __func__, qp, crp);
1379 ccp_do_pst_to_lsb(struct ccp_queue *qp, uint32_t lsbaddr, const void *src,
1384 	sglist_reset(qp->cq_sg_ulptx);
1385 	error = sglist_append(qp->cq_sg_ulptx, __DECONST(void *, src), len);
1389 	error = ccp_passthrough_sgl(qp, lsbaddr, true, qp->cq_sg_ulptx, len,
1395 ccp_do_xts(struct ccp_queue *qp, struct ccp_session *s, struct cryptop *crp,
1405 	dev = qp->cq_softc->dev;
1416 	for (i = 0; i < qp->cq_sg_ulptx->sg_nseg; i++) {
1419 		seg = &qp->cq_sg_ulptx->sg_segs[i];
1421 		desc = &qp->desc_ring[qp->cq_tail];
1424 		desc->eom = (i == qp->cq_sg_ulptx->sg_nseg - 1);
1427 		    __func__, qp->cq_tail, (int)desc->som, (int)desc->eom,
1431 			memcpy(&qp->completions_ring[qp->cq_tail], cctx,
1439 		    qp->cq_tail, (unsigned)desc->aes_xts.type,
1452 		desc->key_lo = ccp_queue_lsb_address(qp, LSB_ENTRY_KEY);
1456 		desc->lsb_ctx_id = ccp_queue_lsb_entry(qp, LSB_ENTRY_IV);
1458 		qp->cq_tail = (qp->cq_tail + 1) %
1459 		    (1 << qp->cq_softc->ring_size_order);
1465 ccp_do_blkcipher(struct ccp_queue *qp, struct ccp_session *s,
1477 	dev = qp->cq_softc->dev;
1494 	for (i = 0; i < qp->cq_sg_crp->sg_nseg; i++)
1495 		if ((qp->cq_sg_crp->sg_segs[i].ss_len % AES_BLOCK_LEN) != 0) {
1497 			    qp->cq_sg_crp->sg_segs[i].ss_len);
1514 	error = ccp_do_pst_to_lsb(qp, ccp_queue_lsb_address(qp, LSB_ENTRY_IV),
1564 		error = ccp_do_pst_to_lsb(qp,
1565 		    ccp_queue_lsb_address(qp, LSB_ENTRY_KEY + 1), keydata,
1569 		error = ccp_do_pst_to_lsb(qp,
1570 		    ccp_queue_lsb_address(qp, LSB_ENTRY_KEY),
1577 			error = ccp_do_pst_to_lsb(qp,
1578 			    ccp_queue_lsb_address(qp, LSB_ENTRY_KEY) +
1582 			error = ccp_do_pst_to_lsb(qp,
1583 			    ccp_queue_lsb_address(qp, LSB_ENTRY_KEY + 1) +
1587 		error = ccp_do_pst_to_lsb(qp,
1588 		    ccp_queue_lsb_address(qp, LSB_ENTRY_KEY), keydata,
1597 	sglist_reset(qp->cq_sg_ulptx);
1598 	error = sglist_append_sglist(qp->cq_sg_ulptx, qp->cq_sg_crp,
1604 	    (void *)PHYS_TO_DMAP(qp->cq_sg_ulptx->sg_segs[0].ss_paddr), " ");
1606 	DPRINTF(dev, "%s: starting AES ops @ %u\n", __func__, qp->cq_tail);
1608 	if (ccp_queue_get_ring_space(qp) < qp->cq_sg_ulptx->sg_nseg)
1612 		return (ccp_do_xts(qp, s, crp, dir, cctx));
1614 	for (i = 0; i < qp->cq_sg_ulptx->sg_nseg; i++) {
1617 		seg = &qp->cq_sg_ulptx->sg_segs[i];
1619 		desc = &qp->desc_ring[qp->cq_tail];
1622 		desc->eom = (i == qp->cq_sg_ulptx->sg_nseg - 1);
1625 		    __func__, qp->cq_tail, (int)desc->som, (int)desc->eom,
1629 			memcpy(&qp->completions_ring[qp->cq_tail], cctx,
1643 		    qp->cq_tail, (unsigned)desc->aes.mode,
1656 		desc->key_lo = ccp_queue_lsb_address(qp, LSB_ENTRY_KEY);
1660 		desc->lsb_ctx_id = ccp_queue_lsb_entry(qp, LSB_ENTRY_IV);
1662 		qp->cq_tail = (qp->cq_tail + 1) %
1663 		    (1 << qp->cq_softc->ring_size_order);
1669 ccp_blkcipher(struct ccp_queue *qp, struct ccp_session *s, struct cryptop *crp)
1677 	return (ccp_do_blkcipher(qp, s, crp, &ctx));
1681 ccp_authenc_done(struct ccp_queue *qp, struct ccp_session *s, void *vcrp,
1690 	ccp_do_hmac_done(qp, s, crp, error);
1694 ccp_authenc(struct ccp_queue *qp, struct ccp_session *s, struct cryptop *crp)
1705 		error = ccp_do_blkcipher(qp, s, crp, NULL);
1707 		error = ccp_do_hmac(qp, s, crp, NULL);
1713 		error = ccp_do_hmac(qp, s, crp, &ctx);
1715 		error = ccp_do_blkcipher(qp, s, crp, &ctx);
1720 ccp_do_ghash_aad(struct ccp_queue *qp, struct ccp_session *s)
1726 	if (ccp_queue_get_ring_space(qp) < qp->cq_sg_ulptx->sg_nseg)
1729 	for (i = 0; i < qp->cq_sg_ulptx->sg_nseg; i++) {
1730 		seg = &qp->cq_sg_ulptx->sg_segs[i];
1732 		desc = &qp->desc_ring[qp->cq_tail];
1746 		desc->lsb_ctx_id = ccp_queue_lsb_entry(qp, LSB_ENTRY_IV);
1748 		desc->key_lo = ccp_queue_lsb_address(qp, LSB_ENTRY_KEY);
1751 		qp->cq_tail = (qp->cq_tail + 1) %
1752 		    (1 << qp->cq_softc->ring_size_order);
1758 ccp_do_gctr(struct ccp_queue *qp, struct ccp_session *s,
1763 	if (ccp_queue_get_ring_space(qp) == 0)
1766 	desc = &qp->desc_ring[qp->cq_tail];
1784 	desc->lsb_ctx_id = ccp_queue_lsb_entry(qp, LSB_ENTRY_IV);
1786 	desc->key_lo = ccp_queue_lsb_address(qp, LSB_ENTRY_KEY);
1789 	qp->cq_tail = (qp->cq_tail + 1) %
1790 	    (1 << qp->cq_softc->ring_size_order);
1795 ccp_do_ghash_final(struct ccp_queue *qp, struct ccp_session *s)
1799 	if (ccp_queue_get_ring_space(qp) == 0)
1802 	desc = &qp->desc_ring[qp->cq_tail];
1811 	desc->src_lo = ccp_queue_lsb_address(qp, LSB_ENTRY_GHASH_IN);
1814 	desc->lsb_ctx_id = ccp_queue_lsb_entry(qp, LSB_ENTRY_IV);
1816 	desc->key_lo = ccp_queue_lsb_address(qp, LSB_ENTRY_KEY);
1819 	desc->dst_lo = ccp_queue_lsb_address(qp, LSB_ENTRY_GHASH);
1822 	qp->cq_tail = (qp->cq_tail + 1) %
1823 	    (1 << qp->cq_softc->ring_size_order);
1828 ccp_gcm_done(struct ccp_queue *qp, struct ccp_session *s, void *vcrp,
1861 ccp_gcm(struct ccp_queue *qp, struct ccp_session *s, struct cryptop *crp)
1873 	dev = qp->cq_softc->dev;
1895 	error = ccp_do_pst_to_lsb(qp, ccp_queue_lsb_address(qp, LSB_ENTRY_IV),
1899 	error = ccp_do_pst_to_lsb(qp, ccp_queue_lsb_address(qp, LSB_ENTRY_KEY),
1903 	error = ccp_do_pst_to_lsb(qp,
1904 	    ccp_queue_lsb_address(qp, LSB_ENTRY_GHASH_IN), s->gmac.final_block,
1911 		sglist_reset(qp->cq_sg_ulptx);
1912 		error = sglist_append_sglist(qp->cq_sg_ulptx, qp->cq_sg_crp,
1918 		for (i = 0; i < qp->cq_sg_ulptx->sg_nseg; i++)
1919 			if ((qp->cq_sg_ulptx->sg_segs[i].ss_len %
1923 				    qp->cq_sg_ulptx->sg_segs[i].ss_len);
1927 		error = ccp_do_ghash_aad(qp, s);
1933 	sglist_reset(qp->cq_sg_ulptx);
1934 	error = sglist_append_sglist(qp->cq_sg_ulptx, qp->cq_sg_crp,
1949 	for (i = 0; i < qp->cq_sg_ulptx->sg_nseg; i++)
1950 		if ((qp->cq_sg_ulptx->sg_segs[i].ss_len % AES_BLOCK_LEN) != 0) {
1952 			    qp->cq_sg_ulptx->sg_segs[i].ss_len);
1956 	for (i = 0; i < qp->cq_sg_ulptx->sg_nseg; i++) {
1959 		seg = &qp->cq_sg_ulptx->sg_segs[i];
1960 		error = ccp_do_gctr(qp, s, dir, seg,
1962 		    i == (qp->cq_sg_ulptx->sg_nseg - 1));
1968 	error = ccp_do_pst_to_lsb(qp, ccp_queue_lsb_address(qp, LSB_ENTRY_IV),
1978 	error = ccp_do_ghash_final(qp, s);
1983 	sglist_reset(qp->cq_sg_ulptx);
1985 		error = sglist_append_sglist(qp->cq_sg_ulptx, qp->cq_sg_crp,
1992 		error = sglist_append(qp->cq_sg_ulptx, s->gmac.final_block,
1996 	error = ccp_passthrough_sgl(qp,
1997 	    ccp_queue_lsb_address(qp, LSB_ENTRY_GHASH), false, qp->cq_sg_ulptx,
2051 db_ccp_show_queue_hw(struct ccp_queue *qp)
2058 	sc = qp->cq_softc;
2059 	q = qp->cq_qindex;
2100 	i = (headlo - (uint32_t)qp->desc_ring_bus_addr) / Q_DESC_SIZE;
2102 	    (void *)&qp->desc_ring[i], " ");