11541Srgrimes// SPDX-License-Identifier: GPL-2.0-only 21541Srgrimes/* 31541Srgrimes * Intel Keem Bay OCS HCU Crypto Driver. 41541Srgrimes * 51541Srgrimes * Copyright (C) 2018-2020 Intel Corporation 61541Srgrimes */ 71541Srgrimes 81541Srgrimes#include <crypto/engine.h> 91541Srgrimes#include <crypto/hmac.h> 101541Srgrimes#include <crypto/internal/hash.h> 111541Srgrimes#include <crypto/scatterwalk.h> 121541Srgrimes#include <crypto/sha2.h> 131541Srgrimes#include <crypto/sm3.h> 141541Srgrimes#include <linux/completion.h> 151541Srgrimes#include <linux/dma-mapping.h> 161541Srgrimes#include <linux/err.h> 171541Srgrimes#include <linux/interrupt.h> 181541Srgrimes#include <linux/kernel.h> 191541Srgrimes#include <linux/mod_devicetable.h> 201541Srgrimes#include <linux/module.h> 211541Srgrimes#include <linux/platform_device.h> 221541Srgrimes#include <linux/string.h> 231541Srgrimes 241541Srgrimes#include "ocs-hcu.h" 251541Srgrimes 261541Srgrimes#define DRV_NAME "keembay-ocs-hcu" 271541Srgrimes 281541Srgrimes/* Flag marking a final request. */ 291541Srgrimes#define REQ_FINAL BIT(0) 301541Srgrimes/* Flag marking a HMAC request. */ 311541Srgrimes#define REQ_FLAGS_HMAC BIT(1) 321541Srgrimes/* Flag set when HW HMAC is being used. */ 331541Srgrimes#define REQ_FLAGS_HMAC_HW BIT(2) 341541Srgrimes/* Flag set when SW HMAC is being used. */ 351541Srgrimes#define REQ_FLAGS_HMAC_SW BIT(3) 3636503Speter 3750477Speter/** 381541Srgrimes * struct ocs_hcu_ctx: OCS HCU Transform context. 391541Srgrimes * @hcu_dev: The OCS HCU device used by the transformation. 401541Srgrimes * @key: The key (used only for HMAC transformations). 419336Sdfr * @key_len: The length of the key. 421541Srgrimes * @is_sm3_tfm: Whether or not this is an SM3 transformation. 431541Srgrimes * @is_hmac_tfm: Whether or not this is a HMAC transformation. 441541Srgrimes */ 451541Srgrimesstruct ocs_hcu_ctx { 461541Srgrimes struct ocs_hcu_dev *hcu_dev; 471541Srgrimes u8 key[SHA512_BLOCK_SIZE]; 481541Srgrimes size_t key_len; 491541Srgrimes bool is_sm3_tfm; 501541Srgrimes bool is_hmac_tfm; 511541Srgrimes}; 521541Srgrimes 531541Srgrimes/** 541541Srgrimes * struct ocs_hcu_rctx - Context for the request. 551541Srgrimes * @hcu_dev: OCS HCU device to be used to service the request. 569336Sdfr * @flags: Flags tracking request status. 579336Sdfr * @algo: Algorithm to use for the request. 5848125Sjulian * @blk_sz: Block size of the transformation / request. 5948125Sjulian * @dig_sz: Digest size of the transformation / request. 6048125Sjulian * @dma_list: OCS DMA linked list. 6148125Sjulian * @hash_ctx: OCS HCU hashing context. 6248125Sjulian * @buffer: Buffer to store: partial block of data and SW HMAC 6348125Sjulian * artifacts (ipad, opad, etc.). 6448125Sjulian * @buf_cnt: Number of bytes currently stored in the buffer. 6548125Sjulian * @buf_dma_addr: The DMA address of @buffer (when mapped). 6648125Sjulian * @buf_dma_count: The number of bytes in @buffer currently DMA-mapped. 6748125Sjulian * @sg: Head of the scatterlist entries containing data. 6848125Sjulian * @sg_data_total: Total data in the SG list at any time. 6948125Sjulian * @sg_data_offset: Offset into the data of the current individual SG node. 7048125Sjulian * @sg_dma_nents: Number of sg entries mapped in dma_list. 7153131Seivind */ 7248125Sjulianstruct ocs_hcu_rctx { 7348125Sjulian struct ocs_hcu_dev *hcu_dev; 7448125Sjulian u32 flags; 7548125Sjulian enum ocs_hcu_algo algo; 7648125Sjulian size_t blk_sz; 7748125Sjulian size_t dig_sz; 7848125Sjulian struct ocs_hcu_dma_list *dma_list; 791541Srgrimes struct ocs_hcu_hash_ctx hash_ctx; 801541Srgrimes /* 811541Srgrimes * Buffer is double the block size because we need space for SW HMAC 821541Srgrimes * artifacts, i.e: 831541Srgrimes * - ipad (1 block) + a possible partial block of data. 841541Srgrimes * - opad (1 block) + digest of H(k ^ ipad || m) 8518020Sbde */ 861541Srgrimes u8 buffer[2 * SHA512_BLOCK_SIZE]; 871541Srgrimes size_t buf_cnt; 889336Sdfr dma_addr_t buf_dma_addr; 899336Sdfr size_t buf_dma_count; 9029024Sbde struct scatterlist *sg; 911541Srgrimes unsigned int sg_data_total; 921541Srgrimes unsigned int sg_data_offset; 931541Srgrimes unsigned int sg_dma_nents; 949336Sdfr}; 9512911Sphk 9651344Sdillon/** 971541Srgrimes * struct ocs_hcu_drv - Driver data 981541Srgrimes * @dev_list: The list of HCU devices. 9912662Sdg * @lock: The lock protecting dev_list. 10032011Sbde */ 10132937Sdysonstruct ocs_hcu_drv { 1021541Srgrimes struct list_head dev_list; 1039336Sdfr spinlock_t lock; /* Protects dev_list. */ 1041541Srgrimes}; 1051541Srgrimes 1061541Srgrimesstatic struct ocs_hcu_drv ocs_hcu = { 1071541Srgrimes .dev_list = LIST_HEAD_INIT(ocs_hcu.dev_list), 1081541Srgrimes .lock = __SPIN_LOCK_UNLOCKED(ocs_hcu.lock), 1091541Srgrimes}; 11048125Sjulian 11148125Sjulian/* 11248125Sjulian * Return the total amount of data in the request; that is: the data in the 11348125Sjulian * request buffer + the data in the sg list. 11448125Sjulian */ 11548125Sjulianstatic inline unsigned int kmb_get_total_data(struct ocs_hcu_rctx *rctx) 11651344Sdillon{ 11751344Sdillon return rctx->sg_data_total + rctx->buf_cnt; 11854567Sdillon} 11954567Sdillon 12054567Sdillon/* Move remaining content of scatter-gather list to context buffer. */ 12154567Sdillonstatic int flush_sg_to_ocs_buffer(struct ocs_hcu_rctx *rctx) 12254567Sdillon{ 12354567Sdillon size_t count; 12454567Sdillon 12554567Sdillon if (rctx->sg_data_total > (sizeof(rctx->buffer) - rctx->buf_cnt)) { 12654567Sdillon WARN(1, "%s: sg data does not fit in buffer\n", __func__); 12754567Sdillon return -EINVAL; 12854567Sdillon } 12954567Sdillon 13013416Sphk while (rctx->sg_data_total) { 13113416Sphk if (!rctx->sg) { 13213416Sphk WARN(1, "%s: unexpected NULL sg\n", __func__); 13313416Sphk return -EINVAL; 13413416Sphk } 1351541Srgrimes /* 13636541Speter * If current sg has been fully processed, skip to the next 13736541Speter * one. 1389336Sdfr */ 1399336Sdfr if (rctx->sg_data_offset == rctx->sg->length) { 14013416Sphk rctx->sg = sg_next(rctx->sg); 1419336Sdfr rctx->sg_data_offset = 0; 14225664Sdfr continue; 1431541Srgrimes } 14444078Sdfr /* 14544078Sdfr * Determine the maximum data available to copy from the node. 14633181Seivind * Minimum of the length left in the sg node, or the total data 14715479Sbde * in the request. 14851344Sdillon */ 14951344Sdillon count = min(rctx->sg->length - rctx->sg_data_offset, 15051344Sdillon rctx->sg_data_total); 15151344Sdillon /* Copy from scatter-list entry to context buffer. */ 15210224Sdg scatterwalk_map_and_copy(&rctx->buffer[rctx->buf_cnt], 15312911Sphk rctx->sg, rctx->sg_data_offset, 15436251Speter count, 0); 15512911Sphk 15612911Sphk rctx->sg_data_offset += count; 15712911Sphk rctx->sg_data_total -= count; 1581541Srgrimes rctx->buf_cnt += count; 15948125Sjulian } 16048125Sjulian 16148125Sjulian return 0; 16248125Sjulian} 16348125Sjulian 16448125Sjulianstatic struct ocs_hcu_dev *kmb_ocs_hcu_find_dev(struct ahash_request *req) 16548125Sjulian{ 16648125Sjulian struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 16748125Sjulian struct ocs_hcu_ctx *tctx = crypto_ahash_ctx(tfm); 16848125Sjulian 16948125Sjulian /* If the HCU device for the request was previously set, return it. */ 17048125Sjulian if (tctx->hcu_dev) 17148125Sjulian return tctx->hcu_dev; 17248125Sjulian 17348125Sjulian /* 1749336Sdfr * Otherwise, get the first HCU device available (there should be one 1751541Srgrimes * and only one device). 1761549Srgrimes */ 1779336Sdfr spin_lock_bh(&ocs_hcu.lock); 1789336Sdfr tctx->hcu_dev = list_first_entry_or_null(&ocs_hcu.dev_list, 1799336Sdfr struct ocs_hcu_dev, 1809336Sdfr list); 1819336Sdfr spin_unlock_bh(&ocs_hcu.lock); 1821541Srgrimes 1839336Sdfr return tctx->hcu_dev; 18428270Swollman} 1859336Sdfr 1869336Sdfr/* Free OCS DMA linked list and DMA-able context buffer. */ 18748125Sjulianstatic void kmb_ocs_hcu_dma_cleanup(struct ahash_request *req, 1889336Sdfr struct ocs_hcu_rctx *rctx) 1891541Srgrimes{ 19036541Speter struct ocs_hcu_dev *hcu_dev = rctx->hcu_dev; 19136541Speter struct device *dev = hcu_dev->dev; 1921541Srgrimes 1939336Sdfr /* Unmap rctx->buffer (if mapped). */ 1941541Srgrimes if (rctx->buf_dma_count) { 1959336Sdfr dma_unmap_single(dev, rctx->buf_dma_addr, rctx->buf_dma_count, 1969336Sdfr DMA_TO_DEVICE); 1979336Sdfr rctx->buf_dma_count = 0; 1981541Srgrimes } 1991541Srgrimes 20048125Sjulian /* Unmap req->src (if mapped). */ 2019336Sdfr if (rctx->sg_dma_nents) { 2029336Sdfr dma_unmap_sg(dev, req->src, rctx->sg_dma_nents, DMA_TO_DEVICE); 2039336Sdfr rctx->sg_dma_nents = 0; 2041541Srgrimes } 2051541Srgrimes 20636541Speter /* Free dma_list (if allocated). */ 20736503Speter if (rctx->dma_list) { 20836503Speter ocs_hcu_dma_list_free(hcu_dev, rctx->dma_list); 20936503Speter rctx->dma_list = NULL; 2109336Sdfr } 2119336Sdfr} 21248125Sjulian 21348125Sjulian/* 2149336Sdfr * Prepare for DMA operation: 21536541Speter * - DMA-map request context buffer (if needed) 2169336Sdfr * - DMA-map SG list (only the entries to be processed, see note below) 21736251Speter * - Allocate OCS HCU DMA linked list (number of elements = SG entries to 2189336Sdfr * process + context buffer (if not empty)). 2199336Sdfr * - Add DMA-mapped request context buffer to OCS HCU DMA list. 2209336Sdfr * - Add SG entries to DMA list. 2219336Sdfr * 2229336Sdfr * Note: if this is a final request, we process all the data in the SG list, 2239336Sdfr * otherwise we can only process up to the maximum amount of block-aligned data 2249336Sdfr * (the remainder will be put into the context buffer and processed in the next 22536251Speter * request). 2269336Sdfr */ 2279336Sdfrstatic int kmb_ocs_dma_prepare(struct ahash_request *req) 2289336Sdfr{ 2299336Sdfr struct ocs_hcu_rctx *rctx = ahash_request_ctx_dma(req); 2309336Sdfr struct device *dev = rctx->hcu_dev->dev; 2319336Sdfr unsigned int remainder = 0; 23236251Speter unsigned int total; 2339336Sdfr size_t nents; 2349336Sdfr size_t count; 23517761Sdyson int rc; 23648125Sjulian int i; 2379336Sdfr 2389336Sdfr /* This function should be called only when there is data to process. */ 23936541Speter total = kmb_get_total_data(rctx); 2409336Sdfr if (!total) 24148125Sjulian return -EINVAL; 24248125Sjulian 24348125Sjulian /* 24448125Sjulian * If this is not a final DMA (terminated DMA), the data passed to the 2451541Srgrimes * HCU must be aligned to the block size; compute the remainder data to 2461541Srgrimes * be processed in the next request. 2471541Srgrimes */ 2481541Srgrimes if (!(rctx->flags & REQ_FINAL)) 2491541Srgrimes remainder = total % rctx->blk_sz; 2501549Srgrimes 2519336Sdfr /* Determine the number of scatter gather list entries to process. */ 2529336Sdfr nents = sg_nents_for_len(req->src, rctx->sg_data_total - remainder); 2539336Sdfr 2549336Sdfr /* If there are entries to process, map them. */ 2559336Sdfr if (nents) { 2561541Srgrimes rctx->sg_dma_nents = dma_map_sg(dev, req->src, nents, 2579336Sdfr DMA_TO_DEVICE); 25828270Swollman if (!rctx->sg_dma_nents) { 2599336Sdfr dev_err(dev, "Failed to MAP SG\n"); 2609336Sdfr rc = -ENOMEM; 2619336Sdfr goto cleanup; 2621541Srgrimes } 2631541Srgrimes /* 26454485Sdillon * The value returned by dma_map_sg() can be < nents; so update 2659336Sdfr * nents accordingly. 2661541Srgrimes */ 26736541Speter nents = rctx->sg_dma_nents; 26836541Speter } 2691541Srgrimes 2701541Srgrimes /* 2711541Srgrimes * If context buffer is not empty, map it and add extra DMA entry for 2721541Srgrimes * it. 2731541Srgrimes */ 2741541Srgrimes if (rctx->buf_cnt) { 27548125Sjulian rctx->buf_dma_addr = dma_map_single(dev, rctx->buffer, 2761541Srgrimes rctx->buf_cnt, 2771541Srgrimes DMA_TO_DEVICE); 27836503Speter if (dma_mapping_error(dev, rctx->buf_dma_addr)) { 27936503Speter dev_err(dev, "Failed to map request context buffer\n"); 28036503Speter rc = -ENOMEM; 2811541Srgrimes goto cleanup; 28248125Sjulian } 28348125Sjulian rctx->buf_dma_count = rctx->buf_cnt; 2849336Sdfr /* Increase number of dma entries. */ 2859336Sdfr nents++; 2869336Sdfr } 28717761Sdyson 28848125Sjulian /* Allocate OCS HCU DMA list. */ 2899336Sdfr rctx->dma_list = ocs_hcu_dma_list_alloc(rctx->hcu_dev, nents); 29048125Sjulian if (!rctx->dma_list) { 29148125Sjulian rc = -ENOMEM; 29248125Sjulian goto cleanup; 29348125Sjulian } 2949336Sdfr 2959336Sdfr /* Add request context buffer (if previously DMA-mapped) */ 29648125Sjulian if (rctx->buf_dma_count) { 29748125Sjulian rc = ocs_hcu_dma_list_add_tail(rctx->hcu_dev, rctx->dma_list, 29848125Sjulian rctx->buf_dma_addr, 29948125Sjulian rctx->buf_dma_count); 30048125Sjulian if (rc) 30148125Sjulian goto cleanup; 3021541Srgrimes } 3031541Srgrimes 3041541Srgrimes /* Add the SG nodes to be processed to the DMA linked list. */ 3051541Srgrimes for_each_sg(req->src, rctx->sg, rctx->sg_dma_nents, i) { 3061541Srgrimes /* 3071549Srgrimes * The number of bytes to add to the list entry is the minimum 3089336Sdfr * between: 3099336Sdfr * - The DMA length of the SG entry. 3109336Sdfr * - The data left to be processed. 3119336Sdfr */ 3129336Sdfr count = min(rctx->sg_data_total - remainder, 3131541Srgrimes sg_dma_len(rctx->sg) - rctx->sg_data_offset); 3149336Sdfr /* 31528270Swollman * Do not create a zero length DMA descriptor. Check in case of 3169336Sdfr * zero length SG node. 3179336Sdfr */ 3189336Sdfr if (count == 0) 3191541Srgrimes continue; 3201541Srgrimes /* Add sg to HCU DMA list. */ 3219336Sdfr rc = ocs_hcu_dma_list_add_tail(rctx->hcu_dev, 32248125Sjulian rctx->dma_list, 3239336Sdfr rctx->sg->dma_address, 3241541Srgrimes count); 32536541Speter if (rc) 32636541Speter goto cleanup; 3271541Srgrimes 3289336Sdfr /* Update amount of data remaining in SG list. */ 3299336Sdfr rctx->sg_data_total -= count; 3301541Srgrimes 3311541Srgrimes /* 3329336Sdfr * If remaining data is equal to remainder (note: 'less than' 3339336Sdfr * case should never happen in practice), we are done: update 3341541Srgrimes * offset and exit the loop. 33548125Sjulian */ 3361541Srgrimes if (rctx->sg_data_total <= remainder) { 3371541Srgrimes WARN_ON(rctx->sg_data_total < remainder); 3381541Srgrimes rctx->sg_data_offset += count; 3399336Sdfr break; 3409336Sdfr } 34136541Speter 3429336Sdfr /* 3439336Sdfr * If we get here is because we need to process the next sg in 34436541Speter * the list; set offset within the sg to 0. 3459336Sdfr */ 3469336Sdfr rctx->sg_data_offset = 0; 3479336Sdfr } 3489336Sdfr 3499336Sdfr return 0; 3509336Sdfrcleanup: 3519336Sdfr dev_err(dev, "Failed to prepare DMA.\n"); 3529336Sdfr kmb_ocs_hcu_dma_cleanup(req, rctx); 3539336Sdfr 3549336Sdfr return rc; 3559336Sdfr} 3569336Sdfr 3579336Sdfrstatic void kmb_ocs_hcu_secure_cleanup(struct ahash_request *req) 3589336Sdfr{ 3599336Sdfr struct ocs_hcu_rctx *rctx = ahash_request_ctx_dma(req); 3609336Sdfr 3619336Sdfr /* Clear buffer of any data. */ 3629336Sdfr memzero_explicit(rctx->buffer, sizeof(rctx->buffer)); 3639336Sdfr} 3649336Sdfr 3651541Srgrimesstatic int kmb_ocs_hcu_handle_queue(struct ahash_request *req) 3669336Sdfr{ 3671541Srgrimes struct ocs_hcu_dev *hcu_dev = kmb_ocs_hcu_find_dev(req); 36818397Snate 36936541Speter if (!hcu_dev) 37018397Snate return -ENOENT; 3711541Srgrimes 3721541Srgrimes return crypto_transfer_hash_request_to_engine(hcu_dev->engine, req); 3739336Sdfr} 3749336Sdfr 3759336Sdfrstatic int prepare_ipad(struct ahash_request *req) 3761541Srgrimes{ 3771541Srgrimes struct ocs_hcu_rctx *rctx = ahash_request_ctx_dma(req); 3781541Srgrimes struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 3799336Sdfr struct ocs_hcu_ctx *ctx = crypto_ahash_ctx(tfm); 3809336Sdfr int i; 38136503Speter 38236503Speter WARN(rctx->buf_cnt, "%s: Context buffer is not empty\n", __func__); 38336503Speter WARN(!(rctx->flags & REQ_FLAGS_HMAC_SW), 3849336Sdfr "%s: HMAC_SW flag is not set\n", __func__); 3859336Sdfr /* 38648125Sjulian * Key length must be equal to block size. If key is shorter, 38748125Sjulian * we pad it with zero (note: key cannot be longer, since 3889336Sdfr * longer keys are hashed by kmb_ocs_hcu_setkey()). 38948125Sjulian */ 39048125Sjulian if (ctx->key_len > rctx->blk_sz) { 39148125Sjulian WARN(1, "%s: Invalid key length in tfm context\n", __func__); 39248125Sjulian return -EINVAL; 39348125Sjulian } 3949336Sdfr memzero_explicit(&ctx->key[ctx->key_len], 3959336Sdfr rctx->blk_sz - ctx->key_len); 3969336Sdfr ctx->key_len = rctx->blk_sz; 3979336Sdfr /* 39818397Snate * Prepare IPAD for HMAC. Only done for first block. 39918397Snate * HMAC(k,m) = H(k ^ opad || H(k ^ ipad || m)) 4009336Sdfr * k ^ ipad will be first hashed block. 4019336Sdfr * k ^ opad will be calculated in the final request. 40217761Sdyson * Only needed if not using HW HMAC. 40348125Sjulian */ 4049336Sdfr for (i = 0; i < rctx->blk_sz; i++) 4059336Sdfr rctx->buffer[i] = ctx->key[i] ^ HMAC_IPAD_VALUE; 40648125Sjulian rctx->buf_cnt = rctx->blk_sz; 40748125Sjulian 4089336Sdfr return 0; 4099336Sdfr} 4109336Sdfr 4119336Sdfrstatic int kmb_ocs_hcu_do_one_request(struct crypto_engine *engine, void *areq) 4121541Srgrimes{ 4131541Srgrimes struct ahash_request *req = container_of(areq, struct ahash_request, 4141541Srgrimes base); 4151541Srgrimes struct ocs_hcu_dev *hcu_dev = kmb_ocs_hcu_find_dev(req); 4161541Srgrimes struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 4171541Srgrimes struct ocs_hcu_rctx *rctx = ahash_request_ctx_dma(req); 4181541Srgrimes struct ocs_hcu_ctx *tctx = crypto_ahash_ctx(tfm); 4191541Srgrimes int rc; 4201541Srgrimes int i; 4211541Srgrimes 4221541Srgrimes if (!hcu_dev) { 4231541Srgrimes rc = -ENOENT; 42436503Speter goto error; 42536503Speter } 4269336Sdfr 4271541Srgrimes /* 4289336Sdfr * If hardware HMAC flag is set, perform HMAC in hardware. 4299336Sdfr * 4309336Sdfr * NOTE: this flag implies REQ_FINAL && kmb_get_total_data(rctx) 4319336Sdfr */ 4321541Srgrimes if (rctx->flags & REQ_FLAGS_HMAC_HW) { 43317761Sdyson /* Map input data into the HCU DMA linked list. */ 43448125Sjulian rc = kmb_ocs_dma_prepare(req); 4359336Sdfr if (rc) 4369336Sdfr goto error; 4379336Sdfr 43848125Sjulian rc = ocs_hcu_hmac(hcu_dev, rctx->algo, tctx->key, tctx->key_len, 43948125Sjulian rctx->dma_list, req->result, rctx->dig_sz); 4409336Sdfr 4419336Sdfr /* Unmap data and free DMA list regardless of return code. */ 4429336Sdfr kmb_ocs_hcu_dma_cleanup(req, rctx); 4431541Srgrimes 44448125Sjulian /* Process previous return code. */ 44548125Sjulian if (rc) 44648125Sjulian goto error; 44748125Sjulian 44848125Sjulian goto done; 44948125Sjulian } 4501541Srgrimes 4511541Srgrimes /* Handle update request case. */ 4521541Srgrimes if (!(rctx->flags & REQ_FINAL)) { 4531541Srgrimes /* Update should always have input data. */ 4541541Srgrimes if (!kmb_get_total_data(rctx)) 4551549Srgrimes return -EINVAL; 4569336Sdfr 4579336Sdfr /* Map input data into the HCU DMA linked list. */ 4589336Sdfr rc = kmb_ocs_dma_prepare(req); 4599336Sdfr if (rc) 4609336Sdfr goto error; 4611541Srgrimes 4629336Sdfr /* Do hashing step. */ 46328270Swollman rc = ocs_hcu_hash_update(hcu_dev, &rctx->hash_ctx, 4649336Sdfr rctx->dma_list); 4659336Sdfr 4669336Sdfr /* Unmap data and free DMA list regardless of return code. */ 46727446Sdfr kmb_ocs_hcu_dma_cleanup(req, rctx); 46854485Sdillon 4699336Sdfr /* Process previous return code. */ 4701541Srgrimes if (rc) 4711541Srgrimes goto error; 47236541Speter 47336541Speter /* 4741541Srgrimes * Reset request buffer count (data in the buffer was just 4759336Sdfr * processed). 47627446Sdfr */ 4771541Srgrimes rctx->buf_cnt = 0; 4781541Srgrimes /* 4799336Sdfr * Move remaining sg data into the request buffer, so that it 4809336Sdfr * will be processed during the next request. 4811541Srgrimes * 48248125Sjulian * NOTE: we have remaining data if kmb_get_total_data() was not 48348125Sjulian * a multiple of block size. 48448125Sjulian */ 4851541Srgrimes rc = flush_sg_to_ocs_buffer(rctx); 4861541Srgrimes if (rc) 4879336Sdfr goto error; 48827446Sdfr 48927446Sdfr goto done; 49027446Sdfr } 4911541Srgrimes 4921541Srgrimes /* If we get here, this is a final request. */ 4931541Srgrimes 4949336Sdfr /* If there is data to process, use finup. */ 49527446Sdfr if (kmb_get_total_data(rctx)) { 49627446Sdfr /* Map input data into the HCU DMA linked list. */ 49748125Sjulian rc = kmb_ocs_dma_prepare(req); 49848125Sjulian if (rc) 49948125Sjulian goto error; 50048125Sjulian 50148125Sjulian /* Do hashing step. */ 50248125Sjulian rc = ocs_hcu_hash_finup(hcu_dev, &rctx->hash_ctx, 50348125Sjulian rctx->dma_list, 50448125Sjulian req->result, rctx->dig_sz); 50548125Sjulian /* Free DMA list regardless of return code. */ 50648125Sjulian kmb_ocs_hcu_dma_cleanup(req, rctx); 50748125Sjulian 50848125Sjulian /* Process previous return code. */ 50948125Sjulian if (rc) 51048125Sjulian goto error; 51148125Sjulian 51248125Sjulian } else { /* Otherwise (if we have no data), use final. */ 51348125Sjulian rc = ocs_hcu_hash_final(hcu_dev, &rctx->hash_ctx, req->result, 51448125Sjulian rctx->dig_sz); 51548125Sjulian if (rc) 51648125Sjulian goto error; 51748125Sjulian } 51848125Sjulian 51948125Sjulian /* 52048125Sjulian * If we are finalizing a SW HMAC request, we just computed the result 52148125Sjulian * of: H(k ^ ipad || m). 52248125Sjulian * 52327446Sdfr * We now need to complete the HMAC calculation with the OPAD step, 52427446Sdfr * that is, we need to compute H(k ^ opad || digest), where digest is 52527446Sdfr * the digest we just obtained, i.e., H(k ^ ipad || m). 52627446Sdfr */ 52748125Sjulian if (rctx->flags & REQ_FLAGS_HMAC_SW) { 52848125Sjulian /* 52948125Sjulian * Compute k ^ opad and store it in the request buffer (which 53048125Sjulian * is not used anymore at this point). 53148125Sjulian * Note: key has been padded / hashed already (so keylen == 53248125Sjulian * blksz) . 53348125Sjulian */ 53448125Sjulian WARN_ON(tctx->key_len != rctx->blk_sz); 53548125Sjulian for (i = 0; i < rctx->blk_sz; i++) 53627446Sdfr rctx->buffer[i] = tctx->key[i] ^ HMAC_OPAD_VALUE; 53727446Sdfr /* Now append the digest to the rest of the buffer. */ 53827446Sdfr for (i = 0; (i < rctx->dig_sz); i++) 53927446Sdfr rctx->buffer[rctx->blk_sz + i] = req->result[i]; 54027446Sdfr 54127446Sdfr /* Now hash the buffer to obtain the final HMAC. */ 54227446Sdfr rc = ocs_hcu_digest(hcu_dev, rctx->algo, rctx->buffer, 54327446Sdfr rctx->blk_sz + rctx->dig_sz, req->result, 54448125Sjulian rctx->dig_sz); 54527446Sdfr if (rc) 54648125Sjulian goto error; 54748125Sjulian } 54848125Sjulian 54927446Sdfr /* Perform secure clean-up. */ 55027446Sdfr kmb_ocs_hcu_secure_cleanup(req); 55148125Sjuliandone: 55227446Sdfr crypto_finalize_hash_request(hcu_dev->engine, req, 0); 55327446Sdfr 55427446Sdfr return 0; 55527446Sdfr 55648125Sjulianerror: 55727446Sdfr kmb_ocs_hcu_secure_cleanup(req); 55848125Sjulian return rc; 55927446Sdfr} 56048125Sjulian 56148125Sjulianstatic int kmb_ocs_hcu_init(struct ahash_request *req) 56227446Sdfr{ 56327446Sdfr struct ocs_hcu_dev *hcu_dev = kmb_ocs_hcu_find_dev(req); 56427446Sdfr struct ocs_hcu_rctx *rctx = ahash_request_ctx_dma(req); 56527446Sdfr struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 56648125Sjulian struct ocs_hcu_ctx *ctx = crypto_ahash_ctx(tfm); 56748125Sjulian 56827446Sdfr if (!hcu_dev) 56927446Sdfr return -ENOENT; 57048125Sjulian 57127446Sdfr /* Initialize entire request context to zero. */ 57248125Sjulian memset(rctx, 0, sizeof(*rctx)); 57327446Sdfr 57427446Sdfr rctx->hcu_dev = hcu_dev; 57527446Sdfr rctx->dig_sz = crypto_ahash_digestsize(tfm); 57627446Sdfr 5779336Sdfr switch (rctx->dig_sz) { 5789336Sdfr#ifdef CONFIG_CRYPTO_DEV_KEEMBAY_OCS_HCU_HMAC_SHA224 5799336Sdfr case SHA224_DIGEST_SIZE: 5809336Sdfr rctx->blk_sz = SHA224_BLOCK_SIZE; 58117761Sdyson rctx->algo = OCS_HCU_ALGO_SHA224; 58248125Sjulian break; 5839336Sdfr#endif /* CONFIG_CRYPTO_DEV_KEEMBAY_OCS_HCU_HMAC_SHA224 */ 58427446Sdfr case SHA256_DIGEST_SIZE: 58548125Sjulian rctx->blk_sz = SHA256_BLOCK_SIZE; 58648125Sjulian /* 58748125Sjulian * SHA256 and SM3 have the same digest size: use info from tfm 58848125Sjulian * context to find out which one we should use. 58948125Sjulian */ 59048125Sjulian rctx->algo = ctx->is_sm3_tfm ? OCS_HCU_ALGO_SM3 : 5919336Sdfr OCS_HCU_ALGO_SHA256; 5929336Sdfr break; 5939336Sdfr case SHA384_DIGEST_SIZE: 59448125Sjulian rctx->blk_sz = SHA384_BLOCK_SIZE; 59548125Sjulian rctx->algo = OCS_HCU_ALGO_SHA384; 5969336Sdfr break; 59727446Sdfr case SHA512_DIGEST_SIZE: 59827446Sdfr rctx->blk_sz = SHA512_BLOCK_SIZE; 59948125Sjulian rctx->algo = OCS_HCU_ALGO_SHA512; 60048125Sjulian break; 60148125Sjulian default: 60248125Sjulian return -EINVAL; 60348125Sjulian } 60448125Sjulian 60527446Sdfr /* Initialize intermediate data. */ 60648125Sjulian ocs_hcu_hash_init(&rctx->hash_ctx, rctx->algo); 60729653Sdyson 60848125Sjulian /* If this a HMAC request, set HMAC flag. */ 60948125Sjulian if (ctx->is_hmac_tfm) 61048125Sjulian rctx->flags |= REQ_FLAGS_HMAC; 61148125Sjulian 61248125Sjulian return 0; 61348125Sjulian} 61427446Sdfr 6151541Srgrimesstatic int kmb_ocs_hcu_update(struct ahash_request *req) 6161541Srgrimes{ 6173305Sphk struct ocs_hcu_rctx *rctx = ahash_request_ctx_dma(req); 6189336Sdfr int rc; 6199336Sdfr 62048125Sjulian if (!req->nbytes) 62117761Sdyson return 0; 62248125Sjulian 6239336Sdfr rctx->sg_data_total = req->nbytes; 6243305Sphk rctx->sg_data_offset = 0; 6259336Sdfr rctx->sg = req->src; 62648125Sjulian 62748125Sjulian /* 6281541Srgrimes * If we are doing HMAC, then we must use SW-assisted HMAC, since HW 6299336Sdfr * HMAC does not support context switching (there it can only be used 6309336Sdfr * with finup() or digest()). 6319336Sdfr */ 6329336Sdfr if (rctx->flags & REQ_FLAGS_HMAC && 6339336Sdfr !(rctx->flags & REQ_FLAGS_HMAC_SW)) { 6349336Sdfr rctx->flags |= REQ_FLAGS_HMAC_SW; 6359336Sdfr rc = prepare_ipad(req); 6361541Srgrimes if (rc) 63748125Sjulian return rc; 63848125Sjulian } 63948125Sjulian 64048125Sjulian /* 64148125Sjulian * If remaining sg_data fits into ctx buffer, just copy it there; we'll 64248125Sjulian * process it at the next update() or final(). 64348125Sjulian */ 64448125Sjulian if (rctx->sg_data_total <= (sizeof(rctx->buffer) - rctx->buf_cnt)) 64548125Sjulian return flush_sg_to_ocs_buffer(rctx); 64648125Sjulian 64748125Sjulian return kmb_ocs_hcu_handle_queue(req); 6481541Srgrimes} 6491541Srgrimes 6501541Srgrimes/* Common logic for kmb_ocs_hcu_final() and kmb_ocs_hcu_finup(). */ 6511541Srgrimesstatic int kmb_ocs_hcu_fin_common(struct ahash_request *req) 6521541Srgrimes{ 6531549Srgrimes struct ocs_hcu_rctx *rctx = ahash_request_ctx_dma(req); 6549336Sdfr struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 6559336Sdfr struct ocs_hcu_ctx *ctx = crypto_ahash_ctx(tfm); 6569336Sdfr int rc; 6579336Sdfr 6589336Sdfr rctx->flags |= REQ_FINAL; 6591541Srgrimes 6609336Sdfr /* 66128270Swollman * If this is a HMAC request and, so far, we didn't have to switch to 6629336Sdfr * SW HMAC, check if we can use HW HMAC. 6639336Sdfr */ 6641541Srgrimes if (rctx->flags & REQ_FLAGS_HMAC && 6651541Srgrimes !(rctx->flags & REQ_FLAGS_HMAC_SW)) { 6661541Srgrimes /* 66736541Speter * If we are here, it means we never processed any data so far, 66836541Speter * so we can use HW HMAC, but only if there is some data to 6691541Srgrimes * process (since OCS HW MAC does not support zero-length 6709336Sdfr * messages) and the key length is supported by the hardware 6719336Sdfr * (OCS HCU HW only supports length <= 64); if HW HMAC cannot 6721541Srgrimes * be used, fall back to SW-assisted HMAC. 6739336Sdfr */ 67448125Sjulian if (kmb_get_total_data(rctx) && 6759336Sdfr ctx->key_len <= OCS_HCU_HW_KEY_LEN) { 6769336Sdfr rctx->flags |= REQ_FLAGS_HMAC_HW; 6771541Srgrimes } else { 6781541Srgrimes rctx->flags |= REQ_FLAGS_HMAC_SW; 6791541Srgrimes rc = prepare_ipad(req); 6801541Srgrimes if (rc) 68148125Sjulian return rc; 6829336Sdfr } 68348125Sjulian } 6849336Sdfr 68548125Sjulian return kmb_ocs_hcu_handle_queue(req); 6861541Srgrimes} 6871541Srgrimes 6881541Srgrimesstatic int kmb_ocs_hcu_final(struct ahash_request *req) 6891541Srgrimes{ 6901541Srgrimes struct ocs_hcu_rctx *rctx = ahash_request_ctx_dma(req); 6911541Srgrimes 6921541Srgrimes rctx->sg_data_total = 0; 6931541Srgrimes rctx->sg_data_offset = 0; 6941541Srgrimes rctx->sg = NULL; 6951541Srgrimes 6961541Srgrimes return kmb_ocs_hcu_fin_common(req); 6971541Srgrimes} 6981541Srgrimes 6991541Srgrimesstatic int kmb_ocs_hcu_finup(struct ahash_request *req) 7001541Srgrimes{ 7011541Srgrimes struct ocs_hcu_rctx *rctx = ahash_request_ctx_dma(req); 7021541Srgrimes 7031541Srgrimes rctx->sg_data_total = req->nbytes; 7041541Srgrimes rctx->sg_data_offset = 0; 7051541Srgrimes rctx->sg = req->src; 7061541Srgrimes 7071541Srgrimes return kmb_ocs_hcu_fin_common(req); 7081541Srgrimes} 7091541Srgrimes 7101541Srgrimesstatic int kmb_ocs_hcu_digest(struct ahash_request *req) 7111541Srgrimes{ 7121541Srgrimes int rc = 0; 7131541Srgrimes struct ocs_hcu_dev *hcu_dev = kmb_ocs_hcu_find_dev(req); 7141541Srgrimes 7151541Srgrimes if (!hcu_dev) 7161541Srgrimes return -ENOENT; 71736503Speter 71836503Speter rc = kmb_ocs_hcu_init(req); 71936503Speter if (rc) 7209336Sdfr return rc; 7219336Sdfr 72248125Sjulian rc = kmb_ocs_hcu_finup(req); 72348125Sjulian 7241541Srgrimes return rc; 7251541Srgrimes} 7269336Sdfr 7279336Sdfrstatic int kmb_ocs_hcu_export(struct ahash_request *req, void *out) 7289336Sdfr{ 7299336Sdfr struct ocs_hcu_rctx *rctx = ahash_request_ctx_dma(req); 7301541Srgrimes 7311541Srgrimes /* Intermediate data is always stored and applied per request. */ 7329336Sdfr memcpy(out, rctx, sizeof(*rctx)); 7331541Srgrimes 7341541Srgrimes return 0; 7359336Sdfr} 73617761Sdyson 73748125Sjulianstatic int kmb_ocs_hcu_import(struct ahash_request *req, const void *in) 7389336Sdfr{ 7399336Sdfr struct ocs_hcu_rctx *rctx = ahash_request_ctx_dma(req); 7409336Sdfr 74148125Sjulian /* Intermediate data is always stored and applied per request. */ 74248125Sjulian memcpy(rctx, in, sizeof(*rctx)); 74348125Sjulian 74448125Sjulian return 0; 7459336Sdfr} 7461541Srgrimes 7471541Srgrimesstatic int kmb_ocs_hcu_setkey(struct crypto_ahash *tfm, const u8 *key, 7481541Srgrimes unsigned int keylen) 7491541Srgrimes{ 7501541Srgrimes unsigned int digestsize = crypto_ahash_digestsize(tfm); 75136541Speter struct ocs_hcu_ctx *ctx = crypto_ahash_ctx(tfm); 7521541Srgrimes size_t blk_sz = crypto_ahash_blocksize(tfm); 7531541Srgrimes struct crypto_ahash *ahash_tfm; 75448125Sjulian struct ahash_request *req; 75548125Sjulian struct crypto_wait wait; 75648125Sjulian struct scatterlist sg; 75748125Sjulian const char *alg_name; 75848125Sjulian int rc; 75948125Sjulian 76048125Sjulian /* 7611541Srgrimes * Key length must be equal to block size: 7621541Srgrimes * - If key is shorter, we are done for now (the key will be padded 7631541Srgrimes * later on); this is to maximize the use of HW HMAC (which works 7641541Srgrimes * only for keys <= 64 bytes). 7651541Srgrimes * - If key is longer, we hash it. 7661549Srgrimes */ 7679336Sdfr if (keylen <= blk_sz) { 7689336Sdfr memcpy(ctx->key, key, keylen); 7699336Sdfr ctx->key_len = keylen; 7709336Sdfr return 0; 7719336Sdfr } 7721541Srgrimes 7739336Sdfr switch (digestsize) { 77428270Swollman#ifdef CONFIG_CRYPTO_DEV_KEEMBAY_OCS_HCU_HMAC_SHA224 7759336Sdfr case SHA224_DIGEST_SIZE: 7769336Sdfr alg_name = "sha224-keembay-ocs"; 7771541Srgrimes break; 7781541Srgrimes#endif /* CONFIG_CRYPTO_DEV_KEEMBAY_OCS_HCU_HMAC_SHA224 */ 7791541Srgrimes case SHA256_DIGEST_SIZE: 7809336Sdfr alg_name = ctx->is_sm3_tfm ? "sm3-keembay-ocs" : 78136541Speter "sha256-keembay-ocs"; 78236541Speter break; 7839336Sdfr case SHA384_DIGEST_SIZE: 7841541Srgrimes alg_name = "sha384-keembay-ocs"; 7859336Sdfr break; 7869336Sdfr case SHA512_DIGEST_SIZE: 7871541Srgrimes alg_name = "sha512-keembay-ocs"; 7881541Srgrimes break; 7891541Srgrimes default: 79054485Sdillon return -EINVAL; 7919336Sdfr } 7921541Srgrimes 7931541Srgrimes ahash_tfm = crypto_alloc_ahash(alg_name, 0, 0); 7941541Srgrimes if (IS_ERR(ahash_tfm)) 79554567Sdillon return PTR_ERR(ahash_tfm); 7961541Srgrimes 79754567Sdillon req = ahash_request_alloc(ahash_tfm, GFP_KERNEL); 7981541Srgrimes if (!req) { 7991541Srgrimes rc = -ENOMEM; 80048125Sjulian goto err_free_ahash; 8011541Srgrimes } 8021541Srgrimes 8039336Sdfr crypto_init_wait(&wait); 80436541Speter ahash_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG, 80547751Speter crypto_req_done, &wait); 8069336Sdfr crypto_ahash_clear_flags(ahash_tfm, ~0); 80736541Speter 80836541Speter sg_init_one(&sg, key, keylen); 8091541Srgrimes ahash_request_set_crypt(req, &sg, ctx->key, keylen); 8109336Sdfr 81148125Sjulian rc = crypto_wait_req(crypto_ahash_digest(req), &wait); 81248125Sjulian if (rc == 0) 81348125Sjulian ctx->key_len = digestsize; 81448125Sjulian 81548125Sjulian ahash_request_free(req); 81648125Sjulianerr_free_ahash: 81748125Sjulian crypto_free_ahash(ahash_tfm); 81836503Speter 81936503Speter return rc; 82036503Speter} 82148125Sjulian 8229336Sdfr/* Set request size and initialize tfm context. */ 8239336Sdfrstatic void __cra_init(struct crypto_tfm *tfm, struct ocs_hcu_ctx *ctx) 82448125Sjulian{ 82548125Sjulian crypto_ahash_set_reqsize_dma(__crypto_ahash_cast(tfm), 8269336Sdfr sizeof(struct ocs_hcu_rctx)); 82748125Sjulian} 8281541Srgrimes 8299336Sdfrstatic int kmb_ocs_hcu_sha_cra_init(struct crypto_tfm *tfm) 8309336Sdfr{ 8319336Sdfr struct ocs_hcu_ctx *ctx = crypto_tfm_ctx(tfm); 8329336Sdfr 8331541Srgrimes __cra_init(tfm, ctx); 8349336Sdfr 8359336Sdfr return 0; 83636503Speter} 83736251Speter 8381541Srgrimesstatic int kmb_ocs_hcu_sm3_cra_init(struct crypto_tfm *tfm) 8399336Sdfr{ 8409336Sdfr struct ocs_hcu_ctx *ctx = crypto_tfm_ctx(tfm); 8419336Sdfr 8423305Sphk __cra_init(tfm, ctx); 84317761Sdyson 84448125Sjulian ctx->is_sm3_tfm = true; 8459336Sdfr 8469336Sdfr return 0; 84748125Sjulian} 84848125Sjulian 8491541Srgrimesstatic int kmb_ocs_hcu_hmac_sm3_cra_init(struct crypto_tfm *tfm) 85054567Sdillon{ 85154567Sdillon struct ocs_hcu_ctx *ctx = crypto_tfm_ctx(tfm); 85254567Sdillon 85354567Sdillon __cra_init(tfm, ctx); 85454567Sdillon 8551541Srgrimes ctx->is_sm3_tfm = true; 8561541Srgrimes ctx->is_hmac_tfm = true; 8579336Sdfr 85847751Speter return 0; 8599336Sdfr} 8609336Sdfr 86154567Sdillonstatic int kmb_ocs_hcu_hmac_cra_init(struct crypto_tfm *tfm) 86254567Sdillon{ 86354567Sdillon struct ocs_hcu_ctx *ctx = crypto_tfm_ctx(tfm); 86454567Sdillon 86554567Sdillon __cra_init(tfm, ctx); 86654567Sdillon 86754567Sdillon ctx->is_hmac_tfm = true; 86854567Sdillon 86954567Sdillon return 0; 87054567Sdillon} 87154567Sdillon 87254567Sdillon/* Function called when 'tfm' is de-initialized. */ 87354567Sdillonstatic void kmb_ocs_hcu_hmac_cra_exit(struct crypto_tfm *tfm) 87454567Sdillon{ 87554567Sdillon struct ocs_hcu_ctx *ctx = crypto_tfm_ctx(tfm); 87654567Sdillon 87754567Sdillon /* Clear the key. */ 87854567Sdillon memzero_explicit(ctx->key, sizeof(ctx->key)); 87954567Sdillon} 88054567Sdillon 88154567Sdillonstatic struct ahash_engine_alg ocs_hcu_algs[] = { 88254567Sdillon#ifdef CONFIG_CRYPTO_DEV_KEEMBAY_OCS_HCU_HMAC_SHA224 88354567Sdillon{ 88454567Sdillon .base.init = kmb_ocs_hcu_init, 88554567Sdillon .base.update = kmb_ocs_hcu_update, 88654567Sdillon .base.final = kmb_ocs_hcu_final, 88754567Sdillon .base.finup = kmb_ocs_hcu_finup, 88854567Sdillon .base.digest = kmb_ocs_hcu_digest, 88954567Sdillon .base.export = kmb_ocs_hcu_export, 89054567Sdillon .base.import = kmb_ocs_hcu_import, 89154567Sdillon .base.halg = { 89254567Sdillon .digestsize = SHA224_DIGEST_SIZE, 89354567Sdillon .statesize = sizeof(struct ocs_hcu_rctx), 89454567Sdillon .base = { 89554567Sdillon .cra_name = "sha224", 89654567Sdillon .cra_driver_name = "sha224-keembay-ocs", 89754567Sdillon .cra_priority = 255, 89854567Sdillon .cra_flags = CRYPTO_ALG_ASYNC, 89954567Sdillon .cra_blocksize = SHA224_BLOCK_SIZE, 90054567Sdillon .cra_ctxsize = sizeof(struct ocs_hcu_ctx), 90154567Sdillon .cra_alignmask = 0, 90254567Sdillon .cra_module = THIS_MODULE, 90354567Sdillon .cra_init = kmb_ocs_hcu_sha_cra_init, 90454567Sdillon } 90554567Sdillon }, 90654567Sdillon .op.do_one_request = kmb_ocs_hcu_do_one_request, 90754567Sdillon}, 90854567Sdillon{ 90954567Sdillon .base.init = kmb_ocs_hcu_init, 91054567Sdillon .base.update = kmb_ocs_hcu_update, 91154567Sdillon .base.final = kmb_ocs_hcu_final, 91254567Sdillon .base.finup = kmb_ocs_hcu_finup, 91354567Sdillon .base.digest = kmb_ocs_hcu_digest, 91454567Sdillon .base.export = kmb_ocs_hcu_export, 91554567Sdillon .base.import = kmb_ocs_hcu_import, 91654567Sdillon .base.setkey = kmb_ocs_hcu_setkey, 9179336Sdfr .base.halg = { 9189336Sdfr .digestsize = SHA224_DIGEST_SIZE, 91936541Speter .statesize = sizeof(struct ocs_hcu_rctx), 9209336Sdfr .base = { 9219336Sdfr .cra_name = "hmac(sha224)", 92236541Speter .cra_driver_name = "hmac-sha224-keembay-ocs", 9239336Sdfr .cra_priority = 255, 92436541Speter .cra_flags = CRYPTO_ALG_ASYNC, 9259336Sdfr .cra_blocksize = SHA224_BLOCK_SIZE, 92636541Speter .cra_ctxsize = sizeof(struct ocs_hcu_ctx), 9279336Sdfr .cra_alignmask = 0, 92847751Speter .cra_module = THIS_MODULE, 9291541Srgrimes .cra_init = kmb_ocs_hcu_hmac_cra_init, 9301541Srgrimes .cra_exit = kmb_ocs_hcu_hmac_cra_exit, 9311541Srgrimes } 9321541Srgrimes }, 9331541Srgrimes .op.do_one_request = kmb_ocs_hcu_do_one_request, 9341541Srgrimes}, 9351541Srgrimes#endif /* CONFIG_CRYPTO_DEV_KEEMBAY_OCS_HCU_HMAC_SHA224 */ 9361541Srgrimes{ 9371541Srgrimes .base.init = kmb_ocs_hcu_init, 9389336Sdfr .base.update = kmb_ocs_hcu_update, 9391541Srgrimes .base.final = kmb_ocs_hcu_final, 9401541Srgrimes .base.finup = kmb_ocs_hcu_finup, 9411541Srgrimes .base.digest = kmb_ocs_hcu_digest, 9421541Srgrimes .base.export = kmb_ocs_hcu_export, 9431541Srgrimes .base.import = kmb_ocs_hcu_import, 9441541Srgrimes .base.halg = { 9451541Srgrimes .digestsize = SHA256_DIGEST_SIZE, 9461541Srgrimes .statesize = sizeof(struct ocs_hcu_rctx), 9471541Srgrimes .base = { 9481541Srgrimes .cra_name = "sha256", 9499336Sdfr .cra_driver_name = "sha256-keembay-ocs", 9509336Sdfr .cra_priority = 255, 9519336Sdfr .cra_flags = CRYPTO_ALG_ASYNC, 9529336Sdfr .cra_blocksize = SHA256_BLOCK_SIZE, 95347751Speter .cra_ctxsize = sizeof(struct ocs_hcu_ctx), 9549336Sdfr .cra_alignmask = 0, 9559336Sdfr .cra_module = THIS_MODULE, 9569336Sdfr .cra_init = kmb_ocs_hcu_sha_cra_init, 9579336Sdfr } 9589336Sdfr }, 9599336Sdfr .op.do_one_request = kmb_ocs_hcu_do_one_request, 9609336Sdfr}, 9619336Sdfr{ 9629336Sdfr .base.init = kmb_ocs_hcu_init, 9639336Sdfr .base.update = kmb_ocs_hcu_update, 9649336Sdfr .base.final = kmb_ocs_hcu_final, 9659336Sdfr .base.finup = kmb_ocs_hcu_finup, 9669336Sdfr .base.digest = kmb_ocs_hcu_digest, 9679336Sdfr .base.export = kmb_ocs_hcu_export, 9689336Sdfr .base.import = kmb_ocs_hcu_import, 9691541Srgrimes .base.setkey = kmb_ocs_hcu_setkey, 9701541Srgrimes .base.halg = { 97147751Speter .digestsize = SHA256_DIGEST_SIZE, 9721541Srgrimes .statesize = sizeof(struct ocs_hcu_rctx), 9731541Srgrimes .base = { 97454567Sdillon .cra_name = "hmac(sha256)", 9751541Srgrimes .cra_driver_name = "hmac-sha256-keembay-ocs", 97654567Sdillon .cra_priority = 255, 9771541Srgrimes .cra_flags = CRYPTO_ALG_ASYNC, 9789336Sdfr .cra_blocksize = SHA256_BLOCK_SIZE, 9799336Sdfr .cra_ctxsize = sizeof(struct ocs_hcu_ctx), 9809336Sdfr .cra_alignmask = 0, 9811541Srgrimes .cra_module = THIS_MODULE, 98217761Sdyson .cra_init = kmb_ocs_hcu_hmac_cra_init, 98348125Sjulian .cra_exit = kmb_ocs_hcu_hmac_cra_exit, 9849336Sdfr } 9859336Sdfr }, 98648125Sjulian .op.do_one_request = kmb_ocs_hcu_do_one_request, 98748125Sjulian}, 9881541Srgrimes{ 98948125Sjulian .base.init = kmb_ocs_hcu_init, 9901541Srgrimes .base.update = kmb_ocs_hcu_update, 99148125Sjulian .base.final = kmb_ocs_hcu_final, 99217761Sdyson .base.finup = kmb_ocs_hcu_finup, 99348125Sjulian .base.digest = kmb_ocs_hcu_digest, 9949336Sdfr .base.export = kmb_ocs_hcu_export, 99547751Speter .base.import = kmb_ocs_hcu_import, 99647751Speter .base.halg = { 99747751Speter .digestsize = SM3_DIGEST_SIZE, 99847751Speter .statesize = sizeof(struct ocs_hcu_rctx), 99947751Speter .base = { 10009336Sdfr .cra_name = "sm3", 100147751Speter .cra_driver_name = "sm3-keembay-ocs", 10029336Sdfr .cra_priority = 255, 10039336Sdfr .cra_flags = CRYPTO_ALG_ASYNC, 10049336Sdfr .cra_blocksize = SM3_BLOCK_SIZE, 10059336Sdfr .cra_ctxsize = sizeof(struct ocs_hcu_ctx), 10069336Sdfr .cra_alignmask = 0, 100747751Speter .cra_module = THIS_MODULE, 100848125Sjulian .cra_init = kmb_ocs_hcu_sm3_cra_init, 100948125Sjulian } 101048125Sjulian }, 101148125Sjulian .op.do_one_request = kmb_ocs_hcu_do_one_request, 10121541Srgrimes}, 10131541Srgrimes{ 10141541Srgrimes .base.init = kmb_ocs_hcu_init, 10151541Srgrimes .base.update = kmb_ocs_hcu_update, 10161541Srgrimes .base.final = kmb_ocs_hcu_final, 10171549Srgrimes .base.finup = kmb_ocs_hcu_finup, 10189336Sdfr .base.digest = kmb_ocs_hcu_digest, 10199336Sdfr .base.export = kmb_ocs_hcu_export, 10209336Sdfr .base.import = kmb_ocs_hcu_import, 10219336Sdfr .base.setkey = kmb_ocs_hcu_setkey, 10229336Sdfr .base.halg = { 10231541Srgrimes .digestsize = SM3_DIGEST_SIZE, 10249336Sdfr .statesize = sizeof(struct ocs_hcu_rctx), 102528270Swollman .base = { 10269336Sdfr .cra_name = "hmac(sm3)", 10279336Sdfr .cra_driver_name = "hmac-sm3-keembay-ocs", 10281541Srgrimes .cra_priority = 255, 10299336Sdfr .cra_flags = CRYPTO_ALG_ASYNC, 10301541Srgrimes .cra_blocksize = SM3_BLOCK_SIZE, 10319336Sdfr .cra_ctxsize = sizeof(struct ocs_hcu_ctx), 10329336Sdfr .cra_alignmask = 0, 10339336Sdfr .cra_module = THIS_MODULE, 10341541Srgrimes .cra_init = kmb_ocs_hcu_hmac_sm3_cra_init, 103536541Speter .cra_exit = kmb_ocs_hcu_hmac_cra_exit, 103636541Speter } 10371541Srgrimes }, 103811921Sphk .op.do_one_request = kmb_ocs_hcu_do_one_request, 10399336Sdfr}, 10409336Sdfr{ 10419336Sdfr .base.init = kmb_ocs_hcu_init, 10421541Srgrimes .base.update = kmb_ocs_hcu_update, 10431541Srgrimes .base.final = kmb_ocs_hcu_final, 104448125Sjulian .base.finup = kmb_ocs_hcu_finup, 10459336Sdfr .base.digest = kmb_ocs_hcu_digest, 10461541Srgrimes .base.export = kmb_ocs_hcu_export, 10471541Srgrimes .base.import = kmb_ocs_hcu_import, 10481541Srgrimes .base.halg = { 10491541Srgrimes .digestsize = SHA384_DIGEST_SIZE, 10501541Srgrimes .statesize = sizeof(struct ocs_hcu_rctx), 105148125Sjulian .base = { 10529336Sdfr .cra_name = "sha384", 10539336Sdfr .cra_driver_name = "sha384-keembay-ocs", 105448125Sjulian .cra_priority = 255, 105548125Sjulian .cra_flags = CRYPTO_ALG_ASYNC, 10569336Sdfr .cra_blocksize = SHA384_BLOCK_SIZE, 10571541Srgrimes .cra_ctxsize = sizeof(struct ocs_hcu_ctx), 10581541Srgrimes .cra_alignmask = 0, 10599336Sdfr .cra_module = THIS_MODULE, 106036541Speter .cra_init = kmb_ocs_hcu_sha_cra_init, 106147751Speter } 10629336Sdfr }, 10639336Sdfr .op.do_one_request = kmb_ocs_hcu_do_one_request, 10649336Sdfr}, 106536541Speter{ 106636541Speter .base.init = kmb_ocs_hcu_init, 10671541Srgrimes .base.update = kmb_ocs_hcu_update, 106810224Sdg .base.final = kmb_ocs_hcu_final, 106910224Sdg .base.finup = kmb_ocs_hcu_finup, 10701541Srgrimes .base.digest = kmb_ocs_hcu_digest, 107136541Speter .base.export = kmb_ocs_hcu_export, 10729336Sdfr .base.import = kmb_ocs_hcu_import, 10739336Sdfr .base.setkey = kmb_ocs_hcu_setkey, 10749336Sdfr .base.halg = { 10759336Sdfr .digestsize = SHA384_DIGEST_SIZE, 10769336Sdfr .statesize = sizeof(struct ocs_hcu_rctx), 10779336Sdfr .base = { 10789336Sdfr .cra_name = "hmac(sha384)", 10799336Sdfr .cra_driver_name = "hmac-sha384-keembay-ocs", 10809336Sdfr .cra_priority = 255, 10819336Sdfr .cra_flags = CRYPTO_ALG_ASYNC, 10829336Sdfr .cra_blocksize = SHA384_BLOCK_SIZE, 10839336Sdfr .cra_ctxsize = sizeof(struct ocs_hcu_ctx), 10849336Sdfr .cra_alignmask = 0, 10859336Sdfr .cra_module = THIS_MODULE, 10869336Sdfr .cra_init = kmb_ocs_hcu_hmac_cra_init, 10879336Sdfr .cra_exit = kmb_ocs_hcu_hmac_cra_exit, 10889336Sdfr } 10891541Srgrimes }, 10909336Sdfr .op.do_one_request = kmb_ocs_hcu_do_one_request, 10919336Sdfr}, 10929336Sdfr{ 10939336Sdfr .base.init = kmb_ocs_hcu_init, 10949336Sdfr .base.update = kmb_ocs_hcu_update, 10959336Sdfr .base.final = kmb_ocs_hcu_final, 10969336Sdfr .base.finup = kmb_ocs_hcu_finup, 10979336Sdfr .base.digest = kmb_ocs_hcu_digest, 10989336Sdfr .base.export = kmb_ocs_hcu_export, 10999336Sdfr .base.import = kmb_ocs_hcu_import, 11009336Sdfr .base.halg = { 11019336Sdfr .digestsize = SHA512_DIGEST_SIZE, 11029336Sdfr .statesize = sizeof(struct ocs_hcu_rctx), 11031541Srgrimes .base = { 11049336Sdfr .cra_name = "sha512", 11059336Sdfr .cra_driver_name = "sha512-keembay-ocs", 11069336Sdfr .cra_priority = 255, 11079336Sdfr .cra_flags = CRYPTO_ALG_ASYNC, 110848125Sjulian .cra_blocksize = SHA512_BLOCK_SIZE, 110948125Sjulian .cra_ctxsize = sizeof(struct ocs_hcu_ctx), 11109336Sdfr .cra_alignmask = 0, 111136503Speter .cra_module = THIS_MODULE, 111236503Speter .cra_init = kmb_ocs_hcu_sha_cra_init, 111336503Speter } 111448125Sjulian }, 11159336Sdfr .op.do_one_request = kmb_ocs_hcu_do_one_request, 11169336Sdfr}, 111748125Sjulian{ 111848125Sjulian .base.init = kmb_ocs_hcu_init, 11199336Sdfr .base.update = kmb_ocs_hcu_update, 11209336Sdfr .base.final = kmb_ocs_hcu_final, 11219336Sdfr .base.finup = kmb_ocs_hcu_finup, 11221541Srgrimes .base.digest = kmb_ocs_hcu_digest, 11239336Sdfr .base.export = kmb_ocs_hcu_export, 11249336Sdfr .base.import = kmb_ocs_hcu_import, 11259336Sdfr .base.setkey = kmb_ocs_hcu_setkey, 11269336Sdfr .base.halg = { 11271541Srgrimes .digestsize = SHA512_DIGEST_SIZE, 11289336Sdfr .statesize = sizeof(struct ocs_hcu_rctx), 11299336Sdfr .base = { 113036251Speter .cra_name = "hmac(sha512)", 11319336Sdfr .cra_driver_name = "hmac-sha512-keembay-ocs", 11323305Sphk .cra_priority = 255, 113317761Sdyson .cra_flags = CRYPTO_ALG_ASYNC, 113448125Sjulian .cra_blocksize = SHA512_BLOCK_SIZE, 11359336Sdfr .cra_ctxsize = sizeof(struct ocs_hcu_ctx), 11369336Sdfr .cra_alignmask = 0, 113748125Sjulian .cra_module = THIS_MODULE, 113848125Sjulian .cra_init = kmb_ocs_hcu_hmac_cra_init, 11391541Srgrimes .cra_exit = kmb_ocs_hcu_hmac_cra_exit, 11409336Sdfr } 11419336Sdfr }, 11429336Sdfr .op.do_one_request = kmb_ocs_hcu_do_one_request, 11439336Sdfr}, 11449336Sdfr}; 11459336Sdfr 11469336Sdfr/* Device tree driver match. */ 11479336Sdfrstatic const struct of_device_id kmb_ocs_hcu_of_match[] = { 11489336Sdfr { 11499336Sdfr .compatible = "intel,keembay-ocs-hcu", 11509336Sdfr }, 11519336Sdfr {} 11529336Sdfr}; 11539336Sdfr 11549336Sdfrstatic void kmb_ocs_hcu_remove(struct platform_device *pdev) 11559336Sdfr{ 11569336Sdfr struct ocs_hcu_dev *hcu_dev = platform_get_drvdata(pdev); 11579336Sdfr 11589336Sdfr crypto_engine_unregister_ahashes(ocs_hcu_algs, ARRAY_SIZE(ocs_hcu_algs)); 11599336Sdfr 11609336Sdfr crypto_engine_exit(hcu_dev->engine); 11619336Sdfr 11629336Sdfr spin_lock_bh(&ocs_hcu.lock); 11639336Sdfr list_del(&hcu_dev->list); 11649336Sdfr spin_unlock_bh(&ocs_hcu.lock); 11659336Sdfr} 11669336Sdfr 11679336Sdfrstatic int kmb_ocs_hcu_probe(struct platform_device *pdev) 11689336Sdfr{ 11699336Sdfr struct device *dev = &pdev->dev; 11709336Sdfr struct ocs_hcu_dev *hcu_dev; 11719336Sdfr int rc; 11729336Sdfr 11739336Sdfr hcu_dev = devm_kzalloc(dev, sizeof(*hcu_dev), GFP_KERNEL); 11749336Sdfr if (!hcu_dev) 11759336Sdfr return -ENOMEM; 11769336Sdfr 11779336Sdfr hcu_dev->dev = dev; 11789336Sdfr 117917761Sdyson platform_set_drvdata(pdev, hcu_dev); 118048125Sjulian rc = dma_set_mask_and_coherent(&pdev->dev, OCS_HCU_DMA_BIT_MASK); 11819336Sdfr if (rc) 11829336Sdfr return rc; 11839336Sdfr 11849336Sdfr hcu_dev->io_base = devm_platform_ioremap_resource(pdev, 0); 11859336Sdfr if (IS_ERR(hcu_dev->io_base)) 11869336Sdfr return PTR_ERR(hcu_dev->io_base); 118748125Sjulian 118848125Sjulian init_completion(&hcu_dev->irq_done); 118948125Sjulian 119048125Sjulian /* Get and request IRQ. */ 119136541Speter hcu_dev->irq = platform_get_irq(pdev, 0); 11929336Sdfr if (hcu_dev->irq < 0) 119326418Sdfr return hcu_dev->irq; 119426418Sdfr 119526418Sdfr rc = devm_request_threaded_irq(&pdev->dev, hcu_dev->irq, 119626418Sdfr ocs_hcu_irq_handler, NULL, 0, 11979336Sdfr "keembay-ocs-hcu", hcu_dev); 11989336Sdfr if (rc < 0) { 11999336Sdfr dev_err(dev, "Could not request IRQ.\n"); 12009336Sdfr return rc; 12019336Sdfr } 12029336Sdfr 12039336Sdfr INIT_LIST_HEAD(&hcu_dev->list); 12049336Sdfr 12059336Sdfr spin_lock_bh(&ocs_hcu.lock); 12069336Sdfr list_add_tail(&hcu_dev->list, &ocs_hcu.dev_list); 12079336Sdfr spin_unlock_bh(&ocs_hcu.lock); 12089336Sdfr 12099336Sdfr /* Initialize crypto engine */ 12109336Sdfr hcu_dev->engine = crypto_engine_alloc_init(dev, 1); 121148125Sjulian if (!hcu_dev->engine) { 121248125Sjulian rc = -ENOMEM; 121348125Sjulian goto list_del; 121448125Sjulian } 12159336Sdfr 12169336Sdfr rc = crypto_engine_start(hcu_dev->engine); 12179336Sdfr if (rc) { 12189336Sdfr dev_err(dev, "Could not start engine.\n"); 12199336Sdfr goto cleanup; 12209336Sdfr } 12219336Sdfr 12229336Sdfr /* Security infrastructure guarantees OCS clock is enabled. */ 12239336Sdfr 12249336Sdfr rc = crypto_engine_register_ahashes(ocs_hcu_algs, ARRAY_SIZE(ocs_hcu_algs)); 12259336Sdfr if (rc) { 12269336Sdfr dev_err(dev, "Could not register algorithms.\n"); 12279336Sdfr goto cleanup; 12289336Sdfr } 12299336Sdfr 12309336Sdfr return 0; 12319336Sdfr 12329336Sdfrcleanup: 12339336Sdfr crypto_engine_exit(hcu_dev->engine); 12349336Sdfrlist_del: 12359336Sdfr spin_lock_bh(&ocs_hcu.lock); 12369336Sdfr list_del(&hcu_dev->list); 12379336Sdfr spin_unlock_bh(&ocs_hcu.lock); 12389336Sdfr 12399336Sdfr return rc; 124036541Speter} 124136541Speter 12429336Sdfr/* The OCS driver is a platform device. */ 12439336Sdfrstatic struct platform_driver kmb_ocs_hcu_driver = { 12449336Sdfr .probe = kmb_ocs_hcu_probe, 12459336Sdfr .remove_new = kmb_ocs_hcu_remove, 12469336Sdfr .driver = { 124754485Sdillon .name = DRV_NAME, 12489336Sdfr .of_match_table = kmb_ocs_hcu_of_match, 12499336Sdfr }, 12509336Sdfr}; 125148125Sjulian 12529336Sdfrmodule_platform_driver(kmb_ocs_hcu_driver); 12539336Sdfr 12549336SdfrMODULE_LICENSE("GPL"); 12559336Sdfr