1 2/* SPDX-License-Identifier: GPL-2.0-only */ 3/* 4 * Copyright 2016 Broadcom 5 */ 6 7#ifndef _CIPHER_H 8#define _CIPHER_H 9 10#include <linux/atomic.h> 11#include <linux/mailbox/brcm-message.h> 12#include <linux/mailbox_client.h> 13#include <crypto/aes.h> 14#include <crypto/internal/hash.h> 15#include <crypto/internal/skcipher.h> 16#include <crypto/aead.h> 17#include <crypto/arc4.h> 18#include <crypto/gcm.h> 19#include <crypto/sha1.h> 20#include <crypto/sha2.h> 21#include <crypto/sha3.h> 22 23#include "spu.h" 24#include "spum.h" 25#include "spu2.h" 26 27/* Driver supports up to MAX_SPUS SPU blocks */ 28#define MAX_SPUS 16 29 30#define ARC4_STATE_SIZE 4 31 32#define CCM_AES_IV_SIZE 16 33#define CCM_ESP_IV_SIZE 8 34#define RFC4543_ICV_SIZE 16 35 36#define MAX_KEY_SIZE ARC4_MAX_KEY_SIZE 37#define MAX_IV_SIZE AES_BLOCK_SIZE 38#define MAX_DIGEST_SIZE SHA3_512_DIGEST_SIZE 39#define MAX_ASSOC_SIZE 512 40 41/* size of salt value for AES-GCM-ESP and AES-CCM-ESP */ 42#define GCM_ESP_SALT_SIZE 4 43#define CCM_ESP_SALT_SIZE 3 44#define MAX_SALT_SIZE GCM_ESP_SALT_SIZE 45#define GCM_ESP_SALT_OFFSET 0 46#define CCM_ESP_SALT_OFFSET 1 47 48#define GCM_ESP_DIGESTSIZE 16 49 50#define MAX_HASH_BLOCK_SIZE SHA512_BLOCK_SIZE 51 52/* 53 * Maximum number of bytes from a non-final hash request that can be deferred 54 * until more data is available. With new crypto API framework, this 55 * can be no more than one block of data. 56 */ 57#define HASH_CARRY_MAX MAX_HASH_BLOCK_SIZE 58 59/* Force at least 4-byte alignment of all SPU message fields */ 60#define SPU_MSG_ALIGN 4 61 62/* Number of times to resend mailbox message if mb queue is full */ 63#define SPU_MB_RETRY_MAX 1000 64 65/* op_counts[] indexes */ 66enum op_type { 67 SPU_OP_CIPHER, 68 SPU_OP_HASH, 69 SPU_OP_HMAC, 70 SPU_OP_AEAD, 71 SPU_OP_NUM 72}; 73 74enum spu_spu_type { 75 SPU_TYPE_SPUM, 76 SPU_TYPE_SPU2, 77}; 78 79/* 80 * SPUM_NS2 and SPUM_NSP are the SPU-M block on Northstar 2 and Northstar Plus, 81 * respectively. 82 */ 83enum spu_spu_subtype { 84 SPU_SUBTYPE_SPUM_NS2, 85 SPU_SUBTYPE_SPUM_NSP, 86 SPU_SUBTYPE_SPU2_V1, 87 SPU_SUBTYPE_SPU2_V2 88}; 89 90struct spu_type_subtype { 91 enum spu_spu_type type; 92 enum spu_spu_subtype subtype; 93}; 94 95struct cipher_op { 96 enum spu_cipher_alg alg; 97 enum spu_cipher_mode mode; 98}; 99 100struct auth_op { 101 enum hash_alg alg; 102 enum hash_mode mode; 103}; 104 105struct iproc_alg_s { 106 u32 type; 107 union { 108 struct skcipher_alg skcipher; 109 struct ahash_alg hash; 110 struct aead_alg aead; 111 } alg; 112 struct cipher_op cipher_info; 113 struct auth_op auth_info; 114 bool auth_first; 115 bool registered; 116}; 117 118/* 119 * Buffers for a SPU request/reply message pair. All part of one structure to 120 * allow a single alloc per request. 121 */ 122struct spu_msg_buf { 123 /* Request message fragments */ 124 125 /* 126 * SPU request message header. For SPU-M, holds MH, EMH, SCTX, BDESC, 127 * and BD header. For SPU2, holds FMD, OMD. 128 */ 129 u8 bcm_spu_req_hdr[ALIGN(SPU2_HEADER_ALLOC_LEN, SPU_MSG_ALIGN)]; 130 131 /* IV or counter. Size to include salt. Also used for XTS tweek. */ 132 u8 iv_ctr[ALIGN(2 * AES_BLOCK_SIZE, SPU_MSG_ALIGN)]; 133 134 /* Hash digest. request and response. */ 135 u8 digest[ALIGN(MAX_DIGEST_SIZE, SPU_MSG_ALIGN)]; 136 137 /* SPU request message padding */ 138 u8 spu_req_pad[ALIGN(SPU_PAD_LEN_MAX, SPU_MSG_ALIGN)]; 139 140 /* SPU-M request message STATUS field */ 141 u8 tx_stat[ALIGN(SPU_TX_STATUS_LEN, SPU_MSG_ALIGN)]; 142 143 /* Response message fragments */ 144 145 /* SPU response message header */ 146 u8 spu_resp_hdr[ALIGN(SPU2_HEADER_ALLOC_LEN, SPU_MSG_ALIGN)]; 147 148 /* SPU response message STATUS field padding */ 149 u8 rx_stat_pad[ALIGN(SPU_STAT_PAD_MAX, SPU_MSG_ALIGN)]; 150 151 /* SPU response message STATUS field */ 152 u8 rx_stat[ALIGN(SPU_RX_STATUS_LEN, SPU_MSG_ALIGN)]; 153 154 union { 155 /* Buffers only used for skcipher */ 156 struct { 157 /* 158 * Field used for either SUPDT when RC4 is used 159 * -OR- tweak value when XTS/AES is used 160 */ 161 u8 supdt_tweak[ALIGN(SPU_SUPDT_LEN, SPU_MSG_ALIGN)]; 162 } c; 163 164 /* Buffers only used for aead */ 165 struct { 166 /* SPU response pad for GCM data */ 167 u8 gcmpad[ALIGN(AES_BLOCK_SIZE, SPU_MSG_ALIGN)]; 168 169 /* SPU request msg padding for GCM AAD */ 170 u8 req_aad_pad[ALIGN(SPU_PAD_LEN_MAX, SPU_MSG_ALIGN)]; 171 172 /* SPU response data to be discarded */ 173 u8 resp_aad[ALIGN(MAX_ASSOC_SIZE + MAX_IV_SIZE, 174 SPU_MSG_ALIGN)]; 175 } a; 176 }; 177}; 178 179struct iproc_ctx_s { 180 u8 enckey[MAX_KEY_SIZE + ARC4_STATE_SIZE]; 181 unsigned int enckeylen; 182 183 u8 authkey[MAX_KEY_SIZE + ARC4_STATE_SIZE]; 184 unsigned int authkeylen; 185 186 u8 salt[MAX_SALT_SIZE]; 187 unsigned int salt_len; 188 unsigned int salt_offset; 189 u8 iv[MAX_IV_SIZE]; 190 191 unsigned int digestsize; 192 193 struct iproc_alg_s *alg; 194 bool is_esp; 195 196 struct cipher_op cipher; 197 enum spu_cipher_type cipher_type; 198 199 struct auth_op auth; 200 bool auth_first; 201 202 /* 203 * The maximum length in bytes of the payload in a SPU message for this 204 * context. For SPU-M, the payload is the combination of AAD and data. 205 * For SPU2, the payload is just data. A value of SPU_MAX_PAYLOAD_INF 206 * indicates that there is no limit to the length of the SPU message 207 * payload. 208 */ 209 unsigned int max_payload; 210 211 struct crypto_aead *fallback_cipher; 212 213 /* auth_type is determined during processing of request */ 214 215 u8 ipad[MAX_HASH_BLOCK_SIZE]; 216 u8 opad[MAX_HASH_BLOCK_SIZE]; 217 218 /* 219 * Buffer to hold SPU message header template. Template is created at 220 * setkey time for skcipher requests, since most of the fields in the 221 * header are known at that time. At request time, just fill in a few 222 * missing pieces related to length of data in the request and IVs, etc. 223 */ 224 u8 bcm_spu_req_hdr[ALIGN(SPU2_HEADER_ALLOC_LEN, SPU_MSG_ALIGN)]; 225 226 /* Length of SPU request header */ 227 u16 spu_req_hdr_len; 228 229 /* Expected length of SPU response header */ 230 u16 spu_resp_hdr_len; 231 232 /* 233 * shash descriptor - needed to perform incremental hashing in 234 * software, when hw doesn't support it. 235 */ 236 struct shash_desc *shash; 237 238 bool is_rfc4543; /* RFC 4543 style of GMAC */ 239}; 240 241/* state from iproc_reqctx_s necessary for hash state export/import */ 242struct spu_hash_export_s { 243 unsigned int total_todo; 244 unsigned int total_sent; 245 u8 hash_carry[HASH_CARRY_MAX]; 246 unsigned int hash_carry_len; 247 u8 incr_hash[MAX_DIGEST_SIZE]; 248 bool is_sw_hmac; 249}; 250 251struct iproc_reqctx_s { 252 /* general context */ 253 struct crypto_async_request *parent; 254 255 /* only valid after enqueue() */ 256 struct iproc_ctx_s *ctx; 257 258 u8 chan_idx; /* Mailbox channel to be used to submit this request */ 259 260 /* total todo, rx'd, and sent for this request */ 261 unsigned int total_todo; 262 unsigned int total_received; /* only valid for skcipher */ 263 unsigned int total_sent; 264 265 /* 266 * num bytes sent to hw from the src sg in this request. This can differ 267 * from total_sent for incremental hashing. total_sent includes previous 268 * init() and update() data. src_sent does not. 269 */ 270 unsigned int src_sent; 271 272 /* 273 * For AEAD requests, start of associated data. This will typically 274 * point to the beginning of the src scatterlist from the request, 275 * since assoc data is at the beginning of the src scatterlist rather 276 * than in its own sg. 277 */ 278 struct scatterlist *assoc; 279 280 /* 281 * scatterlist entry and offset to start of data for next chunk. Crypto 282 * API src scatterlist for AEAD starts with AAD, if present. For first 283 * chunk, src_sg is sg entry at beginning of input data (after AAD). 284 * src_skip begins at the offset in that sg entry where data begins. 285 */ 286 struct scatterlist *src_sg; 287 int src_nents; /* Number of src entries with data */ 288 u32 src_skip; /* bytes of current sg entry already used */ 289 290 /* 291 * Same for destination. For AEAD, if there is AAD, output data must 292 * be written at offset following AAD. 293 */ 294 struct scatterlist *dst_sg; 295 int dst_nents; /* Number of dst entries with data */ 296 u32 dst_skip; /* bytes of current sg entry already written */ 297 298 /* Mailbox message used to send this request to PDC driver */ 299 struct brcm_message mb_mssg; 300 301 bool bd_suppress; /* suppress BD field in SPU response? */ 302 303 /* cipher context */ 304 bool is_encrypt; 305 306 /* 307 * CBC mode: IV. CTR mode: counter. Else empty. Used as a DMA 308 * buffer for AEAD requests. So allocate as DMAable memory. If IV 309 * concatenated with salt, includes the salt. 310 */ 311 u8 *iv_ctr; 312 /* Length of IV or counter, in bytes */ 313 unsigned int iv_ctr_len; 314 315 /* 316 * Hash requests can be of any size, whether initial, update, or final. 317 * A non-final request must be submitted to the SPU as an integral 318 * number of blocks. This may leave data at the end of the request 319 * that is not a full block. Since the request is non-final, it cannot 320 * be padded. So, we write the remainder to this hash_carry buffer and 321 * hold it until the next request arrives. The carry data is then 322 * submitted at the beginning of the data in the next SPU msg. 323 * hash_carry_len is the number of bytes currently in hash_carry. These 324 * fields are only used for ahash requests. 325 */ 326 u8 hash_carry[HASH_CARRY_MAX]; 327 unsigned int hash_carry_len; 328 unsigned int is_final; /* is this the final for the hash op? */ 329 330 /* 331 * Digest from incremental hash is saved here to include in next hash 332 * operation. Cannot be stored in req->result for truncated hashes, 333 * since result may be sized for final digest. Cannot be saved in 334 * msg_buf because that gets deleted between incremental hash ops 335 * and is not saved as part of export(). 336 */ 337 u8 incr_hash[MAX_DIGEST_SIZE]; 338 339 /* hmac context */ 340 bool is_sw_hmac; 341 342 gfp_t gfp; 343 344 /* Buffers used to build SPU request and response messages */ 345 struct spu_msg_buf msg_buf; 346 347 struct aead_request req; 348}; 349 350/* 351 * Structure encapsulates a set of function pointers specific to the type of 352 * SPU hardware running. These functions handling creation and parsing of 353 * SPU request messages and SPU response messages. Includes hardware-specific 354 * values read from device tree. 355 */ 356struct spu_hw { 357 void (*spu_dump_msg_hdr)(u8 *buf, unsigned int buf_len); 358 u32 (*spu_ctx_max_payload)(enum spu_cipher_alg cipher_alg, 359 enum spu_cipher_mode cipher_mode, 360 unsigned int blocksize); 361 u32 (*spu_payload_length)(u8 *spu_hdr); 362 u16 (*spu_response_hdr_len)(u16 auth_key_len, u16 enc_key_len, 363 bool is_hash); 364 u16 (*spu_hash_pad_len)(enum hash_alg hash_alg, 365 enum hash_mode hash_mode, u32 chunksize, 366 u16 hash_block_size); 367 u32 (*spu_gcm_ccm_pad_len)(enum spu_cipher_mode cipher_mode, 368 unsigned int data_size); 369 u32 (*spu_assoc_resp_len)(enum spu_cipher_mode cipher_mode, 370 unsigned int assoc_len, 371 unsigned int iv_len, bool is_encrypt); 372 u8 (*spu_aead_ivlen)(enum spu_cipher_mode cipher_mode, 373 u16 iv_len); 374 enum hash_type (*spu_hash_type)(u32 src_sent); 375 u32 (*spu_digest_size)(u32 digest_size, enum hash_alg alg, 376 enum hash_type); 377 u32 (*spu_create_request)(u8 *spu_hdr, 378 struct spu_request_opts *req_opts, 379 struct spu_cipher_parms *cipher_parms, 380 struct spu_hash_parms *hash_parms, 381 struct spu_aead_parms *aead_parms, 382 unsigned int data_size); 383 u16 (*spu_cipher_req_init)(u8 *spu_hdr, 384 struct spu_cipher_parms *cipher_parms); 385 void (*spu_cipher_req_finish)(u8 *spu_hdr, 386 u16 spu_req_hdr_len, 387 unsigned int is_inbound, 388 struct spu_cipher_parms *cipher_parms, 389 unsigned int data_size); 390 void (*spu_request_pad)(u8 *pad_start, u32 gcm_padding, 391 u32 hash_pad_len, enum hash_alg auth_alg, 392 enum hash_mode auth_mode, 393 unsigned int total_sent, u32 status_padding); 394 u8 (*spu_xts_tweak_in_payload)(void); 395 u8 (*spu_tx_status_len)(void); 396 u8 (*spu_rx_status_len)(void); 397 int (*spu_status_process)(u8 *statp); 398 void (*spu_ccm_update_iv)(unsigned int digestsize, 399 struct spu_cipher_parms *cipher_parms, 400 unsigned int assoclen, unsigned int chunksize, 401 bool is_encrypt, bool is_esp); 402 u32 (*spu_wordalign_padlen)(u32 data_size); 403 404 /* The base virtual address of the SPU hw registers */ 405 void __iomem *reg_vbase[MAX_SPUS]; 406 407 /* Version of the SPU hardware */ 408 enum spu_spu_type spu_type; 409 410 /* Sub-version of the SPU hardware */ 411 enum spu_spu_subtype spu_subtype; 412 413 /* The number of SPUs on this platform */ 414 u32 num_spu; 415 416 /* The number of SPU channels on this platform */ 417 u32 num_chan; 418}; 419 420struct bcm_device_private { 421 struct platform_device *pdev; 422 423 struct spu_hw spu; 424 425 atomic_t session_count; /* number of streams active */ 426 atomic_t stream_count; /* monotonic counter for streamID's */ 427 428 /* Length of BCM header. Set to 0 when hw does not expect BCM HEADER. */ 429 u8 bcm_hdr_len; 430 431 /* The index of the channel to use for the next crypto request */ 432 atomic_t next_chan; 433 434 struct dentry *debugfs_dir; 435 struct dentry *debugfs_stats; 436 437 /* Number of request bytes processed and result bytes returned */ 438 atomic64_t bytes_in; 439 atomic64_t bytes_out; 440 441 /* Number of operations of each type */ 442 atomic_t op_counts[SPU_OP_NUM]; 443 444 atomic_t cipher_cnt[CIPHER_ALG_LAST][CIPHER_MODE_LAST]; 445 atomic_t hash_cnt[HASH_ALG_LAST]; 446 atomic_t hmac_cnt[HASH_ALG_LAST]; 447 atomic_t aead_cnt[AEAD_TYPE_LAST]; 448 449 /* Number of calls to setkey() for each operation type */ 450 atomic_t setkey_cnt[SPU_OP_NUM]; 451 452 /* Number of times request was resubmitted because mb was full */ 453 atomic_t mb_no_spc; 454 455 /* Number of mailbox send failures */ 456 atomic_t mb_send_fail; 457 458 /* Number of ICV check failures for AEAD messages */ 459 atomic_t bad_icv; 460 461 struct mbox_client mcl; 462 463 /* Array of mailbox channel pointers, one for each channel */ 464 struct mbox_chan **mbox; 465}; 466 467extern struct bcm_device_private iproc_priv; 468 469#endif 470