1/*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (c) 2020 Rubicon Communications, LLC (Netgate) 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 16 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 17 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 18 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 19 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 20 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 21 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 22 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 23 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 24 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25 * 26 * $FreeBSD$ 27 */ 28 29#ifndef _SAFEXCEL_VAR_H_ 30#define _SAFEXCEL_VAR_H_ 31 32#include <sys/counter.h> 33 34#define SAFEXCEL_MAX_RINGS 4 35#define SAFEXCEL_MAX_BATCH_SIZE 64 36#define SAFEXCEL_MAX_FRAGMENTS 64 37#define SAFEXCEL_MAX_IV_LEN 16 38#define SAFEXCEL_MAX_REQUEST_SIZE 65535 39 40#define SAFEXCEL_RING_SIZE 512 41#define SAFEXCEL_MAX_ITOKENS 4 42#define SAFEXCEL_MAX_ATOKENS 16 43#define SAFEXCEL_FETCH_COUNT 1 44#define SAFEXCEL_MAX_KEY_LEN 32 45#define SAFEXCEL_MAX_RING_AIC 14 46 47/* 48 * Context Record format. 49 * 50 * In this driver the context control words are always set in the control data. 51 * This helps optimize fetching of the context record. This is configured by 52 * setting SAFEXCEL_OPTION_CTX_CTRL_IN_CMD. 53 */ 54struct safexcel_context_record { 55 uint32_t control0; /* Unused. */ 56 uint32_t control1; /* Unused. */ 57 uint32_t data[40]; /* Key material. */ 58} __packed; 59 60/* Processing Engine Control Data format. */ 61struct safexcel_control_data { 62 uint32_t packet_length : 17; 63 uint32_t options : 13; 64 uint32_t type : 2; 65 66 uint16_t application_id; 67 uint16_t rsvd; 68 69 uint32_t context_lo; 70 uint32_t context_hi; 71 72 uint32_t control0; 73 uint32_t control1; 74 75 /* Inline instructions or IV. */ 76 uint32_t token[SAFEXCEL_MAX_ITOKENS]; 77} __packed; 78 79/* 80 * Basic Command Descriptor. 81 * 82 * The Processing Engine and driver cooperate to maintain a set of command 83 * rings, representing outstanding crypto operation requests. Each descriptor 84 * corresponds to an input data segment, and thus a single crypto(9) request may 85 * span several contiguous command descriptors. 86 * 87 * The first command descriptor for a request stores the input token, which 88 * encodes data specific to the requested operation, such as the encryption 89 * mode. For some operations data is passed outside the descriptor, in a 90 * context record (e.g., encryption keys), or in an "additional token data" 91 * region (e.g., instructions). 92 */ 93struct safexcel_cmd_descr { 94 uint32_t particle_size : 17; 95 uint32_t rsvd0 : 5; 96 uint32_t last_seg : 1; 97 uint32_t first_seg : 1; 98 uint32_t additional_cdata_size : 8; 99 uint32_t rsvd1; 100 101 uint32_t data_lo; 102 uint32_t data_hi; 103 104 uint32_t atok_lo; 105 uint32_t atok_hi; 106 107 struct safexcel_control_data control_data; 108} __packed; 109 110/* Context control word 0 fields. */ 111#define SAFEXCEL_CONTROL0_TYPE_NULL_OUT 0x0 112#define SAFEXCEL_CONTROL0_TYPE_NULL_IN 0x1 113#define SAFEXCEL_CONTROL0_TYPE_HASH_OUT 0x2 114#define SAFEXCEL_CONTROL0_TYPE_HASH_IN 0x3 115#define SAFEXCEL_CONTROL0_TYPE_CRYPTO_OUT 0x4 116#define SAFEXCEL_CONTROL0_TYPE_CRYPTO_IN 0x5 117#define SAFEXCEL_CONTROL0_TYPE_ENCRYPT_HASH_OUT 0x6 118#define SAFEXCEL_CONTROL0_TYPE_DECRYPT_HASH_IN 0x7 119#define SAFEXCEL_CONTROL0_TYPE_HASH_ENCRYPT_OUT 0xe 120#define SAFEXCEL_CONTROL0_TYPE_HASH_DECRYPT_IN 0xf 121#define SAFEXCEL_CONTROL0_RESTART_HASH (1 << 4) 122#define SAFEXCEL_CONTROL0_NO_FINISH_HASH (1 << 5) 123#define SAFEXCEL_CONTROL0_SIZE(n) (((n) & 0xff) << 8) 124#define SAFEXCEL_CONTROL0_KEY_EN (1 << 16) 125#define SAFEXCEL_CONTROL0_CRYPTO_ALG_AES128 (0x5 << 17) 126#define SAFEXCEL_CONTROL0_CRYPTO_ALG_AES192 (0x6 << 17) 127#define SAFEXCEL_CONTROL0_CRYPTO_ALG_AES256 (0x7 << 17) 128#define SAFEXCEL_CONTROL0_DIGEST_PRECOMPUTED (0x1 << 21) 129#define SAFEXCEL_CONTROL0_DIGEST_CCM (0x2 << 21) 130#define SAFEXCEL_CONTROL0_DIGEST_GMAC (0x2 << 21) 131#define SAFEXCEL_CONTROL0_DIGEST_HMAC (0x3 << 21) 132#define SAFEXCEL_CONTROL0_HASH_ALG_SHA1 (0x2 << 23) 133#define SAFEXCEL_CONTROL0_HASH_ALG_SHA224 (0x4 << 23) 134#define SAFEXCEL_CONTROL0_HASH_ALG_SHA256 (0x3 << 23) 135#define SAFEXCEL_CONTROL0_HASH_ALG_SHA384 (0x6 << 23) 136#define SAFEXCEL_CONTROL0_HASH_ALG_SHA512 (0x5 << 23) 137#define SAFEXCEL_CONTROL0_HASH_ALG_XCBC128 (0x1 << 23) 138#define SAFEXCEL_CONTROL0_HASH_ALG_XCBC192 (0x2 << 23) 139#define SAFEXCEL_CONTROL0_HASH_ALG_XCBC256 (0x3 << 23) 140#define SAFEXCEL_CONTROL0_HASH_ALG_GHASH (0x4 << 23) 141#define SAFEXCEL_CONTROL0_INV_FR (0x5 << 24) 142#define SAFEXCEL_CONTROL0_INV_TR (0x6 << 24) 143 144/* Context control word 1 fields. */ 145#define SAFEXCEL_CONTROL1_CRYPTO_MODE_ECB 0x0 146#define SAFEXCEL_CONTROL1_CRYPTO_MODE_CBC 0x1 147#define SAFEXCEL_CONTROL1_CRYPTO_MODE_ICM 0x3 148#define SAFEXCEL_CONTROL1_CRYPTO_MODE_OFB 0x4 149#define SAFEXCEL_CONTROL1_CRYPTO_MODE_CFB128 0x5 150#define SAFEXCEL_CONTROL1_CRYPTO_MODE_CTR 0x6 151#define SAFEXCEL_CONTROL1_CRYPTO_MODE_XTS 0x7 152#define SAFEXCEL_CONTROL1_CRYPTO_MODE_CCM (0x6 | (1 << 17)) 153#define SAFEXCEL_CONTROL1_CRYPTO_MODE_GCM (0x6 | (1 << 17)) 154#define SAFEXCEL_CONTROL1_IV0 (1u << 5) 155#define SAFEXCEL_CONTROL1_IV1 (1u << 6) 156#define SAFEXCEL_CONTROL1_IV2 (1u << 7) 157#define SAFEXCEL_CONTROL1_IV3 (1u << 8) 158#define SAFEXCEL_CONTROL1_DIGEST_CNT (1u << 9) 159#define SAFEXCEL_CONTROL1_COUNTER_MODE (1u << 10) 160#define SAFEXCEL_CONTROL1_ENCRYPT_HASH_RES (1u << 17) 161#define SAFEXCEL_CONTROL1_HASH_STORE (1u << 19) 162 163/* Control options. */ 164#define SAFEXCEL_OPTION_IP (1u << 0) /* must be set */ 165#define SAFEXCEL_OPTION_CP (1u << 1) /* 64-bit ctx addr */ 166#define SAFEXCEL_OPTION_RC_AUTO (2u << 3) /* auto ctx reuse */ 167#define SAFEXCEL_OPTION_CTX_CTRL_IN_CMD (1u << 8) /* ctx ctrl */ 168#define SAFEXCEL_OPTION_4_TOKEN_IV_CMD 0xe00 /* IV in bypass */ 169 170struct safexcel_res_data { 171 uint32_t packet_length : 17; 172 uint32_t error_code : 15; 173 174 uint32_t bypass_length : 4; 175 uint32_t e15 : 1; 176 uint32_t rsvd0 : 16; 177 uint32_t hash_bytes : 1; 178 uint32_t hash_length : 6; 179 uint32_t generic_bytes : 1; 180 uint32_t checksum : 1; 181 uint32_t next_header : 1; 182 uint32_t length : 1; 183 184 uint16_t application_id; 185 uint16_t rsvd1; 186 187 uint32_t rsvd2; 188}; 189 190/* Basic Result Descriptor format */ 191struct safexcel_res_descr { 192 uint32_t particle_size : 17; 193 uint32_t rsvd0 : 3; 194 uint32_t descriptor_overflow : 1; 195 uint32_t buffer_overflow : 1; 196 uint32_t last_seg : 1; 197 uint32_t first_seg : 1; 198 uint32_t result_size : 8; 199 200 uint32_t rsvd1; 201 202 uint32_t data_lo; 203 uint32_t data_hi; 204 205 struct safexcel_res_data result_data; 206} __packed; 207 208/* Result data error codes. */ 209#define SAFEXCEL_RESULT_ERR_PACKET_LEN (1u << 0) 210#define SAFEXCEL_RESULT_ERR_TOKEN_ERROR (1u << 1) 211#define SAFEXCEL_RESULT_ERR_BYPASS (1u << 2) 212#define SAFEXCEL_RESULT_ERR_CRYPTO_BLOCK_SIZE (1u << 3) 213#define SAFEXCEL_RESULT_ERR_HASH_BLOCK_SIZE (1u << 4) 214#define SAFEXCEL_RESULT_ERR_INVALID_CMD (1u << 5) 215#define SAFEXCEL_RESULT_ERR_PROHIBITED_ALGO (1u << 6) 216#define SAFEXCEL_RESULT_ERR_HASH_INPUT_OVERFLOW (1u << 7) 217#define SAFEXCEL_RESULT_ERR_TTL_UNDERFLOW (1u << 8) 218#define SAFEXCEL_RESULT_ERR_AUTH_FAILED (1u << 9) 219#define SAFEXCEL_RESULT_ERR_SEQNO_CHECK_FAILED (1u << 10) 220#define SAFEXCEL_RESULT_ERR_SPI_CHECK (1u << 11) 221#define SAFEXCEL_RESULT_ERR_CHECKSUM (1u << 12) 222#define SAFEXCEL_RESULT_ERR_PAD_VERIFICATION (1u << 13) 223#define SAFEXCEL_RESULT_ERR_TIMEOUT (1u << 14) 224#define SAFEXCEL_RESULT_ERR_OUTPUT_DMA (1u << 15) 225 226/* 227 * The EIP-96 (crypto transform engine) is programmed using a set of 228 * data processing instructions with the encodings defined below. 229 */ 230struct safexcel_instr { 231 uint32_t length : 17; /* bytes to be processed */ 232 uint32_t status : 2; /* stream status */ 233 uint32_t instructions : 9; 234 uint32_t opcode : 4; 235} __packed; 236 237/* Type 1, operational data instructions. */ 238#define SAFEXCEL_INSTR_OPCODE_DIRECTION 0x0 239#define SAFEXCEL_INSTR_OPCODE_PRE_CHECKSUM 0x1 240#define SAFEXCEL_INSTR_OPCODE_INSERT 0x2 241#define SAFEXCEL_INSTR_OPCODE_INSERT_CTX 0x9 242#define SAFEXCEL_INSTR_OPCODE_REPLACE 0x3 243#define SAFEXCEL_INSTR_OPCODE_RETRIEVE 0x4 244#define SAFEXCEL_INSTR_OPCODE_MUTE 0x5 245/* Type 2, IP header instructions. */ 246#define SAFEXCEL_INSTR_OPCODE_IPV4 0x7 247#define SAFEXCEL_INSTR_OPCODE_IPV4_CHECKSUM 0x6 248#define SAFEXCEL_INSTR_OPCODE_IPV6 0x8 249/* Type 3, postprocessing instructions. */ 250#define SAFEXCEL_INSTR_OPCODE_INSERT_REMOVE_RESULT 0xa 251#define SAFEXCEL_INSTR_OPCODE_REPLACE_BYTE 0xb 252/* Type 4, result instructions. */ 253#define SAFEXCEL_INSTR_OPCODE_VERIFY_FIELDS 0xd 254/* Type 5, context control instructions. */ 255#define SAFEXCEL_INSTR_OPCODE_CONTEXT_ACCESS 0xe 256/* Type 6, context control instructions. */ 257#define SAFEXCEL_INSTR_OPCODE_BYPASS_TOKEN_DATA 0xf 258 259/* Status bits for type 1 and 2 instructions. */ 260#define SAFEXCEL_INSTR_STATUS_LAST_HASH (1u << 0) 261#define SAFEXCEL_INSTR_STATUS_LAST_PACKET (1u << 1) 262/* Status bits for type 3 instructions. */ 263#define SAFEXCEL_INSTR_STATUS_NO_CKSUM_MOD (1u << 0) 264 265/* Instruction-dependent flags. */ 266#define SAFEXCEL_INSTR_INSERT_HASH_DIGEST 0x1c 267#define SAFEXCEL_INSTR_INSERT_IMMEDIATE 0x1b 268#define SAFEXCEL_INSTR_DEST_OUTPUT (1u << 5) 269#define SAFEXCEL_INSTR_DEST_HASH (1u << 6) 270#define SAFEXCEL_INSTR_DEST_CRYPTO (1u << 7) 271#define SAFEXCEL_INSTR_INS_LAST (1u << 8) 272 273#define SAFEXCEL_INSTR_VERIFY_HASH (1u << 16) 274#define SAFEXCEL_INSTR_VERIFY_PADDING (1u << 5) 275 276#define SAFEXCEL_INSTR_CTX_ACCESS_WRITE (1u << 11) 277#define SAFEXCEL_INSTR_CTX_ACCESS_PASS (1u << 12) 278#define SAFEXCEL_INSTR_CTX_ACCESS_FAIL (1u << 13) 279#define SAFEXCEL_INSTR_CTX_ACCESS_HASHRES (0x1c) 280 281#define SAFEXCEL_TOKEN_TYPE_BYPASS 0x0 282#define SAFEXCEL_TOKEN_TYPE_AUTONOMOUS 0x3 283 284#define SAFEXCEL_CONTEXT_SMALL 0x2 285#define SAFEXCEL_CONTEXT_LARGE 0x3 286 287struct safexcel_reg_offsets { 288 uint32_t hia_aic; 289 uint32_t hia_aic_g; 290 uint32_t hia_aic_r; 291 uint32_t hia_aic_xdr; 292 uint32_t hia_dfe; 293 uint32_t hia_dfe_thr; 294 uint32_t hia_dse; 295 uint32_t hia_dse_thr; 296 uint32_t hia_gen_cfg; 297 uint32_t pe; 298}; 299 300struct safexcel_config { 301 uint32_t hdw; /* Host interface Data Width. */ 302 uint32_t aic_rings; /* Number of AIC rings. */ 303 uint32_t pes; /* Number of PEs. */ 304 uint32_t rings; /* Number of rings. */ 305 306 uint32_t cd_size; /* CDR descriptor size. */ 307 uint32_t cd_offset; /* CDR offset (size + alignment). */ 308 309 uint32_t rd_size; /* RDR descriptor size. */ 310 uint32_t rd_offset; /* RDR offset. */ 311 312 uint32_t atok_offset; /* Additional token offset. */ 313 314 uint32_t caps; /* Device capabilities. */ 315}; 316 317#define SAFEXCEL_DPRINTF(sc, lvl, ...) do { \ 318 if ((sc)->sc_debug >= (lvl)) \ 319 device_printf((sc)->sc_dev, __VA_ARGS__); \ 320} while (0) 321 322struct safexcel_dma_mem { 323 caddr_t vaddr; 324 bus_addr_t paddr; 325 bus_dma_tag_t tag; 326 bus_dmamap_t map; 327}; 328 329struct safexcel_cmd_descr_ring { 330 struct safexcel_dma_mem dma; 331 struct safexcel_cmd_descr *desc; 332 int write; 333 int read; 334}; 335 336struct safexcel_res_descr_ring { 337 struct safexcel_dma_mem dma; 338 struct safexcel_res_descr *desc; 339 int write; 340 int read; 341}; 342 343struct safexcel_session { 344 uint32_t alg; /* cipher algorithm */ 345 uint32_t digest; /* digest type */ 346 uint32_t hash; /* hash algorithm */ 347 uint32_t mode; /* cipher mode of operation */ 348 unsigned int digestlen; /* digest length */ 349 unsigned int statelen; /* HMAC hash state length */ 350 uint8_t key[AES_MAX_KEY * 2]; 351 unsigned int klen; /* cipher key length */ 352 unsigned int ivlen; 353 union { 354 uint32_t ghash_key[AES_BLOCK_LEN / sizeof(uint32_t)]; 355 uint32_t xcbc_key[(AES_BLOCK_LEN * 2 + AES_MAX_KEY) / 356 sizeof(uint32_t)]; 357 uint8_t tweak_key[AES_MAX_KEY]; 358 }; 359 struct { 360 uint8_t hmac_ipad[HMAC_MAX_BLOCK_LEN]; 361 uint8_t hmac_opad[HMAC_MAX_BLOCK_LEN]; 362 }; 363}; 364 365struct safexcel_softc; 366 367struct safexcel_request { 368 STAILQ_ENTRY(safexcel_request) link; 369 bool dmap_loaded; 370 int ringidx; 371 bus_dmamap_t dmap; 372 int error; 373 int cdescs, rdescs; 374 uint8_t iv[SAFEXCEL_MAX_IV_LEN]; 375 struct safexcel_cmd_descr *cdesc; 376 struct safexcel_dma_mem ctx; 377 struct safexcel_session *sess; 378 struct cryptop *crp; 379 struct cryptodesc *enc, *mac; 380 struct safexcel_softc *sc; 381}; 382 383struct safexcel_ring { 384 struct mtx mtx; 385 struct sglist *cmd_data; 386 struct safexcel_cmd_descr_ring cdr; 387 struct sglist *res_data; 388 struct safexcel_res_descr_ring rdr; 389 390 /* Shadows the command descriptor ring. */ 391 struct safexcel_request requests[SAFEXCEL_RING_SIZE]; 392 393 /* Count of requests pending submission. */ 394 int pending; 395 int pending_cdesc, pending_rdesc; 396 397 /* Count of outstanding requests. */ 398 int queued; 399 400 /* Requests were deferred due to a resource shortage. */ 401 int blocked; 402 403 struct safexcel_dma_mem dma_atok; 404 bus_dma_tag_t data_dtag; 405 406 char lockname[32]; 407}; 408 409struct safexcel_intr_handle { 410 struct safexcel_softc *sc; 411 void *handle; 412 int ring; 413}; 414 415struct safexcel_softc { 416 device_t sc_dev; 417 uint32_t sc_type; /* EIP-97 or 197 */ 418 int sc_debug; 419 420 struct resource *sc_res; 421 struct resource *sc_intr[SAFEXCEL_MAX_RINGS]; 422 struct safexcel_intr_handle sc_ih[SAFEXCEL_MAX_RINGS]; 423 424 counter_u64_t sc_req_alloc_failures; 425 counter_u64_t sc_cdesc_alloc_failures; 426 counter_u64_t sc_rdesc_alloc_failures; 427 428 struct safexcel_ring sc_ring[SAFEXCEL_MAX_RINGS]; 429 430 int32_t sc_cid; 431 struct safexcel_reg_offsets sc_offsets; 432 struct safexcel_config sc_config; 433}; 434 435#define SAFEXCEL_WRITE(sc, off, val) bus_write_4((sc)->sc_res, (off), (val)) 436#define SAFEXCEL_READ(sc, off) bus_read_4((sc)->sc_res, (off)) 437 438#define SAFEXCEL_ADDR_LO(addr) ((uint64_t)(addr) & 0xffffffffu) 439#define SAFEXCEL_ADDR_HI(addr) (((uint64_t)(addr) >> 32) & 0xffffffffu) 440 441#endif /* _SAFEXCEL_VAR_H_ */ 442