1// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB 2// Copyright (c) 2019 Mellanox Technologies. 3 4#include "mlx5_core.h" 5#include "lib/crypto.h" 6 7#define MLX5_CRYPTO_DEK_POOLS_NUM (MLX5_ACCEL_OBJ_TYPE_KEY_NUM - 1) 8#define type2idx(type) ((type) - 1) 9 10#define MLX5_CRYPTO_DEK_POOL_SYNC_THRESH 128 11 12/* calculate the num of DEKs, which are freed by any user 13 * (for example, TLS) after last revalidation in a pool or a bulk. 14 */ 15#define MLX5_CRYPTO_DEK_CALC_FREED(a) \ 16 ({ typeof(a) _a = (a); \ 17 _a->num_deks - _a->avail_deks - _a->in_use_deks; }) 18 19#define MLX5_CRYPTO_DEK_POOL_CALC_FREED(pool) MLX5_CRYPTO_DEK_CALC_FREED(pool) 20#define MLX5_CRYPTO_DEK_BULK_CALC_FREED(bulk) MLX5_CRYPTO_DEK_CALC_FREED(bulk) 21 22#define MLX5_CRYPTO_DEK_BULK_IDLE(bulk) \ 23 ({ typeof(bulk) _bulk = (bulk); \ 24 _bulk->avail_deks == _bulk->num_deks; }) 25 26enum { 27 MLX5_CRYPTO_DEK_ALL_TYPE = BIT(0), 28}; 29 30struct mlx5_crypto_dek_pool { 31 struct mlx5_core_dev *mdev; 32 u32 key_purpose; 33 int num_deks; /* the total number of keys in this pool */ 34 int avail_deks; /* the number of available keys in this pool */ 35 int in_use_deks; /* the number of being used keys in this pool */ 36 struct mutex lock; /* protect the following lists, and the bulks */ 37 struct list_head partial_list; /* some of keys are available */ 38 struct list_head full_list; /* no available keys */ 39 struct list_head avail_list; /* all keys are available to use */ 40 41 /* No in-used keys, and all need to be synced. 42 * These bulks will be put to avail list after sync. 43 */ 44 struct list_head sync_list; 45 46 bool syncing; 47 struct list_head wait_for_free; 48 struct work_struct sync_work; 49 50 spinlock_t destroy_lock; /* protect destroy_list */ 51 struct list_head destroy_list; 52 struct work_struct destroy_work; 53}; 54 55struct mlx5_crypto_dek_bulk { 56 struct mlx5_core_dev *mdev; 57 int base_obj_id; 58 int avail_start; /* the bit to start search */ 59 int num_deks; /* the total number of keys in a bulk */ 60 int avail_deks; /* the number of keys available, with need_sync bit 0 */ 61 int in_use_deks; /* the number of keys being used, with in_use bit 1 */ 62 struct list_head entry; 63 64 /* 0: not being used by any user, 1: otherwise */ 65 unsigned long *in_use; 66 67 /* The bits are set when they are used, and reset after crypto_sync 68 * is executed. So, the value 0 means the key is newly created, or not 69 * used after sync, and 1 means it is in use, or freed but not synced 70 */ 71 unsigned long *need_sync; 72}; 73 74struct mlx5_crypto_dek_priv { 75 struct mlx5_core_dev *mdev; 76 int log_dek_obj_range; 77}; 78 79struct mlx5_crypto_dek { 80 struct mlx5_crypto_dek_bulk *bulk; 81 struct list_head entry; 82 u32 obj_id; 83}; 84 85u32 mlx5_crypto_dek_get_id(struct mlx5_crypto_dek *dek) 86{ 87 return dek->obj_id; 88} 89 90static int mlx5_crypto_dek_get_key_sz(struct mlx5_core_dev *mdev, 91 u32 sz_bytes, u8 *key_sz_p) 92{ 93 u32 sz_bits = sz_bytes * BITS_PER_BYTE; 94 95 switch (sz_bits) { 96 case 128: 97 *key_sz_p = MLX5_GENERAL_OBJECT_TYPE_ENCRYPTION_KEY_KEY_SIZE_128; 98 break; 99 case 256: 100 *key_sz_p = MLX5_GENERAL_OBJECT_TYPE_ENCRYPTION_KEY_KEY_SIZE_256; 101 break; 102 default: 103 mlx5_core_err(mdev, "Crypto offload error, invalid key size (%u bits)\n", 104 sz_bits); 105 return -EINVAL; 106 } 107 108 return 0; 109} 110 111static int mlx5_crypto_dek_fill_key(struct mlx5_core_dev *mdev, u8 *key_obj, 112 const void *key, u32 sz_bytes) 113{ 114 void *dst; 115 u8 key_sz; 116 int err; 117 118 err = mlx5_crypto_dek_get_key_sz(mdev, sz_bytes, &key_sz); 119 if (err) 120 return err; 121 122 MLX5_SET(encryption_key_obj, key_obj, key_size, key_sz); 123 124 if (sz_bytes == 16) 125 /* For key size of 128b the MSBs are reserved. */ 126 dst = MLX5_ADDR_OF(encryption_key_obj, key_obj, key[1]); 127 else 128 dst = MLX5_ADDR_OF(encryption_key_obj, key_obj, key); 129 130 memcpy(dst, key, sz_bytes); 131 132 return 0; 133} 134 135static int mlx5_crypto_cmd_sync_crypto(struct mlx5_core_dev *mdev, 136 int crypto_type) 137{ 138 u32 in[MLX5_ST_SZ_DW(sync_crypto_in)] = {}; 139 int err; 140 141 mlx5_core_dbg(mdev, 142 "Execute SYNC_CRYPTO command with crypto_type(0x%x)\n", 143 crypto_type); 144 145 MLX5_SET(sync_crypto_in, in, opcode, MLX5_CMD_OP_SYNC_CRYPTO); 146 MLX5_SET(sync_crypto_in, in, crypto_type, crypto_type); 147 148 err = mlx5_cmd_exec_in(mdev, sync_crypto, in); 149 if (err) 150 mlx5_core_err(mdev, 151 "Failed to exec sync crypto, type=%d, err=%d\n", 152 crypto_type, err); 153 154 return err; 155} 156 157static int mlx5_crypto_create_dek_bulk(struct mlx5_core_dev *mdev, 158 u32 key_purpose, int log_obj_range, 159 u32 *obj_id) 160{ 161 u32 in[MLX5_ST_SZ_DW(create_encryption_key_in)] = {}; 162 u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)]; 163 void *obj, *param; 164 int err; 165 166 MLX5_SET(general_obj_in_cmd_hdr, in, opcode, 167 MLX5_CMD_OP_CREATE_GENERAL_OBJECT); 168 MLX5_SET(general_obj_in_cmd_hdr, in, obj_type, 169 MLX5_GENERAL_OBJECT_TYPES_ENCRYPTION_KEY); 170 param = MLX5_ADDR_OF(general_obj_in_cmd_hdr, in, op_param); 171 MLX5_SET(general_obj_create_param, param, log_obj_range, log_obj_range); 172 173 obj = MLX5_ADDR_OF(create_encryption_key_in, in, encryption_key_object); 174 MLX5_SET(encryption_key_obj, obj, key_purpose, key_purpose); 175 MLX5_SET(encryption_key_obj, obj, pd, mdev->mlx5e_res.hw_objs.pdn); 176 177 err = mlx5_cmd_exec(mdev, in, sizeof(in), out, sizeof(out)); 178 if (err) 179 return err; 180 181 *obj_id = MLX5_GET(general_obj_out_cmd_hdr, out, obj_id); 182 mlx5_core_dbg(mdev, "DEK objects created, bulk=%d, obj_id=%d\n", 183 1 << log_obj_range, *obj_id); 184 185 return 0; 186} 187 188static int mlx5_crypto_modify_dek_key(struct mlx5_core_dev *mdev, 189 const void *key, u32 sz_bytes, u32 key_purpose, 190 u32 obj_id, u32 obj_offset) 191{ 192 u32 in[MLX5_ST_SZ_DW(modify_encryption_key_in)] = {}; 193 u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)]; 194 void *obj, *param; 195 int err; 196 197 MLX5_SET(general_obj_in_cmd_hdr, in, opcode, 198 MLX5_CMD_OP_MODIFY_GENERAL_OBJECT); 199 MLX5_SET(general_obj_in_cmd_hdr, in, obj_type, 200 MLX5_GENERAL_OBJECT_TYPES_ENCRYPTION_KEY); 201 MLX5_SET(general_obj_in_cmd_hdr, in, obj_id, obj_id); 202 203 param = MLX5_ADDR_OF(general_obj_in_cmd_hdr, in, op_param); 204 MLX5_SET(general_obj_query_param, param, obj_offset, obj_offset); 205 206 obj = MLX5_ADDR_OF(modify_encryption_key_in, in, encryption_key_object); 207 MLX5_SET64(encryption_key_obj, obj, modify_field_select, 1); 208 MLX5_SET(encryption_key_obj, obj, key_purpose, key_purpose); 209 MLX5_SET(encryption_key_obj, obj, pd, mdev->mlx5e_res.hw_objs.pdn); 210 211 err = mlx5_crypto_dek_fill_key(mdev, obj, key, sz_bytes); 212 if (err) 213 return err; 214 215 err = mlx5_cmd_exec(mdev, in, sizeof(in), out, sizeof(out)); 216 217 /* avoid leaking key on the stack */ 218 memzero_explicit(in, sizeof(in)); 219 220 return err; 221} 222 223static int mlx5_crypto_create_dek_key(struct mlx5_core_dev *mdev, 224 const void *key, u32 sz_bytes, 225 u32 key_purpose, u32 *p_key_id) 226{ 227 u32 in[MLX5_ST_SZ_DW(create_encryption_key_in)] = {}; 228 u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)]; 229 u64 general_obj_types; 230 void *obj; 231 int err; 232 233 general_obj_types = MLX5_CAP_GEN_64(mdev, general_obj_types); 234 if (!(general_obj_types & 235 MLX5_HCA_CAP_GENERAL_OBJECT_TYPES_ENCRYPTION_KEY)) 236 return -EINVAL; 237 238 MLX5_SET(general_obj_in_cmd_hdr, in, opcode, 239 MLX5_CMD_OP_CREATE_GENERAL_OBJECT); 240 MLX5_SET(general_obj_in_cmd_hdr, in, obj_type, 241 MLX5_GENERAL_OBJECT_TYPES_ENCRYPTION_KEY); 242 243 obj = MLX5_ADDR_OF(create_encryption_key_in, in, encryption_key_object); 244 MLX5_SET(encryption_key_obj, obj, key_purpose, key_purpose); 245 MLX5_SET(encryption_key_obj, obj, pd, mdev->mlx5e_res.hw_objs.pdn); 246 247 err = mlx5_crypto_dek_fill_key(mdev, obj, key, sz_bytes); 248 if (err) 249 return err; 250 251 err = mlx5_cmd_exec(mdev, in, sizeof(in), out, sizeof(out)); 252 if (!err) 253 *p_key_id = MLX5_GET(general_obj_out_cmd_hdr, out, obj_id); 254 255 /* avoid leaking key on the stack */ 256 memzero_explicit(in, sizeof(in)); 257 258 return err; 259} 260 261static void mlx5_crypto_destroy_dek_key(struct mlx5_core_dev *mdev, u32 key_id) 262{ 263 u32 in[MLX5_ST_SZ_DW(general_obj_in_cmd_hdr)] = {}; 264 u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)]; 265 266 MLX5_SET(general_obj_in_cmd_hdr, in, opcode, 267 MLX5_CMD_OP_DESTROY_GENERAL_OBJECT); 268 MLX5_SET(general_obj_in_cmd_hdr, in, obj_type, 269 MLX5_GENERAL_OBJECT_TYPES_ENCRYPTION_KEY); 270 MLX5_SET(general_obj_in_cmd_hdr, in, obj_id, key_id); 271 272 mlx5_cmd_exec(mdev, in, sizeof(in), out, sizeof(out)); 273} 274 275int mlx5_create_encryption_key(struct mlx5_core_dev *mdev, 276 const void *key, u32 sz_bytes, 277 u32 key_type, u32 *p_key_id) 278{ 279 return mlx5_crypto_create_dek_key(mdev, key, sz_bytes, key_type, p_key_id); 280} 281 282void mlx5_destroy_encryption_key(struct mlx5_core_dev *mdev, u32 key_id) 283{ 284 mlx5_crypto_destroy_dek_key(mdev, key_id); 285} 286 287static struct mlx5_crypto_dek_bulk * 288mlx5_crypto_dek_bulk_create(struct mlx5_crypto_dek_pool *pool) 289{ 290 struct mlx5_crypto_dek_priv *dek_priv = pool->mdev->mlx5e_res.dek_priv; 291 struct mlx5_core_dev *mdev = pool->mdev; 292 struct mlx5_crypto_dek_bulk *bulk; 293 int num_deks, base_obj_id; 294 int err; 295 296 bulk = kzalloc(sizeof(*bulk), GFP_KERNEL); 297 if (!bulk) 298 return ERR_PTR(-ENOMEM); 299 300 num_deks = 1 << dek_priv->log_dek_obj_range; 301 bulk->need_sync = bitmap_zalloc(num_deks, GFP_KERNEL); 302 if (!bulk->need_sync) { 303 err = -ENOMEM; 304 goto err_out; 305 } 306 307 bulk->in_use = bitmap_zalloc(num_deks, GFP_KERNEL); 308 if (!bulk->in_use) { 309 err = -ENOMEM; 310 goto err_out; 311 } 312 313 err = mlx5_crypto_create_dek_bulk(mdev, pool->key_purpose, 314 dek_priv->log_dek_obj_range, 315 &base_obj_id); 316 if (err) 317 goto err_out; 318 319 bulk->base_obj_id = base_obj_id; 320 bulk->num_deks = num_deks; 321 bulk->avail_deks = num_deks; 322 bulk->mdev = mdev; 323 324 return bulk; 325 326err_out: 327 bitmap_free(bulk->in_use); 328 bitmap_free(bulk->need_sync); 329 kfree(bulk); 330 return ERR_PTR(err); 331} 332 333static struct mlx5_crypto_dek_bulk * 334mlx5_crypto_dek_pool_add_bulk(struct mlx5_crypto_dek_pool *pool) 335{ 336 struct mlx5_crypto_dek_bulk *bulk; 337 338 bulk = mlx5_crypto_dek_bulk_create(pool); 339 if (IS_ERR(bulk)) 340 return bulk; 341 342 pool->avail_deks += bulk->num_deks; 343 pool->num_deks += bulk->num_deks; 344 list_add(&bulk->entry, &pool->partial_list); 345 346 return bulk; 347} 348 349static void mlx5_crypto_dek_bulk_free(struct mlx5_crypto_dek_bulk *bulk) 350{ 351 mlx5_crypto_destroy_dek_key(bulk->mdev, bulk->base_obj_id); 352 bitmap_free(bulk->need_sync); 353 bitmap_free(bulk->in_use); 354 kfree(bulk); 355} 356 357static void mlx5_crypto_dek_pool_remove_bulk(struct mlx5_crypto_dek_pool *pool, 358 struct mlx5_crypto_dek_bulk *bulk, 359 bool delay) 360{ 361 pool->num_deks -= bulk->num_deks; 362 pool->avail_deks -= bulk->avail_deks; 363 pool->in_use_deks -= bulk->in_use_deks; 364 list_del(&bulk->entry); 365 if (!delay) 366 mlx5_crypto_dek_bulk_free(bulk); 367} 368 369static struct mlx5_crypto_dek_bulk * 370mlx5_crypto_dek_pool_pop(struct mlx5_crypto_dek_pool *pool, u32 *obj_offset) 371{ 372 struct mlx5_crypto_dek_bulk *bulk; 373 int pos; 374 375 mutex_lock(&pool->lock); 376 bulk = list_first_entry_or_null(&pool->partial_list, 377 struct mlx5_crypto_dek_bulk, entry); 378 379 if (bulk) { 380 pos = find_next_zero_bit(bulk->need_sync, bulk->num_deks, 381 bulk->avail_start); 382 if (pos == bulk->num_deks) { 383 mlx5_core_err(pool->mdev, "Wrong DEK bulk avail_start.\n"); 384 pos = find_first_zero_bit(bulk->need_sync, bulk->num_deks); 385 } 386 WARN_ON(pos == bulk->num_deks); 387 } else { 388 bulk = list_first_entry_or_null(&pool->avail_list, 389 struct mlx5_crypto_dek_bulk, 390 entry); 391 if (bulk) { 392 list_move(&bulk->entry, &pool->partial_list); 393 } else { 394 bulk = mlx5_crypto_dek_pool_add_bulk(pool); 395 if (IS_ERR(bulk)) 396 goto out; 397 } 398 pos = 0; 399 } 400 401 *obj_offset = pos; 402 bitmap_set(bulk->need_sync, pos, 1); 403 bitmap_set(bulk->in_use, pos, 1); 404 bulk->in_use_deks++; 405 bulk->avail_deks--; 406 if (!bulk->avail_deks) { 407 list_move(&bulk->entry, &pool->full_list); 408 bulk->avail_start = bulk->num_deks; 409 } else { 410 bulk->avail_start = pos + 1; 411 } 412 pool->avail_deks--; 413 pool->in_use_deks++; 414 415out: 416 mutex_unlock(&pool->lock); 417 return bulk; 418} 419 420static bool mlx5_crypto_dek_need_sync(struct mlx5_crypto_dek_pool *pool) 421{ 422 return !pool->syncing && 423 MLX5_CRYPTO_DEK_POOL_CALC_FREED(pool) > MLX5_CRYPTO_DEK_POOL_SYNC_THRESH; 424} 425 426static int mlx5_crypto_dek_free_locked(struct mlx5_crypto_dek_pool *pool, 427 struct mlx5_crypto_dek *dek) 428{ 429 struct mlx5_crypto_dek_bulk *bulk = dek->bulk; 430 int obj_offset; 431 bool old_val; 432 int err = 0; 433 434 obj_offset = dek->obj_id - bulk->base_obj_id; 435 old_val = test_and_clear_bit(obj_offset, bulk->in_use); 436 WARN_ON_ONCE(!old_val); 437 if (!old_val) { 438 err = -ENOENT; 439 goto out_free; 440 } 441 pool->in_use_deks--; 442 bulk->in_use_deks--; 443 if (!bulk->avail_deks && !bulk->in_use_deks) 444 list_move(&bulk->entry, &pool->sync_list); 445 446 if (mlx5_crypto_dek_need_sync(pool) && schedule_work(&pool->sync_work)) 447 pool->syncing = true; 448 449out_free: 450 kfree(dek); 451 return err; 452} 453 454static int mlx5_crypto_dek_pool_push(struct mlx5_crypto_dek_pool *pool, 455 struct mlx5_crypto_dek *dek) 456{ 457 int err = 0; 458 459 mutex_lock(&pool->lock); 460 if (pool->syncing) 461 list_add(&dek->entry, &pool->wait_for_free); 462 else 463 err = mlx5_crypto_dek_free_locked(pool, dek); 464 mutex_unlock(&pool->lock); 465 466 return err; 467} 468 469/* Update the bits for a bulk while sync, and avail_next for search. 470 * As the combinations of (need_sync, in_use) of one DEK are 471 * - (0,0) means the key is ready for use, 472 * - (1,1) means the key is currently being used by a user, 473 * - (1,0) means the key is freed, and waiting for being synced, 474 * - (0,1) is invalid state. 475 * the number of revalidated DEKs can be calculated by 476 * hweight_long(need_sync XOR in_use), and the need_sync bits can be reset 477 * by simply copying from in_use bits. 478 */ 479static void mlx5_crypto_dek_bulk_reset_synced(struct mlx5_crypto_dek_pool *pool, 480 struct mlx5_crypto_dek_bulk *bulk) 481{ 482 unsigned long *need_sync = bulk->need_sync; 483 unsigned long *in_use = bulk->in_use; 484 int i, freed, reused, avail_next; 485 bool first = true; 486 487 freed = MLX5_CRYPTO_DEK_BULK_CALC_FREED(bulk); 488 489 for (i = 0; freed && i < BITS_TO_LONGS(bulk->num_deks); 490 i++, need_sync++, in_use++) { 491 reused = hweight_long((*need_sync) ^ (*in_use)); 492 if (!reused) 493 continue; 494 495 bulk->avail_deks += reused; 496 pool->avail_deks += reused; 497 *need_sync = *in_use; 498 if (first) { 499 avail_next = i * BITS_PER_TYPE(long); 500 if (bulk->avail_start > avail_next) 501 bulk->avail_start = avail_next; 502 first = false; 503 } 504 505 freed -= reused; 506 } 507} 508 509/* Return true if the bulk is reused, false if destroyed with delay */ 510static bool mlx5_crypto_dek_bulk_handle_avail(struct mlx5_crypto_dek_pool *pool, 511 struct mlx5_crypto_dek_bulk *bulk, 512 struct list_head *destroy_list) 513{ 514 if (list_empty(&pool->avail_list)) { 515 list_move(&bulk->entry, &pool->avail_list); 516 return true; 517 } 518 519 mlx5_crypto_dek_pool_remove_bulk(pool, bulk, true); 520 list_add(&bulk->entry, destroy_list); 521 return false; 522} 523 524static void mlx5_crypto_dek_pool_splice_destroy_list(struct mlx5_crypto_dek_pool *pool, 525 struct list_head *list, 526 struct list_head *head) 527{ 528 spin_lock(&pool->destroy_lock); 529 list_splice_init(list, head); 530 spin_unlock(&pool->destroy_lock); 531} 532 533static void mlx5_crypto_dek_pool_free_wait_keys(struct mlx5_crypto_dek_pool *pool) 534{ 535 struct mlx5_crypto_dek *dek, *next; 536 537 list_for_each_entry_safe(dek, next, &pool->wait_for_free, entry) { 538 list_del(&dek->entry); 539 mlx5_crypto_dek_free_locked(pool, dek); 540 } 541} 542 543/* For all the bulks in each list, reset the bits while sync. 544 * Move them to different lists according to the number of available DEKs. 545 * Destrory all the idle bulks, except one for quick service. 546 * And free DEKs in the waiting list at the end of this func. 547 */ 548static void mlx5_crypto_dek_pool_reset_synced(struct mlx5_crypto_dek_pool *pool) 549{ 550 struct mlx5_crypto_dek_bulk *bulk, *tmp; 551 LIST_HEAD(destroy_list); 552 553 list_for_each_entry_safe(bulk, tmp, &pool->partial_list, entry) { 554 mlx5_crypto_dek_bulk_reset_synced(pool, bulk); 555 if (MLX5_CRYPTO_DEK_BULK_IDLE(bulk)) 556 mlx5_crypto_dek_bulk_handle_avail(pool, bulk, &destroy_list); 557 } 558 559 list_for_each_entry_safe(bulk, tmp, &pool->full_list, entry) { 560 mlx5_crypto_dek_bulk_reset_synced(pool, bulk); 561 562 if (!bulk->avail_deks) 563 continue; 564 565 if (MLX5_CRYPTO_DEK_BULK_IDLE(bulk)) 566 mlx5_crypto_dek_bulk_handle_avail(pool, bulk, &destroy_list); 567 else 568 list_move(&bulk->entry, &pool->partial_list); 569 } 570 571 list_for_each_entry_safe(bulk, tmp, &pool->sync_list, entry) { 572 bulk->avail_deks = bulk->num_deks; 573 pool->avail_deks += bulk->num_deks; 574 if (mlx5_crypto_dek_bulk_handle_avail(pool, bulk, &destroy_list)) { 575 memset(bulk->need_sync, 0, BITS_TO_BYTES(bulk->num_deks)); 576 bulk->avail_start = 0; 577 } 578 } 579 580 mlx5_crypto_dek_pool_free_wait_keys(pool); 581 582 if (!list_empty(&destroy_list)) { 583 mlx5_crypto_dek_pool_splice_destroy_list(pool, &destroy_list, 584 &pool->destroy_list); 585 schedule_work(&pool->destroy_work); 586 } 587} 588 589static void mlx5_crypto_dek_sync_work_fn(struct work_struct *work) 590{ 591 struct mlx5_crypto_dek_pool *pool = 592 container_of(work, struct mlx5_crypto_dek_pool, sync_work); 593 int err; 594 595 err = mlx5_crypto_cmd_sync_crypto(pool->mdev, BIT(pool->key_purpose)); 596 mutex_lock(&pool->lock); 597 if (!err) 598 mlx5_crypto_dek_pool_reset_synced(pool); 599 pool->syncing = false; 600 mutex_unlock(&pool->lock); 601} 602 603struct mlx5_crypto_dek *mlx5_crypto_dek_create(struct mlx5_crypto_dek_pool *dek_pool, 604 const void *key, u32 sz_bytes) 605{ 606 struct mlx5_crypto_dek_priv *dek_priv = dek_pool->mdev->mlx5e_res.dek_priv; 607 struct mlx5_core_dev *mdev = dek_pool->mdev; 608 u32 key_purpose = dek_pool->key_purpose; 609 struct mlx5_crypto_dek_bulk *bulk; 610 struct mlx5_crypto_dek *dek; 611 int obj_offset; 612 int err; 613 614 dek = kzalloc(sizeof(*dek), GFP_KERNEL); 615 if (!dek) 616 return ERR_PTR(-ENOMEM); 617 618 if (!dek_priv) { 619 err = mlx5_crypto_create_dek_key(mdev, key, sz_bytes, 620 key_purpose, &dek->obj_id); 621 goto out; 622 } 623 624 bulk = mlx5_crypto_dek_pool_pop(dek_pool, &obj_offset); 625 if (IS_ERR(bulk)) { 626 err = PTR_ERR(bulk); 627 goto out; 628 } 629 630 dek->bulk = bulk; 631 dek->obj_id = bulk->base_obj_id + obj_offset; 632 err = mlx5_crypto_modify_dek_key(mdev, key, sz_bytes, key_purpose, 633 bulk->base_obj_id, obj_offset); 634 if (err) { 635 mlx5_crypto_dek_pool_push(dek_pool, dek); 636 return ERR_PTR(err); 637 } 638 639out: 640 if (err) { 641 kfree(dek); 642 return ERR_PTR(err); 643 } 644 645 return dek; 646} 647 648void mlx5_crypto_dek_destroy(struct mlx5_crypto_dek_pool *dek_pool, 649 struct mlx5_crypto_dek *dek) 650{ 651 struct mlx5_crypto_dek_priv *dek_priv = dek_pool->mdev->mlx5e_res.dek_priv; 652 struct mlx5_core_dev *mdev = dek_pool->mdev; 653 654 if (!dek_priv) { 655 mlx5_crypto_destroy_dek_key(mdev, dek->obj_id); 656 kfree(dek); 657 } else { 658 mlx5_crypto_dek_pool_push(dek_pool, dek); 659 } 660} 661 662static void mlx5_crypto_dek_free_destroy_list(struct list_head *destroy_list) 663{ 664 struct mlx5_crypto_dek_bulk *bulk, *tmp; 665 666 list_for_each_entry_safe(bulk, tmp, destroy_list, entry) 667 mlx5_crypto_dek_bulk_free(bulk); 668} 669 670static void mlx5_crypto_dek_destroy_work_fn(struct work_struct *work) 671{ 672 struct mlx5_crypto_dek_pool *pool = 673 container_of(work, struct mlx5_crypto_dek_pool, destroy_work); 674 LIST_HEAD(destroy_list); 675 676 mlx5_crypto_dek_pool_splice_destroy_list(pool, &pool->destroy_list, 677 &destroy_list); 678 mlx5_crypto_dek_free_destroy_list(&destroy_list); 679} 680 681struct mlx5_crypto_dek_pool * 682mlx5_crypto_dek_pool_create(struct mlx5_core_dev *mdev, int key_purpose) 683{ 684 struct mlx5_crypto_dek_pool *pool; 685 686 pool = kzalloc(sizeof(*pool), GFP_KERNEL); 687 if (!pool) 688 return ERR_PTR(-ENOMEM); 689 690 pool->mdev = mdev; 691 pool->key_purpose = key_purpose; 692 693 mutex_init(&pool->lock); 694 INIT_LIST_HEAD(&pool->avail_list); 695 INIT_LIST_HEAD(&pool->partial_list); 696 INIT_LIST_HEAD(&pool->full_list); 697 INIT_LIST_HEAD(&pool->sync_list); 698 INIT_LIST_HEAD(&pool->wait_for_free); 699 INIT_WORK(&pool->sync_work, mlx5_crypto_dek_sync_work_fn); 700 spin_lock_init(&pool->destroy_lock); 701 INIT_LIST_HEAD(&pool->destroy_list); 702 INIT_WORK(&pool->destroy_work, mlx5_crypto_dek_destroy_work_fn); 703 704 return pool; 705} 706 707void mlx5_crypto_dek_pool_destroy(struct mlx5_crypto_dek_pool *pool) 708{ 709 struct mlx5_crypto_dek_bulk *bulk, *tmp; 710 711 cancel_work_sync(&pool->sync_work); 712 cancel_work_sync(&pool->destroy_work); 713 714 mlx5_crypto_dek_pool_free_wait_keys(pool); 715 716 list_for_each_entry_safe(bulk, tmp, &pool->avail_list, entry) 717 mlx5_crypto_dek_pool_remove_bulk(pool, bulk, false); 718 719 list_for_each_entry_safe(bulk, tmp, &pool->full_list, entry) 720 mlx5_crypto_dek_pool_remove_bulk(pool, bulk, false); 721 722 list_for_each_entry_safe(bulk, tmp, &pool->sync_list, entry) 723 mlx5_crypto_dek_pool_remove_bulk(pool, bulk, false); 724 725 list_for_each_entry_safe(bulk, tmp, &pool->partial_list, entry) 726 mlx5_crypto_dek_pool_remove_bulk(pool, bulk, false); 727 728 mlx5_crypto_dek_free_destroy_list(&pool->destroy_list); 729 730 mutex_destroy(&pool->lock); 731 732 kfree(pool); 733} 734 735void mlx5_crypto_dek_cleanup(struct mlx5_crypto_dek_priv *dek_priv) 736{ 737 if (!dek_priv) 738 return; 739 740 kfree(dek_priv); 741} 742 743struct mlx5_crypto_dek_priv *mlx5_crypto_dek_init(struct mlx5_core_dev *mdev) 744{ 745 struct mlx5_crypto_dek_priv *dek_priv; 746 int err; 747 748 if (!MLX5_CAP_CRYPTO(mdev, log_dek_max_alloc)) 749 return NULL; 750 751 dek_priv = kzalloc(sizeof(*dek_priv), GFP_KERNEL); 752 if (!dek_priv) 753 return ERR_PTR(-ENOMEM); 754 755 dek_priv->mdev = mdev; 756 dek_priv->log_dek_obj_range = min_t(int, 12, 757 MLX5_CAP_CRYPTO(mdev, log_dek_max_alloc)); 758 759 /* sync all types of objects */ 760 err = mlx5_crypto_cmd_sync_crypto(mdev, MLX5_CRYPTO_DEK_ALL_TYPE); 761 if (err) 762 goto err_sync_crypto; 763 764 mlx5_core_dbg(mdev, "Crypto DEK enabled, %d deks per alloc (max %d), total %d\n", 765 1 << dek_priv->log_dek_obj_range, 766 1 << MLX5_CAP_CRYPTO(mdev, log_dek_max_alloc), 767 1 << MLX5_CAP_CRYPTO(mdev, log_max_num_deks)); 768 769 return dek_priv; 770 771err_sync_crypto: 772 kfree(dek_priv); 773 return ERR_PTR(err); 774} 775