1// SPDX-License-Identifier: GPL-2.0-only 2/* 3 * Copyright (c) 2015-2021, 2023 Linaro Limited 4 */ 5#include <linux/device.h> 6#include <linux/err.h> 7#include <linux/errno.h> 8#include <linux/mm.h> 9#include <linux/slab.h> 10#include <linux/tee_drv.h> 11#include <linux/types.h> 12#include "optee_private.h" 13 14#define MAX_ARG_PARAM_COUNT 6 15 16/* 17 * How much memory we allocate for each entry. This doesn't have to be a 18 * single page, but it makes sense to keep at least keep it as multiples of 19 * the page size. 20 */ 21#define SHM_ENTRY_SIZE PAGE_SIZE 22 23/* 24 * We need to have a compile time constant to be able to determine the 25 * maximum needed size of the bit field. 26 */ 27#define MIN_ARG_SIZE OPTEE_MSG_GET_ARG_SIZE(MAX_ARG_PARAM_COUNT) 28#define MAX_ARG_COUNT_PER_ENTRY (SHM_ENTRY_SIZE / MIN_ARG_SIZE) 29 30/* 31 * Shared memory for argument structs are cached here. The number of 32 * arguments structs that can fit is determined at runtime depending on the 33 * needed RPC parameter count reported by secure world 34 * (optee->rpc_param_count). 35 */ 36struct optee_shm_arg_entry { 37 struct list_head list_node; 38 struct tee_shm *shm; 39 DECLARE_BITMAP(map, MAX_ARG_COUNT_PER_ENTRY); 40}; 41 42void optee_cq_init(struct optee_call_queue *cq, int thread_count) 43{ 44 mutex_init(&cq->mutex); 45 INIT_LIST_HEAD(&cq->waiters); 46 47 /* 48 * If cq->total_thread_count is 0 then we're not trying to keep 49 * track of how many free threads we have, instead we're relying on 50 * the secure world to tell us when we're out of thread and have to 51 * wait for another thread to become available. 52 */ 53 cq->total_thread_count = thread_count; 54 cq->free_thread_count = thread_count; 55} 56 57void optee_cq_wait_init(struct optee_call_queue *cq, 58 struct optee_call_waiter *w, bool sys_thread) 59{ 60 unsigned int free_thread_threshold; 61 bool need_wait = false; 62 63 memset(w, 0, sizeof(*w)); 64 65 /* 66 * We're preparing to make a call to secure world. In case we can't 67 * allocate a thread in secure world we'll end up waiting in 68 * optee_cq_wait_for_completion(). 69 * 70 * Normally if there's no contention in secure world the call will 71 * complete and we can cleanup directly with optee_cq_wait_final(). 72 */ 73 mutex_lock(&cq->mutex); 74 75 /* 76 * We add ourselves to the queue, but we don't wait. This 77 * guarantees that we don't lose a completion if secure world 78 * returns busy and another thread just exited and try to complete 79 * someone. 80 */ 81 init_completion(&w->c); 82 list_add_tail(&w->list_node, &cq->waiters); 83 w->sys_thread = sys_thread; 84 85 if (cq->total_thread_count) { 86 if (sys_thread || !cq->sys_thread_req_count) 87 free_thread_threshold = 0; 88 else 89 free_thread_threshold = 1; 90 91 if (cq->free_thread_count > free_thread_threshold) 92 cq->free_thread_count--; 93 else 94 need_wait = true; 95 } 96 97 mutex_unlock(&cq->mutex); 98 99 while (need_wait) { 100 optee_cq_wait_for_completion(cq, w); 101 mutex_lock(&cq->mutex); 102 103 if (sys_thread || !cq->sys_thread_req_count) 104 free_thread_threshold = 0; 105 else 106 free_thread_threshold = 1; 107 108 if (cq->free_thread_count > free_thread_threshold) { 109 cq->free_thread_count--; 110 need_wait = false; 111 } 112 113 mutex_unlock(&cq->mutex); 114 } 115} 116 117void optee_cq_wait_for_completion(struct optee_call_queue *cq, 118 struct optee_call_waiter *w) 119{ 120 wait_for_completion(&w->c); 121 122 mutex_lock(&cq->mutex); 123 124 /* Move to end of list to get out of the way for other waiters */ 125 list_del(&w->list_node); 126 reinit_completion(&w->c); 127 list_add_tail(&w->list_node, &cq->waiters); 128 129 mutex_unlock(&cq->mutex); 130} 131 132static void optee_cq_complete_one(struct optee_call_queue *cq) 133{ 134 struct optee_call_waiter *w; 135 136 /* Wake a waiting system session if any, prior to a normal session */ 137 list_for_each_entry(w, &cq->waiters, list_node) { 138 if (w->sys_thread && !completion_done(&w->c)) { 139 complete(&w->c); 140 return; 141 } 142 } 143 144 list_for_each_entry(w, &cq->waiters, list_node) { 145 if (!completion_done(&w->c)) { 146 complete(&w->c); 147 break; 148 } 149 } 150} 151 152void optee_cq_wait_final(struct optee_call_queue *cq, 153 struct optee_call_waiter *w) 154{ 155 /* 156 * We're done with the call to secure world. The thread in secure 157 * world that was used for this call is now available for some 158 * other task to use. 159 */ 160 mutex_lock(&cq->mutex); 161 162 /* Get out of the list */ 163 list_del(&w->list_node); 164 165 cq->free_thread_count++; 166 167 /* Wake up one eventual waiting task */ 168 optee_cq_complete_one(cq); 169 170 /* 171 * If we're completed we've got a completion from another task that 172 * was just done with its call to secure world. Since yet another 173 * thread now is available in secure world wake up another eventual 174 * waiting task. 175 */ 176 if (completion_done(&w->c)) 177 optee_cq_complete_one(cq); 178 179 mutex_unlock(&cq->mutex); 180} 181 182/* Count registered system sessions to reserved a system thread or not */ 183static bool optee_cq_incr_sys_thread_count(struct optee_call_queue *cq) 184{ 185 if (cq->total_thread_count <= 1) 186 return false; 187 188 mutex_lock(&cq->mutex); 189 cq->sys_thread_req_count++; 190 mutex_unlock(&cq->mutex); 191 192 return true; 193} 194 195static void optee_cq_decr_sys_thread_count(struct optee_call_queue *cq) 196{ 197 mutex_lock(&cq->mutex); 198 cq->sys_thread_req_count--; 199 /* If there's someone waiting, let it resume */ 200 optee_cq_complete_one(cq); 201 mutex_unlock(&cq->mutex); 202} 203 204/* Requires the filpstate mutex to be held */ 205static struct optee_session *find_session(struct optee_context_data *ctxdata, 206 u32 session_id) 207{ 208 struct optee_session *sess; 209 210 list_for_each_entry(sess, &ctxdata->sess_list, list_node) 211 if (sess->session_id == session_id) 212 return sess; 213 214 return NULL; 215} 216 217void optee_shm_arg_cache_init(struct optee *optee, u32 flags) 218{ 219 INIT_LIST_HEAD(&optee->shm_arg_cache.shm_args); 220 mutex_init(&optee->shm_arg_cache.mutex); 221 optee->shm_arg_cache.flags = flags; 222} 223 224void optee_shm_arg_cache_uninit(struct optee *optee) 225{ 226 struct list_head *head = &optee->shm_arg_cache.shm_args; 227 struct optee_shm_arg_entry *entry; 228 229 mutex_destroy(&optee->shm_arg_cache.mutex); 230 while (!list_empty(head)) { 231 entry = list_first_entry(head, struct optee_shm_arg_entry, 232 list_node); 233 list_del(&entry->list_node); 234 if (find_first_bit(entry->map, MAX_ARG_COUNT_PER_ENTRY) != 235 MAX_ARG_COUNT_PER_ENTRY) { 236 pr_err("Freeing non-free entry\n"); 237 } 238 tee_shm_free(entry->shm); 239 kfree(entry); 240 } 241} 242 243size_t optee_msg_arg_size(size_t rpc_param_count) 244{ 245 size_t sz = OPTEE_MSG_GET_ARG_SIZE(MAX_ARG_PARAM_COUNT); 246 247 if (rpc_param_count) 248 sz += OPTEE_MSG_GET_ARG_SIZE(rpc_param_count); 249 250 return sz; 251} 252 253/** 254 * optee_get_msg_arg() - Provide shared memory for argument struct 255 * @ctx: Caller TEE context 256 * @num_params: Number of parameter to store 257 * @entry_ret: Entry pointer, needed when freeing the buffer 258 * @shm_ret: Shared memory buffer 259 * @offs_ret: Offset of argument strut in shared memory buffer 260 * 261 * @returns a pointer to the argument struct in memory, else an ERR_PTR 262 */ 263struct optee_msg_arg *optee_get_msg_arg(struct tee_context *ctx, 264 size_t num_params, 265 struct optee_shm_arg_entry **entry_ret, 266 struct tee_shm **shm_ret, 267 u_int *offs_ret) 268{ 269 struct optee *optee = tee_get_drvdata(ctx->teedev); 270 size_t sz = optee_msg_arg_size(optee->rpc_param_count); 271 struct optee_shm_arg_entry *entry; 272 struct optee_msg_arg *ma; 273 size_t args_per_entry; 274 u_long bit; 275 u_int offs; 276 void *res; 277 278 if (num_params > MAX_ARG_PARAM_COUNT) 279 return ERR_PTR(-EINVAL); 280 281 if (optee->shm_arg_cache.flags & OPTEE_SHM_ARG_SHARED) 282 args_per_entry = SHM_ENTRY_SIZE / sz; 283 else 284 args_per_entry = 1; 285 286 mutex_lock(&optee->shm_arg_cache.mutex); 287 list_for_each_entry(entry, &optee->shm_arg_cache.shm_args, list_node) { 288 bit = find_first_zero_bit(entry->map, MAX_ARG_COUNT_PER_ENTRY); 289 if (bit < args_per_entry) 290 goto have_entry; 291 } 292 293 /* 294 * No entry was found, let's allocate a new. 295 */ 296 entry = kzalloc(sizeof(*entry), GFP_KERNEL); 297 if (!entry) { 298 res = ERR_PTR(-ENOMEM); 299 goto out; 300 } 301 302 if (optee->shm_arg_cache.flags & OPTEE_SHM_ARG_ALLOC_PRIV) 303 res = tee_shm_alloc_priv_buf(ctx, SHM_ENTRY_SIZE); 304 else 305 res = tee_shm_alloc_kernel_buf(ctx, SHM_ENTRY_SIZE); 306 307 if (IS_ERR(res)) { 308 kfree(entry); 309 goto out; 310 } 311 entry->shm = res; 312 list_add(&entry->list_node, &optee->shm_arg_cache.shm_args); 313 bit = 0; 314 315have_entry: 316 offs = bit * sz; 317 res = tee_shm_get_va(entry->shm, offs); 318 if (IS_ERR(res)) 319 goto out; 320 ma = res; 321 set_bit(bit, entry->map); 322 memset(ma, 0, sz); 323 ma->num_params = num_params; 324 *entry_ret = entry; 325 *shm_ret = entry->shm; 326 *offs_ret = offs; 327out: 328 mutex_unlock(&optee->shm_arg_cache.mutex); 329 return res; 330} 331 332/** 333 * optee_free_msg_arg() - Free previsouly obtained shared memory 334 * @ctx: Caller TEE context 335 * @entry: Pointer returned when the shared memory was obtained 336 * @offs: Offset of shared memory buffer to free 337 * 338 * This function frees the shared memory obtained with optee_get_msg_arg(). 339 */ 340void optee_free_msg_arg(struct tee_context *ctx, 341 struct optee_shm_arg_entry *entry, u_int offs) 342{ 343 struct optee *optee = tee_get_drvdata(ctx->teedev); 344 size_t sz = optee_msg_arg_size(optee->rpc_param_count); 345 u_long bit; 346 347 if (offs > SHM_ENTRY_SIZE || offs % sz) { 348 pr_err("Invalid offs %u\n", offs); 349 return; 350 } 351 bit = offs / sz; 352 353 mutex_lock(&optee->shm_arg_cache.mutex); 354 355 if (!test_bit(bit, entry->map)) 356 pr_err("Bit pos %lu is already free\n", bit); 357 clear_bit(bit, entry->map); 358 359 mutex_unlock(&optee->shm_arg_cache.mutex); 360} 361 362int optee_open_session(struct tee_context *ctx, 363 struct tee_ioctl_open_session_arg *arg, 364 struct tee_param *param) 365{ 366 struct optee *optee = tee_get_drvdata(ctx->teedev); 367 struct optee_context_data *ctxdata = ctx->data; 368 struct optee_shm_arg_entry *entry; 369 struct tee_shm *shm; 370 struct optee_msg_arg *msg_arg; 371 struct optee_session *sess = NULL; 372 uuid_t client_uuid; 373 u_int offs; 374 int rc; 375 376 /* +2 for the meta parameters added below */ 377 msg_arg = optee_get_msg_arg(ctx, arg->num_params + 2, 378 &entry, &shm, &offs); 379 if (IS_ERR(msg_arg)) 380 return PTR_ERR(msg_arg); 381 382 msg_arg->cmd = OPTEE_MSG_CMD_OPEN_SESSION; 383 msg_arg->cancel_id = arg->cancel_id; 384 385 /* 386 * Initialize and add the meta parameters needed when opening a 387 * session. 388 */ 389 msg_arg->params[0].attr = OPTEE_MSG_ATTR_TYPE_VALUE_INPUT | 390 OPTEE_MSG_ATTR_META; 391 msg_arg->params[1].attr = OPTEE_MSG_ATTR_TYPE_VALUE_INPUT | 392 OPTEE_MSG_ATTR_META; 393 memcpy(&msg_arg->params[0].u.value, arg->uuid, sizeof(arg->uuid)); 394 msg_arg->params[1].u.value.c = arg->clnt_login; 395 396 rc = tee_session_calc_client_uuid(&client_uuid, arg->clnt_login, 397 arg->clnt_uuid); 398 if (rc) 399 goto out; 400 export_uuid(msg_arg->params[1].u.octets, &client_uuid); 401 402 rc = optee->ops->to_msg_param(optee, msg_arg->params + 2, 403 arg->num_params, param); 404 if (rc) 405 goto out; 406 407 sess = kzalloc(sizeof(*sess), GFP_KERNEL); 408 if (!sess) { 409 rc = -ENOMEM; 410 goto out; 411 } 412 413 if (optee->ops->do_call_with_arg(ctx, shm, offs, 414 sess->use_sys_thread)) { 415 msg_arg->ret = TEEC_ERROR_COMMUNICATION; 416 msg_arg->ret_origin = TEEC_ORIGIN_COMMS; 417 } 418 419 if (msg_arg->ret == TEEC_SUCCESS) { 420 /* A new session has been created, add it to the list. */ 421 sess->session_id = msg_arg->session; 422 mutex_lock(&ctxdata->mutex); 423 list_add(&sess->list_node, &ctxdata->sess_list); 424 mutex_unlock(&ctxdata->mutex); 425 } else { 426 kfree(sess); 427 } 428 429 if (optee->ops->from_msg_param(optee, param, arg->num_params, 430 msg_arg->params + 2)) { 431 arg->ret = TEEC_ERROR_COMMUNICATION; 432 arg->ret_origin = TEEC_ORIGIN_COMMS; 433 /* Close session again to avoid leakage */ 434 optee_close_session(ctx, msg_arg->session); 435 } else { 436 arg->session = msg_arg->session; 437 arg->ret = msg_arg->ret; 438 arg->ret_origin = msg_arg->ret_origin; 439 } 440out: 441 optee_free_msg_arg(ctx, entry, offs); 442 443 return rc; 444} 445 446int optee_system_session(struct tee_context *ctx, u32 session) 447{ 448 struct optee *optee = tee_get_drvdata(ctx->teedev); 449 struct optee_context_data *ctxdata = ctx->data; 450 struct optee_session *sess; 451 int rc = -EINVAL; 452 453 mutex_lock(&ctxdata->mutex); 454 455 sess = find_session(ctxdata, session); 456 if (sess && (sess->use_sys_thread || 457 optee_cq_incr_sys_thread_count(&optee->call_queue))) { 458 sess->use_sys_thread = true; 459 rc = 0; 460 } 461 462 mutex_unlock(&ctxdata->mutex); 463 464 return rc; 465} 466 467int optee_close_session_helper(struct tee_context *ctx, u32 session, 468 bool system_thread) 469{ 470 struct optee *optee = tee_get_drvdata(ctx->teedev); 471 struct optee_shm_arg_entry *entry; 472 struct optee_msg_arg *msg_arg; 473 struct tee_shm *shm; 474 u_int offs; 475 476 msg_arg = optee_get_msg_arg(ctx, 0, &entry, &shm, &offs); 477 if (IS_ERR(msg_arg)) 478 return PTR_ERR(msg_arg); 479 480 msg_arg->cmd = OPTEE_MSG_CMD_CLOSE_SESSION; 481 msg_arg->session = session; 482 optee->ops->do_call_with_arg(ctx, shm, offs, system_thread); 483 484 optee_free_msg_arg(ctx, entry, offs); 485 486 if (system_thread) 487 optee_cq_decr_sys_thread_count(&optee->call_queue); 488 489 return 0; 490} 491 492int optee_close_session(struct tee_context *ctx, u32 session) 493{ 494 struct optee_context_data *ctxdata = ctx->data; 495 struct optee_session *sess; 496 bool system_thread; 497 498 /* Check that the session is valid and remove it from the list */ 499 mutex_lock(&ctxdata->mutex); 500 sess = find_session(ctxdata, session); 501 if (sess) 502 list_del(&sess->list_node); 503 mutex_unlock(&ctxdata->mutex); 504 if (!sess) 505 return -EINVAL; 506 system_thread = sess->use_sys_thread; 507 kfree(sess); 508 509 return optee_close_session_helper(ctx, session, system_thread); 510} 511 512int optee_invoke_func(struct tee_context *ctx, struct tee_ioctl_invoke_arg *arg, 513 struct tee_param *param) 514{ 515 struct optee *optee = tee_get_drvdata(ctx->teedev); 516 struct optee_context_data *ctxdata = ctx->data; 517 struct optee_shm_arg_entry *entry; 518 struct optee_msg_arg *msg_arg; 519 struct optee_session *sess; 520 struct tee_shm *shm; 521 bool system_thread; 522 u_int offs; 523 int rc; 524 525 /* Check that the session is valid */ 526 mutex_lock(&ctxdata->mutex); 527 sess = find_session(ctxdata, arg->session); 528 if (sess) 529 system_thread = sess->use_sys_thread; 530 mutex_unlock(&ctxdata->mutex); 531 if (!sess) 532 return -EINVAL; 533 534 msg_arg = optee_get_msg_arg(ctx, arg->num_params, 535 &entry, &shm, &offs); 536 if (IS_ERR(msg_arg)) 537 return PTR_ERR(msg_arg); 538 msg_arg->cmd = OPTEE_MSG_CMD_INVOKE_COMMAND; 539 msg_arg->func = arg->func; 540 msg_arg->session = arg->session; 541 msg_arg->cancel_id = arg->cancel_id; 542 543 rc = optee->ops->to_msg_param(optee, msg_arg->params, arg->num_params, 544 param); 545 if (rc) 546 goto out; 547 548 if (optee->ops->do_call_with_arg(ctx, shm, offs, system_thread)) { 549 msg_arg->ret = TEEC_ERROR_COMMUNICATION; 550 msg_arg->ret_origin = TEEC_ORIGIN_COMMS; 551 } 552 553 if (optee->ops->from_msg_param(optee, param, arg->num_params, 554 msg_arg->params)) { 555 msg_arg->ret = TEEC_ERROR_COMMUNICATION; 556 msg_arg->ret_origin = TEEC_ORIGIN_COMMS; 557 } 558 559 arg->ret = msg_arg->ret; 560 arg->ret_origin = msg_arg->ret_origin; 561out: 562 optee_free_msg_arg(ctx, entry, offs); 563 return rc; 564} 565 566int optee_cancel_req(struct tee_context *ctx, u32 cancel_id, u32 session) 567{ 568 struct optee *optee = tee_get_drvdata(ctx->teedev); 569 struct optee_context_data *ctxdata = ctx->data; 570 struct optee_shm_arg_entry *entry; 571 struct optee_msg_arg *msg_arg; 572 struct optee_session *sess; 573 bool system_thread; 574 struct tee_shm *shm; 575 u_int offs; 576 577 /* Check that the session is valid */ 578 mutex_lock(&ctxdata->mutex); 579 sess = find_session(ctxdata, session); 580 if (sess) 581 system_thread = sess->use_sys_thread; 582 mutex_unlock(&ctxdata->mutex); 583 if (!sess) 584 return -EINVAL; 585 586 msg_arg = optee_get_msg_arg(ctx, 0, &entry, &shm, &offs); 587 if (IS_ERR(msg_arg)) 588 return PTR_ERR(msg_arg); 589 590 msg_arg->cmd = OPTEE_MSG_CMD_CANCEL; 591 msg_arg->session = session; 592 msg_arg->cancel_id = cancel_id; 593 optee->ops->do_call_with_arg(ctx, shm, offs, system_thread); 594 595 optee_free_msg_arg(ctx, entry, offs); 596 return 0; 597} 598 599static bool is_normal_memory(pgprot_t p) 600{ 601#if defined(CONFIG_ARM) 602 return (((pgprot_val(p) & L_PTE_MT_MASK) == L_PTE_MT_WRITEALLOC) || 603 ((pgprot_val(p) & L_PTE_MT_MASK) == L_PTE_MT_WRITEBACK)); 604#elif defined(CONFIG_ARM64) 605 return (pgprot_val(p) & PTE_ATTRINDX_MASK) == PTE_ATTRINDX(MT_NORMAL); 606#else 607#error "Unsupported architecture" 608#endif 609} 610 611static int __check_mem_type(struct mm_struct *mm, unsigned long start, 612 unsigned long end) 613{ 614 struct vm_area_struct *vma; 615 VMA_ITERATOR(vmi, mm, start); 616 617 for_each_vma_range(vmi, vma, end) { 618 if (!is_normal_memory(vma->vm_page_prot)) 619 return -EINVAL; 620 } 621 622 return 0; 623} 624 625int optee_check_mem_type(unsigned long start, size_t num_pages) 626{ 627 struct mm_struct *mm = current->mm; 628 int rc; 629 630 /* 631 * Allow kernel address to register with OP-TEE as kernel 632 * pages are configured as normal memory only. 633 */ 634 if (virt_addr_valid((void *)start) || is_vmalloc_addr((void *)start)) 635 return 0; 636 637 mmap_read_lock(mm); 638 rc = __check_mem_type(mm, start, start + num_pages * PAGE_SIZE); 639 mmap_read_unlock(mm); 640 641 return rc; 642} 643 644static int simple_call_with_arg(struct tee_context *ctx, u32 cmd) 645{ 646 struct optee *optee = tee_get_drvdata(ctx->teedev); 647 struct optee_shm_arg_entry *entry; 648 struct optee_msg_arg *msg_arg; 649 struct tee_shm *shm; 650 u_int offs; 651 652 msg_arg = optee_get_msg_arg(ctx, 0, &entry, &shm, &offs); 653 if (IS_ERR(msg_arg)) 654 return PTR_ERR(msg_arg); 655 656 msg_arg->cmd = cmd; 657 optee->ops->do_call_with_arg(ctx, shm, offs, false); 658 659 optee_free_msg_arg(ctx, entry, offs); 660 return 0; 661} 662 663int optee_do_bottom_half(struct tee_context *ctx) 664{ 665 return simple_call_with_arg(ctx, OPTEE_MSG_CMD_DO_BOTTOM_HALF); 666} 667 668int optee_stop_async_notif(struct tee_context *ctx) 669{ 670 return simple_call_with_arg(ctx, OPTEE_MSG_CMD_STOP_ASYNC_NOTIF); 671} 672