1// SPDX-License-Identifier: GPL-2.0 2/* 3 * Copyright 2020-2021 Amazon.com, Inc. or its affiliates. All Rights Reserved. 4 */ 5 6/** 7 * DOC: Enclave lifetime management driver for Nitro Enclaves (NE). 8 * Nitro is a hypervisor that has been developed by Amazon. 9 */ 10 11#include <linux/anon_inodes.h> 12#include <linux/capability.h> 13#include <linux/cpu.h> 14#include <linux/device.h> 15#include <linux/file.h> 16#include <linux/hugetlb.h> 17#include <linux/limits.h> 18#include <linux/list.h> 19#include <linux/miscdevice.h> 20#include <linux/mm.h> 21#include <linux/mman.h> 22#include <linux/module.h> 23#include <linux/mutex.h> 24#include <linux/nitro_enclaves.h> 25#include <linux/pci.h> 26#include <linux/poll.h> 27#include <linux/range.h> 28#include <linux/slab.h> 29#include <linux/types.h> 30#include <uapi/linux/vm_sockets.h> 31 32#include "ne_misc_dev.h" 33#include "ne_pci_dev.h" 34 35/** 36 * NE_CPUS_SIZE - Size for max 128 CPUs, for now, in a cpu-list string, comma 37 * separated. The NE CPU pool includes CPUs from a single NUMA 38 * node. 39 */ 40#define NE_CPUS_SIZE (512) 41 42/** 43 * NE_EIF_LOAD_OFFSET - The offset where to copy the Enclave Image Format (EIF) 44 * image in enclave memory. 45 */ 46#define NE_EIF_LOAD_OFFSET (8 * 1024UL * 1024UL) 47 48/** 49 * NE_MIN_ENCLAVE_MEM_SIZE - The minimum memory size an enclave can be launched 50 * with. 51 */ 52#define NE_MIN_ENCLAVE_MEM_SIZE (64 * 1024UL * 1024UL) 53 54/** 55 * NE_MIN_MEM_REGION_SIZE - The minimum size of an enclave memory region. 56 */ 57#define NE_MIN_MEM_REGION_SIZE (2 * 1024UL * 1024UL) 58 59/** 60 * NE_PARENT_VM_CID - The CID for the vsock device of the primary / parent VM. 61 */ 62#define NE_PARENT_VM_CID (3) 63 64static long ne_ioctl(struct file *file, unsigned int cmd, unsigned long arg); 65 66static const struct file_operations ne_fops = { 67 .owner = THIS_MODULE, 68 .llseek = noop_llseek, 69 .unlocked_ioctl = ne_ioctl, 70}; 71 72static struct miscdevice ne_misc_dev = { 73 .minor = MISC_DYNAMIC_MINOR, 74 .name = "nitro_enclaves", 75 .fops = &ne_fops, 76 .mode = 0660, 77}; 78 79struct ne_devs ne_devs = { 80 .ne_misc_dev = &ne_misc_dev, 81}; 82 83/* 84 * TODO: Update logic to create new sysfs entries instead of using 85 * a kernel parameter e.g. if multiple sysfs files needed. 86 */ 87static int ne_set_kernel_param(const char *val, const struct kernel_param *kp); 88 89static const struct kernel_param_ops ne_cpu_pool_ops = { 90 .get = param_get_string, 91 .set = ne_set_kernel_param, 92}; 93 94static char ne_cpus[NE_CPUS_SIZE]; 95static struct kparam_string ne_cpus_arg = { 96 .maxlen = sizeof(ne_cpus), 97 .string = ne_cpus, 98}; 99 100module_param_cb(ne_cpus, &ne_cpu_pool_ops, &ne_cpus_arg, 0644); 101/* https://www.kernel.org/doc/html/latest/admin-guide/kernel-parameters.html#cpu-lists */ 102MODULE_PARM_DESC(ne_cpus, "<cpu-list> - CPU pool used for Nitro Enclaves"); 103 104/** 105 * struct ne_cpu_pool - CPU pool used for Nitro Enclaves. 106 * @avail_threads_per_core: Available full CPU cores to be dedicated to 107 * enclave(s). The cpumasks from the array, indexed 108 * by core id, contain all the threads from the 109 * available cores, that are not set for created 110 * enclave(s). The full CPU cores are part of the 111 * NE CPU pool. 112 * @mutex: Mutex for the access to the NE CPU pool. 113 * @nr_parent_vm_cores : The size of the available threads per core array. 114 * The total number of CPU cores available on the 115 * primary / parent VM. 116 * @nr_threads_per_core: The number of threads that a full CPU core has. 117 * @numa_node: NUMA node of the CPUs in the pool. 118 */ 119struct ne_cpu_pool { 120 cpumask_var_t *avail_threads_per_core; 121 struct mutex mutex; 122 unsigned int nr_parent_vm_cores; 123 unsigned int nr_threads_per_core; 124 int numa_node; 125}; 126 127static struct ne_cpu_pool ne_cpu_pool; 128 129/** 130 * struct ne_phys_contig_mem_regions - Contiguous physical memory regions. 131 * @num: The number of regions that currently has. 132 * @regions: The array of physical memory regions. 133 */ 134struct ne_phys_contig_mem_regions { 135 unsigned long num; 136 struct range *regions; 137}; 138 139/** 140 * ne_check_enclaves_created() - Verify if at least one enclave has been created. 141 * @void: No parameters provided. 142 * 143 * Context: Process context. 144 * Return: 145 * * True if at least one enclave is created. 146 * * False otherwise. 147 */ 148static bool ne_check_enclaves_created(void) 149{ 150 struct ne_pci_dev *ne_pci_dev = ne_devs.ne_pci_dev; 151 bool ret = false; 152 153 if (!ne_pci_dev) 154 return ret; 155 156 mutex_lock(&ne_pci_dev->enclaves_list_mutex); 157 158 if (!list_empty(&ne_pci_dev->enclaves_list)) 159 ret = true; 160 161 mutex_unlock(&ne_pci_dev->enclaves_list_mutex); 162 163 return ret; 164} 165 166/** 167 * ne_setup_cpu_pool() - Set the NE CPU pool after handling sanity checks such 168 * as not sharing CPU cores with the primary / parent VM 169 * or not using CPU 0, which should remain available for 170 * the primary / parent VM. Offline the CPUs from the 171 * pool after the checks passed. 172 * @ne_cpu_list: The CPU list used for setting NE CPU pool. 173 * 174 * Context: Process context. 175 * Return: 176 * * 0 on success. 177 * * Negative return value on failure. 178 */ 179static int ne_setup_cpu_pool(const char *ne_cpu_list) 180{ 181 int core_id = -1; 182 unsigned int cpu = 0; 183 cpumask_var_t cpu_pool; 184 unsigned int cpu_sibling = 0; 185 unsigned int i = 0; 186 int numa_node = -1; 187 int rc = -EINVAL; 188 189 if (!zalloc_cpumask_var(&cpu_pool, GFP_KERNEL)) 190 return -ENOMEM; 191 192 mutex_lock(&ne_cpu_pool.mutex); 193 194 rc = cpulist_parse(ne_cpu_list, cpu_pool); 195 if (rc < 0) { 196 pr_err("%s: Error in cpulist parse [rc=%d]\n", ne_misc_dev.name, rc); 197 198 goto free_pool_cpumask; 199 } 200 201 cpu = cpumask_any(cpu_pool); 202 if (cpu >= nr_cpu_ids) { 203 pr_err("%s: No CPUs available in CPU pool\n", ne_misc_dev.name); 204 205 rc = -EINVAL; 206 207 goto free_pool_cpumask; 208 } 209 210 /* 211 * Check if the CPUs are online, to further get info about them 212 * e.g. numa node, core id, siblings. 213 */ 214 for_each_cpu(cpu, cpu_pool) 215 if (cpu_is_offline(cpu)) { 216 pr_err("%s: CPU %d is offline, has to be online to get its metadata\n", 217 ne_misc_dev.name, cpu); 218 219 rc = -EINVAL; 220 221 goto free_pool_cpumask; 222 } 223 224 /* 225 * Check if the CPUs from the NE CPU pool are from the same NUMA node. 226 */ 227 for_each_cpu(cpu, cpu_pool) 228 if (numa_node < 0) { 229 numa_node = cpu_to_node(cpu); 230 if (numa_node < 0) { 231 pr_err("%s: Invalid NUMA node %d\n", 232 ne_misc_dev.name, numa_node); 233 234 rc = -EINVAL; 235 236 goto free_pool_cpumask; 237 } 238 } else { 239 if (numa_node != cpu_to_node(cpu)) { 240 pr_err("%s: CPUs with different NUMA nodes\n", 241 ne_misc_dev.name); 242 243 rc = -EINVAL; 244 245 goto free_pool_cpumask; 246 } 247 } 248 249 /* 250 * Check if CPU 0 and its siblings are included in the provided CPU pool 251 * They should remain available for the primary / parent VM. 252 */ 253 if (cpumask_test_cpu(0, cpu_pool)) { 254 pr_err("%s: CPU 0 has to remain available\n", ne_misc_dev.name); 255 256 rc = -EINVAL; 257 258 goto free_pool_cpumask; 259 } 260 261 for_each_cpu(cpu_sibling, topology_sibling_cpumask(0)) { 262 if (cpumask_test_cpu(cpu_sibling, cpu_pool)) { 263 pr_err("%s: CPU sibling %d for CPU 0 is in CPU pool\n", 264 ne_misc_dev.name, cpu_sibling); 265 266 rc = -EINVAL; 267 268 goto free_pool_cpumask; 269 } 270 } 271 272 /* 273 * Check if CPU siblings are included in the provided CPU pool. The 274 * expectation is that full CPU cores are made available in the CPU pool 275 * for enclaves. 276 */ 277 for_each_cpu(cpu, cpu_pool) { 278 for_each_cpu(cpu_sibling, topology_sibling_cpumask(cpu)) { 279 if (!cpumask_test_cpu(cpu_sibling, cpu_pool)) { 280 pr_err("%s: CPU %d is not in CPU pool\n", 281 ne_misc_dev.name, cpu_sibling); 282 283 rc = -EINVAL; 284 285 goto free_pool_cpumask; 286 } 287 } 288 } 289 290 /* Calculate the number of threads from a full CPU core. */ 291 cpu = cpumask_any(cpu_pool); 292 for_each_cpu(cpu_sibling, topology_sibling_cpumask(cpu)) 293 ne_cpu_pool.nr_threads_per_core++; 294 295 ne_cpu_pool.nr_parent_vm_cores = nr_cpu_ids / ne_cpu_pool.nr_threads_per_core; 296 297 ne_cpu_pool.avail_threads_per_core = kcalloc(ne_cpu_pool.nr_parent_vm_cores, 298 sizeof(*ne_cpu_pool.avail_threads_per_core), 299 GFP_KERNEL); 300 if (!ne_cpu_pool.avail_threads_per_core) { 301 rc = -ENOMEM; 302 303 goto free_pool_cpumask; 304 } 305 306 for (i = 0; i < ne_cpu_pool.nr_parent_vm_cores; i++) 307 if (!zalloc_cpumask_var(&ne_cpu_pool.avail_threads_per_core[i], GFP_KERNEL)) { 308 rc = -ENOMEM; 309 310 goto free_cores_cpumask; 311 } 312 313 /* 314 * Split the NE CPU pool in threads per core to keep the CPU topology 315 * after offlining the CPUs. 316 */ 317 for_each_cpu(cpu, cpu_pool) { 318 core_id = topology_core_id(cpu); 319 if (core_id < 0 || core_id >= ne_cpu_pool.nr_parent_vm_cores) { 320 pr_err("%s: Invalid core id %d for CPU %d\n", 321 ne_misc_dev.name, core_id, cpu); 322 323 rc = -EINVAL; 324 325 goto clear_cpumask; 326 } 327 328 cpumask_set_cpu(cpu, ne_cpu_pool.avail_threads_per_core[core_id]); 329 } 330 331 /* 332 * CPUs that are given to enclave(s) should not be considered online 333 * by Linux anymore, as the hypervisor will degrade them to floating. 334 * The physical CPUs (full cores) are carved out of the primary / parent 335 * VM and given to the enclave VM. The same number of vCPUs would run 336 * on less pCPUs for the primary / parent VM. 337 * 338 * We offline them here, to not degrade performance and expose correct 339 * topology to Linux and user space. 340 */ 341 for_each_cpu(cpu, cpu_pool) { 342 rc = remove_cpu(cpu); 343 if (rc != 0) { 344 pr_err("%s: CPU %d is not offlined [rc=%d]\n", 345 ne_misc_dev.name, cpu, rc); 346 347 goto online_cpus; 348 } 349 } 350 351 free_cpumask_var(cpu_pool); 352 353 ne_cpu_pool.numa_node = numa_node; 354 355 mutex_unlock(&ne_cpu_pool.mutex); 356 357 return 0; 358 359online_cpus: 360 for_each_cpu(cpu, cpu_pool) 361 add_cpu(cpu); 362clear_cpumask: 363 for (i = 0; i < ne_cpu_pool.nr_parent_vm_cores; i++) 364 cpumask_clear(ne_cpu_pool.avail_threads_per_core[i]); 365free_cores_cpumask: 366 for (i = 0; i < ne_cpu_pool.nr_parent_vm_cores; i++) 367 free_cpumask_var(ne_cpu_pool.avail_threads_per_core[i]); 368 kfree(ne_cpu_pool.avail_threads_per_core); 369free_pool_cpumask: 370 free_cpumask_var(cpu_pool); 371 ne_cpu_pool.nr_parent_vm_cores = 0; 372 ne_cpu_pool.nr_threads_per_core = 0; 373 ne_cpu_pool.numa_node = -1; 374 mutex_unlock(&ne_cpu_pool.mutex); 375 376 return rc; 377} 378 379/** 380 * ne_teardown_cpu_pool() - Online the CPUs from the NE CPU pool and cleanup the 381 * CPU pool. 382 * @void: No parameters provided. 383 * 384 * Context: Process context. 385 */ 386static void ne_teardown_cpu_pool(void) 387{ 388 unsigned int cpu = 0; 389 unsigned int i = 0; 390 int rc = -EINVAL; 391 392 mutex_lock(&ne_cpu_pool.mutex); 393 394 if (!ne_cpu_pool.nr_parent_vm_cores) { 395 mutex_unlock(&ne_cpu_pool.mutex); 396 397 return; 398 } 399 400 for (i = 0; i < ne_cpu_pool.nr_parent_vm_cores; i++) { 401 for_each_cpu(cpu, ne_cpu_pool.avail_threads_per_core[i]) { 402 rc = add_cpu(cpu); 403 if (rc != 0) 404 pr_err("%s: CPU %d is not onlined [rc=%d]\n", 405 ne_misc_dev.name, cpu, rc); 406 } 407 408 cpumask_clear(ne_cpu_pool.avail_threads_per_core[i]); 409 410 free_cpumask_var(ne_cpu_pool.avail_threads_per_core[i]); 411 } 412 413 kfree(ne_cpu_pool.avail_threads_per_core); 414 ne_cpu_pool.nr_parent_vm_cores = 0; 415 ne_cpu_pool.nr_threads_per_core = 0; 416 ne_cpu_pool.numa_node = -1; 417 418 mutex_unlock(&ne_cpu_pool.mutex); 419} 420 421/** 422 * ne_set_kernel_param() - Set the NE CPU pool value via the NE kernel parameter. 423 * @val: NE CPU pool string value. 424 * @kp : NE kernel parameter associated with the NE CPU pool. 425 * 426 * Context: Process context. 427 * Return: 428 * * 0 on success. 429 * * Negative return value on failure. 430 */ 431static int ne_set_kernel_param(const char *val, const struct kernel_param *kp) 432{ 433 char error_val[] = ""; 434 int rc = -EINVAL; 435 436 if (!capable(CAP_SYS_ADMIN)) 437 return -EPERM; 438 439 if (ne_check_enclaves_created()) { 440 pr_err("%s: The CPU pool is used by enclave(s)\n", ne_misc_dev.name); 441 442 return -EPERM; 443 } 444 445 ne_teardown_cpu_pool(); 446 447 rc = ne_setup_cpu_pool(val); 448 if (rc < 0) { 449 pr_err("%s: Error in setup CPU pool [rc=%d]\n", ne_misc_dev.name, rc); 450 451 param_set_copystring(error_val, kp); 452 453 return rc; 454 } 455 456 rc = param_set_copystring(val, kp); 457 if (rc < 0) { 458 pr_err("%s: Error in param set copystring [rc=%d]\n", ne_misc_dev.name, rc); 459 460 ne_teardown_cpu_pool(); 461 462 param_set_copystring(error_val, kp); 463 464 return rc; 465 } 466 467 return 0; 468} 469 470/** 471 * ne_donated_cpu() - Check if the provided CPU is already used by the enclave. 472 * @ne_enclave : Private data associated with the current enclave. 473 * @cpu: CPU to check if already used. 474 * 475 * Context: Process context. This function is called with the ne_enclave mutex held. 476 * Return: 477 * * True if the provided CPU is already used by the enclave. 478 * * False otherwise. 479 */ 480static bool ne_donated_cpu(struct ne_enclave *ne_enclave, unsigned int cpu) 481{ 482 if (cpumask_test_cpu(cpu, ne_enclave->vcpu_ids)) 483 return true; 484 485 return false; 486} 487 488/** 489 * ne_get_unused_core_from_cpu_pool() - Get the id of a full core from the 490 * NE CPU pool. 491 * @void: No parameters provided. 492 * 493 * Context: Process context. This function is called with the ne_enclave and 494 * ne_cpu_pool mutexes held. 495 * Return: 496 * * Core id. 497 * * -1 if no CPU core available in the pool. 498 */ 499static int ne_get_unused_core_from_cpu_pool(void) 500{ 501 int core_id = -1; 502 unsigned int i = 0; 503 504 for (i = 0; i < ne_cpu_pool.nr_parent_vm_cores; i++) 505 if (!cpumask_empty(ne_cpu_pool.avail_threads_per_core[i])) { 506 core_id = i; 507 508 break; 509 } 510 511 return core_id; 512} 513 514/** 515 * ne_set_enclave_threads_per_core() - Set the threads of the provided core in 516 * the enclave data structure. 517 * @ne_enclave : Private data associated with the current enclave. 518 * @core_id: Core id to get its threads from the NE CPU pool. 519 * @vcpu_id: vCPU id part of the provided core. 520 * 521 * Context: Process context. This function is called with the ne_enclave and 522 * ne_cpu_pool mutexes held. 523 * Return: 524 * * 0 on success. 525 * * Negative return value on failure. 526 */ 527static int ne_set_enclave_threads_per_core(struct ne_enclave *ne_enclave, 528 int core_id, u32 vcpu_id) 529{ 530 unsigned int cpu = 0; 531 532 if (core_id < 0 && vcpu_id == 0) { 533 dev_err_ratelimited(ne_misc_dev.this_device, 534 "No CPUs available in NE CPU pool\n"); 535 536 return -NE_ERR_NO_CPUS_AVAIL_IN_POOL; 537 } 538 539 if (core_id < 0) { 540 dev_err_ratelimited(ne_misc_dev.this_device, 541 "CPU %d is not in NE CPU pool\n", vcpu_id); 542 543 return -NE_ERR_VCPU_NOT_IN_CPU_POOL; 544 } 545 546 if (core_id >= ne_enclave->nr_parent_vm_cores) { 547 dev_err_ratelimited(ne_misc_dev.this_device, 548 "Invalid core id %d - ne_enclave\n", core_id); 549 550 return -NE_ERR_VCPU_INVALID_CPU_CORE; 551 } 552 553 for_each_cpu(cpu, ne_cpu_pool.avail_threads_per_core[core_id]) 554 cpumask_set_cpu(cpu, ne_enclave->threads_per_core[core_id]); 555 556 cpumask_clear(ne_cpu_pool.avail_threads_per_core[core_id]); 557 558 return 0; 559} 560 561/** 562 * ne_get_cpu_from_cpu_pool() - Get a CPU from the NE CPU pool, either from the 563 * remaining sibling(s) of a CPU core or the first 564 * sibling of a new CPU core. 565 * @ne_enclave : Private data associated with the current enclave. 566 * @vcpu_id: vCPU to get from the NE CPU pool. 567 * 568 * Context: Process context. This function is called with the ne_enclave mutex held. 569 * Return: 570 * * 0 on success. 571 * * Negative return value on failure. 572 */ 573static int ne_get_cpu_from_cpu_pool(struct ne_enclave *ne_enclave, u32 *vcpu_id) 574{ 575 int core_id = -1; 576 unsigned int cpu = 0; 577 unsigned int i = 0; 578 int rc = -EINVAL; 579 580 /* 581 * If previously allocated a thread of a core to this enclave, first 582 * check remaining sibling(s) for new CPU allocations, so that full 583 * CPU cores are used for the enclave. 584 */ 585 for (i = 0; i < ne_enclave->nr_parent_vm_cores; i++) 586 for_each_cpu(cpu, ne_enclave->threads_per_core[i]) 587 if (!ne_donated_cpu(ne_enclave, cpu)) { 588 *vcpu_id = cpu; 589 590 return 0; 591 } 592 593 mutex_lock(&ne_cpu_pool.mutex); 594 595 /* 596 * If no remaining siblings, get a core from the NE CPU pool and keep 597 * track of all the threads in the enclave threads per core data structure. 598 */ 599 core_id = ne_get_unused_core_from_cpu_pool(); 600 601 rc = ne_set_enclave_threads_per_core(ne_enclave, core_id, *vcpu_id); 602 if (rc < 0) 603 goto unlock_mutex; 604 605 *vcpu_id = cpumask_any(ne_enclave->threads_per_core[core_id]); 606 607 rc = 0; 608 609unlock_mutex: 610 mutex_unlock(&ne_cpu_pool.mutex); 611 612 return rc; 613} 614 615/** 616 * ne_get_vcpu_core_from_cpu_pool() - Get from the NE CPU pool the id of the 617 * core associated with the provided vCPU. 618 * @vcpu_id: Provided vCPU id to get its associated core id. 619 * 620 * Context: Process context. This function is called with the ne_enclave and 621 * ne_cpu_pool mutexes held. 622 * Return: 623 * * Core id. 624 * * -1 if the provided vCPU is not in the pool. 625 */ 626static int ne_get_vcpu_core_from_cpu_pool(u32 vcpu_id) 627{ 628 int core_id = -1; 629 unsigned int i = 0; 630 631 for (i = 0; i < ne_cpu_pool.nr_parent_vm_cores; i++) 632 if (cpumask_test_cpu(vcpu_id, ne_cpu_pool.avail_threads_per_core[i])) { 633 core_id = i; 634 635 break; 636 } 637 638 return core_id; 639} 640 641/** 642 * ne_check_cpu_in_cpu_pool() - Check if the given vCPU is in the available CPUs 643 * from the pool. 644 * @ne_enclave : Private data associated with the current enclave. 645 * @vcpu_id: ID of the vCPU to check if available in the NE CPU pool. 646 * 647 * Context: Process context. This function is called with the ne_enclave mutex held. 648 * Return: 649 * * 0 on success. 650 * * Negative return value on failure. 651 */ 652static int ne_check_cpu_in_cpu_pool(struct ne_enclave *ne_enclave, u32 vcpu_id) 653{ 654 int core_id = -1; 655 unsigned int i = 0; 656 int rc = -EINVAL; 657 658 if (ne_donated_cpu(ne_enclave, vcpu_id)) { 659 dev_err_ratelimited(ne_misc_dev.this_device, 660 "CPU %d already used\n", vcpu_id); 661 662 return -NE_ERR_VCPU_ALREADY_USED; 663 } 664 665 /* 666 * If previously allocated a thread of a core to this enclave, but not 667 * the full core, first check remaining sibling(s). 668 */ 669 for (i = 0; i < ne_enclave->nr_parent_vm_cores; i++) 670 if (cpumask_test_cpu(vcpu_id, ne_enclave->threads_per_core[i])) 671 return 0; 672 673 mutex_lock(&ne_cpu_pool.mutex); 674 675 /* 676 * If no remaining siblings, get from the NE CPU pool the core 677 * associated with the vCPU and keep track of all the threads in the 678 * enclave threads per core data structure. 679 */ 680 core_id = ne_get_vcpu_core_from_cpu_pool(vcpu_id); 681 682 rc = ne_set_enclave_threads_per_core(ne_enclave, core_id, vcpu_id); 683 if (rc < 0) 684 goto unlock_mutex; 685 686 rc = 0; 687 688unlock_mutex: 689 mutex_unlock(&ne_cpu_pool.mutex); 690 691 return rc; 692} 693 694/** 695 * ne_add_vcpu_ioctl() - Add a vCPU to the slot associated with the current 696 * enclave. 697 * @ne_enclave : Private data associated with the current enclave. 698 * @vcpu_id: ID of the CPU to be associated with the given slot, 699 * apic id on x86. 700 * 701 * Context: Process context. This function is called with the ne_enclave mutex held. 702 * Return: 703 * * 0 on success. 704 * * Negative return value on failure. 705 */ 706static int ne_add_vcpu_ioctl(struct ne_enclave *ne_enclave, u32 vcpu_id) 707{ 708 struct ne_pci_dev_cmd_reply cmd_reply = {}; 709 struct pci_dev *pdev = ne_devs.ne_pci_dev->pdev; 710 int rc = -EINVAL; 711 struct slot_add_vcpu_req slot_add_vcpu_req = {}; 712 713 if (ne_enclave->mm != current->mm) 714 return -EIO; 715 716 slot_add_vcpu_req.slot_uid = ne_enclave->slot_uid; 717 slot_add_vcpu_req.vcpu_id = vcpu_id; 718 719 rc = ne_do_request(pdev, SLOT_ADD_VCPU, 720 &slot_add_vcpu_req, sizeof(slot_add_vcpu_req), 721 &cmd_reply, sizeof(cmd_reply)); 722 if (rc < 0) { 723 dev_err_ratelimited(ne_misc_dev.this_device, 724 "Error in slot add vCPU [rc=%d]\n", rc); 725 726 return rc; 727 } 728 729 cpumask_set_cpu(vcpu_id, ne_enclave->vcpu_ids); 730 731 ne_enclave->nr_vcpus++; 732 733 return 0; 734} 735 736/** 737 * ne_sanity_check_user_mem_region() - Sanity check the user space memory 738 * region received during the set user 739 * memory region ioctl call. 740 * @ne_enclave : Private data associated with the current enclave. 741 * @mem_region : User space memory region to be sanity checked. 742 * 743 * Context: Process context. This function is called with the ne_enclave mutex held. 744 * Return: 745 * * 0 on success. 746 * * Negative return value on failure. 747 */ 748static int ne_sanity_check_user_mem_region(struct ne_enclave *ne_enclave, 749 struct ne_user_memory_region mem_region) 750{ 751 struct ne_mem_region *ne_mem_region = NULL; 752 753 if (ne_enclave->mm != current->mm) 754 return -EIO; 755 756 if (mem_region.memory_size & (NE_MIN_MEM_REGION_SIZE - 1)) { 757 dev_err_ratelimited(ne_misc_dev.this_device, 758 "User space memory size is not multiple of 2 MiB\n"); 759 760 return -NE_ERR_INVALID_MEM_REGION_SIZE; 761 } 762 763 if (!IS_ALIGNED(mem_region.userspace_addr, NE_MIN_MEM_REGION_SIZE)) { 764 dev_err_ratelimited(ne_misc_dev.this_device, 765 "User space address is not 2 MiB aligned\n"); 766 767 return -NE_ERR_UNALIGNED_MEM_REGION_ADDR; 768 } 769 770 if ((mem_region.userspace_addr & (NE_MIN_MEM_REGION_SIZE - 1)) || 771 !access_ok((void __user *)(unsigned long)mem_region.userspace_addr, 772 mem_region.memory_size)) { 773 dev_err_ratelimited(ne_misc_dev.this_device, 774 "Invalid user space address range\n"); 775 776 return -NE_ERR_INVALID_MEM_REGION_ADDR; 777 } 778 779 list_for_each_entry(ne_mem_region, &ne_enclave->mem_regions_list, 780 mem_region_list_entry) { 781 u64 memory_size = ne_mem_region->memory_size; 782 u64 userspace_addr = ne_mem_region->userspace_addr; 783 784 if ((userspace_addr <= mem_region.userspace_addr && 785 mem_region.userspace_addr < (userspace_addr + memory_size)) || 786 (mem_region.userspace_addr <= userspace_addr && 787 (mem_region.userspace_addr + mem_region.memory_size) > userspace_addr)) { 788 dev_err_ratelimited(ne_misc_dev.this_device, 789 "User space memory region already used\n"); 790 791 return -NE_ERR_MEM_REGION_ALREADY_USED; 792 } 793 } 794 795 return 0; 796} 797 798/** 799 * ne_sanity_check_user_mem_region_page() - Sanity check a page from the user space 800 * memory region received during the set 801 * user memory region ioctl call. 802 * @ne_enclave : Private data associated with the current enclave. 803 * @mem_region_page: Page from the user space memory region to be sanity checked. 804 * 805 * Context: Process context. This function is called with the ne_enclave mutex held. 806 * Return: 807 * * 0 on success. 808 * * Negative return value on failure. 809 */ 810static int ne_sanity_check_user_mem_region_page(struct ne_enclave *ne_enclave, 811 struct page *mem_region_page) 812{ 813 if (!PageHuge(mem_region_page)) { 814 dev_err_ratelimited(ne_misc_dev.this_device, 815 "Not a hugetlbfs page\n"); 816 817 return -NE_ERR_MEM_NOT_HUGE_PAGE; 818 } 819 820 if (page_size(mem_region_page) & (NE_MIN_MEM_REGION_SIZE - 1)) { 821 dev_err_ratelimited(ne_misc_dev.this_device, 822 "Page size not multiple of 2 MiB\n"); 823 824 return -NE_ERR_INVALID_PAGE_SIZE; 825 } 826 827 if (ne_enclave->numa_node != page_to_nid(mem_region_page)) { 828 dev_err_ratelimited(ne_misc_dev.this_device, 829 "Page is not from NUMA node %d\n", 830 ne_enclave->numa_node); 831 832 return -NE_ERR_MEM_DIFFERENT_NUMA_NODE; 833 } 834 835 return 0; 836} 837 838/** 839 * ne_sanity_check_phys_mem_region() - Sanity check the start address and the size 840 * of a physical memory region. 841 * @phys_mem_region_paddr : Physical start address of the region to be sanity checked. 842 * @phys_mem_region_size : Length of the region to be sanity checked. 843 * 844 * Context: Process context. This function is called with the ne_enclave mutex held. 845 * Return: 846 * * 0 on success. 847 * * Negative return value on failure. 848 */ 849static int ne_sanity_check_phys_mem_region(u64 phys_mem_region_paddr, 850 u64 phys_mem_region_size) 851{ 852 if (phys_mem_region_size & (NE_MIN_MEM_REGION_SIZE - 1)) { 853 dev_err_ratelimited(ne_misc_dev.this_device, 854 "Physical mem region size is not multiple of 2 MiB\n"); 855 856 return -EINVAL; 857 } 858 859 if (!IS_ALIGNED(phys_mem_region_paddr, NE_MIN_MEM_REGION_SIZE)) { 860 dev_err_ratelimited(ne_misc_dev.this_device, 861 "Physical mem region address is not 2 MiB aligned\n"); 862 863 return -EINVAL; 864 } 865 866 return 0; 867} 868 869/** 870 * ne_merge_phys_contig_memory_regions() - Add a memory region and merge the adjacent 871 * regions if they are physically contiguous. 872 * @phys_contig_regions : Private data associated with the contiguous physical memory regions. 873 * @page_paddr : Physical start address of the region to be added. 874 * @page_size : Length of the region to be added. 875 * 876 * Context: Process context. This function is called with the ne_enclave mutex held. 877 * Return: 878 * * 0 on success. 879 * * Negative return value on failure. 880 */ 881static int 882ne_merge_phys_contig_memory_regions(struct ne_phys_contig_mem_regions *phys_contig_regions, 883 u64 page_paddr, u64 page_size) 884{ 885 unsigned long num = phys_contig_regions->num; 886 int rc = 0; 887 888 rc = ne_sanity_check_phys_mem_region(page_paddr, page_size); 889 if (rc < 0) 890 return rc; 891 892 /* Physically contiguous, just merge */ 893 if (num && (phys_contig_regions->regions[num - 1].end + 1) == page_paddr) { 894 phys_contig_regions->regions[num - 1].end += page_size; 895 } else { 896 phys_contig_regions->regions[num].start = page_paddr; 897 phys_contig_regions->regions[num].end = page_paddr + page_size - 1; 898 phys_contig_regions->num++; 899 } 900 901 return 0; 902} 903 904/** 905 * ne_set_user_memory_region_ioctl() - Add user space memory region to the slot 906 * associated with the current enclave. 907 * @ne_enclave : Private data associated with the current enclave. 908 * @mem_region : User space memory region to be associated with the given slot. 909 * 910 * Context: Process context. This function is called with the ne_enclave mutex held. 911 * Return: 912 * * 0 on success. 913 * * Negative return value on failure. 914 */ 915static int ne_set_user_memory_region_ioctl(struct ne_enclave *ne_enclave, 916 struct ne_user_memory_region mem_region) 917{ 918 long gup_rc = 0; 919 unsigned long i = 0; 920 unsigned long max_nr_pages = 0; 921 unsigned long memory_size = 0; 922 struct ne_mem_region *ne_mem_region = NULL; 923 struct pci_dev *pdev = ne_devs.ne_pci_dev->pdev; 924 struct ne_phys_contig_mem_regions phys_contig_mem_regions = {}; 925 int rc = -EINVAL; 926 927 rc = ne_sanity_check_user_mem_region(ne_enclave, mem_region); 928 if (rc < 0) 929 return rc; 930 931 ne_mem_region = kzalloc(sizeof(*ne_mem_region), GFP_KERNEL); 932 if (!ne_mem_region) 933 return -ENOMEM; 934 935 max_nr_pages = mem_region.memory_size / NE_MIN_MEM_REGION_SIZE; 936 937 ne_mem_region->pages = kcalloc(max_nr_pages, sizeof(*ne_mem_region->pages), 938 GFP_KERNEL); 939 if (!ne_mem_region->pages) { 940 rc = -ENOMEM; 941 942 goto free_mem_region; 943 } 944 945 phys_contig_mem_regions.regions = kcalloc(max_nr_pages, 946 sizeof(*phys_contig_mem_regions.regions), 947 GFP_KERNEL); 948 if (!phys_contig_mem_regions.regions) { 949 rc = -ENOMEM; 950 951 goto free_mem_region; 952 } 953 954 do { 955 i = ne_mem_region->nr_pages; 956 957 if (i == max_nr_pages) { 958 dev_err_ratelimited(ne_misc_dev.this_device, 959 "Reached max nr of pages in the pages data struct\n"); 960 961 rc = -ENOMEM; 962 963 goto put_pages; 964 } 965 966 gup_rc = get_user_pages_unlocked(mem_region.userspace_addr + memory_size, 1, 967 ne_mem_region->pages + i, FOLL_GET); 968 969 if (gup_rc < 0) { 970 rc = gup_rc; 971 972 dev_err_ratelimited(ne_misc_dev.this_device, 973 "Error in get user pages [rc=%d]\n", rc); 974 975 goto put_pages; 976 } 977 978 rc = ne_sanity_check_user_mem_region_page(ne_enclave, ne_mem_region->pages[i]); 979 if (rc < 0) 980 goto put_pages; 981 982 rc = ne_merge_phys_contig_memory_regions(&phys_contig_mem_regions, 983 page_to_phys(ne_mem_region->pages[i]), 984 page_size(ne_mem_region->pages[i])); 985 if (rc < 0) 986 goto put_pages; 987 988 memory_size += page_size(ne_mem_region->pages[i]); 989 990 ne_mem_region->nr_pages++; 991 } while (memory_size < mem_region.memory_size); 992 993 if ((ne_enclave->nr_mem_regions + phys_contig_mem_regions.num) > 994 ne_enclave->max_mem_regions) { 995 dev_err_ratelimited(ne_misc_dev.this_device, 996 "Reached max memory regions %lld\n", 997 ne_enclave->max_mem_regions); 998 999 rc = -NE_ERR_MEM_MAX_REGIONS; 1000 1001 goto put_pages; 1002 } 1003 1004 for (i = 0; i < phys_contig_mem_regions.num; i++) { 1005 u64 phys_region_addr = phys_contig_mem_regions.regions[i].start; 1006 u64 phys_region_size = range_len(&phys_contig_mem_regions.regions[i]); 1007 1008 rc = ne_sanity_check_phys_mem_region(phys_region_addr, phys_region_size); 1009 if (rc < 0) 1010 goto put_pages; 1011 } 1012 1013 ne_mem_region->memory_size = mem_region.memory_size; 1014 ne_mem_region->userspace_addr = mem_region.userspace_addr; 1015 1016 list_add(&ne_mem_region->mem_region_list_entry, &ne_enclave->mem_regions_list); 1017 1018 for (i = 0; i < phys_contig_mem_regions.num; i++) { 1019 struct ne_pci_dev_cmd_reply cmd_reply = {}; 1020 struct slot_add_mem_req slot_add_mem_req = {}; 1021 1022 slot_add_mem_req.slot_uid = ne_enclave->slot_uid; 1023 slot_add_mem_req.paddr = phys_contig_mem_regions.regions[i].start; 1024 slot_add_mem_req.size = range_len(&phys_contig_mem_regions.regions[i]); 1025 1026 rc = ne_do_request(pdev, SLOT_ADD_MEM, 1027 &slot_add_mem_req, sizeof(slot_add_mem_req), 1028 &cmd_reply, sizeof(cmd_reply)); 1029 if (rc < 0) { 1030 dev_err_ratelimited(ne_misc_dev.this_device, 1031 "Error in slot add mem [rc=%d]\n", rc); 1032 1033 kfree(phys_contig_mem_regions.regions); 1034 1035 /* 1036 * Exit here without put pages as memory regions may 1037 * already been added. 1038 */ 1039 return rc; 1040 } 1041 1042 ne_enclave->mem_size += slot_add_mem_req.size; 1043 ne_enclave->nr_mem_regions++; 1044 } 1045 1046 kfree(phys_contig_mem_regions.regions); 1047 1048 return 0; 1049 1050put_pages: 1051 for (i = 0; i < ne_mem_region->nr_pages; i++) 1052 put_page(ne_mem_region->pages[i]); 1053free_mem_region: 1054 kfree(phys_contig_mem_regions.regions); 1055 kfree(ne_mem_region->pages); 1056 kfree(ne_mem_region); 1057 1058 return rc; 1059} 1060 1061/** 1062 * ne_start_enclave_ioctl() - Trigger enclave start after the enclave resources, 1063 * such as memory and CPU, have been set. 1064 * @ne_enclave : Private data associated with the current enclave. 1065 * @enclave_start_info : Enclave info that includes enclave cid and flags. 1066 * 1067 * Context: Process context. This function is called with the ne_enclave mutex held. 1068 * Return: 1069 * * 0 on success. 1070 * * Negative return value on failure. 1071 */ 1072static int ne_start_enclave_ioctl(struct ne_enclave *ne_enclave, 1073 struct ne_enclave_start_info *enclave_start_info) 1074{ 1075 struct ne_pci_dev_cmd_reply cmd_reply = {}; 1076 unsigned int cpu = 0; 1077 struct enclave_start_req enclave_start_req = {}; 1078 unsigned int i = 0; 1079 struct pci_dev *pdev = ne_devs.ne_pci_dev->pdev; 1080 int rc = -EINVAL; 1081 1082 if (!ne_enclave->nr_mem_regions) { 1083 dev_err_ratelimited(ne_misc_dev.this_device, 1084 "Enclave has no mem regions\n"); 1085 1086 return -NE_ERR_NO_MEM_REGIONS_ADDED; 1087 } 1088 1089 if (ne_enclave->mem_size < NE_MIN_ENCLAVE_MEM_SIZE) { 1090 dev_err_ratelimited(ne_misc_dev.this_device, 1091 "Enclave memory is less than %ld\n", 1092 NE_MIN_ENCLAVE_MEM_SIZE); 1093 1094 return -NE_ERR_ENCLAVE_MEM_MIN_SIZE; 1095 } 1096 1097 if (!ne_enclave->nr_vcpus) { 1098 dev_err_ratelimited(ne_misc_dev.this_device, 1099 "Enclave has no vCPUs\n"); 1100 1101 return -NE_ERR_NO_VCPUS_ADDED; 1102 } 1103 1104 for (i = 0; i < ne_enclave->nr_parent_vm_cores; i++) 1105 for_each_cpu(cpu, ne_enclave->threads_per_core[i]) 1106 if (!cpumask_test_cpu(cpu, ne_enclave->vcpu_ids)) { 1107 dev_err_ratelimited(ne_misc_dev.this_device, 1108 "Full CPU cores not used\n"); 1109 1110 return -NE_ERR_FULL_CORES_NOT_USED; 1111 } 1112 1113 enclave_start_req.enclave_cid = enclave_start_info->enclave_cid; 1114 enclave_start_req.flags = enclave_start_info->flags; 1115 enclave_start_req.slot_uid = ne_enclave->slot_uid; 1116 1117 rc = ne_do_request(pdev, ENCLAVE_START, 1118 &enclave_start_req, sizeof(enclave_start_req), 1119 &cmd_reply, sizeof(cmd_reply)); 1120 if (rc < 0) { 1121 dev_err_ratelimited(ne_misc_dev.this_device, 1122 "Error in enclave start [rc=%d]\n", rc); 1123 1124 return rc; 1125 } 1126 1127 ne_enclave->state = NE_STATE_RUNNING; 1128 1129 enclave_start_info->enclave_cid = cmd_reply.enclave_cid; 1130 1131 return 0; 1132} 1133 1134/** 1135 * ne_enclave_ioctl() - Ioctl function provided by the enclave file. 1136 * @file: File associated with this ioctl function. 1137 * @cmd: The command that is set for the ioctl call. 1138 * @arg: The argument that is provided for the ioctl call. 1139 * 1140 * Context: Process context. 1141 * Return: 1142 * * 0 on success. 1143 * * Negative return value on failure. 1144 */ 1145static long ne_enclave_ioctl(struct file *file, unsigned int cmd, unsigned long arg) 1146{ 1147 struct ne_enclave *ne_enclave = file->private_data; 1148 1149 switch (cmd) { 1150 case NE_ADD_VCPU: { 1151 int rc = -EINVAL; 1152 u32 vcpu_id = 0; 1153 1154 if (copy_from_user(&vcpu_id, (void __user *)arg, sizeof(vcpu_id))) 1155 return -EFAULT; 1156 1157 mutex_lock(&ne_enclave->enclave_info_mutex); 1158 1159 if (ne_enclave->state != NE_STATE_INIT) { 1160 dev_err_ratelimited(ne_misc_dev.this_device, 1161 "Enclave is not in init state\n"); 1162 1163 mutex_unlock(&ne_enclave->enclave_info_mutex); 1164 1165 return -NE_ERR_NOT_IN_INIT_STATE; 1166 } 1167 1168 if (vcpu_id >= (ne_enclave->nr_parent_vm_cores * 1169 ne_enclave->nr_threads_per_core)) { 1170 dev_err_ratelimited(ne_misc_dev.this_device, 1171 "vCPU id higher than max CPU id\n"); 1172 1173 mutex_unlock(&ne_enclave->enclave_info_mutex); 1174 1175 return -NE_ERR_INVALID_VCPU; 1176 } 1177 1178 if (!vcpu_id) { 1179 /* Use the CPU pool for choosing a CPU for the enclave. */ 1180 rc = ne_get_cpu_from_cpu_pool(ne_enclave, &vcpu_id); 1181 if (rc < 0) { 1182 dev_err_ratelimited(ne_misc_dev.this_device, 1183 "Error in get CPU from pool [rc=%d]\n", 1184 rc); 1185 1186 mutex_unlock(&ne_enclave->enclave_info_mutex); 1187 1188 return rc; 1189 } 1190 } else { 1191 /* Check if the provided vCPU is available in the NE CPU pool. */ 1192 rc = ne_check_cpu_in_cpu_pool(ne_enclave, vcpu_id); 1193 if (rc < 0) { 1194 dev_err_ratelimited(ne_misc_dev.this_device, 1195 "Error in check CPU %d in pool [rc=%d]\n", 1196 vcpu_id, rc); 1197 1198 mutex_unlock(&ne_enclave->enclave_info_mutex); 1199 1200 return rc; 1201 } 1202 } 1203 1204 rc = ne_add_vcpu_ioctl(ne_enclave, vcpu_id); 1205 if (rc < 0) { 1206 mutex_unlock(&ne_enclave->enclave_info_mutex); 1207 1208 return rc; 1209 } 1210 1211 mutex_unlock(&ne_enclave->enclave_info_mutex); 1212 1213 if (copy_to_user((void __user *)arg, &vcpu_id, sizeof(vcpu_id))) 1214 return -EFAULT; 1215 1216 return 0; 1217 } 1218 1219 case NE_GET_IMAGE_LOAD_INFO: { 1220 struct ne_image_load_info image_load_info = {}; 1221 1222 if (copy_from_user(&image_load_info, (void __user *)arg, sizeof(image_load_info))) 1223 return -EFAULT; 1224 1225 mutex_lock(&ne_enclave->enclave_info_mutex); 1226 1227 if (ne_enclave->state != NE_STATE_INIT) { 1228 dev_err_ratelimited(ne_misc_dev.this_device, 1229 "Enclave is not in init state\n"); 1230 1231 mutex_unlock(&ne_enclave->enclave_info_mutex); 1232 1233 return -NE_ERR_NOT_IN_INIT_STATE; 1234 } 1235 1236 mutex_unlock(&ne_enclave->enclave_info_mutex); 1237 1238 if (!image_load_info.flags || 1239 image_load_info.flags >= NE_IMAGE_LOAD_MAX_FLAG_VAL) { 1240 dev_err_ratelimited(ne_misc_dev.this_device, 1241 "Incorrect flag in enclave image load info\n"); 1242 1243 return -NE_ERR_INVALID_FLAG_VALUE; 1244 } 1245 1246 if (image_load_info.flags == NE_EIF_IMAGE) 1247 image_load_info.memory_offset = NE_EIF_LOAD_OFFSET; 1248 1249 if (copy_to_user((void __user *)arg, &image_load_info, sizeof(image_load_info))) 1250 return -EFAULT; 1251 1252 return 0; 1253 } 1254 1255 case NE_SET_USER_MEMORY_REGION: { 1256 struct ne_user_memory_region mem_region = {}; 1257 int rc = -EINVAL; 1258 1259 if (copy_from_user(&mem_region, (void __user *)arg, sizeof(mem_region))) 1260 return -EFAULT; 1261 1262 if (mem_region.flags >= NE_MEMORY_REGION_MAX_FLAG_VAL) { 1263 dev_err_ratelimited(ne_misc_dev.this_device, 1264 "Incorrect flag for user memory region\n"); 1265 1266 return -NE_ERR_INVALID_FLAG_VALUE; 1267 } 1268 1269 mutex_lock(&ne_enclave->enclave_info_mutex); 1270 1271 if (ne_enclave->state != NE_STATE_INIT) { 1272 dev_err_ratelimited(ne_misc_dev.this_device, 1273 "Enclave is not in init state\n"); 1274 1275 mutex_unlock(&ne_enclave->enclave_info_mutex); 1276 1277 return -NE_ERR_NOT_IN_INIT_STATE; 1278 } 1279 1280 rc = ne_set_user_memory_region_ioctl(ne_enclave, mem_region); 1281 if (rc < 0) { 1282 mutex_unlock(&ne_enclave->enclave_info_mutex); 1283 1284 return rc; 1285 } 1286 1287 mutex_unlock(&ne_enclave->enclave_info_mutex); 1288 1289 return 0; 1290 } 1291 1292 case NE_START_ENCLAVE: { 1293 struct ne_enclave_start_info enclave_start_info = {}; 1294 int rc = -EINVAL; 1295 1296 if (copy_from_user(&enclave_start_info, (void __user *)arg, 1297 sizeof(enclave_start_info))) 1298 return -EFAULT; 1299 1300 if (enclave_start_info.flags >= NE_ENCLAVE_START_MAX_FLAG_VAL) { 1301 dev_err_ratelimited(ne_misc_dev.this_device, 1302 "Incorrect flag in enclave start info\n"); 1303 1304 return -NE_ERR_INVALID_FLAG_VALUE; 1305 } 1306 1307 /* 1308 * Do not use well-known CIDs - 0, 1, 2 - for enclaves. 1309 * VMADDR_CID_ANY = -1U 1310 * VMADDR_CID_HYPERVISOR = 0 1311 * VMADDR_CID_LOCAL = 1 1312 * VMADDR_CID_HOST = 2 1313 * Note: 0 is used as a placeholder to auto-generate an enclave CID. 1314 * http://man7.org/linux/man-pages/man7/vsock.7.html 1315 */ 1316 if (enclave_start_info.enclave_cid > 0 && 1317 enclave_start_info.enclave_cid <= VMADDR_CID_HOST) { 1318 dev_err_ratelimited(ne_misc_dev.this_device, 1319 "Well-known CID value, not to be used for enclaves\n"); 1320 1321 return -NE_ERR_INVALID_ENCLAVE_CID; 1322 } 1323 1324 if (enclave_start_info.enclave_cid == U32_MAX) { 1325 dev_err_ratelimited(ne_misc_dev.this_device, 1326 "Well-known CID value, not to be used for enclaves\n"); 1327 1328 return -NE_ERR_INVALID_ENCLAVE_CID; 1329 } 1330 1331 /* 1332 * Do not use the CID of the primary / parent VM for enclaves. 1333 */ 1334 if (enclave_start_info.enclave_cid == NE_PARENT_VM_CID) { 1335 dev_err_ratelimited(ne_misc_dev.this_device, 1336 "CID of the parent VM, not to be used for enclaves\n"); 1337 1338 return -NE_ERR_INVALID_ENCLAVE_CID; 1339 } 1340 1341 /* 64-bit CIDs are not yet supported for the vsock device. */ 1342 if (enclave_start_info.enclave_cid > U32_MAX) { 1343 dev_err_ratelimited(ne_misc_dev.this_device, 1344 "64-bit CIDs not yet supported for the vsock device\n"); 1345 1346 return -NE_ERR_INVALID_ENCLAVE_CID; 1347 } 1348 1349 mutex_lock(&ne_enclave->enclave_info_mutex); 1350 1351 if (ne_enclave->state != NE_STATE_INIT) { 1352 dev_err_ratelimited(ne_misc_dev.this_device, 1353 "Enclave is not in init state\n"); 1354 1355 mutex_unlock(&ne_enclave->enclave_info_mutex); 1356 1357 return -NE_ERR_NOT_IN_INIT_STATE; 1358 } 1359 1360 rc = ne_start_enclave_ioctl(ne_enclave, &enclave_start_info); 1361 if (rc < 0) { 1362 mutex_unlock(&ne_enclave->enclave_info_mutex); 1363 1364 return rc; 1365 } 1366 1367 mutex_unlock(&ne_enclave->enclave_info_mutex); 1368 1369 if (copy_to_user((void __user *)arg, &enclave_start_info, 1370 sizeof(enclave_start_info))) 1371 return -EFAULT; 1372 1373 return 0; 1374 } 1375 1376 default: 1377 return -ENOTTY; 1378 } 1379 1380 return 0; 1381} 1382 1383/** 1384 * ne_enclave_remove_all_mem_region_entries() - Remove all memory region entries 1385 * from the enclave data structure. 1386 * @ne_enclave : Private data associated with the current enclave. 1387 * 1388 * Context: Process context. This function is called with the ne_enclave mutex held. 1389 */ 1390static void ne_enclave_remove_all_mem_region_entries(struct ne_enclave *ne_enclave) 1391{ 1392 unsigned long i = 0; 1393 struct ne_mem_region *ne_mem_region = NULL; 1394 struct ne_mem_region *ne_mem_region_tmp = NULL; 1395 1396 list_for_each_entry_safe(ne_mem_region, ne_mem_region_tmp, 1397 &ne_enclave->mem_regions_list, 1398 mem_region_list_entry) { 1399 list_del(&ne_mem_region->mem_region_list_entry); 1400 1401 for (i = 0; i < ne_mem_region->nr_pages; i++) 1402 put_page(ne_mem_region->pages[i]); 1403 1404 kfree(ne_mem_region->pages); 1405 1406 kfree(ne_mem_region); 1407 } 1408} 1409 1410/** 1411 * ne_enclave_remove_all_vcpu_id_entries() - Remove all vCPU id entries from 1412 * the enclave data structure. 1413 * @ne_enclave : Private data associated with the current enclave. 1414 * 1415 * Context: Process context. This function is called with the ne_enclave mutex held. 1416 */ 1417static void ne_enclave_remove_all_vcpu_id_entries(struct ne_enclave *ne_enclave) 1418{ 1419 unsigned int cpu = 0; 1420 unsigned int i = 0; 1421 1422 mutex_lock(&ne_cpu_pool.mutex); 1423 1424 for (i = 0; i < ne_enclave->nr_parent_vm_cores; i++) { 1425 for_each_cpu(cpu, ne_enclave->threads_per_core[i]) 1426 /* Update the available NE CPU pool. */ 1427 cpumask_set_cpu(cpu, ne_cpu_pool.avail_threads_per_core[i]); 1428 1429 free_cpumask_var(ne_enclave->threads_per_core[i]); 1430 } 1431 1432 mutex_unlock(&ne_cpu_pool.mutex); 1433 1434 kfree(ne_enclave->threads_per_core); 1435 1436 free_cpumask_var(ne_enclave->vcpu_ids); 1437} 1438 1439/** 1440 * ne_pci_dev_remove_enclave_entry() - Remove the enclave entry from the data 1441 * structure that is part of the NE PCI 1442 * device private data. 1443 * @ne_enclave : Private data associated with the current enclave. 1444 * @ne_pci_dev : Private data associated with the PCI device. 1445 * 1446 * Context: Process context. This function is called with the ne_pci_dev enclave 1447 * mutex held. 1448 */ 1449static void ne_pci_dev_remove_enclave_entry(struct ne_enclave *ne_enclave, 1450 struct ne_pci_dev *ne_pci_dev) 1451{ 1452 struct ne_enclave *ne_enclave_entry = NULL; 1453 struct ne_enclave *ne_enclave_entry_tmp = NULL; 1454 1455 list_for_each_entry_safe(ne_enclave_entry, ne_enclave_entry_tmp, 1456 &ne_pci_dev->enclaves_list, enclave_list_entry) { 1457 if (ne_enclave_entry->slot_uid == ne_enclave->slot_uid) { 1458 list_del(&ne_enclave_entry->enclave_list_entry); 1459 1460 break; 1461 } 1462 } 1463} 1464 1465/** 1466 * ne_enclave_release() - Release function provided by the enclave file. 1467 * @inode: Inode associated with this file release function. 1468 * @file: File associated with this release function. 1469 * 1470 * Context: Process context. 1471 * Return: 1472 * * 0 on success. 1473 * * Negative return value on failure. 1474 */ 1475static int ne_enclave_release(struct inode *inode, struct file *file) 1476{ 1477 struct ne_pci_dev_cmd_reply cmd_reply = {}; 1478 struct enclave_stop_req enclave_stop_request = {}; 1479 struct ne_enclave *ne_enclave = file->private_data; 1480 struct ne_pci_dev *ne_pci_dev = ne_devs.ne_pci_dev; 1481 struct pci_dev *pdev = ne_pci_dev->pdev; 1482 int rc = -EINVAL; 1483 struct slot_free_req slot_free_req = {}; 1484 1485 if (!ne_enclave) 1486 return 0; 1487 1488 /* 1489 * Early exit in case there is an error in the enclave creation logic 1490 * and fput() is called on the cleanup path. 1491 */ 1492 if (!ne_enclave->slot_uid) 1493 return 0; 1494 1495 /* 1496 * Acquire the enclave list mutex before the enclave mutex 1497 * in order to avoid deadlocks with @ref ne_event_work_handler. 1498 */ 1499 mutex_lock(&ne_pci_dev->enclaves_list_mutex); 1500 mutex_lock(&ne_enclave->enclave_info_mutex); 1501 1502 if (ne_enclave->state != NE_STATE_INIT && ne_enclave->state != NE_STATE_STOPPED) { 1503 enclave_stop_request.slot_uid = ne_enclave->slot_uid; 1504 1505 rc = ne_do_request(pdev, ENCLAVE_STOP, 1506 &enclave_stop_request, sizeof(enclave_stop_request), 1507 &cmd_reply, sizeof(cmd_reply)); 1508 if (rc < 0) { 1509 dev_err_ratelimited(ne_misc_dev.this_device, 1510 "Error in enclave stop [rc=%d]\n", rc); 1511 1512 goto unlock_mutex; 1513 } 1514 1515 memset(&cmd_reply, 0, sizeof(cmd_reply)); 1516 } 1517 1518 slot_free_req.slot_uid = ne_enclave->slot_uid; 1519 1520 rc = ne_do_request(pdev, SLOT_FREE, 1521 &slot_free_req, sizeof(slot_free_req), 1522 &cmd_reply, sizeof(cmd_reply)); 1523 if (rc < 0) { 1524 dev_err_ratelimited(ne_misc_dev.this_device, 1525 "Error in slot free [rc=%d]\n", rc); 1526 1527 goto unlock_mutex; 1528 } 1529 1530 ne_pci_dev_remove_enclave_entry(ne_enclave, ne_pci_dev); 1531 ne_enclave_remove_all_mem_region_entries(ne_enclave); 1532 ne_enclave_remove_all_vcpu_id_entries(ne_enclave); 1533 1534 mutex_unlock(&ne_enclave->enclave_info_mutex); 1535 mutex_unlock(&ne_pci_dev->enclaves_list_mutex); 1536 1537 kfree(ne_enclave); 1538 1539 return 0; 1540 1541unlock_mutex: 1542 mutex_unlock(&ne_enclave->enclave_info_mutex); 1543 mutex_unlock(&ne_pci_dev->enclaves_list_mutex); 1544 1545 return rc; 1546} 1547 1548/** 1549 * ne_enclave_poll() - Poll functionality used for enclave out-of-band events. 1550 * @file: File associated with this poll function. 1551 * @wait: Poll table data structure. 1552 * 1553 * Context: Process context. 1554 * Return: 1555 * * Poll mask. 1556 */ 1557static __poll_t ne_enclave_poll(struct file *file, poll_table *wait) 1558{ 1559 __poll_t mask = 0; 1560 struct ne_enclave *ne_enclave = file->private_data; 1561 1562 poll_wait(file, &ne_enclave->eventq, wait); 1563 1564 if (ne_enclave->has_event) 1565 mask |= EPOLLHUP; 1566 1567 return mask; 1568} 1569 1570static const struct file_operations ne_enclave_fops = { 1571 .owner = THIS_MODULE, 1572 .llseek = noop_llseek, 1573 .poll = ne_enclave_poll, 1574 .unlocked_ioctl = ne_enclave_ioctl, 1575 .release = ne_enclave_release, 1576}; 1577 1578/** 1579 * ne_create_vm_ioctl() - Alloc slot to be associated with an enclave. Create 1580 * enclave file descriptor to be further used for enclave 1581 * resources handling e.g. memory regions and CPUs. 1582 * @ne_pci_dev : Private data associated with the PCI device. 1583 * @slot_uid: User pointer to store the generated unique slot id 1584 * associated with an enclave to. 1585 * 1586 * Context: Process context. This function is called with the ne_pci_dev enclave 1587 * mutex held. 1588 * Return: 1589 * * Enclave fd on success. 1590 * * Negative return value on failure. 1591 */ 1592static int ne_create_vm_ioctl(struct ne_pci_dev *ne_pci_dev, u64 __user *slot_uid) 1593{ 1594 struct ne_pci_dev_cmd_reply cmd_reply = {}; 1595 int enclave_fd = -1; 1596 struct file *enclave_file = NULL; 1597 unsigned int i = 0; 1598 struct ne_enclave *ne_enclave = NULL; 1599 struct pci_dev *pdev = ne_pci_dev->pdev; 1600 int rc = -EINVAL; 1601 struct slot_alloc_req slot_alloc_req = {}; 1602 1603 mutex_lock(&ne_cpu_pool.mutex); 1604 1605 for (i = 0; i < ne_cpu_pool.nr_parent_vm_cores; i++) 1606 if (!cpumask_empty(ne_cpu_pool.avail_threads_per_core[i])) 1607 break; 1608 1609 if (i == ne_cpu_pool.nr_parent_vm_cores) { 1610 dev_err_ratelimited(ne_misc_dev.this_device, 1611 "No CPUs available in CPU pool\n"); 1612 1613 mutex_unlock(&ne_cpu_pool.mutex); 1614 1615 return -NE_ERR_NO_CPUS_AVAIL_IN_POOL; 1616 } 1617 1618 mutex_unlock(&ne_cpu_pool.mutex); 1619 1620 ne_enclave = kzalloc(sizeof(*ne_enclave), GFP_KERNEL); 1621 if (!ne_enclave) 1622 return -ENOMEM; 1623 1624 mutex_lock(&ne_cpu_pool.mutex); 1625 1626 ne_enclave->nr_parent_vm_cores = ne_cpu_pool.nr_parent_vm_cores; 1627 ne_enclave->nr_threads_per_core = ne_cpu_pool.nr_threads_per_core; 1628 ne_enclave->numa_node = ne_cpu_pool.numa_node; 1629 1630 mutex_unlock(&ne_cpu_pool.mutex); 1631 1632 ne_enclave->threads_per_core = kcalloc(ne_enclave->nr_parent_vm_cores, 1633 sizeof(*ne_enclave->threads_per_core), 1634 GFP_KERNEL); 1635 if (!ne_enclave->threads_per_core) { 1636 rc = -ENOMEM; 1637 1638 goto free_ne_enclave; 1639 } 1640 1641 for (i = 0; i < ne_enclave->nr_parent_vm_cores; i++) 1642 if (!zalloc_cpumask_var(&ne_enclave->threads_per_core[i], GFP_KERNEL)) { 1643 rc = -ENOMEM; 1644 1645 goto free_cpumask; 1646 } 1647 1648 if (!zalloc_cpumask_var(&ne_enclave->vcpu_ids, GFP_KERNEL)) { 1649 rc = -ENOMEM; 1650 1651 goto free_cpumask; 1652 } 1653 1654 enclave_fd = get_unused_fd_flags(O_CLOEXEC); 1655 if (enclave_fd < 0) { 1656 rc = enclave_fd; 1657 1658 dev_err_ratelimited(ne_misc_dev.this_device, 1659 "Error in getting unused fd [rc=%d]\n", rc); 1660 1661 goto free_cpumask; 1662 } 1663 1664 enclave_file = anon_inode_getfile("ne-vm", &ne_enclave_fops, ne_enclave, O_RDWR); 1665 if (IS_ERR(enclave_file)) { 1666 rc = PTR_ERR(enclave_file); 1667 1668 dev_err_ratelimited(ne_misc_dev.this_device, 1669 "Error in anon inode get file [rc=%d]\n", rc); 1670 1671 goto put_fd; 1672 } 1673 1674 rc = ne_do_request(pdev, SLOT_ALLOC, 1675 &slot_alloc_req, sizeof(slot_alloc_req), 1676 &cmd_reply, sizeof(cmd_reply)); 1677 if (rc < 0) { 1678 dev_err_ratelimited(ne_misc_dev.this_device, 1679 "Error in slot alloc [rc=%d]\n", rc); 1680 1681 goto put_file; 1682 } 1683 1684 init_waitqueue_head(&ne_enclave->eventq); 1685 ne_enclave->has_event = false; 1686 mutex_init(&ne_enclave->enclave_info_mutex); 1687 ne_enclave->max_mem_regions = cmd_reply.mem_regions; 1688 INIT_LIST_HEAD(&ne_enclave->mem_regions_list); 1689 ne_enclave->mm = current->mm; 1690 ne_enclave->slot_uid = cmd_reply.slot_uid; 1691 ne_enclave->state = NE_STATE_INIT; 1692 1693 list_add(&ne_enclave->enclave_list_entry, &ne_pci_dev->enclaves_list); 1694 1695 if (copy_to_user(slot_uid, &ne_enclave->slot_uid, sizeof(ne_enclave->slot_uid))) { 1696 /* 1697 * As we're holding the only reference to 'enclave_file', fput() 1698 * will call ne_enclave_release() which will do a proper cleanup 1699 * of all so far allocated resources, leaving only the unused fd 1700 * for us to free. 1701 */ 1702 fput(enclave_file); 1703 put_unused_fd(enclave_fd); 1704 1705 return -EFAULT; 1706 } 1707 1708 fd_install(enclave_fd, enclave_file); 1709 1710 return enclave_fd; 1711 1712put_file: 1713 fput(enclave_file); 1714put_fd: 1715 put_unused_fd(enclave_fd); 1716free_cpumask: 1717 free_cpumask_var(ne_enclave->vcpu_ids); 1718 for (i = 0; i < ne_enclave->nr_parent_vm_cores; i++) 1719 free_cpumask_var(ne_enclave->threads_per_core[i]); 1720 kfree(ne_enclave->threads_per_core); 1721free_ne_enclave: 1722 kfree(ne_enclave); 1723 1724 return rc; 1725} 1726 1727/** 1728 * ne_ioctl() - Ioctl function provided by the NE misc device. 1729 * @file: File associated with this ioctl function. 1730 * @cmd: The command that is set for the ioctl call. 1731 * @arg: The argument that is provided for the ioctl call. 1732 * 1733 * Context: Process context. 1734 * Return: 1735 * * Ioctl result (e.g. enclave file descriptor) on success. 1736 * * Negative return value on failure. 1737 */ 1738static long ne_ioctl(struct file *file, unsigned int cmd, unsigned long arg) 1739{ 1740 switch (cmd) { 1741 case NE_CREATE_VM: { 1742 int enclave_fd = -1; 1743 struct ne_pci_dev *ne_pci_dev = ne_devs.ne_pci_dev; 1744 u64 __user *slot_uid = (void __user *)arg; 1745 1746 mutex_lock(&ne_pci_dev->enclaves_list_mutex); 1747 enclave_fd = ne_create_vm_ioctl(ne_pci_dev, slot_uid); 1748 mutex_unlock(&ne_pci_dev->enclaves_list_mutex); 1749 1750 return enclave_fd; 1751 } 1752 1753 default: 1754 return -ENOTTY; 1755 } 1756 1757 return 0; 1758} 1759 1760#if defined(CONFIG_NITRO_ENCLAVES_MISC_DEV_TEST) 1761#include "ne_misc_dev_test.c" 1762#endif 1763 1764static int __init ne_init(void) 1765{ 1766 mutex_init(&ne_cpu_pool.mutex); 1767 1768 return pci_register_driver(&ne_pci_driver); 1769} 1770 1771static void __exit ne_exit(void) 1772{ 1773 pci_unregister_driver(&ne_pci_driver); 1774 1775 ne_teardown_cpu_pool(); 1776} 1777 1778module_init(ne_init); 1779module_exit(ne_exit); 1780 1781MODULE_AUTHOR("Amazon.com, Inc. or its affiliates"); 1782MODULE_DESCRIPTION("Nitro Enclaves Driver"); 1783MODULE_LICENSE("GPL v2"); 1784