1/* 2 * Copyright 2019, Data61, CSIRO (ABN 41 687 119 230) 3 * 4 * SPDX-License-Identifier: BSD-2-Clause 5 */ 6 7#include <stdio.h> 8#include <stdlib.h> 9#include <string.h> 10 11#include <utils/sglib.h> 12 13#include <sel4vm/guest_vm.h> 14#include <sel4vm/guest_memory.h> 15 16#include "guest_memory.h" 17 18typedef enum reservation_type { 19 MEM_REGULAR_RES, 20 MEM_ANON_RES 21} reservation_type_t; 22 23/* VM Memory reservation object: Represents a reservation in the guest VM's memory */ 24struct vm_memory_reservation { 25 /* Base address of reserved memory region */ 26 uintptr_t addr; 27 /* Size of memory region */ 28 size_t size; 29 /* Callback to be invoked if memory region is faulted on*/ 30 memory_fault_callback_fn fault_callback; 31 /* Iterator to be invoked for performing a map on the reservation region */ 32 memory_map_iterator_fn memory_map_iterator; 33 /* If the reservation is pending to be mapped into the vm's address space */ 34 bool is_mapped; 35 /* Cookies to pass onto callback and iterator functions */ 36 void *fault_callback_cookie; 37 void *memory_iterator_cookie; 38 /* The reservation in the vm's vspace object */ 39 reservation_t vspace_reservation; 40 /* The type of reservation i.e regular, anonymous */ 41 reservation_type_t res_type; 42}; 43 44typedef struct anon_region { 45 uintptr_t addr; 46 size_t size; 47 uintptr_t alloc_addr; 48 reservation_t vspace_reservation; 49 int num_reservations; 50 vm_memory_reservation_t **reservations; 51} anon_region_t; 52 53typedef struct res_tree { 54 uintptr_t addr; 55 size_t size; 56 reservation_type_t res_type; 57 void *data; 58 char color_field; 59 struct res_tree *left; 60 struct res_tree *right; 61} res_tree; 62 63static inline int reservation_node_cmp(res_tree *x, res_tree *y) 64{ 65 if (x->addr < y->addr) { 66 if (x->addr + x->size > y->addr) { 67 /* The two regions intersect */ 68 return 0; 69 } else { 70 return -1; 71 } 72 } 73 if (x->addr < y->addr + y->size) { 74 /* The two regions intersect */ 75 return 0; 76 } 77 return 1; 78} 79 80SGLIB_DEFINE_RBTREE_PROTOTYPES(res_tree, left, right, color_field, reservation_node_cmp); 81SGLIB_DEFINE_RBTREE_FUNCTIONS(res_tree, left, right, color_field, reservation_node_cmp); 82 83struct vm_memory_reservation_cookie { 84 struct res_tree *regular_res_tree; 85 struct res_tree *anon_res_tree; 86}; 87 88static res_tree *find_memory_reservation_by_addr(vm_t *vm, uintptr_t addr) 89{ 90 res_tree *result_node; 91 vm_memory_reservation_cookie_t *res_cookie = vm->mem.reservation_cookie; 92 if (!res_cookie) { 93 ZF_LOGE("Failed to find memory reservation: VM memory backend not initialised"); 94 return NULL; 95 } 96 res_tree search_node; 97 search_node.addr = addr; 98 search_node.size = 0; 99 /* Search the regular reservation tree to begin with */ 100 result_node = sglib_res_tree_find_member(res_cookie->regular_res_tree, &search_node); 101 if (result_node) { 102 return result_node; 103 } 104 /* Search the anonymous reservation tree if nothing found */ 105 result_node = sglib_res_tree_find_member(res_cookie->anon_res_tree, &search_node); 106 return result_node; 107} 108 109static void remove_memory_reservation_node(vm_t *vm, uintptr_t addr, size_t size, reservation_type_t res_type) 110{ 111 res_tree *tree; 112 res_tree *result_node; 113 vm_memory_reservation_cookie_t *res_cookie = vm->mem.reservation_cookie; 114 if (!res_cookie) { 115 ZF_LOGE("Failed to find memory reservation: VM memory backend not initialised"); 116 return; 117 } 118 if (res_type == MEM_REGULAR_RES) { 119 tree = res_cookie->regular_res_tree; 120 } else { 121 tree = res_cookie->anon_res_tree; 122 } 123 res_tree search_node; 124 search_node.addr = addr; 125 search_node.size = size; 126 result_node = sglib_res_tree_find_member(tree, &search_node); 127 if (!result_node) { 128 /* No node found */ 129 return; 130 } 131 /* Region needs to be an exact match */ 132 if ((result_node->addr == search_node.addr) && (result_node->size == search_node.size)) { 133 sglib_res_tree_delete(&tree, result_node); 134 if (res_type == MEM_REGULAR_RES) { 135 res_cookie->regular_res_tree = tree; 136 } else { 137 res_cookie->anon_res_tree = tree; 138 } 139 } 140 return; 141} 142 143static int add_memory_reservation_node(vm_t *vm, uintptr_t addr, size_t size, reservation_type_t res_type, void *data) 144{ 145 int err; 146 res_tree *tree; 147 res_tree *result_node; 148 ps_io_ops_t *ops = vm->io_ops; 149 vm_memory_reservation_cookie_t *res_cookie = vm->mem.reservation_cookie; 150 if (!res_cookie) { 151 ZF_LOGE("Failed to find memory reservation: VM memory backend not initialised"); 152 return -1; 153 } 154 if (res_type == MEM_REGULAR_RES) { 155 tree = res_cookie->regular_res_tree; 156 } else { 157 tree = res_cookie->anon_res_tree; 158 } 159 res_tree search_node; 160 search_node.addr = addr; 161 search_node.size = size; 162 163 res_tree *found_node = sglib_res_tree_find_member(tree, &search_node); 164 if (found_node == NULL) { 165 err = ps_calloc(&ops->malloc_ops, 1, sizeof(res_tree), (void **)&result_node); 166 if (err) { 167 ZF_LOGE("Failed to add memory reservation: Unable to allocate new reservation node"); 168 return -1; 169 } 170 result_node->addr = addr; 171 result_node->size = size; 172 result_node->res_type = res_type; 173 result_node->data = data; 174 sglib_res_tree_add(&tree, result_node); 175 if (res_type == MEM_REGULAR_RES) { 176 res_cookie->regular_res_tree = tree; 177 } else { 178 res_cookie->anon_res_tree = tree; 179 } 180 } else { 181 ZF_LOGE("Failed to add memory reservation: Reservation region already exists"); 182 return -1; 183 } 184 return 0; 185} 186 187static anon_region_t *find_allocable_anon_region(vm_t *vm, size_t size) 188{ 189 res_tree *anon_tree; 190 res_tree *anon_node; 191 anon_region_t *ret_region = NULL; 192 struct sglib_res_tree_iterator it; 193 vm_memory_reservation_cookie_t *res_cookie = vm->mem.reservation_cookie; 194 if (!res_cookie) { 195 return NULL; 196 } 197 anon_tree = res_cookie->anon_res_tree; 198 for (anon_node = sglib_res_tree_it_init_inorder(&it, anon_tree); anon_node != NULL; 199 anon_node = sglib_res_tree_it_next(&it)) { 200 anon_region_t *curr_region = (anon_region_t *)anon_node->data; 201 size_t free_area_size = curr_region->size - (curr_region->alloc_addr - curr_region->addr); 202 if (size <= free_area_size) { 203 ret_region = curr_region; 204 break; 205 } 206 } 207 return ret_region; 208} 209 210static void free_vm_reservation(vm_t *vm, vm_memory_reservation_t *reservation) 211{ 212 if (!vm || !reservation) { 213 return; 214 } 215 ps_io_ops_t *ops = vm->io_ops; 216 ps_free(&ops->malloc_ops, sizeof(vm_memory_reservation_t), reservation); 217} 218 219static vm_memory_reservation_t *allocate_vm_reservation(vm_t *vm, uintptr_t addr, size_t size, 220 reservation_t vspace_reservation) 221{ 222 int err; 223 ps_io_ops_t *ops = vm->io_ops; 224 vm_memory_reservation_t *new_reservation; 225 err = ps_calloc(&ops->malloc_ops, 1, sizeof(vm_memory_reservation_t), (void **)&new_reservation); 226 if (err) { 227 ZF_LOGE("Failed to allocate vm reservation: Unable to allocate new vm memory reservation"); 228 return NULL; 229 } 230 231 new_reservation->addr = addr; 232 new_reservation->size = size; 233 new_reservation->is_mapped = false; 234 new_reservation->vspace_reservation = vspace_reservation; 235 return new_reservation; 236} 237 238static vm_memory_reservation_t *find_anon_reservation_by_addr(uintptr_t addr, size_t size, 239 anon_region_t *anon_region) 240{ 241 int num_anon_reservations; 242 vm_memory_reservation_t **reservations; 243 244 if (!anon_region) { 245 ZF_LOGE("Failed to find anonymous reservation: anon region NULL"); 246 return NULL; 247 } 248 num_anon_reservations = anon_region->num_reservations; 249 reservations = anon_region->reservations; 250 251 if (!reservations) { 252 ZF_LOGE("Failed to find anonymous reservation: anon region has no reservations"); 253 return NULL; 254 } 255 256 for (int i = 0; i < num_anon_reservations; i++) { 257 vm_memory_reservation_t *curr_res = reservations[i]; 258 if (curr_res->addr <= addr && curr_res->addr + curr_res->size >= addr + size) { 259 return curr_res; 260 } 261 } 262 263 return NULL; 264} 265 266memory_fault_result_t vm_memory_handle_fault(vm_t *vm, vm_vcpu_t *vcpu, uintptr_t addr, size_t size) 267{ 268 int err; 269 res_tree *reservation_node = find_memory_reservation_by_addr(vm, addr); 270 vm_memory_reservation_t *fault_reservation; 271 272 if (!reservation_node) { 273 ZF_LOGW("Unable to find reservation for addr: 0x%x, memory fault left unhandled", addr); 274 return FAULT_UNHANDLED; 275 } 276 277 if ((reservation_node->addr + size) > (reservation_node->addr + reservation_node->size)) { 278 ZF_LOGE("Failed to handle memory fault: Invalid fault region"); 279 return FAULT_ERROR; 280 } 281 282 if (reservation_node->res_type == MEM_REGULAR_RES) { 283 fault_reservation = (vm_memory_reservation_t *)reservation_node->data; 284 } else { 285 fault_reservation = find_anon_reservation_by_addr(addr, size, 286 (anon_region_t *)reservation_node->data); 287 if (!fault_reservation) { 288 ZF_LOGW("Unable to find anoymous reservation for addr: 0x%x, memory fault left unhandled", addr); 289 return FAULT_UNHANDLED; 290 } 291 } 292 293 if (!fault_reservation->is_mapped && fault_reservation->memory_map_iterator) { 294 /* Deferred mapping */ 295 err = map_vm_memory_reservation(vm, fault_reservation, 296 fault_reservation->memory_map_iterator, fault_reservation->memory_iterator_cookie); 297 if (err) { 298 ZF_LOGE("Unable to handle memory fault: Failed to map memory"); 299 return FAULT_ERROR; 300 } 301 return FAULT_RESTART; 302 } 303 304 if (!fault_reservation->fault_callback) { 305 return FAULT_ERROR; 306 } 307 308 return fault_reservation->fault_callback(vm, vcpu, addr, size, fault_reservation->fault_callback_cookie); 309} 310 311vm_memory_reservation_t *vm_reserve_memory_at(vm_t *vm, uintptr_t addr, size_t size, 312 memory_fault_callback_fn fault_callback, void *cookie) 313{ 314 int err; 315 vm_memory_reservation_t *new_reservation; 316 317 if (!fault_callback) { 318 ZF_LOGE("Failed to reserve vm reservation: NULL fault callback"); 319 return NULL; 320 } 321 322 /* A placeholder reservation to ensure nothing else takes the range 323 * We will update the reservation with the correct rights on mapping */ 324 reservation_t vspace_reservation = vspace_reserve_deferred_rights_range_at(&vm->mem.vm_vspace, (void *)addr, 325 size, 1); 326 if (!vspace_reservation.res) { 327 ZF_LOGE("Failed to allocate vm reservation: Unable to create vspace reservation at address 0x%x of size %zu", 328 addr, size); 329 return NULL; 330 } 331 new_reservation = allocate_vm_reservation(vm, addr, size, vspace_reservation); 332 if (!new_reservation) { 333 ZF_LOGE("Failed to reserve vm memory: Unable to allocate new vm reservation"); 334 vspace_free_reservation(&vm->mem.vm_vspace, vspace_reservation); 335 return NULL; 336 } 337 new_reservation->fault_callback = fault_callback; 338 new_reservation->fault_callback_cookie = cookie; 339 new_reservation->res_type = MEM_REGULAR_RES; 340 341 err = add_memory_reservation_node(vm, addr, size, MEM_REGULAR_RES, (void *)new_reservation); 342 if (err) { 343 ZF_LOGE("Failed to reserve vm memory: Unable to add vm memory reservation to list"); 344 vspace_free_reservation(&vm->mem.vm_vspace, vspace_reservation); 345 free_vm_reservation(vm, new_reservation); 346 return NULL; 347 } 348 return new_reservation; 349} 350 351int vm_memory_make_anon(vm_t *vm, uintptr_t addr, size_t size) 352{ 353 int err; 354 reservation_t vspace_reservation = vspace_reserve_deferred_rights_range_at(&vm->mem.vm_vspace, (void *)addr, 355 size, 1); 356 if (!vspace_reservation.res) { 357 ZF_LOGE("Failed to make anonymous memory region: Unable to create vspace reservation of size %zu", size); 358 return -1; 359 } 360 361 anon_region_t *region_data; 362 ps_io_ops_t *ops = vm->io_ops; 363 err = ps_calloc(&ops->malloc_ops, 1, sizeof(anon_region_t), (void **)®ion_data); 364 if (err) { 365 ZF_LOGE("Failed to make anonymous memory region : Unable to allocate anonymous region"); 366 return -1; 367 } 368 region_data->addr = addr; 369 region_data->size = size; 370 region_data->alloc_addr = addr; 371 region_data->vspace_reservation = vspace_reservation; 372 region_data->num_reservations = 0; 373 374 err = add_memory_reservation_node(vm, addr, size, MEM_ANON_RES, (void *)region_data); 375 if (err) { 376 ZF_LOGE("Failed to reserve vm memory: Unable to add vm memory reservation to list"); 377 vspace_free_reservation(&vm->mem.vm_vspace, vspace_reservation); 378 ps_free(&ops->malloc_ops, sizeof(anon_region_t), region_data); 379 return -1; 380 } 381 return 0; 382} 383 384vm_memory_reservation_t *vm_reserve_anon_memory(vm_t *vm, size_t size, 385 memory_fault_callback_fn fault_callback, void *cookie, uintptr_t *addr) 386{ 387 int err; 388 vm_memory_reservation_t *new_reservation; 389 anon_region_t *allocable_region; 390 uintptr_t reservation_addr; 391 392 if (!fault_callback) { 393 ZF_LOGE("Failed to reserve anon memory region: NULL fault callback"); 394 return NULL; 395 } 396 397 allocable_region = find_allocable_anon_region(vm, ROUND_UP(size, BIT(seL4_PageBits))); 398 if (!allocable_region) { 399 ZF_LOGE("Failed to reserve anon memory: No anonymous memory available to cater reservation size"); 400 return NULL; 401 } 402 403 reservation_addr = allocable_region->alloc_addr; 404 405 /* Make a sub-reservation token. */ 406 new_reservation = allocate_vm_reservation(vm, reservation_addr, size, allocable_region->vspace_reservation); 407 if (!new_reservation) { 408 ZF_LOGE("Failed to reserve vm memory: Unable to allocate new vm reservation"); 409 return NULL; 410 } 411 new_reservation->fault_callback = fault_callback; 412 new_reservation->fault_callback_cookie = cookie; 413 new_reservation->res_type = MEM_ANON_RES; 414 415 /* Register the sub-reservation token into the region - It will need to iterate over it on faults */ 416 vm_memory_reservation_t **extended_reservations = realloc(allocable_region->reservations, 417 sizeof(vm_memory_reservation_t *) * (allocable_region->num_reservations + 1)); 418 if (!extended_reservations) { 419 free_vm_reservation(vm, new_reservation); 420 return NULL; 421 } 422 allocable_region->reservations = extended_reservations; 423 424 allocable_region->reservations[allocable_region->num_reservations] = new_reservation; 425 allocable_region->alloc_addr += ROUND_UP(size, BIT(seL4_PageBits)); 426 allocable_region->num_reservations += 1; 427 428 *addr = reservation_addr; 429 return new_reservation; 430} 431 432int vm_free_reserved_memory(vm_t *vm, vm_memory_reservation_t *reservation) 433{ 434 ps_io_ops_t *ops = vm->io_ops; 435 if (!reservation) { 436 ZF_LOGE("Failed to free reserved memory: Invalid reservation"); 437 return -1; 438 } 439 440 if (reservation->res_type == MEM_ANON_RES) { 441 /* We current don't support free'ing anonymous reservations */ 442 /* TODO: Support freeing anonymous memory reservations */ 443 ZF_LOGE("Failed to free reserved memory: Cannot free anonymous memory reservations"); 444 return -1; 445 } 446 447 remove_memory_reservation_node(vm, reservation->addr, reservation->size, reservation->res_type); 448 if (reservation->is_mapped) { 449 int page_size = seL4_PageBits; 450 int num_pages = ROUND_UP(reservation->size, BIT(page_size)) >> page_size; 451 vspace_unmap_pages(&vm->mem.vm_vspace, (void *)reservation->addr, num_pages, page_size, vm->vka); 452 } 453 vspace_free_reservation(&vm->mem.vm_vspace, reservation->vspace_reservation); 454 free_vm_reservation(vm, reservation); 455 return 0; 456} 457 458int map_vm_memory_reservation(vm_t *vm, vm_memory_reservation_t *vm_reservation, 459 memory_map_iterator_fn map_iterator, void *map_cookie) 460{ 461 int err; 462 uintptr_t reservation_addr = vm_reservation->addr; 463 size_t reservation_size = vm_reservation->size; 464 uintptr_t current_addr = vm_reservation->addr; 465 466 while (current_addr < reservation_addr + reservation_size) { 467 vm_frame_t reservation_frame = map_iterator(current_addr, map_cookie); 468 if (reservation_frame.cptr == seL4_CapNull) { 469 ZF_LOGE("Failed to get frame for reservation address 0x%lx", current_addr); 470 break; 471 } 472 int ret = vspace_deferred_rights_map_pages_at_vaddr(&vm->mem.vm_vspace, &reservation_frame.cptr, NULL, 473 (void *)reservation_frame.vaddr, 1, reservation_frame.size_bits, 474 reservation_frame.rights, vm_reservation->vspace_reservation); 475 if (ret) { 476 ZF_LOGE("Failed to map address 0x%x into guest vm vspace", reservation_frame.vaddr); 477 return -1; 478 } 479 current_addr += BIT(reservation_frame.size_bits); 480 } 481 vm_reservation->memory_map_iterator = NULL; 482 vm_reservation->memory_iterator_cookie = NULL; 483 vm_reservation->is_mapped = true; 484 return 0; 485} 486 487int vm_map_reservation(vm_t *vm, vm_memory_reservation_t *reservation, 488 memory_map_iterator_fn map_iterator, void *cookie) 489{ 490 int err; 491 if (!vm) { 492 ZF_LOGE("Failed to map vm reservation: Invalid NULL VM handle given"); 493 return -1; 494 } else if (!reservation) { 495 ZF_LOGE("Failed to map vm reservation: Invalid NULL reservation given"); 496 return -1; 497 } else if (!map_iterator) { 498 ZF_LOGE("Failed to map vm reservation: Invalid map iterator given"); 499 return -1; 500 } 501 502 reservation->memory_map_iterator = map_iterator; 503 reservation->memory_iterator_cookie = cookie; 504 if (!config_set(CONFIG_LIB_SEL4VM_DEFER_MEMORY_MAP)) { 505 err = map_vm_memory_reservation(vm, reservation, map_iterator, cookie); 506 /* We remove the iterator after attempting the mapping (regardless of success or fail) 507 * If failed its left to the caller to update the memory map iterator */ 508 if (err) { 509 ZF_LOGE("Failed to map vm reservation: Error when mapping into VM's vspace"); 510 return -1; 511 } 512 } 513 514 return 0; 515} 516 517void vm_get_reservation_memory_region(vm_memory_reservation_t *reservation, uintptr_t *addr, size_t *size) 518{ 519 *addr = reservation->addr; 520 *size = reservation->size; 521} 522 523int vm_memory_init(vm_t *vm) 524{ 525 ps_io_ops_t *ops = vm->io_ops; 526 vm_memory_reservation_cookie_t *cookie; 527 int err = ps_calloc(&ops->malloc_ops, 1, sizeof(vm_memory_reservation_cookie_t), (void **)&cookie); 528 if (err) { 529 ZF_LOGE("Failed to initialise vm memory backend: Unable to allocate vm memory reservation cookie"); 530 return -1; 531 } 532 vm->mem.reservation_cookie = cookie; 533 return 0; 534} 535