1// SPDX-License-Identifier: MIT 2/* 3 * Copyright �� 2022 Intel Corporation 4 */ 5 6#include "xe_gt_pagefault.h" 7 8#include <linux/bitfield.h> 9#include <linux/circ_buf.h> 10 11#include <drm/drm_exec.h> 12#include <drm/drm_managed.h> 13#include <drm/ttm/ttm_execbuf_util.h> 14 15#include "abi/guc_actions_abi.h" 16#include "xe_bo.h" 17#include "xe_gt.h" 18#include "xe_gt_tlb_invalidation.h" 19#include "xe_guc.h" 20#include "xe_guc_ct.h" 21#include "xe_migrate.h" 22#include "xe_pt.h" 23#include "xe_trace.h" 24#include "xe_vm.h" 25 26struct pagefault { 27 u64 page_addr; 28 u32 asid; 29 u16 pdata; 30 u8 vfid; 31 u8 access_type; 32 u8 fault_type; 33 u8 fault_level; 34 u8 engine_class; 35 u8 engine_instance; 36 u8 fault_unsuccessful; 37 bool trva_fault; 38}; 39 40enum access_type { 41 ACCESS_TYPE_READ = 0, 42 ACCESS_TYPE_WRITE = 1, 43 ACCESS_TYPE_ATOMIC = 2, 44 ACCESS_TYPE_RESERVED = 3, 45}; 46 47enum fault_type { 48 NOT_PRESENT = 0, 49 WRITE_ACCESS_VIOLATION = 1, 50 ATOMIC_ACCESS_VIOLATION = 2, 51}; 52 53struct acc { 54 u64 va_range_base; 55 u32 asid; 56 u32 sub_granularity; 57 u8 granularity; 58 u8 vfid; 59 u8 access_type; 60 u8 engine_class; 61 u8 engine_instance; 62}; 63 64static bool access_is_atomic(enum access_type access_type) 65{ 66 return access_type == ACCESS_TYPE_ATOMIC; 67} 68 69static bool vma_is_valid(struct xe_tile *tile, struct xe_vma *vma) 70{ 71 return BIT(tile->id) & vma->tile_present && 72 !(BIT(tile->id) & vma->tile_invalidated); 73} 74 75static bool vma_matches(struct xe_vma *vma, u64 page_addr) 76{ 77 if (page_addr > xe_vma_end(vma) - 1 || 78 page_addr + SZ_4K - 1 < xe_vma_start(vma)) 79 return false; 80 81 return true; 82} 83 84static struct xe_vma *lookup_vma(struct xe_vm *vm, u64 page_addr) 85{ 86 struct xe_vma *vma = NULL; 87 88 if (vm->usm.last_fault_vma) { /* Fast lookup */ 89 if (vma_matches(vm->usm.last_fault_vma, page_addr)) 90 vma = vm->usm.last_fault_vma; 91 } 92 if (!vma) 93 vma = xe_vm_find_overlapping_vma(vm, page_addr, SZ_4K); 94 95 return vma; 96} 97 98static int xe_pf_begin(struct drm_exec *exec, struct xe_vma *vma, 99 bool atomic, unsigned int id) 100{ 101 struct xe_bo *bo = xe_vma_bo(vma); 102 struct xe_vm *vm = xe_vma_vm(vma); 103 int err; 104 105 err = xe_vm_lock_vma(exec, vma); 106 if (err) 107 return err; 108 109 if (atomic && IS_DGFX(vm->xe)) { 110 if (xe_vma_is_userptr(vma)) { 111 err = -EACCES; 112 return err; 113 } 114 115 /* Migrate to VRAM, move should invalidate the VMA first */ 116 err = xe_bo_migrate(bo, XE_PL_VRAM0 + id); 117 if (err) 118 return err; 119 } else if (bo) { 120 /* Create backing store if needed */ 121 err = xe_bo_validate(bo, vm, true); 122 if (err) 123 return err; 124 } 125 126 return 0; 127} 128 129static int handle_pagefault(struct xe_gt *gt, struct pagefault *pf) 130{ 131 struct xe_device *xe = gt_to_xe(gt); 132 struct xe_tile *tile = gt_to_tile(gt); 133 struct drm_exec exec; 134 struct xe_vm *vm; 135 struct xe_vma *vma = NULL; 136 struct dma_fence *fence; 137 bool write_locked; 138 int ret = 0; 139 bool atomic; 140 141 /* SW isn't expected to handle TRTT faults */ 142 if (pf->trva_fault) 143 return -EFAULT; 144 145 /* ASID to VM */ 146 mutex_lock(&xe->usm.lock); 147 vm = xa_load(&xe->usm.asid_to_vm, pf->asid); 148 if (vm && xe_vm_in_fault_mode(vm)) 149 xe_vm_get(vm); 150 else 151 vm = NULL; 152 mutex_unlock(&xe->usm.lock); 153 if (!vm) 154 return -EINVAL; 155 156retry_userptr: 157 /* 158 * TODO: Avoid exclusive lock if VM doesn't have userptrs, or 159 * start out read-locked? 160 */ 161 down_write(&vm->lock); 162 write_locked = true; 163 vma = lookup_vma(vm, pf->page_addr); 164 if (!vma) { 165 ret = -EINVAL; 166 goto unlock_vm; 167 } 168 169 if (!xe_vma_is_userptr(vma) || 170 !xe_vma_userptr_check_repin(to_userptr_vma(vma))) { 171 downgrade_write(&vm->lock); 172 write_locked = false; 173 } 174 175 trace_xe_vma_pagefault(vma); 176 177 atomic = access_is_atomic(pf->access_type); 178 179 /* Check if VMA is valid */ 180 if (vma_is_valid(tile, vma) && !atomic) 181 goto unlock_vm; 182 183 /* TODO: Validate fault */ 184 185 if (xe_vma_is_userptr(vma) && write_locked) { 186 struct xe_userptr_vma *uvma = to_userptr_vma(vma); 187 188 spin_lock(&vm->userptr.invalidated_lock); 189 list_del_init(&uvma->userptr.invalidate_link); 190 spin_unlock(&vm->userptr.invalidated_lock); 191 192 ret = xe_vma_userptr_pin_pages(uvma); 193 if (ret) 194 goto unlock_vm; 195 196 downgrade_write(&vm->lock); 197 write_locked = false; 198 } 199 200 /* Lock VM and BOs dma-resv */ 201 drm_exec_init(&exec, 0, 0); 202 drm_exec_until_all_locked(&exec) { 203 ret = xe_pf_begin(&exec, vma, atomic, tile->id); 204 drm_exec_retry_on_contention(&exec); 205 if (ret) 206 goto unlock_dma_resv; 207 } 208 209 /* Bind VMA only to the GT that has faulted */ 210 trace_xe_vma_pf_bind(vma); 211 fence = __xe_pt_bind_vma(tile, vma, xe_tile_migrate_engine(tile), NULL, 0, 212 vma->tile_present & BIT(tile->id)); 213 if (IS_ERR(fence)) { 214 ret = PTR_ERR(fence); 215 goto unlock_dma_resv; 216 } 217 218 /* 219 * XXX: Should we drop the lock before waiting? This only helps if doing 220 * GPU binds which is currently only done if we have to wait for more 221 * than 10ms on a move. 222 */ 223 dma_fence_wait(fence, false); 224 dma_fence_put(fence); 225 226 if (xe_vma_is_userptr(vma)) 227 ret = xe_vma_userptr_check_repin(to_userptr_vma(vma)); 228 vma->tile_invalidated &= ~BIT(tile->id); 229 230unlock_dma_resv: 231 drm_exec_fini(&exec); 232unlock_vm: 233 if (!ret) 234 vm->usm.last_fault_vma = vma; 235 if (write_locked) 236 up_write(&vm->lock); 237 else 238 up_read(&vm->lock); 239 if (ret == -EAGAIN) 240 goto retry_userptr; 241 242 if (!ret) { 243 ret = xe_gt_tlb_invalidation_vma(gt, NULL, vma); 244 if (ret >= 0) 245 ret = 0; 246 } 247 xe_vm_put(vm); 248 249 return ret; 250} 251 252static int send_pagefault_reply(struct xe_guc *guc, 253 struct xe_guc_pagefault_reply *reply) 254{ 255 u32 action[] = { 256 XE_GUC_ACTION_PAGE_FAULT_RES_DESC, 257 reply->dw0, 258 reply->dw1, 259 }; 260 261 return xe_guc_ct_send(&guc->ct, action, ARRAY_SIZE(action), 0, 0); 262} 263 264static void print_pagefault(struct xe_device *xe, struct pagefault *pf) 265{ 266 drm_dbg(&xe->drm, "\n\tASID: %d\n" 267 "\tVFID: %d\n" 268 "\tPDATA: 0x%04x\n" 269 "\tFaulted Address: 0x%08x%08x\n" 270 "\tFaultType: %d\n" 271 "\tAccessType: %d\n" 272 "\tFaultLevel: %d\n" 273 "\tEngineClass: %d\n" 274 "\tEngineInstance: %d\n", 275 pf->asid, pf->vfid, pf->pdata, upper_32_bits(pf->page_addr), 276 lower_32_bits(pf->page_addr), 277 pf->fault_type, pf->access_type, pf->fault_level, 278 pf->engine_class, pf->engine_instance); 279} 280 281#define PF_MSG_LEN_DW 4 282 283static bool get_pagefault(struct pf_queue *pf_queue, struct pagefault *pf) 284{ 285 const struct xe_guc_pagefault_desc *desc; 286 bool ret = false; 287 288 spin_lock_irq(&pf_queue->lock); 289 if (pf_queue->tail != pf_queue->head) { 290 desc = (const struct xe_guc_pagefault_desc *) 291 (pf_queue->data + pf_queue->tail); 292 293 pf->fault_level = FIELD_GET(PFD_FAULT_LEVEL, desc->dw0); 294 pf->trva_fault = FIELD_GET(XE2_PFD_TRVA_FAULT, desc->dw0); 295 pf->engine_class = FIELD_GET(PFD_ENG_CLASS, desc->dw0); 296 pf->engine_instance = FIELD_GET(PFD_ENG_INSTANCE, desc->dw0); 297 pf->pdata = FIELD_GET(PFD_PDATA_HI, desc->dw1) << 298 PFD_PDATA_HI_SHIFT; 299 pf->pdata |= FIELD_GET(PFD_PDATA_LO, desc->dw0); 300 pf->asid = FIELD_GET(PFD_ASID, desc->dw1); 301 pf->vfid = FIELD_GET(PFD_VFID, desc->dw2); 302 pf->access_type = FIELD_GET(PFD_ACCESS_TYPE, desc->dw2); 303 pf->fault_type = FIELD_GET(PFD_FAULT_TYPE, desc->dw2); 304 pf->page_addr = (u64)(FIELD_GET(PFD_VIRTUAL_ADDR_HI, desc->dw3)) << 305 PFD_VIRTUAL_ADDR_HI_SHIFT; 306 pf->page_addr |= FIELD_GET(PFD_VIRTUAL_ADDR_LO, desc->dw2) << 307 PFD_VIRTUAL_ADDR_LO_SHIFT; 308 309 pf_queue->tail = (pf_queue->tail + PF_MSG_LEN_DW) % 310 PF_QUEUE_NUM_DW; 311 ret = true; 312 } 313 spin_unlock_irq(&pf_queue->lock); 314 315 return ret; 316} 317 318static bool pf_queue_full(struct pf_queue *pf_queue) 319{ 320 lockdep_assert_held(&pf_queue->lock); 321 322 return CIRC_SPACE(pf_queue->head, pf_queue->tail, PF_QUEUE_NUM_DW) <= 323 PF_MSG_LEN_DW; 324} 325 326int xe_guc_pagefault_handler(struct xe_guc *guc, u32 *msg, u32 len) 327{ 328 struct xe_gt *gt = guc_to_gt(guc); 329 struct xe_device *xe = gt_to_xe(gt); 330 struct pf_queue *pf_queue; 331 unsigned long flags; 332 u32 asid; 333 bool full; 334 335 /* 336 * The below logic doesn't work unless PF_QUEUE_NUM_DW % PF_MSG_LEN_DW == 0 337 */ 338 BUILD_BUG_ON(PF_QUEUE_NUM_DW % PF_MSG_LEN_DW); 339 340 if (unlikely(len != PF_MSG_LEN_DW)) 341 return -EPROTO; 342 343 asid = FIELD_GET(PFD_ASID, msg[1]); 344 pf_queue = gt->usm.pf_queue + (asid % NUM_PF_QUEUE); 345 346 spin_lock_irqsave(&pf_queue->lock, flags); 347 full = pf_queue_full(pf_queue); 348 if (!full) { 349 memcpy(pf_queue->data + pf_queue->head, msg, len * sizeof(u32)); 350 pf_queue->head = (pf_queue->head + len) % PF_QUEUE_NUM_DW; 351 queue_work(gt->usm.pf_wq, &pf_queue->worker); 352 } else { 353 drm_warn(&xe->drm, "PF Queue full, shouldn't be possible"); 354 } 355 spin_unlock_irqrestore(&pf_queue->lock, flags); 356 357 return full ? -ENOSPC : 0; 358} 359 360#define USM_QUEUE_MAX_RUNTIME_MS 20 361 362static void pf_queue_work_func(struct work_struct *w) 363{ 364 struct pf_queue *pf_queue = container_of(w, struct pf_queue, worker); 365 struct xe_gt *gt = pf_queue->gt; 366 struct xe_device *xe = gt_to_xe(gt); 367 struct xe_guc_pagefault_reply reply = {}; 368 struct pagefault pf = {}; 369 unsigned long threshold; 370 int ret; 371 372 threshold = jiffies + msecs_to_jiffies(USM_QUEUE_MAX_RUNTIME_MS); 373 374 while (get_pagefault(pf_queue, &pf)) { 375 ret = handle_pagefault(gt, &pf); 376 if (unlikely(ret)) { 377 print_pagefault(xe, &pf); 378 pf.fault_unsuccessful = 1; 379 drm_dbg(&xe->drm, "Fault response: Unsuccessful %d\n", ret); 380 } 381 382 reply.dw0 = FIELD_PREP(PFR_VALID, 1) | 383 FIELD_PREP(PFR_SUCCESS, pf.fault_unsuccessful) | 384 FIELD_PREP(PFR_REPLY, PFR_ACCESS) | 385 FIELD_PREP(PFR_DESC_TYPE, FAULT_RESPONSE_DESC) | 386 FIELD_PREP(PFR_ASID, pf.asid); 387 388 reply.dw1 = FIELD_PREP(PFR_VFID, pf.vfid) | 389 FIELD_PREP(PFR_ENG_INSTANCE, pf.engine_instance) | 390 FIELD_PREP(PFR_ENG_CLASS, pf.engine_class) | 391 FIELD_PREP(PFR_PDATA, pf.pdata); 392 393 send_pagefault_reply(>->uc.guc, &reply); 394 395 if (time_after(jiffies, threshold) && 396 pf_queue->tail != pf_queue->head) { 397 queue_work(gt->usm.pf_wq, w); 398 break; 399 } 400 } 401} 402 403static void acc_queue_work_func(struct work_struct *w); 404 405int xe_gt_pagefault_init(struct xe_gt *gt) 406{ 407 struct xe_device *xe = gt_to_xe(gt); 408 int i; 409 410 if (!xe->info.has_usm) 411 return 0; 412 413 for (i = 0; i < NUM_PF_QUEUE; ++i) { 414 gt->usm.pf_queue[i].gt = gt; 415 spin_lock_init(>->usm.pf_queue[i].lock); 416 INIT_WORK(>->usm.pf_queue[i].worker, pf_queue_work_func); 417 } 418 for (i = 0; i < NUM_ACC_QUEUE; ++i) { 419 gt->usm.acc_queue[i].gt = gt; 420 spin_lock_init(>->usm.acc_queue[i].lock); 421 INIT_WORK(>->usm.acc_queue[i].worker, acc_queue_work_func); 422 } 423 424 gt->usm.pf_wq = alloc_workqueue("xe_gt_page_fault_work_queue", 425 WQ_UNBOUND | WQ_HIGHPRI, NUM_PF_QUEUE); 426 if (!gt->usm.pf_wq) 427 return -ENOMEM; 428 429 gt->usm.acc_wq = alloc_workqueue("xe_gt_access_counter_work_queue", 430 WQ_UNBOUND | WQ_HIGHPRI, 431 NUM_ACC_QUEUE); 432 if (!gt->usm.acc_wq) 433 return -ENOMEM; 434 435 return 0; 436} 437 438void xe_gt_pagefault_reset(struct xe_gt *gt) 439{ 440 struct xe_device *xe = gt_to_xe(gt); 441 int i; 442 443 if (!xe->info.has_usm) 444 return; 445 446 for (i = 0; i < NUM_PF_QUEUE; ++i) { 447 spin_lock_irq(>->usm.pf_queue[i].lock); 448 gt->usm.pf_queue[i].head = 0; 449 gt->usm.pf_queue[i].tail = 0; 450 spin_unlock_irq(>->usm.pf_queue[i].lock); 451 } 452 453 for (i = 0; i < NUM_ACC_QUEUE; ++i) { 454 spin_lock(>->usm.acc_queue[i].lock); 455 gt->usm.acc_queue[i].head = 0; 456 gt->usm.acc_queue[i].tail = 0; 457 spin_unlock(>->usm.acc_queue[i].lock); 458 } 459} 460 461static int granularity_in_byte(int val) 462{ 463 switch (val) { 464 case 0: 465 return SZ_128K; 466 case 1: 467 return SZ_2M; 468 case 2: 469 return SZ_16M; 470 case 3: 471 return SZ_64M; 472 default: 473 return 0; 474 } 475} 476 477static int sub_granularity_in_byte(int val) 478{ 479 return (granularity_in_byte(val) / 32); 480} 481 482static void print_acc(struct xe_device *xe, struct acc *acc) 483{ 484 drm_warn(&xe->drm, "Access counter request:\n" 485 "\tType: %s\n" 486 "\tASID: %d\n" 487 "\tVFID: %d\n" 488 "\tEngine: %d:%d\n" 489 "\tGranularity: 0x%x KB Region/ %d KB sub-granularity\n" 490 "\tSub_Granularity Vector: 0x%08x\n" 491 "\tVA Range base: 0x%016llx\n", 492 acc->access_type ? "AC_NTFY_VAL" : "AC_TRIG_VAL", 493 acc->asid, acc->vfid, acc->engine_class, acc->engine_instance, 494 granularity_in_byte(acc->granularity) / SZ_1K, 495 sub_granularity_in_byte(acc->granularity) / SZ_1K, 496 acc->sub_granularity, acc->va_range_base); 497} 498 499static struct xe_vma *get_acc_vma(struct xe_vm *vm, struct acc *acc) 500{ 501 u64 page_va = acc->va_range_base + (ffs(acc->sub_granularity) - 1) * 502 sub_granularity_in_byte(acc->granularity); 503 504 return xe_vm_find_overlapping_vma(vm, page_va, SZ_4K); 505} 506 507static int handle_acc(struct xe_gt *gt, struct acc *acc) 508{ 509 struct xe_device *xe = gt_to_xe(gt); 510 struct xe_tile *tile = gt_to_tile(gt); 511 struct drm_exec exec; 512 struct xe_vm *vm; 513 struct xe_vma *vma; 514 int ret = 0; 515 516 /* We only support ACC_TRIGGER at the moment */ 517 if (acc->access_type != ACC_TRIGGER) 518 return -EINVAL; 519 520 /* ASID to VM */ 521 mutex_lock(&xe->usm.lock); 522 vm = xa_load(&xe->usm.asid_to_vm, acc->asid); 523 if (vm) 524 xe_vm_get(vm); 525 mutex_unlock(&xe->usm.lock); 526 if (!vm || !xe_vm_in_fault_mode(vm)) 527 return -EINVAL; 528 529 down_read(&vm->lock); 530 531 /* Lookup VMA */ 532 vma = get_acc_vma(vm, acc); 533 if (!vma) { 534 ret = -EINVAL; 535 goto unlock_vm; 536 } 537 538 trace_xe_vma_acc(vma); 539 540 /* Userptr or null can't be migrated, nothing to do */ 541 if (xe_vma_has_no_bo(vma)) 542 goto unlock_vm; 543 544 /* Lock VM and BOs dma-resv */ 545 drm_exec_init(&exec, 0, 0); 546 drm_exec_until_all_locked(&exec) { 547 ret = xe_pf_begin(&exec, vma, true, tile->id); 548 drm_exec_retry_on_contention(&exec); 549 if (ret) 550 break; 551 } 552 553 drm_exec_fini(&exec); 554unlock_vm: 555 up_read(&vm->lock); 556 xe_vm_put(vm); 557 558 return ret; 559} 560 561#define make_u64(hi__, low__) ((u64)(hi__) << 32 | (u64)(low__)) 562 563#define ACC_MSG_LEN_DW 4 564 565static bool get_acc(struct acc_queue *acc_queue, struct acc *acc) 566{ 567 const struct xe_guc_acc_desc *desc; 568 bool ret = false; 569 570 spin_lock(&acc_queue->lock); 571 if (acc_queue->tail != acc_queue->head) { 572 desc = (const struct xe_guc_acc_desc *) 573 (acc_queue->data + acc_queue->tail); 574 575 acc->granularity = FIELD_GET(ACC_GRANULARITY, desc->dw2); 576 acc->sub_granularity = FIELD_GET(ACC_SUBG_HI, desc->dw1) << 31 | 577 FIELD_GET(ACC_SUBG_LO, desc->dw0); 578 acc->engine_class = FIELD_GET(ACC_ENG_CLASS, desc->dw1); 579 acc->engine_instance = FIELD_GET(ACC_ENG_INSTANCE, desc->dw1); 580 acc->asid = FIELD_GET(ACC_ASID, desc->dw1); 581 acc->vfid = FIELD_GET(ACC_VFID, desc->dw2); 582 acc->access_type = FIELD_GET(ACC_TYPE, desc->dw0); 583 acc->va_range_base = make_u64(desc->dw3 & ACC_VIRTUAL_ADDR_RANGE_HI, 584 desc->dw2 & ACC_VIRTUAL_ADDR_RANGE_LO); 585 586 acc_queue->tail = (acc_queue->tail + ACC_MSG_LEN_DW) % 587 ACC_QUEUE_NUM_DW; 588 ret = true; 589 } 590 spin_unlock(&acc_queue->lock); 591 592 return ret; 593} 594 595static void acc_queue_work_func(struct work_struct *w) 596{ 597 struct acc_queue *acc_queue = container_of(w, struct acc_queue, worker); 598 struct xe_gt *gt = acc_queue->gt; 599 struct xe_device *xe = gt_to_xe(gt); 600 struct acc acc = {}; 601 unsigned long threshold; 602 int ret; 603 604 threshold = jiffies + msecs_to_jiffies(USM_QUEUE_MAX_RUNTIME_MS); 605 606 while (get_acc(acc_queue, &acc)) { 607 ret = handle_acc(gt, &acc); 608 if (unlikely(ret)) { 609 print_acc(xe, &acc); 610 drm_warn(&xe->drm, "ACC: Unsuccessful %d\n", ret); 611 } 612 613 if (time_after(jiffies, threshold) && 614 acc_queue->tail != acc_queue->head) { 615 queue_work(gt->usm.acc_wq, w); 616 break; 617 } 618 } 619} 620 621static bool acc_queue_full(struct acc_queue *acc_queue) 622{ 623 lockdep_assert_held(&acc_queue->lock); 624 625 return CIRC_SPACE(acc_queue->head, acc_queue->tail, ACC_QUEUE_NUM_DW) <= 626 ACC_MSG_LEN_DW; 627} 628 629int xe_guc_access_counter_notify_handler(struct xe_guc *guc, u32 *msg, u32 len) 630{ 631 struct xe_gt *gt = guc_to_gt(guc); 632 struct acc_queue *acc_queue; 633 u32 asid; 634 bool full; 635 636 /* 637 * The below logic doesn't work unless ACC_QUEUE_NUM_DW % ACC_MSG_LEN_DW == 0 638 */ 639 BUILD_BUG_ON(ACC_QUEUE_NUM_DW % ACC_MSG_LEN_DW); 640 641 if (unlikely(len != ACC_MSG_LEN_DW)) 642 return -EPROTO; 643 644 asid = FIELD_GET(ACC_ASID, msg[1]); 645 acc_queue = >->usm.acc_queue[asid % NUM_ACC_QUEUE]; 646 647 spin_lock(&acc_queue->lock); 648 full = acc_queue_full(acc_queue); 649 if (!full) { 650 memcpy(acc_queue->data + acc_queue->head, msg, 651 len * sizeof(u32)); 652 acc_queue->head = (acc_queue->head + len) % ACC_QUEUE_NUM_DW; 653 queue_work(gt->usm.acc_wq, &acc_queue->worker); 654 } else { 655 drm_warn(>_to_xe(gt)->drm, "ACC Queue full, dropping ACC"); 656 } 657 spin_unlock(&acc_queue->lock); 658 659 return full ? -ENOSPC : 0; 660} 661