1// SPDX-License-Identifier: MIT 2/* 3 * Copyright �� 2016-2019 Intel Corporation 4 */ 5 6#include <linux/circ_buf.h> 7#include <linux/ktime.h> 8#include <linux/time64.h> 9#include <linux/string_helpers.h> 10#include <linux/timekeeping.h> 11 12#include "i915_drv.h" 13#include "intel_guc_ct.h" 14#include "intel_guc_print.h" 15 16#if IS_ENABLED(CONFIG_DRM_I915_DEBUG_GUC) 17enum { 18 CT_DEAD_ALIVE = 0, 19 CT_DEAD_SETUP, 20 CT_DEAD_WRITE, 21 CT_DEAD_DEADLOCK, 22 CT_DEAD_H2G_HAS_ROOM, 23 CT_DEAD_READ, 24 CT_DEAD_PROCESS_FAILED, 25}; 26 27static void ct_dead_ct_worker_func(struct work_struct *w); 28 29#define CT_DEAD(ct, reason) \ 30 do { \ 31 if (!(ct)->dead_ct_reported) { \ 32 (ct)->dead_ct_reason |= 1 << CT_DEAD_##reason; \ 33 queue_work(system_unbound_wq, &(ct)->dead_ct_worker); \ 34 } \ 35 } while (0) 36#else 37#define CT_DEAD(ct, reason) do { } while (0) 38#endif 39 40static inline struct intel_guc *ct_to_guc(struct intel_guc_ct *ct) 41{ 42 return container_of(ct, struct intel_guc, ct); 43} 44 45#define CT_ERROR(_ct, _fmt, ...) \ 46 guc_err(ct_to_guc(_ct), "CT: " _fmt, ##__VA_ARGS__) 47#ifdef CONFIG_DRM_I915_DEBUG_GUC 48#define CT_DEBUG(_ct, _fmt, ...) \ 49 guc_dbg(ct_to_guc(_ct), "CT: " _fmt, ##__VA_ARGS__) 50#else 51#define CT_DEBUG(...) do { } while (0) 52#endif 53#define CT_PROBE_ERROR(_ct, _fmt, ...) \ 54 guc_probe_error(ct_to_guc(ct), "CT: " _fmt, ##__VA_ARGS__) 55 56/** 57 * DOC: CTB Blob 58 * 59 * We allocate single blob to hold both CTB descriptors and buffers: 60 * 61 * +--------+-----------------------------------------------+------+ 62 * | offset | contents | size | 63 * +========+===============================================+======+ 64 * | 0x0000 | H2G `CTB Descriptor`_ (send) | | 65 * +--------+-----------------------------------------------+ 4K | 66 * | 0x0800 | G2H `CTB Descriptor`_ (recv) | | 67 * +--------+-----------------------------------------------+------+ 68 * | 0x1000 | H2G `CT Buffer`_ (send) | n*4K | 69 * | | | | 70 * +--------+-----------------------------------------------+------+ 71 * | 0x1000 | G2H `CT Buffer`_ (recv) | m*4K | 72 * | + n*4K | | | 73 * +--------+-----------------------------------------------+------+ 74 * 75 * Size of each `CT Buffer`_ must be multiple of 4K. 76 * We don't expect too many messages in flight at any time, unless we are 77 * using the GuC submission. In that case each request requires a minimum 78 * 2 dwords which gives us a maximum 256 queue'd requests. Hopefully this 79 * enough space to avoid backpressure on the driver. We increase the size 80 * of the receive buffer (relative to the send) to ensure a G2H response 81 * CTB has a landing spot. 82 */ 83#define CTB_DESC_SIZE ALIGN(sizeof(struct guc_ct_buffer_desc), SZ_2K) 84#define CTB_H2G_BUFFER_SIZE (SZ_4K) 85#define CTB_G2H_BUFFER_SIZE (4 * CTB_H2G_BUFFER_SIZE) 86#define G2H_ROOM_BUFFER_SIZE (CTB_G2H_BUFFER_SIZE / 4) 87 88struct ct_request { 89 struct list_head link; 90 u32 fence; 91 u32 status; 92 u32 response_len; 93 u32 *response_buf; 94}; 95 96struct ct_incoming_msg { 97 struct list_head link; 98 u32 size; 99 u32 msg[] __counted_by(size); 100}; 101 102enum { CTB_SEND = 0, CTB_RECV = 1 }; 103 104enum { CTB_OWNER_HOST = 0 }; 105 106/* 107 * Some H2G commands involve a synchronous response that the driver needs 108 * to wait for. In such cases, a timeout is required to prevent the driver 109 * from waiting forever in the case of an error (either no error response 110 * is defined in the protocol or something has died and requires a reset). 111 * The specific command may be defined as having a time bound response but 112 * the CT is a queue and that time guarantee only starts from the point 113 * when the command reaches the head of the queue and is processed by GuC. 114 * 115 * Ideally there would be a helper to report the progress of a given 116 * command through the CT. However, that would require a significant 117 * amount of work in the CT layer. In the meantime, provide a reasonable 118 * estimation of the worst case latency it should take for the entire 119 * queue to drain. And therefore, how long a caller should wait before 120 * giving up on their request. The current estimate is based on empirical 121 * measurement of a test that fills the buffer with context creation and 122 * destruction requests as they seem to be the slowest operation. 123 */ 124long intel_guc_ct_max_queue_time_jiffies(void) 125{ 126 /* 127 * A 4KB buffer full of context destroy commands takes a little 128 * over a second to process so bump that to 2s to be super safe. 129 */ 130 return (CTB_H2G_BUFFER_SIZE * HZ) / SZ_2K; 131} 132 133static void ct_receive_tasklet_func(struct tasklet_struct *t); 134static void ct_incoming_request_worker_func(struct work_struct *w); 135 136/** 137 * intel_guc_ct_init_early - Initialize CT state without requiring device access 138 * @ct: pointer to CT struct 139 */ 140void intel_guc_ct_init_early(struct intel_guc_ct *ct) 141{ 142 spin_lock_init(&ct->ctbs.send.lock); 143 spin_lock_init(&ct->ctbs.recv.lock); 144 spin_lock_init(&ct->requests.lock); 145 INIT_LIST_HEAD(&ct->requests.pending); 146 INIT_LIST_HEAD(&ct->requests.incoming); 147#if IS_ENABLED(CONFIG_DRM_I915_DEBUG_GUC) 148 INIT_WORK(&ct->dead_ct_worker, ct_dead_ct_worker_func); 149#endif 150 INIT_WORK(&ct->requests.worker, ct_incoming_request_worker_func); 151 tasklet_setup(&ct->receive_tasklet, ct_receive_tasklet_func); 152 init_waitqueue_head(&ct->wq); 153} 154 155static void guc_ct_buffer_desc_init(struct guc_ct_buffer_desc *desc) 156{ 157 memset(desc, 0, sizeof(*desc)); 158} 159 160static void guc_ct_buffer_reset(struct intel_guc_ct_buffer *ctb) 161{ 162 u32 space; 163 164 ctb->broken = false; 165 ctb->tail = 0; 166 ctb->head = 0; 167 space = CIRC_SPACE(ctb->tail, ctb->head, ctb->size) - ctb->resv_space; 168 atomic_set(&ctb->space, space); 169 170 guc_ct_buffer_desc_init(ctb->desc); 171} 172 173static void guc_ct_buffer_init(struct intel_guc_ct_buffer *ctb, 174 struct guc_ct_buffer_desc *desc, 175 u32 *cmds, u32 size_in_bytes, u32 resv_space) 176{ 177 GEM_BUG_ON(size_in_bytes % 4); 178 179 ctb->desc = desc; 180 ctb->cmds = cmds; 181 ctb->size = size_in_bytes / 4; 182 ctb->resv_space = resv_space / 4; 183 184 guc_ct_buffer_reset(ctb); 185} 186 187static int guc_action_control_ctb(struct intel_guc *guc, u32 control) 188{ 189 u32 request[HOST2GUC_CONTROL_CTB_REQUEST_MSG_LEN] = { 190 FIELD_PREP(GUC_HXG_MSG_0_ORIGIN, GUC_HXG_ORIGIN_HOST) | 191 FIELD_PREP(GUC_HXG_MSG_0_TYPE, GUC_HXG_TYPE_REQUEST) | 192 FIELD_PREP(GUC_HXG_REQUEST_MSG_0_ACTION, GUC_ACTION_HOST2GUC_CONTROL_CTB), 193 FIELD_PREP(HOST2GUC_CONTROL_CTB_REQUEST_MSG_1_CONTROL, control), 194 }; 195 int ret; 196 197 GEM_BUG_ON(control != GUC_CTB_CONTROL_DISABLE && control != GUC_CTB_CONTROL_ENABLE); 198 199 /* CT control must go over MMIO */ 200 ret = intel_guc_send_mmio(guc, request, ARRAY_SIZE(request), NULL, 0); 201 202 return ret > 0 ? -EPROTO : ret; 203} 204 205static int ct_control_enable(struct intel_guc_ct *ct, bool enable) 206{ 207 int err; 208 209 err = guc_action_control_ctb(ct_to_guc(ct), enable ? 210 GUC_CTB_CONTROL_ENABLE : GUC_CTB_CONTROL_DISABLE); 211 if (unlikely(err)) 212 CT_PROBE_ERROR(ct, "Failed to control/%s CTB (%pe)\n", 213 str_enable_disable(enable), ERR_PTR(err)); 214 215 return err; 216} 217 218static int ct_register_buffer(struct intel_guc_ct *ct, bool send, 219 u32 desc_addr, u32 buff_addr, u32 size) 220{ 221 int err; 222 223 err = intel_guc_self_cfg64(ct_to_guc(ct), send ? 224 GUC_KLV_SELF_CFG_H2G_CTB_DESCRIPTOR_ADDR_KEY : 225 GUC_KLV_SELF_CFG_G2H_CTB_DESCRIPTOR_ADDR_KEY, 226 desc_addr); 227 if (unlikely(err)) 228 goto failed; 229 230 err = intel_guc_self_cfg64(ct_to_guc(ct), send ? 231 GUC_KLV_SELF_CFG_H2G_CTB_ADDR_KEY : 232 GUC_KLV_SELF_CFG_G2H_CTB_ADDR_KEY, 233 buff_addr); 234 if (unlikely(err)) 235 goto failed; 236 237 err = intel_guc_self_cfg32(ct_to_guc(ct), send ? 238 GUC_KLV_SELF_CFG_H2G_CTB_SIZE_KEY : 239 GUC_KLV_SELF_CFG_G2H_CTB_SIZE_KEY, 240 size); 241 if (unlikely(err)) 242failed: 243 CT_PROBE_ERROR(ct, "Failed to register %s buffer (%pe)\n", 244 send ? "SEND" : "RECV", ERR_PTR(err)); 245 246 return err; 247} 248 249/** 250 * intel_guc_ct_init - Init buffer-based communication 251 * @ct: pointer to CT struct 252 * 253 * Allocate memory required for buffer-based communication. 254 * 255 * Return: 0 on success, a negative errno code on failure. 256 */ 257int intel_guc_ct_init(struct intel_guc_ct *ct) 258{ 259 struct intel_guc *guc = ct_to_guc(ct); 260 struct guc_ct_buffer_desc *desc; 261 u32 blob_size; 262 u32 cmds_size; 263 u32 resv_space; 264 void *blob; 265 u32 *cmds; 266 int err; 267 268 err = i915_inject_probe_error(guc_to_i915(guc), -ENXIO); 269 if (err) 270 return err; 271 272 GEM_BUG_ON(ct->vma); 273 274 blob_size = 2 * CTB_DESC_SIZE + CTB_H2G_BUFFER_SIZE + CTB_G2H_BUFFER_SIZE; 275 err = intel_guc_allocate_and_map_vma(guc, blob_size, &ct->vma, &blob); 276 if (unlikely(err)) { 277 CT_PROBE_ERROR(ct, "Failed to allocate %u for CTB data (%pe)\n", 278 blob_size, ERR_PTR(err)); 279 return err; 280 } 281 282 CT_DEBUG(ct, "base=%#x size=%u\n", intel_guc_ggtt_offset(guc, ct->vma), blob_size); 283 284 /* store pointers to desc and cmds for send ctb */ 285 desc = blob; 286 cmds = blob + 2 * CTB_DESC_SIZE; 287 cmds_size = CTB_H2G_BUFFER_SIZE; 288 resv_space = 0; 289 CT_DEBUG(ct, "%s desc %#tx cmds %#tx size %u/%u\n", "send", 290 ptrdiff(desc, blob), ptrdiff(cmds, blob), cmds_size, 291 resv_space); 292 293 guc_ct_buffer_init(&ct->ctbs.send, desc, cmds, cmds_size, resv_space); 294 295 /* store pointers to desc and cmds for recv ctb */ 296 desc = blob + CTB_DESC_SIZE; 297 cmds = blob + 2 * CTB_DESC_SIZE + CTB_H2G_BUFFER_SIZE; 298 cmds_size = CTB_G2H_BUFFER_SIZE; 299 resv_space = G2H_ROOM_BUFFER_SIZE; 300 CT_DEBUG(ct, "%s desc %#tx cmds %#tx size %u/%u\n", "recv", 301 ptrdiff(desc, blob), ptrdiff(cmds, blob), cmds_size, 302 resv_space); 303 304 guc_ct_buffer_init(&ct->ctbs.recv, desc, cmds, cmds_size, resv_space); 305 306 return 0; 307} 308 309/** 310 * intel_guc_ct_fini - Fini buffer-based communication 311 * @ct: pointer to CT struct 312 * 313 * Deallocate memory required for buffer-based communication. 314 */ 315void intel_guc_ct_fini(struct intel_guc_ct *ct) 316{ 317 GEM_BUG_ON(ct->enabled); 318 319 tasklet_kill(&ct->receive_tasklet); 320 i915_vma_unpin_and_release(&ct->vma, I915_VMA_RELEASE_MAP); 321 memset(ct, 0, sizeof(*ct)); 322} 323 324/** 325 * intel_guc_ct_enable - Enable buffer based command transport. 326 * @ct: pointer to CT struct 327 * 328 * Return: 0 on success, a negative errno code on failure. 329 */ 330int intel_guc_ct_enable(struct intel_guc_ct *ct) 331{ 332 struct intel_guc *guc = ct_to_guc(ct); 333 u32 base, desc, cmds, size; 334 void *blob; 335 int err; 336 337 GEM_BUG_ON(ct->enabled); 338 339 /* vma should be already allocated and map'ed */ 340 GEM_BUG_ON(!ct->vma); 341 GEM_BUG_ON(!i915_gem_object_has_pinned_pages(ct->vma->obj)); 342 base = intel_guc_ggtt_offset(guc, ct->vma); 343 344 /* blob should start with send descriptor */ 345 blob = __px_vaddr(ct->vma->obj); 346 GEM_BUG_ON(blob != ct->ctbs.send.desc); 347 348 /* (re)initialize descriptors */ 349 guc_ct_buffer_reset(&ct->ctbs.send); 350 guc_ct_buffer_reset(&ct->ctbs.recv); 351 352 /* 353 * Register both CT buffers starting with RECV buffer. 354 * Descriptors are in first half of the blob. 355 */ 356 desc = base + ptrdiff(ct->ctbs.recv.desc, blob); 357 cmds = base + ptrdiff(ct->ctbs.recv.cmds, blob); 358 size = ct->ctbs.recv.size * 4; 359 err = ct_register_buffer(ct, false, desc, cmds, size); 360 if (unlikely(err)) 361 goto err_out; 362 363 desc = base + ptrdiff(ct->ctbs.send.desc, blob); 364 cmds = base + ptrdiff(ct->ctbs.send.cmds, blob); 365 size = ct->ctbs.send.size * 4; 366 err = ct_register_buffer(ct, true, desc, cmds, size); 367 if (unlikely(err)) 368 goto err_out; 369 370 err = ct_control_enable(ct, true); 371 if (unlikely(err)) 372 goto err_out; 373 374 ct->enabled = true; 375 ct->stall_time = KTIME_MAX; 376#if IS_ENABLED(CONFIG_DRM_I915_DEBUG_GUC) 377 ct->dead_ct_reported = false; 378 ct->dead_ct_reason = CT_DEAD_ALIVE; 379#endif 380 381 return 0; 382 383err_out: 384 CT_PROBE_ERROR(ct, "Failed to enable CTB (%pe)\n", ERR_PTR(err)); 385 CT_DEAD(ct, SETUP); 386 return err; 387} 388 389/** 390 * intel_guc_ct_disable - Disable buffer based command transport. 391 * @ct: pointer to CT struct 392 */ 393void intel_guc_ct_disable(struct intel_guc_ct *ct) 394{ 395 struct intel_guc *guc = ct_to_guc(ct); 396 397 GEM_BUG_ON(!ct->enabled); 398 399 ct->enabled = false; 400 401 if (intel_guc_is_fw_running(guc)) { 402 ct_control_enable(ct, false); 403 } 404} 405 406#if IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM) 407static void ct_track_lost_and_found(struct intel_guc_ct *ct, u32 fence, u32 action) 408{ 409 unsigned int lost = fence % ARRAY_SIZE(ct->requests.lost_and_found); 410#if IS_ENABLED(CONFIG_DRM_I915_DEBUG_GUC) 411 unsigned long entries[SZ_32]; 412 unsigned int n; 413 414 n = stack_trace_save(entries, ARRAY_SIZE(entries), 1); 415 416 /* May be called under spinlock, so avoid sleeping */ 417 ct->requests.lost_and_found[lost].stack = stack_depot_save(entries, n, GFP_NOWAIT); 418#endif 419 ct->requests.lost_and_found[lost].fence = fence; 420 ct->requests.lost_and_found[lost].action = action; 421} 422#endif 423 424static u32 ct_get_next_fence(struct intel_guc_ct *ct) 425{ 426 /* For now it's trivial */ 427 return ++ct->requests.last_fence; 428} 429 430static int ct_write(struct intel_guc_ct *ct, 431 const u32 *action, 432 u32 len /* in dwords */, 433 u32 fence, u32 flags) 434{ 435 struct intel_guc_ct_buffer *ctb = &ct->ctbs.send; 436 struct guc_ct_buffer_desc *desc = ctb->desc; 437 u32 tail = ctb->tail; 438 u32 size = ctb->size; 439 u32 header; 440 u32 hxg; 441 u32 type; 442 u32 *cmds = ctb->cmds; 443 unsigned int i; 444 445 if (unlikely(desc->status)) 446 goto corrupted; 447 448 GEM_BUG_ON(tail > size); 449 450#ifdef CONFIG_DRM_I915_DEBUG_GUC 451 if (unlikely(tail != READ_ONCE(desc->tail))) { 452 CT_ERROR(ct, "Tail was modified %u != %u\n", 453 desc->tail, tail); 454 desc->status |= GUC_CTB_STATUS_MISMATCH; 455 goto corrupted; 456 } 457 if (unlikely(READ_ONCE(desc->head) >= size)) { 458 CT_ERROR(ct, "Invalid head offset %u >= %u)\n", 459 desc->head, size); 460 desc->status |= GUC_CTB_STATUS_OVERFLOW; 461 goto corrupted; 462 } 463#endif 464 465 /* 466 * dw0: CT header (including fence) 467 * dw1: HXG header (including action code) 468 * dw2+: action data 469 */ 470 header = FIELD_PREP(GUC_CTB_MSG_0_FORMAT, GUC_CTB_FORMAT_HXG) | 471 FIELD_PREP(GUC_CTB_MSG_0_NUM_DWORDS, len) | 472 FIELD_PREP(GUC_CTB_MSG_0_FENCE, fence); 473 474 type = (flags & INTEL_GUC_CT_SEND_NB) ? GUC_HXG_TYPE_FAST_REQUEST : 475 GUC_HXG_TYPE_REQUEST; 476 hxg = FIELD_PREP(GUC_HXG_MSG_0_TYPE, type) | 477 FIELD_PREP(GUC_HXG_REQUEST_MSG_0_ACTION | 478 GUC_HXG_REQUEST_MSG_0_DATA0, action[0]); 479 480 CT_DEBUG(ct, "writing (tail %u) %*ph %*ph %*ph\n", 481 tail, 4, &header, 4, &hxg, 4 * (len - 1), &action[1]); 482 483 cmds[tail] = header; 484 tail = (tail + 1) % size; 485 486 cmds[tail] = hxg; 487 tail = (tail + 1) % size; 488 489 for (i = 1; i < len; i++) { 490 cmds[tail] = action[i]; 491 tail = (tail + 1) % size; 492 } 493 GEM_BUG_ON(tail > size); 494 495#if IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM) 496 ct_track_lost_and_found(ct, fence, 497 FIELD_GET(GUC_HXG_EVENT_MSG_0_ACTION, action[0])); 498#endif 499 500 /* 501 * make sure H2G buffer update and LRC tail update (if this triggering a 502 * submission) are visible before updating the descriptor tail 503 */ 504 intel_guc_write_barrier(ct_to_guc(ct)); 505 506 /* update local copies */ 507 ctb->tail = tail; 508 GEM_BUG_ON(atomic_read(&ctb->space) < len + GUC_CTB_HDR_LEN); 509 atomic_sub(len + GUC_CTB_HDR_LEN, &ctb->space); 510 511 /* now update descriptor */ 512 WRITE_ONCE(desc->tail, tail); 513 514 return 0; 515 516corrupted: 517 CT_ERROR(ct, "Corrupted descriptor head=%u tail=%u status=%#x\n", 518 desc->head, desc->tail, desc->status); 519 CT_DEAD(ct, WRITE); 520 ctb->broken = true; 521 return -EPIPE; 522} 523 524/** 525 * wait_for_ct_request_update - Wait for CT request state update. 526 * @ct: pointer to CT 527 * @req: pointer to pending request 528 * @status: placeholder for status 529 * 530 * For each sent request, GuC shall send back CT response message. 531 * Our message handler will update status of tracked request once 532 * response message with given fence is received. Wait here and 533 * check for valid response status value. 534 * 535 * Return: 536 * * 0 response received (status is valid) 537 * * -ETIMEDOUT no response within hardcoded timeout 538 */ 539static int wait_for_ct_request_update(struct intel_guc_ct *ct, struct ct_request *req, u32 *status) 540{ 541 int err; 542 bool ct_enabled; 543 544 /* 545 * Fast commands should complete in less than 10us, so sample quickly 546 * up to that length of time, then switch to a slower sleep-wait loop. 547 * No GuC command should ever take longer than 10ms but many GuC 548 * commands can be inflight at time, so use a 1s timeout on the slower 549 * sleep-wait loop. 550 */ 551#define GUC_CTB_RESPONSE_TIMEOUT_SHORT_MS 10 552#define GUC_CTB_RESPONSE_TIMEOUT_LONG_MS 1000 553#define done \ 554 (!(ct_enabled = intel_guc_ct_enabled(ct)) || \ 555 FIELD_GET(GUC_HXG_MSG_0_ORIGIN, READ_ONCE(req->status)) == \ 556 GUC_HXG_ORIGIN_GUC) 557 err = wait_for_us(done, GUC_CTB_RESPONSE_TIMEOUT_SHORT_MS); 558 if (err) 559 err = wait_for(done, GUC_CTB_RESPONSE_TIMEOUT_LONG_MS); 560#undef done 561 if (!ct_enabled) 562 err = -ENODEV; 563 564 *status = req->status; 565 return err; 566} 567 568#define GUC_CTB_TIMEOUT_MS 1500 569static inline bool ct_deadlocked(struct intel_guc_ct *ct) 570{ 571 long timeout = GUC_CTB_TIMEOUT_MS; 572 bool ret = ktime_ms_delta(ktime_get(), ct->stall_time) > timeout; 573 574 if (unlikely(ret)) { 575 struct guc_ct_buffer_desc *send = ct->ctbs.send.desc; 576 struct guc_ct_buffer_desc *recv = ct->ctbs.send.desc; 577 578 CT_ERROR(ct, "Communication stalled for %lld ms, desc status=%#x,%#x\n", 579 ktime_ms_delta(ktime_get(), ct->stall_time), 580 send->status, recv->status); 581 CT_ERROR(ct, "H2G Space: %u (Bytes)\n", 582 atomic_read(&ct->ctbs.send.space) * 4); 583 CT_ERROR(ct, "Head: %u (Dwords)\n", ct->ctbs.send.desc->head); 584 CT_ERROR(ct, "Tail: %u (Dwords)\n", ct->ctbs.send.desc->tail); 585 CT_ERROR(ct, "G2H Space: %u (Bytes)\n", 586 atomic_read(&ct->ctbs.recv.space) * 4); 587 CT_ERROR(ct, "Head: %u\n (Dwords)", ct->ctbs.recv.desc->head); 588 CT_ERROR(ct, "Tail: %u\n (Dwords)", ct->ctbs.recv.desc->tail); 589 590 CT_DEAD(ct, DEADLOCK); 591 ct->ctbs.send.broken = true; 592 } 593 594 return ret; 595} 596 597static inline bool g2h_has_room(struct intel_guc_ct *ct, u32 g2h_len_dw) 598{ 599 struct intel_guc_ct_buffer *ctb = &ct->ctbs.recv; 600 601 /* 602 * We leave a certain amount of space in the G2H CTB buffer for 603 * unexpected G2H CTBs (e.g. logging, engine hang, etc...) 604 */ 605 return !g2h_len_dw || atomic_read(&ctb->space) >= g2h_len_dw; 606} 607 608static inline void g2h_reserve_space(struct intel_guc_ct *ct, u32 g2h_len_dw) 609{ 610 lockdep_assert_held(&ct->ctbs.send.lock); 611 612 GEM_BUG_ON(!g2h_has_room(ct, g2h_len_dw)); 613 614 if (g2h_len_dw) 615 atomic_sub(g2h_len_dw, &ct->ctbs.recv.space); 616} 617 618static inline void g2h_release_space(struct intel_guc_ct *ct, u32 g2h_len_dw) 619{ 620 atomic_add(g2h_len_dw, &ct->ctbs.recv.space); 621} 622 623static inline bool h2g_has_room(struct intel_guc_ct *ct, u32 len_dw) 624{ 625 struct intel_guc_ct_buffer *ctb = &ct->ctbs.send; 626 struct guc_ct_buffer_desc *desc = ctb->desc; 627 u32 head; 628 u32 space; 629 630 if (atomic_read(&ctb->space) >= len_dw) 631 return true; 632 633 head = READ_ONCE(desc->head); 634 if (unlikely(head > ctb->size)) { 635 CT_ERROR(ct, "Invalid head offset %u >= %u)\n", 636 head, ctb->size); 637 desc->status |= GUC_CTB_STATUS_OVERFLOW; 638 ctb->broken = true; 639 CT_DEAD(ct, H2G_HAS_ROOM); 640 return false; 641 } 642 643 space = CIRC_SPACE(ctb->tail, head, ctb->size); 644 atomic_set(&ctb->space, space); 645 646 return space >= len_dw; 647} 648 649static int has_room_nb(struct intel_guc_ct *ct, u32 h2g_dw, u32 g2h_dw) 650{ 651 bool h2g = h2g_has_room(ct, h2g_dw); 652 bool g2h = g2h_has_room(ct, g2h_dw); 653 654 lockdep_assert_held(&ct->ctbs.send.lock); 655 656 if (unlikely(!h2g || !g2h)) { 657 if (ct->stall_time == KTIME_MAX) 658 ct->stall_time = ktime_get(); 659 660 /* Be paranoid and kick G2H tasklet to free credits */ 661 if (!g2h) 662 tasklet_hi_schedule(&ct->receive_tasklet); 663 664 if (unlikely(ct_deadlocked(ct))) 665 return -EPIPE; 666 else 667 return -EBUSY; 668 } 669 670 ct->stall_time = KTIME_MAX; 671 return 0; 672} 673 674#define G2H_LEN_DW(f) ({ \ 675 typeof(f) f_ = (f); \ 676 FIELD_GET(INTEL_GUC_CT_SEND_G2H_DW_MASK, f_) ? \ 677 FIELD_GET(INTEL_GUC_CT_SEND_G2H_DW_MASK, f_) + \ 678 GUC_CTB_HXG_MSG_MIN_LEN : 0; \ 679}) 680static int ct_send_nb(struct intel_guc_ct *ct, 681 const u32 *action, 682 u32 len, 683 u32 flags) 684{ 685 struct intel_guc_ct_buffer *ctb = &ct->ctbs.send; 686 unsigned long spin_flags; 687 u32 g2h_len_dw = G2H_LEN_DW(flags); 688 u32 fence; 689 int ret; 690 691 spin_lock_irqsave(&ctb->lock, spin_flags); 692 693 ret = has_room_nb(ct, len + GUC_CTB_HDR_LEN, g2h_len_dw); 694 if (unlikely(ret)) 695 goto out; 696 697 fence = ct_get_next_fence(ct); 698 ret = ct_write(ct, action, len, fence, flags); 699 if (unlikely(ret)) 700 goto out; 701 702 g2h_reserve_space(ct, g2h_len_dw); 703 intel_guc_notify(ct_to_guc(ct)); 704 705out: 706 spin_unlock_irqrestore(&ctb->lock, spin_flags); 707 708 return ret; 709} 710 711static int ct_send(struct intel_guc_ct *ct, 712 const u32 *action, 713 u32 len, 714 u32 *response_buf, 715 u32 response_buf_size, 716 u32 *status) 717{ 718 struct intel_guc_ct_buffer *ctb = &ct->ctbs.send; 719 struct ct_request request; 720 unsigned long flags; 721 unsigned int sleep_period_ms = 1; 722 bool send_again; 723 u32 fence; 724 int err; 725 726 GEM_BUG_ON(!ct->enabled); 727 GEM_BUG_ON(!len); 728 GEM_BUG_ON(len > GUC_CTB_HXG_MSG_MAX_LEN - GUC_CTB_HDR_LEN); 729 GEM_BUG_ON(!response_buf && response_buf_size); 730 might_sleep(); 731 732resend: 733 send_again = false; 734 735 /* 736 * We use a lazy spin wait loop here as we believe that if the CT 737 * buffers are sized correctly the flow control condition should be 738 * rare. Reserving the maximum size in the G2H credits as we don't know 739 * how big the response is going to be. 740 */ 741retry: 742 spin_lock_irqsave(&ctb->lock, flags); 743 if (unlikely(!h2g_has_room(ct, len + GUC_CTB_HDR_LEN) || 744 !g2h_has_room(ct, GUC_CTB_HXG_MSG_MAX_LEN))) { 745 if (ct->stall_time == KTIME_MAX) 746 ct->stall_time = ktime_get(); 747 spin_unlock_irqrestore(&ctb->lock, flags); 748 749 if (unlikely(ct_deadlocked(ct))) 750 return -EPIPE; 751 752 if (msleep_interruptible(sleep_period_ms)) 753 return -EINTR; 754 sleep_period_ms = sleep_period_ms << 1; 755 756 goto retry; 757 } 758 759 ct->stall_time = KTIME_MAX; 760 761 fence = ct_get_next_fence(ct); 762 request.fence = fence; 763 request.status = 0; 764 request.response_len = response_buf_size; 765 request.response_buf = response_buf; 766 767 spin_lock(&ct->requests.lock); 768 list_add_tail(&request.link, &ct->requests.pending); 769 spin_unlock(&ct->requests.lock); 770 771 err = ct_write(ct, action, len, fence, 0); 772 g2h_reserve_space(ct, GUC_CTB_HXG_MSG_MAX_LEN); 773 774 spin_unlock_irqrestore(&ctb->lock, flags); 775 776 if (unlikely(err)) 777 goto unlink; 778 779 intel_guc_notify(ct_to_guc(ct)); 780 781 err = wait_for_ct_request_update(ct, &request, status); 782 g2h_release_space(ct, GUC_CTB_HXG_MSG_MAX_LEN); 783 if (unlikely(err)) { 784 if (err == -ENODEV) 785 /* wait_for_ct_request_update returns -ENODEV on reset/suspend in progress. 786 * In this case, output is debug rather than error info 787 */ 788 CT_DEBUG(ct, "Request %#x (fence %u) cancelled as CTB is disabled\n", 789 action[0], request.fence); 790 else 791 CT_ERROR(ct, "No response for request %#x (fence %u)\n", 792 action[0], request.fence); 793 goto unlink; 794 } 795 796 if (FIELD_GET(GUC_HXG_MSG_0_TYPE, *status) == GUC_HXG_TYPE_NO_RESPONSE_RETRY) { 797 CT_DEBUG(ct, "retrying request %#x (%u)\n", *action, 798 FIELD_GET(GUC_HXG_RETRY_MSG_0_REASON, *status)); 799 send_again = true; 800 goto unlink; 801 } 802 803 if (FIELD_GET(GUC_HXG_MSG_0_TYPE, *status) != GUC_HXG_TYPE_RESPONSE_SUCCESS) { 804 err = -EIO; 805 goto unlink; 806 } 807 808 if (response_buf) { 809 /* There shall be no data in the status */ 810 WARN_ON(FIELD_GET(GUC_HXG_RESPONSE_MSG_0_DATA0, request.status)); 811 /* Return actual response len */ 812 err = request.response_len; 813 } else { 814 /* There shall be no response payload */ 815 WARN_ON(request.response_len); 816 /* Return data decoded from the status dword */ 817 err = FIELD_GET(GUC_HXG_RESPONSE_MSG_0_DATA0, *status); 818 } 819 820unlink: 821 spin_lock_irqsave(&ct->requests.lock, flags); 822 list_del(&request.link); 823 spin_unlock_irqrestore(&ct->requests.lock, flags); 824 825 if (unlikely(send_again)) 826 goto resend; 827 828 return err; 829} 830 831/* 832 * Command Transport (CT) buffer based GuC send function. 833 */ 834int intel_guc_ct_send(struct intel_guc_ct *ct, const u32 *action, u32 len, 835 u32 *response_buf, u32 response_buf_size, u32 flags) 836{ 837 u32 status = ~0; /* undefined */ 838 int ret; 839 840 if (unlikely(!ct->enabled)) { 841 struct intel_guc *guc = ct_to_guc(ct); 842 struct intel_uc *uc = container_of(guc, struct intel_uc, guc); 843 844 WARN(!uc->reset_in_progress, "Unexpected send: action=%#x\n", *action); 845 return -ENODEV; 846 } 847 848 if (unlikely(ct->ctbs.send.broken)) 849 return -EPIPE; 850 851 if (flags & INTEL_GUC_CT_SEND_NB) 852 return ct_send_nb(ct, action, len, flags); 853 854 ret = ct_send(ct, action, len, response_buf, response_buf_size, &status); 855 if (unlikely(ret < 0)) { 856 if (ret != -ENODEV) 857 CT_ERROR(ct, "Sending action %#x failed (%pe) status=%#X\n", 858 action[0], ERR_PTR(ret), status); 859 } else if (unlikely(ret)) { 860 CT_DEBUG(ct, "send action %#x returned %d (%#x)\n", 861 action[0], ret, ret); 862 } 863 864 return ret; 865} 866 867static struct ct_incoming_msg *ct_alloc_msg(u32 num_dwords) 868{ 869 struct ct_incoming_msg *msg; 870 871 msg = kmalloc(struct_size(msg, msg, num_dwords), GFP_ATOMIC); 872 if (msg) 873 msg->size = num_dwords; 874 return msg; 875} 876 877static void ct_free_msg(struct ct_incoming_msg *msg) 878{ 879 kfree(msg); 880} 881 882/* 883 * Return: number available remaining dwords to read (0 if empty) 884 * or a negative error code on failure 885 */ 886static int ct_read(struct intel_guc_ct *ct, struct ct_incoming_msg **msg) 887{ 888 struct intel_guc_ct_buffer *ctb = &ct->ctbs.recv; 889 struct guc_ct_buffer_desc *desc = ctb->desc; 890 u32 head = ctb->head; 891 u32 tail = READ_ONCE(desc->tail); 892 u32 size = ctb->size; 893 u32 *cmds = ctb->cmds; 894 s32 available; 895 unsigned int len; 896 unsigned int i; 897 u32 header; 898 899 if (unlikely(ctb->broken)) 900 return -EPIPE; 901 902 if (unlikely(desc->status)) { 903 u32 status = desc->status; 904 905 if (status & GUC_CTB_STATUS_UNUSED) { 906 /* 907 * Potentially valid if a CLIENT_RESET request resulted in 908 * contexts/engines being reset. But should never happen as 909 * no contexts should be active when CLIENT_RESET is sent. 910 */ 911 CT_ERROR(ct, "Unexpected G2H after GuC has stopped!\n"); 912 status &= ~GUC_CTB_STATUS_UNUSED; 913 } 914 915 if (status) 916 goto corrupted; 917 } 918 919 GEM_BUG_ON(head > size); 920 921#ifdef CONFIG_DRM_I915_DEBUG_GUC 922 if (unlikely(head != READ_ONCE(desc->head))) { 923 CT_ERROR(ct, "Head was modified %u != %u\n", 924 desc->head, head); 925 desc->status |= GUC_CTB_STATUS_MISMATCH; 926 goto corrupted; 927 } 928#endif 929 if (unlikely(tail >= size)) { 930 CT_ERROR(ct, "Invalid tail offset %u >= %u)\n", 931 tail, size); 932 desc->status |= GUC_CTB_STATUS_OVERFLOW; 933 goto corrupted; 934 } 935 936 /* tail == head condition indicates empty */ 937 available = tail - head; 938 if (unlikely(available == 0)) { 939 *msg = NULL; 940 return 0; 941 } 942 943 /* beware of buffer wrap case */ 944 if (unlikely(available < 0)) 945 available += size; 946 CT_DEBUG(ct, "available %d (%u:%u:%u)\n", available, head, tail, size); 947 GEM_BUG_ON(available < 0); 948 949 header = cmds[head]; 950 head = (head + 1) % size; 951 952 /* message len with header */ 953 len = FIELD_GET(GUC_CTB_MSG_0_NUM_DWORDS, header) + GUC_CTB_MSG_MIN_LEN; 954 if (unlikely(len > (u32)available)) { 955 CT_ERROR(ct, "Incomplete message %*ph %*ph %*ph\n", 956 4, &header, 957 4 * (head + available - 1 > size ? 958 size - head : available - 1), &cmds[head], 959 4 * (head + available - 1 > size ? 960 available - 1 - size + head : 0), &cmds[0]); 961 desc->status |= GUC_CTB_STATUS_UNDERFLOW; 962 goto corrupted; 963 } 964 965 *msg = ct_alloc_msg(len); 966 if (!*msg) { 967 CT_ERROR(ct, "No memory for message %*ph %*ph %*ph\n", 968 4, &header, 969 4 * (head + available - 1 > size ? 970 size - head : available - 1), &cmds[head], 971 4 * (head + available - 1 > size ? 972 available - 1 - size + head : 0), &cmds[0]); 973 return available; 974 } 975 976 (*msg)->msg[0] = header; 977 978 for (i = 1; i < len; i++) { 979 (*msg)->msg[i] = cmds[head]; 980 head = (head + 1) % size; 981 } 982 CT_DEBUG(ct, "received %*ph\n", 4 * len, (*msg)->msg); 983 984 /* update local copies */ 985 ctb->head = head; 986 987 /* now update descriptor */ 988 WRITE_ONCE(desc->head, head); 989 990 intel_guc_write_barrier(ct_to_guc(ct)); 991 992 return available - len; 993 994corrupted: 995 CT_ERROR(ct, "Corrupted descriptor head=%u tail=%u status=%#x\n", 996 desc->head, desc->tail, desc->status); 997 ctb->broken = true; 998 CT_DEAD(ct, READ); 999 return -EPIPE; 1000} 1001 1002#if IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM) 1003static bool ct_check_lost_and_found(struct intel_guc_ct *ct, u32 fence) 1004{ 1005 unsigned int n; 1006 char *buf = NULL; 1007 bool found = false; 1008 1009 lockdep_assert_held(&ct->requests.lock); 1010 1011 for (n = 0; n < ARRAY_SIZE(ct->requests.lost_and_found); n++) { 1012 if (ct->requests.lost_and_found[n].fence != fence) 1013 continue; 1014 found = true; 1015 1016#if IS_ENABLED(CONFIG_DRM_I915_DEBUG_GUC) 1017 buf = kmalloc(SZ_4K, GFP_NOWAIT); 1018 if (buf && stack_depot_snprint(ct->requests.lost_and_found[n].stack, 1019 buf, SZ_4K, 0)) { 1020 CT_ERROR(ct, "Fence %u was used by action %#04x sent at\n%s", 1021 fence, ct->requests.lost_and_found[n].action, buf); 1022 break; 1023 } 1024#endif 1025 CT_ERROR(ct, "Fence %u was used by action %#04x\n", 1026 fence, ct->requests.lost_and_found[n].action); 1027 break; 1028 } 1029 kfree(buf); 1030 return found; 1031} 1032#else 1033static bool ct_check_lost_and_found(struct intel_guc_ct *ct, u32 fence) 1034{ 1035 return false; 1036} 1037#endif 1038 1039static int ct_handle_response(struct intel_guc_ct *ct, struct ct_incoming_msg *response) 1040{ 1041 u32 len = FIELD_GET(GUC_CTB_MSG_0_NUM_DWORDS, response->msg[0]); 1042 u32 fence = FIELD_GET(GUC_CTB_MSG_0_FENCE, response->msg[0]); 1043 const u32 *hxg = &response->msg[GUC_CTB_MSG_MIN_LEN]; 1044 const u32 *data = &hxg[GUC_HXG_MSG_MIN_LEN]; 1045 u32 datalen = len - GUC_HXG_MSG_MIN_LEN; 1046 struct ct_request *req; 1047 unsigned long flags; 1048 bool found = false; 1049 int err = 0; 1050 1051 GEM_BUG_ON(len < GUC_HXG_MSG_MIN_LEN); 1052 GEM_BUG_ON(FIELD_GET(GUC_HXG_MSG_0_ORIGIN, hxg[0]) != GUC_HXG_ORIGIN_GUC); 1053 GEM_BUG_ON(FIELD_GET(GUC_HXG_MSG_0_TYPE, hxg[0]) != GUC_HXG_TYPE_RESPONSE_SUCCESS && 1054 FIELD_GET(GUC_HXG_MSG_0_TYPE, hxg[0]) != GUC_HXG_TYPE_NO_RESPONSE_RETRY && 1055 FIELD_GET(GUC_HXG_MSG_0_TYPE, hxg[0]) != GUC_HXG_TYPE_RESPONSE_FAILURE); 1056 1057 CT_DEBUG(ct, "response fence %u status %#x\n", fence, hxg[0]); 1058 1059 spin_lock_irqsave(&ct->requests.lock, flags); 1060 list_for_each_entry(req, &ct->requests.pending, link) { 1061 if (unlikely(fence != req->fence)) { 1062 CT_DEBUG(ct, "request %u awaits response\n", 1063 req->fence); 1064 continue; 1065 } 1066 if (unlikely(datalen > req->response_len)) { 1067 CT_ERROR(ct, "Response %u too long (datalen %u > %u)\n", 1068 req->fence, datalen, req->response_len); 1069 datalen = min(datalen, req->response_len); 1070 err = -EMSGSIZE; 1071 } 1072 if (datalen) 1073 memcpy(req->response_buf, data, 4 * datalen); 1074 req->response_len = datalen; 1075 WRITE_ONCE(req->status, hxg[0]); 1076 found = true; 1077 break; 1078 } 1079 1080#ifdef CONFIG_DRM_I915_SELFTEST 1081 if (!found && ct_to_guc(ct)->fast_response_selftest) { 1082 CT_DEBUG(ct, "Assuming unsolicited response due to FAST_REQUEST selftest\n"); 1083 ct_to_guc(ct)->fast_response_selftest++; 1084 found = true; 1085 } 1086#endif 1087 1088 if (!found) { 1089 CT_ERROR(ct, "Unsolicited response message: len %u, data %#x (fence %u, last %u)\n", 1090 len, hxg[0], fence, ct->requests.last_fence); 1091 if (!ct_check_lost_and_found(ct, fence)) { 1092 list_for_each_entry(req, &ct->requests.pending, link) 1093 CT_ERROR(ct, "request %u awaits response\n", 1094 req->fence); 1095 } 1096 err = -ENOKEY; 1097 } 1098 spin_unlock_irqrestore(&ct->requests.lock, flags); 1099 1100 if (unlikely(err)) 1101 return err; 1102 1103 ct_free_msg(response); 1104 return 0; 1105} 1106 1107static int ct_process_request(struct intel_guc_ct *ct, struct ct_incoming_msg *request) 1108{ 1109 struct intel_guc *guc = ct_to_guc(ct); 1110 const u32 *hxg; 1111 const u32 *payload; 1112 u32 hxg_len, action, len; 1113 int ret; 1114 1115 hxg = &request->msg[GUC_CTB_MSG_MIN_LEN]; 1116 hxg_len = request->size - GUC_CTB_MSG_MIN_LEN; 1117 payload = &hxg[GUC_HXG_MSG_MIN_LEN]; 1118 action = FIELD_GET(GUC_HXG_EVENT_MSG_0_ACTION, hxg[0]); 1119 len = hxg_len - GUC_HXG_MSG_MIN_LEN; 1120 1121 CT_DEBUG(ct, "request %x %*ph\n", action, 4 * len, payload); 1122 1123 switch (action) { 1124 case INTEL_GUC_ACTION_DEFAULT: 1125 ret = intel_guc_to_host_process_recv_msg(guc, payload, len); 1126 break; 1127 case INTEL_GUC_ACTION_DEREGISTER_CONTEXT_DONE: 1128 ret = intel_guc_deregister_done_process_msg(guc, payload, 1129 len); 1130 break; 1131 case INTEL_GUC_ACTION_SCHED_CONTEXT_MODE_DONE: 1132 ret = intel_guc_sched_done_process_msg(guc, payload, len); 1133 break; 1134 case INTEL_GUC_ACTION_CONTEXT_RESET_NOTIFICATION: 1135 ret = intel_guc_context_reset_process_msg(guc, payload, len); 1136 break; 1137 case INTEL_GUC_ACTION_STATE_CAPTURE_NOTIFICATION: 1138 ret = intel_guc_error_capture_process_msg(guc, payload, len); 1139 if (unlikely(ret)) 1140 CT_ERROR(ct, "error capture notification failed %x %*ph\n", 1141 action, 4 * len, payload); 1142 break; 1143 case INTEL_GUC_ACTION_ENGINE_FAILURE_NOTIFICATION: 1144 ret = intel_guc_engine_failure_process_msg(guc, payload, len); 1145 break; 1146 case INTEL_GUC_ACTION_NOTIFY_FLUSH_LOG_BUFFER_TO_FILE: 1147 intel_guc_log_handle_flush_event(&guc->log); 1148 ret = 0; 1149 break; 1150 case INTEL_GUC_ACTION_NOTIFY_CRASH_DUMP_POSTED: 1151 case INTEL_GUC_ACTION_NOTIFY_EXCEPTION: 1152 ret = intel_guc_crash_process_msg(guc, action); 1153 break; 1154 case INTEL_GUC_ACTION_TLB_INVALIDATION_DONE: 1155 ret = intel_guc_tlb_invalidation_done(guc, payload, len); 1156 break; 1157 default: 1158 ret = -EOPNOTSUPP; 1159 break; 1160 } 1161 1162 if (unlikely(ret)) { 1163 CT_ERROR(ct, "Failed to process request %04x (%pe)\n", 1164 action, ERR_PTR(ret)); 1165 return ret; 1166 } 1167 1168 ct_free_msg(request); 1169 return 0; 1170} 1171 1172static bool ct_process_incoming_requests(struct intel_guc_ct *ct) 1173{ 1174 unsigned long flags; 1175 struct ct_incoming_msg *request; 1176 bool done; 1177 int err; 1178 1179 spin_lock_irqsave(&ct->requests.lock, flags); 1180 request = list_first_entry_or_null(&ct->requests.incoming, 1181 struct ct_incoming_msg, link); 1182 if (request) 1183 list_del(&request->link); 1184 done = !!list_empty(&ct->requests.incoming); 1185 spin_unlock_irqrestore(&ct->requests.lock, flags); 1186 1187 if (!request) 1188 return true; 1189 1190 err = ct_process_request(ct, request); 1191 if (unlikely(err)) { 1192 CT_ERROR(ct, "Failed to process CT message (%pe) %*ph\n", 1193 ERR_PTR(err), 4 * request->size, request->msg); 1194 CT_DEAD(ct, PROCESS_FAILED); 1195 ct_free_msg(request); 1196 } 1197 1198 return done; 1199} 1200 1201static void ct_incoming_request_worker_func(struct work_struct *w) 1202{ 1203 struct intel_guc_ct *ct = 1204 container_of(w, struct intel_guc_ct, requests.worker); 1205 bool done; 1206 1207 do { 1208 done = ct_process_incoming_requests(ct); 1209 } while (!done); 1210} 1211 1212static int ct_handle_event(struct intel_guc_ct *ct, struct ct_incoming_msg *request) 1213{ 1214 const u32 *hxg = &request->msg[GUC_CTB_MSG_MIN_LEN]; 1215 u32 action = FIELD_GET(GUC_HXG_EVENT_MSG_0_ACTION, hxg[0]); 1216 unsigned long flags; 1217 1218 GEM_BUG_ON(FIELD_GET(GUC_HXG_MSG_0_TYPE, hxg[0]) != GUC_HXG_TYPE_EVENT); 1219 1220 /* 1221 * Adjusting the space must be done in IRQ or deadlock can occur as the 1222 * CTB processing in the below workqueue can send CTBs which creates a 1223 * circular dependency if the space was returned there. 1224 */ 1225 switch (action) { 1226 case INTEL_GUC_ACTION_SCHED_CONTEXT_MODE_DONE: 1227 case INTEL_GUC_ACTION_DEREGISTER_CONTEXT_DONE: 1228 case INTEL_GUC_ACTION_TLB_INVALIDATION_DONE: 1229 g2h_release_space(ct, request->size); 1230 } 1231 1232 /* 1233 * TLB invalidation responses must be handled immediately as processing 1234 * of other G2H notifications may be blocked by an invalidation request. 1235 */ 1236 if (action == INTEL_GUC_ACTION_TLB_INVALIDATION_DONE) 1237 return ct_process_request(ct, request); 1238 1239 spin_lock_irqsave(&ct->requests.lock, flags); 1240 list_add_tail(&request->link, &ct->requests.incoming); 1241 spin_unlock_irqrestore(&ct->requests.lock, flags); 1242 1243 queue_work(system_unbound_wq, &ct->requests.worker); 1244 return 0; 1245} 1246 1247static int ct_handle_hxg(struct intel_guc_ct *ct, struct ct_incoming_msg *msg) 1248{ 1249 u32 origin, type; 1250 u32 *hxg; 1251 int err; 1252 1253 if (unlikely(msg->size < GUC_CTB_HXG_MSG_MIN_LEN)) 1254 return -EBADMSG; 1255 1256 hxg = &msg->msg[GUC_CTB_MSG_MIN_LEN]; 1257 1258 origin = FIELD_GET(GUC_HXG_MSG_0_ORIGIN, hxg[0]); 1259 if (unlikely(origin != GUC_HXG_ORIGIN_GUC)) { 1260 err = -EPROTO; 1261 goto failed; 1262 } 1263 1264 type = FIELD_GET(GUC_HXG_MSG_0_TYPE, hxg[0]); 1265 switch (type) { 1266 case GUC_HXG_TYPE_EVENT: 1267 err = ct_handle_event(ct, msg); 1268 break; 1269 case GUC_HXG_TYPE_RESPONSE_SUCCESS: 1270 case GUC_HXG_TYPE_RESPONSE_FAILURE: 1271 case GUC_HXG_TYPE_NO_RESPONSE_RETRY: 1272 err = ct_handle_response(ct, msg); 1273 break; 1274 default: 1275 err = -EOPNOTSUPP; 1276 } 1277 1278 if (unlikely(err)) { 1279failed: 1280 CT_ERROR(ct, "Failed to handle HXG message (%pe) %*ph\n", 1281 ERR_PTR(err), 4 * GUC_HXG_MSG_MIN_LEN, hxg); 1282 } 1283 return err; 1284} 1285 1286static void ct_handle_msg(struct intel_guc_ct *ct, struct ct_incoming_msg *msg) 1287{ 1288 u32 format = FIELD_GET(GUC_CTB_MSG_0_FORMAT, msg->msg[0]); 1289 int err; 1290 1291 if (format == GUC_CTB_FORMAT_HXG) 1292 err = ct_handle_hxg(ct, msg); 1293 else 1294 err = -EOPNOTSUPP; 1295 1296 if (unlikely(err)) { 1297 CT_ERROR(ct, "Failed to process CT message (%pe) %*ph\n", 1298 ERR_PTR(err), 4 * msg->size, msg->msg); 1299 ct_free_msg(msg); 1300 } 1301} 1302 1303/* 1304 * Return: number available remaining dwords to read (0 if empty) 1305 * or a negative error code on failure 1306 */ 1307static int ct_receive(struct intel_guc_ct *ct) 1308{ 1309 struct ct_incoming_msg *msg = NULL; 1310 unsigned long flags; 1311 int ret; 1312 1313 spin_lock_irqsave(&ct->ctbs.recv.lock, flags); 1314 ret = ct_read(ct, &msg); 1315 spin_unlock_irqrestore(&ct->ctbs.recv.lock, flags); 1316 if (ret < 0) 1317 return ret; 1318 1319 if (msg) 1320 ct_handle_msg(ct, msg); 1321 1322 return ret; 1323} 1324 1325static void ct_try_receive_message(struct intel_guc_ct *ct) 1326{ 1327 int ret; 1328 1329 if (GEM_WARN_ON(!ct->enabled)) 1330 return; 1331 1332 ret = ct_receive(ct); 1333 if (ret > 0) 1334 tasklet_hi_schedule(&ct->receive_tasklet); 1335} 1336 1337static void ct_receive_tasklet_func(struct tasklet_struct *t) 1338{ 1339 struct intel_guc_ct *ct = from_tasklet(ct, t, receive_tasklet); 1340 1341 ct_try_receive_message(ct); 1342} 1343 1344/* 1345 * When we're communicating with the GuC over CT, GuC uses events 1346 * to notify us about new messages being posted on the RECV buffer. 1347 */ 1348void intel_guc_ct_event_handler(struct intel_guc_ct *ct) 1349{ 1350 if (unlikely(!ct->enabled)) { 1351 WARN(1, "Unexpected GuC event received while CT disabled!\n"); 1352 return; 1353 } 1354 1355 ct_try_receive_message(ct); 1356} 1357 1358void intel_guc_ct_print_info(struct intel_guc_ct *ct, 1359 struct drm_printer *p) 1360{ 1361 drm_printf(p, "CT %s\n", str_enabled_disabled(ct->enabled)); 1362 1363 if (!ct->enabled) 1364 return; 1365 1366 drm_printf(p, "H2G Space: %u\n", 1367 atomic_read(&ct->ctbs.send.space) * 4); 1368 drm_printf(p, "Head: %u\n", 1369 ct->ctbs.send.desc->head); 1370 drm_printf(p, "Tail: %u\n", 1371 ct->ctbs.send.desc->tail); 1372 drm_printf(p, "G2H Space: %u\n", 1373 atomic_read(&ct->ctbs.recv.space) * 4); 1374 drm_printf(p, "Head: %u\n", 1375 ct->ctbs.recv.desc->head); 1376 drm_printf(p, "Tail: %u\n", 1377 ct->ctbs.recv.desc->tail); 1378} 1379 1380#if IS_ENABLED(CONFIG_DRM_I915_DEBUG_GUC) 1381static void ct_dead_ct_worker_func(struct work_struct *w) 1382{ 1383 struct intel_guc_ct *ct = container_of(w, struct intel_guc_ct, dead_ct_worker); 1384 struct intel_guc *guc = ct_to_guc(ct); 1385 1386 if (ct->dead_ct_reported) 1387 return; 1388 1389 ct->dead_ct_reported = true; 1390 1391 guc_info(guc, "CTB is dead - reason=0x%X\n", ct->dead_ct_reason); 1392 intel_klog_error_capture(guc_to_gt(guc), (intel_engine_mask_t)~0U); 1393} 1394#endif 1395