1/* SPDX-License-Identifier: MIT */ 2/* 3 * Copyright �� 2014-2019 Intel Corporation 4 */ 5 6#ifndef _INTEL_GUC_H_ 7#define _INTEL_GUC_H_ 8 9#include <linux/delay.h> 10#include <linux/iosys-map.h> 11#include <linux/xarray.h> 12 13#include "intel_guc_ct.h" 14#include "intel_guc_fw.h" 15#include "intel_guc_fwif.h" 16#include "intel_guc_log.h" 17#include "intel_guc_reg.h" 18#include "intel_guc_slpc_types.h" 19#include "intel_uc_fw.h" 20#include "intel_uncore.h" 21#include "i915_utils.h" 22#include "i915_vma.h" 23 24struct __guc_ads_blob; 25struct intel_guc_state_capture; 26 27/** 28 * struct intel_guc - Top level structure of GuC. 29 * 30 * It handles firmware loading and manages client pool. intel_guc owns an 31 * i915_sched_engine for submission. 32 */ 33struct intel_guc { 34 /** @fw: the GuC firmware */ 35 struct intel_uc_fw fw; 36 /** @log: sub-structure containing GuC log related data and objects */ 37 struct intel_guc_log log; 38 /** @ct: the command transport communication channel */ 39 struct intel_guc_ct ct; 40 /** @slpc: sub-structure containing SLPC related data and objects */ 41 struct intel_guc_slpc slpc; 42 /** @capture: the error-state-capture module's data and objects */ 43 struct intel_guc_state_capture *capture; 44 45 /** @dbgfs_node: debugfs node */ 46 struct dentry *dbgfs_node; 47 48 /** @sched_engine: Global engine used to submit requests to GuC */ 49 struct i915_sched_engine *sched_engine; 50 /** 51 * @stalled_request: if GuC can't process a request for any reason, we 52 * save it until GuC restarts processing. No other request can be 53 * submitted until the stalled request is processed. 54 */ 55 struct i915_request *stalled_request; 56 /** 57 * @submission_stall_reason: reason why submission is stalled 58 */ 59 enum { 60 STALL_NONE, 61 STALL_REGISTER_CONTEXT, 62 STALL_MOVE_LRC_TAIL, 63 STALL_ADD_REQUEST, 64 } submission_stall_reason; 65 66 /* intel_guc_recv interrupt related state */ 67 /** @irq_lock: protects GuC irq state */ 68 spinlock_t irq_lock; 69 /** 70 * @msg_enabled_mask: mask of events that are processed when receiving 71 * an INTEL_GUC_ACTION_DEFAULT G2H message. 72 */ 73 unsigned int msg_enabled_mask; 74 75 /** 76 * @outstanding_submission_g2h: number of outstanding GuC to Host 77 * responses related to GuC submission, used to determine if the GT is 78 * idle 79 */ 80 atomic_t outstanding_submission_g2h; 81 82 /** @tlb_lookup: xarray to store all pending TLB invalidation requests */ 83 struct xarray tlb_lookup; 84 85 /** 86 * @serial_slot: id to the initial waiter created in tlb_lookup, 87 * which is used only when failed to allocate new waiter. 88 */ 89 u32 serial_slot; 90 91 /** @next_seqno: the next id (sequence number) to allocate. */ 92 u32 next_seqno; 93 94 /** @interrupts: pointers to GuC interrupt-managing functions. */ 95 struct { 96 bool enabled; 97 void (*reset)(struct intel_guc *guc); 98 void (*enable)(struct intel_guc *guc); 99 void (*disable)(struct intel_guc *guc); 100 } interrupts; 101 102 /** 103 * @submission_state: sub-structure for submission state protected by 104 * single lock 105 */ 106 struct { 107 /** 108 * @submission_state.lock: protects everything in 109 * submission_state, ce->guc_id.id, and ce->guc_id.ref 110 * when transitioning in and out of zero 111 */ 112 spinlock_t lock; 113 /** 114 * @submission_state.guc_ids: used to allocate new 115 * guc_ids, single-lrc 116 */ 117 struct ida guc_ids; 118 /** 119 * @submission_state.num_guc_ids: Number of guc_ids, selftest 120 * feature to be able to reduce this number while testing. 121 */ 122 int num_guc_ids; 123 /** 124 * @submission_state.guc_ids_bitmap: used to allocate 125 * new guc_ids, multi-lrc 126 */ 127 unsigned long *guc_ids_bitmap; 128 /** 129 * @submission_state.guc_id_list: list of intel_context 130 * with valid guc_ids but no refs 131 */ 132 struct list_head guc_id_list; 133 /** 134 * @submission_state.guc_ids_in_use: Number single-lrc 135 * guc_ids in use 136 */ 137 unsigned int guc_ids_in_use; 138 /** 139 * @submission_state.destroyed_contexts: list of contexts 140 * waiting to be destroyed (deregistered with the GuC) 141 */ 142 struct list_head destroyed_contexts; 143 /** 144 * @submission_state.destroyed_worker: worker to deregister 145 * contexts, need as we need to take a GT PM reference and 146 * can't from destroy function as it might be in an atomic 147 * context (no sleeping) 148 */ 149 struct work_struct destroyed_worker; 150 /** 151 * @submission_state.reset_fail_worker: worker to trigger 152 * a GT reset after an engine reset fails 153 */ 154 struct work_struct reset_fail_worker; 155 /** 156 * @submission_state.reset_fail_mask: mask of engines that 157 * failed to reset 158 */ 159 intel_engine_mask_t reset_fail_mask; 160 /** 161 * @submission_state.sched_disable_delay_ms: schedule 162 * disable delay, in ms, for contexts 163 */ 164 unsigned int sched_disable_delay_ms; 165 /** 166 * @submission_state.sched_disable_gucid_threshold: 167 * threshold of min remaining available guc_ids before 168 * we start bypassing the schedule disable delay 169 */ 170 unsigned int sched_disable_gucid_threshold; 171 } submission_state; 172 173 /** 174 * @submission_supported: tracks whether we support GuC submission on 175 * the current platform 176 */ 177 bool submission_supported; 178 /** @submission_selected: tracks whether the user enabled GuC submission */ 179 bool submission_selected; 180 /** @submission_initialized: tracks whether GuC submission has been initialised */ 181 bool submission_initialized; 182 /** @submission_version: Submission API version of the currently loaded firmware */ 183 struct intel_uc_fw_ver submission_version; 184 185 /** 186 * @rc_supported: tracks whether we support GuC rc on the current platform 187 */ 188 bool rc_supported; 189 /** @rc_selected: tracks whether the user enabled GuC rc */ 190 bool rc_selected; 191 192 /** @ads_vma: object allocated to hold the GuC ADS */ 193 struct i915_vma *ads_vma; 194 /** @ads_map: contents of the GuC ADS */ 195 struct iosys_map ads_map; 196 /** @ads_regset_size: size of the save/restore regsets in the ADS */ 197 u32 ads_regset_size; 198 /** 199 * @ads_regset_count: number of save/restore registers in the ADS for 200 * each engine 201 */ 202 u32 ads_regset_count[I915_NUM_ENGINES]; 203 /** @ads_regset: save/restore regsets in the ADS */ 204 struct guc_mmio_reg *ads_regset; 205 /** @ads_golden_ctxt_size: size of the golden contexts in the ADS */ 206 u32 ads_golden_ctxt_size; 207 /** @ads_waklv_size: size of workaround KLVs */ 208 u32 ads_waklv_size; 209 /** @ads_capture_size: size of register lists in the ADS used for error capture */ 210 u32 ads_capture_size; 211 212 /** @lrc_desc_pool_v69: object allocated to hold the GuC LRC descriptor pool */ 213 struct i915_vma *lrc_desc_pool_v69; 214 /** @lrc_desc_pool_vaddr_v69: contents of the GuC LRC descriptor pool */ 215 void *lrc_desc_pool_vaddr_v69; 216 217 /** 218 * @context_lookup: used to resolve intel_context from guc_id, if a 219 * context is present in this structure it is registered with the GuC 220 */ 221 struct xarray context_lookup; 222 223 /** @params: Control params for fw initialization */ 224 u32 params[GUC_CTL_MAX_DWORDS]; 225 226 /** @send_regs: GuC's FW specific registers used for sending MMIO H2G */ 227 struct { 228 u32 base; 229 unsigned int count; 230 enum forcewake_domains fw_domains; 231 } send_regs; 232 233 /** @notify_reg: register used to send interrupts to the GuC FW */ 234 i915_reg_t notify_reg; 235 236 /** 237 * @mmio_msg: notification bitmask that the GuC writes in one of its 238 * registers when the CT channel is disabled, to be processed when the 239 * channel is back up. 240 */ 241 u32 mmio_msg; 242 243 /** @send_mutex: used to serialize the intel_guc_send actions */ 244 struct mutex send_mutex; 245 246 /** 247 * @timestamp: GT timestamp object that stores a copy of the timestamp 248 * and adjusts it for overflow using a worker. 249 */ 250 struct { 251 /** 252 * @timestamp.lock: Lock protecting the below fields and 253 * the engine stats. 254 */ 255 spinlock_t lock; 256 257 /** 258 * @timestamp.gt_stamp: 64-bit extended value of the GT 259 * timestamp. 260 */ 261 u64 gt_stamp; 262 263 /** 264 * @timestamp.ping_delay: Period for polling the GT 265 * timestamp for overflow. 266 */ 267 unsigned long ping_delay; 268 269 /** 270 * @timestamp.work: Periodic work to adjust GT timestamp, 271 * engine and context usage for overflows. 272 */ 273 struct delayed_work work; 274 275 /** 276 * @timestamp.shift: Right shift value for the gpm timestamp 277 */ 278 u32 shift; 279 280 /** 281 * @timestamp.last_stat_jiffies: jiffies at last actual 282 * stats collection time. We use this timestamp to ensure 283 * we don't oversample the stats because runtime power 284 * management events can trigger stats collection at much 285 * higher rates than required. 286 */ 287 unsigned long last_stat_jiffies; 288 } timestamp; 289 290 /** 291 * @dead_guc_worker: Asynchronous worker thread for forcing a GuC reset. 292 * Specifically used when the G2H handler wants to issue a reset. Resets 293 * require flushing the G2H queue. So, the G2H processing itself must not 294 * trigger a reset directly. Instead, go via this worker. 295 */ 296 struct work_struct dead_guc_worker; 297 /** 298 * @last_dead_guc_jiffies: timestamp of previous 'dead guc' occurrance 299 * used to prevent a fundamentally broken system from continuously 300 * reloading the GuC. 301 */ 302 unsigned long last_dead_guc_jiffies; 303 304#ifdef CONFIG_DRM_I915_SELFTEST 305 /** 306 * @number_guc_id_stolen: The number of guc_ids that have been stolen 307 */ 308 int number_guc_id_stolen; 309 /** 310 * @fast_response_selftest: Backdoor to CT handler for fast response selftest 311 */ 312 u32 fast_response_selftest; 313#endif 314}; 315 316struct intel_guc_tlb_wait { 317 struct wait_queue_head wq; 318 bool busy; 319}; 320 321/* 322 * GuC version number components are only 8-bit, so converting to a 32bit 8.8.8 323 * integer works. 324 */ 325#define MAKE_GUC_VER(maj, min, pat) (((maj) << 16) | ((min) << 8) | (pat)) 326#define MAKE_GUC_VER_STRUCT(ver) MAKE_GUC_VER((ver).major, (ver).minor, (ver).patch) 327#define GUC_SUBMIT_VER(guc) MAKE_GUC_VER_STRUCT((guc)->submission_version) 328#define GUC_FIRMWARE_VER(guc) MAKE_GUC_VER_STRUCT((guc)->fw.file_selected.ver) 329 330static inline struct intel_guc *log_to_guc(struct intel_guc_log *log) 331{ 332 return container_of(log, struct intel_guc, log); 333} 334 335static 336inline int intel_guc_send(struct intel_guc *guc, const u32 *action, u32 len) 337{ 338 return intel_guc_ct_send(&guc->ct, action, len, NULL, 0, 0); 339} 340 341static 342inline int intel_guc_send_nb(struct intel_guc *guc, const u32 *action, u32 len, 343 u32 g2h_len_dw) 344{ 345 return intel_guc_ct_send(&guc->ct, action, len, NULL, 0, 346 MAKE_SEND_FLAGS(g2h_len_dw)); 347} 348 349static inline int 350intel_guc_send_and_receive(struct intel_guc *guc, const u32 *action, u32 len, 351 u32 *response_buf, u32 response_buf_size) 352{ 353 return intel_guc_ct_send(&guc->ct, action, len, 354 response_buf, response_buf_size, 0); 355} 356 357static inline int intel_guc_send_busy_loop(struct intel_guc *guc, 358 const u32 *action, 359 u32 len, 360 u32 g2h_len_dw, 361 bool loop) 362{ 363 int err; 364 unsigned int sleep_period_ms = 1; 365 bool not_atomic = !in_atomic() && !irqs_disabled(); 366 367 /* 368 * FIXME: Have caller pass in if we are in an atomic context to avoid 369 * using in_atomic(). It is likely safe here as we check for irqs 370 * disabled which basically all the spin locks in the i915 do but 371 * regardless this should be cleaned up. 372 */ 373 374 /* No sleeping with spin locks, just busy loop */ 375 might_sleep_if(loop && not_atomic); 376 377retry: 378 err = intel_guc_send_nb(guc, action, len, g2h_len_dw); 379 if (unlikely(err == -EBUSY && loop)) { 380 if (likely(not_atomic)) { 381 if (msleep_interruptible(sleep_period_ms)) 382 return -EINTR; 383 sleep_period_ms = sleep_period_ms << 1; 384 } else { 385 cpu_relax(); 386 } 387 goto retry; 388 } 389 390 return err; 391} 392 393/* Only call this from the interrupt handler code */ 394static inline void intel_guc_to_host_event_handler(struct intel_guc *guc) 395{ 396 if (guc->interrupts.enabled) 397 intel_guc_ct_event_handler(&guc->ct); 398} 399 400/* GuC addresses above GUC_GGTT_TOP also don't map through the GTT */ 401#define GUC_GGTT_TOP 0xFEE00000 402 403/** 404 * intel_guc_ggtt_offset() - Get and validate the GGTT offset of @vma 405 * @guc: intel_guc structure. 406 * @vma: i915 graphics virtual memory area. 407 * 408 * GuC does not allow any gfx GGTT address that falls into range 409 * [0, ggtt.pin_bias), which is reserved for Boot ROM, SRAM and WOPCM. 410 * Currently, in order to exclude [0, ggtt.pin_bias) address space from 411 * GGTT, all gfx objects used by GuC are allocated with intel_guc_allocate_vma() 412 * and pinned with PIN_OFFSET_BIAS along with the value of ggtt.pin_bias. 413 * 414 * Return: GGTT offset of the @vma. 415 */ 416static inline u32 intel_guc_ggtt_offset(struct intel_guc *guc, 417 struct i915_vma *vma) 418{ 419 u32 offset = i915_ggtt_offset(vma); 420 421 GEM_BUG_ON(offset < i915_ggtt_pin_bias(vma)); 422 GEM_BUG_ON(range_overflows_t(u64, offset, vma->size, GUC_GGTT_TOP)); 423 424 return offset; 425} 426 427void intel_guc_init_early(struct intel_guc *guc); 428void intel_guc_init_late(struct intel_guc *guc); 429void intel_guc_init_send_regs(struct intel_guc *guc); 430void intel_guc_write_params(struct intel_guc *guc); 431int intel_guc_init(struct intel_guc *guc); 432void intel_guc_fini(struct intel_guc *guc); 433void intel_guc_notify(struct intel_guc *guc); 434int intel_guc_send_mmio(struct intel_guc *guc, const u32 *action, u32 len, 435 u32 *response_buf, u32 response_buf_size); 436int intel_guc_to_host_process_recv_msg(struct intel_guc *guc, 437 const u32 *payload, u32 len); 438int intel_guc_auth_huc(struct intel_guc *guc, u32 rsa_offset); 439int intel_guc_suspend(struct intel_guc *guc); 440int intel_guc_resume(struct intel_guc *guc); 441struct i915_vma *intel_guc_allocate_vma(struct intel_guc *guc, u32 size); 442int intel_guc_allocate_and_map_vma(struct intel_guc *guc, u32 size, 443 struct i915_vma **out_vma, void **out_vaddr); 444int intel_guc_self_cfg32(struct intel_guc *guc, u16 key, u32 value); 445int intel_guc_self_cfg64(struct intel_guc *guc, u16 key, u64 value); 446 447static inline bool intel_guc_is_supported(struct intel_guc *guc) 448{ 449 return intel_uc_fw_is_supported(&guc->fw); 450} 451 452static inline bool intel_guc_is_wanted(struct intel_guc *guc) 453{ 454 return intel_uc_fw_is_enabled(&guc->fw); 455} 456 457static inline bool intel_guc_is_used(struct intel_guc *guc) 458{ 459 GEM_BUG_ON(__intel_uc_fw_status(&guc->fw) == INTEL_UC_FIRMWARE_SELECTED); 460 return intel_uc_fw_is_available(&guc->fw); 461} 462 463static inline bool intel_guc_is_fw_running(struct intel_guc *guc) 464{ 465 return intel_uc_fw_is_running(&guc->fw); 466} 467 468static inline bool intel_guc_is_ready(struct intel_guc *guc) 469{ 470 return intel_guc_is_fw_running(guc) && intel_guc_ct_enabled(&guc->ct); 471} 472 473static inline void intel_guc_reset_interrupts(struct intel_guc *guc) 474{ 475 guc->interrupts.reset(guc); 476} 477 478static inline void intel_guc_enable_interrupts(struct intel_guc *guc) 479{ 480 guc->interrupts.enable(guc); 481} 482 483static inline void intel_guc_disable_interrupts(struct intel_guc *guc) 484{ 485 guc->interrupts.disable(guc); 486} 487 488static inline int intel_guc_sanitize(struct intel_guc *guc) 489{ 490 intel_uc_fw_sanitize(&guc->fw); 491 intel_guc_disable_interrupts(guc); 492 intel_guc_ct_sanitize(&guc->ct); 493 guc->mmio_msg = 0; 494 495 return 0; 496} 497 498static inline void intel_guc_enable_msg(struct intel_guc *guc, u32 mask) 499{ 500 spin_lock_irq(&guc->irq_lock); 501 guc->msg_enabled_mask |= mask; 502 spin_unlock_irq(&guc->irq_lock); 503} 504 505static inline void intel_guc_disable_msg(struct intel_guc *guc, u32 mask) 506{ 507 spin_lock_irq(&guc->irq_lock); 508 guc->msg_enabled_mask &= ~mask; 509 spin_unlock_irq(&guc->irq_lock); 510} 511 512int intel_guc_wait_for_idle(struct intel_guc *guc, long timeout); 513 514int intel_guc_deregister_done_process_msg(struct intel_guc *guc, 515 const u32 *msg, u32 len); 516int intel_guc_sched_done_process_msg(struct intel_guc *guc, 517 const u32 *msg, u32 len); 518int intel_guc_context_reset_process_msg(struct intel_guc *guc, 519 const u32 *msg, u32 len); 520int intel_guc_engine_failure_process_msg(struct intel_guc *guc, 521 const u32 *msg, u32 len); 522int intel_guc_error_capture_process_msg(struct intel_guc *guc, 523 const u32 *msg, u32 len); 524int intel_guc_crash_process_msg(struct intel_guc *guc, u32 action); 525 526struct intel_engine_cs * 527intel_guc_lookup_engine(struct intel_guc *guc, u8 guc_class, u8 instance); 528 529void intel_guc_find_hung_context(struct intel_engine_cs *engine); 530 531int intel_guc_global_policies_update(struct intel_guc *guc); 532 533void intel_guc_context_ban(struct intel_context *ce, struct i915_request *rq); 534 535void intel_guc_submission_reset_prepare(struct intel_guc *guc); 536void intel_guc_submission_reset(struct intel_guc *guc, intel_engine_mask_t stalled); 537void intel_guc_submission_reset_finish(struct intel_guc *guc); 538void intel_guc_submission_cancel_requests(struct intel_guc *guc); 539 540void intel_guc_load_status(struct intel_guc *guc, struct drm_printer *p); 541 542void intel_guc_write_barrier(struct intel_guc *guc); 543 544void intel_guc_dump_time_info(struct intel_guc *guc, struct drm_printer *p); 545 546int intel_guc_sched_disable_gucid_threshold_max(struct intel_guc *guc); 547 548bool intel_guc_tlb_invalidation_is_available(struct intel_guc *guc); 549int intel_guc_invalidate_tlb_engines(struct intel_guc *guc); 550int intel_guc_invalidate_tlb_guc(struct intel_guc *guc); 551int intel_guc_tlb_invalidation_done(struct intel_guc *guc, 552 const u32 *payload, u32 len); 553void wake_up_all_tlb_invalidate(struct intel_guc *guc); 554#endif 555