amdgpu_amdkfd_gfx_v9.c revision 1.2
1/* 2 * Copyright 2014-2018 Advanced Micro Devices, Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 */ 22#include <linux/mmu_context.h> 23 24#include "amdgpu.h" 25#include "amdgpu_amdkfd.h" 26#include "gc/gc_9_0_offset.h" 27#include "gc/gc_9_0_sh_mask.h" 28#include "vega10_enum.h" 29#include "sdma0/sdma0_4_0_offset.h" 30#include "sdma0/sdma0_4_0_sh_mask.h" 31#include "sdma1/sdma1_4_0_offset.h" 32#include "sdma1/sdma1_4_0_sh_mask.h" 33#include "athub/athub_1_0_offset.h" 34#include "athub/athub_1_0_sh_mask.h" 35#include "oss/osssys_4_0_offset.h" 36#include "oss/osssys_4_0_sh_mask.h" 37#include "soc15_common.h" 38#include "v9_structs.h" 39#include "soc15.h" 40#include "soc15d.h" 41#include "mmhub_v1_0.h" 42#include "gfxhub_v1_0.h" 43 44 45enum hqd_dequeue_request_type { 46 NO_ACTION = 0, 47 DRAIN_PIPE, 48 RESET_WAVES 49}; 50 51static inline struct amdgpu_device *get_amdgpu_device(struct kgd_dev *kgd) 52{ 53 return (struct amdgpu_device *)kgd; 54} 55 56static void lock_srbm(struct kgd_dev *kgd, uint32_t mec, uint32_t pipe, 57 uint32_t queue, uint32_t vmid) 58{ 59 struct amdgpu_device *adev = get_amdgpu_device(kgd); 60 61 mutex_lock(&adev->srbm_mutex); 62 soc15_grbm_select(adev, mec, pipe, queue, vmid); 63} 64 65static void unlock_srbm(struct kgd_dev *kgd) 66{ 67 struct amdgpu_device *adev = get_amdgpu_device(kgd); 68 69 soc15_grbm_select(adev, 0, 0, 0, 0); 70 mutex_unlock(&adev->srbm_mutex); 71} 72 73static void acquire_queue(struct kgd_dev *kgd, uint32_t pipe_id, 74 uint32_t queue_id) 75{ 76 struct amdgpu_device *adev = get_amdgpu_device(kgd); 77 78 uint32_t mec = (pipe_id / adev->gfx.mec.num_pipe_per_mec) + 1; 79 uint32_t pipe = (pipe_id % adev->gfx.mec.num_pipe_per_mec); 80 81 lock_srbm(kgd, mec, pipe, queue_id, 0); 82} 83 84static uint64_t get_queue_mask(struct amdgpu_device *adev, 85 uint32_t pipe_id, uint32_t queue_id) 86{ 87 unsigned int bit = pipe_id * adev->gfx.mec.num_queue_per_pipe + 88 queue_id; 89 90 return 1ull << bit; 91} 92 93static void release_queue(struct kgd_dev *kgd) 94{ 95 unlock_srbm(kgd); 96} 97 98void kgd_gfx_v9_program_sh_mem_settings(struct kgd_dev *kgd, uint32_t vmid, 99 uint32_t sh_mem_config, 100 uint32_t sh_mem_ape1_base, 101 uint32_t sh_mem_ape1_limit, 102 uint32_t sh_mem_bases) 103{ 104 struct amdgpu_device *adev = get_amdgpu_device(kgd); 105 106 lock_srbm(kgd, 0, 0, 0, vmid); 107 108 WREG32_RLC(SOC15_REG_OFFSET(GC, 0, mmSH_MEM_CONFIG), sh_mem_config); 109 WREG32_RLC(SOC15_REG_OFFSET(GC, 0, mmSH_MEM_BASES), sh_mem_bases); 110 /* APE1 no longer exists on GFX9 */ 111 112 unlock_srbm(kgd); 113} 114 115int kgd_gfx_v9_set_pasid_vmid_mapping(struct kgd_dev *kgd, unsigned int pasid, 116 unsigned int vmid) 117{ 118 struct amdgpu_device *adev = get_amdgpu_device(kgd); 119 120 /* 121 * We have to assume that there is no outstanding mapping. 122 * The ATC_VMID_PASID_MAPPING_UPDATE_STATUS bit could be 0 because 123 * a mapping is in progress or because a mapping finished 124 * and the SW cleared it. 125 * So the protocol is to always wait & clear. 126 */ 127 uint32_t pasid_mapping = (pasid == 0) ? 0 : (uint32_t)pasid | 128 ATC_VMID0_PASID_MAPPING__VALID_MASK; 129 130 /* 131 * need to do this twice, once for gfx and once for mmhub 132 * for ATC add 16 to VMID for mmhub, for IH different registers. 133 * ATC_VMID0..15 registers are separate from ATC_VMID16..31. 134 */ 135 136 WREG32(SOC15_REG_OFFSET(ATHUB, 0, mmATC_VMID0_PASID_MAPPING) + vmid, 137 pasid_mapping); 138 139 while (!(RREG32(SOC15_REG_OFFSET( 140 ATHUB, 0, 141 mmATC_VMID_PASID_MAPPING_UPDATE_STATUS)) & 142 (1U << vmid))) 143 cpu_relax(); 144 145 WREG32(SOC15_REG_OFFSET(ATHUB, 0, 146 mmATC_VMID_PASID_MAPPING_UPDATE_STATUS), 147 1U << vmid); 148 149 /* Mapping vmid to pasid also for IH block */ 150 WREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_VMID_0_LUT) + vmid, 151 pasid_mapping); 152 153 WREG32(SOC15_REG_OFFSET(ATHUB, 0, mmATC_VMID16_PASID_MAPPING) + vmid, 154 pasid_mapping); 155 156 while (!(RREG32(SOC15_REG_OFFSET( 157 ATHUB, 0, 158 mmATC_VMID_PASID_MAPPING_UPDATE_STATUS)) & 159 (1U << (vmid + 16)))) 160 cpu_relax(); 161 162 WREG32(SOC15_REG_OFFSET(ATHUB, 0, 163 mmATC_VMID_PASID_MAPPING_UPDATE_STATUS), 164 1U << (vmid + 16)); 165 166 /* Mapping vmid to pasid also for IH block */ 167 WREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_VMID_0_LUT_MM) + vmid, 168 pasid_mapping); 169 return 0; 170} 171 172/* TODO - RING0 form of field is obsolete, seems to date back to SI 173 * but still works 174 */ 175 176int kgd_gfx_v9_init_interrupts(struct kgd_dev *kgd, uint32_t pipe_id) 177{ 178 struct amdgpu_device *adev = get_amdgpu_device(kgd); 179 uint32_t mec; 180 uint32_t pipe; 181 182 mec = (pipe_id / adev->gfx.mec.num_pipe_per_mec) + 1; 183 pipe = (pipe_id % adev->gfx.mec.num_pipe_per_mec); 184 185 lock_srbm(kgd, mec, pipe, 0, 0); 186 187 WREG32(SOC15_REG_OFFSET(GC, 0, mmCPC_INT_CNTL), 188 CP_INT_CNTL_RING0__TIME_STAMP_INT_ENABLE_MASK | 189 CP_INT_CNTL_RING0__OPCODE_ERROR_INT_ENABLE_MASK); 190 191 unlock_srbm(kgd); 192 193 return 0; 194} 195 196static uint32_t get_sdma_rlc_reg_offset(struct amdgpu_device *adev, 197 unsigned int engine_id, 198 unsigned int queue_id) 199{ 200 uint32_t sdma_engine_reg_base[2] = { 201 SOC15_REG_OFFSET(SDMA0, 0, 202 mmSDMA0_RLC0_RB_CNTL) - mmSDMA0_RLC0_RB_CNTL, 203 SOC15_REG_OFFSET(SDMA1, 0, 204 mmSDMA1_RLC0_RB_CNTL) - mmSDMA1_RLC0_RB_CNTL 205 }; 206 uint32_t retval = sdma_engine_reg_base[engine_id] 207 + queue_id * (mmSDMA0_RLC1_RB_CNTL - mmSDMA0_RLC0_RB_CNTL); 208 209 pr_debug("RLC register offset for SDMA%d RLC%d: 0x%x\n", engine_id, 210 queue_id, retval); 211 212 return retval; 213} 214 215static inline struct v9_mqd *get_mqd(void *mqd) 216{ 217 return (struct v9_mqd *)mqd; 218} 219 220static inline struct v9_sdma_mqd *get_sdma_mqd(void *mqd) 221{ 222 return (struct v9_sdma_mqd *)mqd; 223} 224 225int kgd_gfx_v9_hqd_load(struct kgd_dev *kgd, void *mqd, uint32_t pipe_id, 226 uint32_t queue_id, uint32_t __user *wptr, 227 uint32_t wptr_shift, uint32_t wptr_mask, 228 struct mm_struct *mm) 229{ 230 struct amdgpu_device *adev = get_amdgpu_device(kgd); 231 struct v9_mqd *m; 232 uint32_t *mqd_hqd; 233 uint32_t reg, hqd_base, data; 234 235 m = get_mqd(mqd); 236 237 acquire_queue(kgd, pipe_id, queue_id); 238 239 /* HQD registers extend from CP_MQD_BASE_ADDR to CP_HQD_EOP_WPTR_MEM. */ 240 mqd_hqd = &m->cp_mqd_base_addr_lo; 241 hqd_base = SOC15_REG_OFFSET(GC, 0, mmCP_MQD_BASE_ADDR); 242 243 for (reg = hqd_base; 244 reg <= SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_WPTR_HI); reg++) 245 WREG32_RLC(reg, mqd_hqd[reg - hqd_base]); 246 247 248 /* Activate doorbell logic before triggering WPTR poll. */ 249 data = REG_SET_FIELD(m->cp_hqd_pq_doorbell_control, 250 CP_HQD_PQ_DOORBELL_CONTROL, DOORBELL_EN, 1); 251 WREG32_RLC(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL), data); 252 253 if (wptr) { 254 /* Don't read wptr with get_user because the user 255 * context may not be accessible (if this function 256 * runs in a work queue). Instead trigger a one-shot 257 * polling read from memory in the CP. This assumes 258 * that wptr is GPU-accessible in the queue's VMID via 259 * ATC or SVM. WPTR==RPTR before starting the poll so 260 * the CP starts fetching new commands from the right 261 * place. 262 * 263 * Guessing a 64-bit WPTR from a 32-bit RPTR is a bit 264 * tricky. Assume that the queue didn't overflow. The 265 * number of valid bits in the 32-bit RPTR depends on 266 * the queue size. The remaining bits are taken from 267 * the saved 64-bit WPTR. If the WPTR wrapped, add the 268 * queue size. 269 */ 270 uint32_t queue_size = 271 2 << REG_GET_FIELD(m->cp_hqd_pq_control, 272 CP_HQD_PQ_CONTROL, QUEUE_SIZE); 273 uint64_t guessed_wptr = m->cp_hqd_pq_rptr & (queue_size - 1); 274 275 if ((m->cp_hqd_pq_wptr_lo & (queue_size - 1)) < guessed_wptr) 276 guessed_wptr += queue_size; 277 guessed_wptr += m->cp_hqd_pq_wptr_lo & ~(queue_size - 1); 278 guessed_wptr += (uint64_t)m->cp_hqd_pq_wptr_hi << 32; 279 280 WREG32_RLC(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_WPTR_LO), 281 lower_32_bits(guessed_wptr)); 282 WREG32_RLC(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_WPTR_HI), 283 upper_32_bits(guessed_wptr)); 284 WREG32_RLC(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_WPTR_POLL_ADDR), 285 lower_32_bits((uintptr_t)wptr)); 286 WREG32_RLC(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_WPTR_POLL_ADDR_HI), 287 upper_32_bits((uintptr_t)wptr)); 288 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_PQ_WPTR_POLL_CNTL1), 289 (uint32_t)get_queue_mask(adev, pipe_id, queue_id)); 290 } 291 292 /* Start the EOP fetcher */ 293 WREG32_RLC(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_EOP_RPTR), 294 REG_SET_FIELD(m->cp_hqd_eop_rptr, 295 CP_HQD_EOP_RPTR, INIT_FETCHER, 1)); 296 297 data = REG_SET_FIELD(m->cp_hqd_active, CP_HQD_ACTIVE, ACTIVE, 1); 298 WREG32_RLC(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_ACTIVE), data); 299 300 release_queue(kgd); 301 302 return 0; 303} 304 305int kgd_gfx_v9_hiq_mqd_load(struct kgd_dev *kgd, void *mqd, 306 uint32_t pipe_id, uint32_t queue_id, 307 uint32_t doorbell_off) 308{ 309 struct amdgpu_device *adev = get_amdgpu_device(kgd); 310 struct amdgpu_ring *kiq_ring = &adev->gfx.kiq.ring; 311 struct v9_mqd *m; 312 uint32_t mec, pipe; 313 int r; 314 315 m = get_mqd(mqd); 316 317 acquire_queue(kgd, pipe_id, queue_id); 318 319 mec = (pipe_id / adev->gfx.mec.num_pipe_per_mec) + 1; 320 pipe = (pipe_id % adev->gfx.mec.num_pipe_per_mec); 321 322 pr_debug("kfd: set HIQ, mec:%d, pipe:%d, queue:%d.\n", 323 mec, pipe, queue_id); 324 325 spin_lock(&adev->gfx.kiq.ring_lock); 326 r = amdgpu_ring_alloc(kiq_ring, 7); 327 if (r) { 328 pr_err("Failed to alloc KIQ (%d).\n", r); 329 goto out_unlock; 330 } 331 332 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_MAP_QUEUES, 5)); 333 amdgpu_ring_write(kiq_ring, 334 PACKET3_MAP_QUEUES_QUEUE_SEL(0) | /* Queue_Sel */ 335 PACKET3_MAP_QUEUES_VMID(m->cp_hqd_vmid) | /* VMID */ 336 PACKET3_MAP_QUEUES_QUEUE(queue_id) | 337 PACKET3_MAP_QUEUES_PIPE(pipe) | 338 PACKET3_MAP_QUEUES_ME((mec - 1)) | 339 PACKET3_MAP_QUEUES_QUEUE_TYPE(0) | /*queue_type: normal compute queue */ 340 PACKET3_MAP_QUEUES_ALLOC_FORMAT(0) | /* alloc format: all_on_one_pipe */ 341 PACKET3_MAP_QUEUES_ENGINE_SEL(1) | /* engine_sel: hiq */ 342 PACKET3_MAP_QUEUES_NUM_QUEUES(1)); /* num_queues: must be 1 */ 343 amdgpu_ring_write(kiq_ring, 344 PACKET3_MAP_QUEUES_DOORBELL_OFFSET(doorbell_off)); 345 amdgpu_ring_write(kiq_ring, m->cp_mqd_base_addr_lo); 346 amdgpu_ring_write(kiq_ring, m->cp_mqd_base_addr_hi); 347 amdgpu_ring_write(kiq_ring, m->cp_hqd_pq_wptr_poll_addr_lo); 348 amdgpu_ring_write(kiq_ring, m->cp_hqd_pq_wptr_poll_addr_hi); 349 amdgpu_ring_commit(kiq_ring); 350 351out_unlock: 352 spin_unlock(&adev->gfx.kiq.ring_lock); 353 release_queue(kgd); 354 355 return r; 356} 357 358int kgd_gfx_v9_hqd_dump(struct kgd_dev *kgd, 359 uint32_t pipe_id, uint32_t queue_id, 360 uint32_t (**dump)[2], uint32_t *n_regs) 361{ 362 struct amdgpu_device *adev = get_amdgpu_device(kgd); 363 uint32_t i = 0, reg; 364#define HQD_N_REGS 56 365#define DUMP_REG(addr) do { \ 366 if (WARN_ON_ONCE(i >= HQD_N_REGS)) \ 367 break; \ 368 (*dump)[i][0] = (addr) << 2; \ 369 (*dump)[i++][1] = RREG32(addr); \ 370 } while (0) 371 372 *dump = kmalloc_array(HQD_N_REGS * 2, sizeof(uint32_t), GFP_KERNEL); 373 if (*dump == NULL) 374 return -ENOMEM; 375 376 acquire_queue(kgd, pipe_id, queue_id); 377 378 for (reg = SOC15_REG_OFFSET(GC, 0, mmCP_MQD_BASE_ADDR); 379 reg <= SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_WPTR_HI); reg++) 380 DUMP_REG(reg); 381 382 release_queue(kgd); 383 384 WARN_ON_ONCE(i != HQD_N_REGS); 385 *n_regs = i; 386 387 return 0; 388} 389 390static int kgd_hqd_sdma_load(struct kgd_dev *kgd, void *mqd, 391 uint32_t __user *wptr, struct mm_struct *mm) 392{ 393 struct amdgpu_device *adev = get_amdgpu_device(kgd); 394 struct v9_sdma_mqd *m; 395 uint32_t sdma_rlc_reg_offset; 396 unsigned long end_jiffies; 397 uint32_t data; 398 uint64_t data64; 399 uint64_t __user *wptr64 = (uint64_t __user *)wptr; 400 401 m = get_sdma_mqd(mqd); 402 sdma_rlc_reg_offset = get_sdma_rlc_reg_offset(adev, m->sdma_engine_id, 403 m->sdma_queue_id); 404 405 WREG32(sdma_rlc_reg_offset + mmSDMA0_RLC0_RB_CNTL, 406 m->sdmax_rlcx_rb_cntl & (~SDMA0_RLC0_RB_CNTL__RB_ENABLE_MASK)); 407 408 end_jiffies = msecs_to_jiffies(2000) + jiffies; 409 while (true) { 410 data = RREG32(sdma_rlc_reg_offset + mmSDMA0_RLC0_CONTEXT_STATUS); 411 if (data & SDMA0_RLC0_CONTEXT_STATUS__IDLE_MASK) 412 break; 413 if (time_after(jiffies, end_jiffies)) { 414 pr_err("SDMA RLC not idle in %s\n", __func__); 415 return -ETIME; 416 } 417 usleep_range(500, 1000); 418 } 419 420 WREG32(sdma_rlc_reg_offset + mmSDMA0_RLC0_DOORBELL_OFFSET, 421 m->sdmax_rlcx_doorbell_offset); 422 423 data = REG_SET_FIELD(m->sdmax_rlcx_doorbell, SDMA0_RLC0_DOORBELL, 424 ENABLE, 1); 425 WREG32(sdma_rlc_reg_offset + mmSDMA0_RLC0_DOORBELL, data); 426 WREG32(sdma_rlc_reg_offset + mmSDMA0_RLC0_RB_RPTR, 427 m->sdmax_rlcx_rb_rptr); 428 WREG32(sdma_rlc_reg_offset + mmSDMA0_RLC0_RB_RPTR_HI, 429 m->sdmax_rlcx_rb_rptr_hi); 430 431 WREG32(sdma_rlc_reg_offset + mmSDMA0_RLC0_MINOR_PTR_UPDATE, 1); 432 if (read_user_wptr(mm, wptr64, data64)) { 433 WREG32(sdma_rlc_reg_offset + mmSDMA0_RLC0_RB_WPTR, 434 lower_32_bits(data64)); 435 WREG32(sdma_rlc_reg_offset + mmSDMA0_RLC0_RB_WPTR_HI, 436 upper_32_bits(data64)); 437 } else { 438 WREG32(sdma_rlc_reg_offset + mmSDMA0_RLC0_RB_WPTR, 439 m->sdmax_rlcx_rb_rptr); 440 WREG32(sdma_rlc_reg_offset + mmSDMA0_RLC0_RB_WPTR_HI, 441 m->sdmax_rlcx_rb_rptr_hi); 442 } 443 WREG32(sdma_rlc_reg_offset + mmSDMA0_RLC0_MINOR_PTR_UPDATE, 0); 444 445 WREG32(sdma_rlc_reg_offset + mmSDMA0_RLC0_RB_BASE, m->sdmax_rlcx_rb_base); 446 WREG32(sdma_rlc_reg_offset + mmSDMA0_RLC0_RB_BASE_HI, 447 m->sdmax_rlcx_rb_base_hi); 448 WREG32(sdma_rlc_reg_offset + mmSDMA0_RLC0_RB_RPTR_ADDR_LO, 449 m->sdmax_rlcx_rb_rptr_addr_lo); 450 WREG32(sdma_rlc_reg_offset + mmSDMA0_RLC0_RB_RPTR_ADDR_HI, 451 m->sdmax_rlcx_rb_rptr_addr_hi); 452 453 data = REG_SET_FIELD(m->sdmax_rlcx_rb_cntl, SDMA0_RLC0_RB_CNTL, 454 RB_ENABLE, 1); 455 WREG32(sdma_rlc_reg_offset + mmSDMA0_RLC0_RB_CNTL, data); 456 457 return 0; 458} 459 460static int kgd_hqd_sdma_dump(struct kgd_dev *kgd, 461 uint32_t engine_id, uint32_t queue_id, 462 uint32_t (**dump)[2], uint32_t *n_regs) 463{ 464 struct amdgpu_device *adev = get_amdgpu_device(kgd); 465 uint32_t sdma_rlc_reg_offset = get_sdma_rlc_reg_offset(adev, 466 engine_id, queue_id); 467 uint32_t i = 0, reg; 468#undef HQD_N_REGS 469#define HQD_N_REGS (19+6+7+10) 470 471 *dump = kmalloc_array(HQD_N_REGS * 2, sizeof(uint32_t), GFP_KERNEL); 472 if (*dump == NULL) 473 return -ENOMEM; 474 475 for (reg = mmSDMA0_RLC0_RB_CNTL; reg <= mmSDMA0_RLC0_DOORBELL; reg++) 476 DUMP_REG(sdma_rlc_reg_offset + reg); 477 for (reg = mmSDMA0_RLC0_STATUS; reg <= mmSDMA0_RLC0_CSA_ADDR_HI; reg++) 478 DUMP_REG(sdma_rlc_reg_offset + reg); 479 for (reg = mmSDMA0_RLC0_IB_SUB_REMAIN; 480 reg <= mmSDMA0_RLC0_MINOR_PTR_UPDATE; reg++) 481 DUMP_REG(sdma_rlc_reg_offset + reg); 482 for (reg = mmSDMA0_RLC0_MIDCMD_DATA0; 483 reg <= mmSDMA0_RLC0_MIDCMD_CNTL; reg++) 484 DUMP_REG(sdma_rlc_reg_offset + reg); 485 486 WARN_ON_ONCE(i != HQD_N_REGS); 487 *n_regs = i; 488 489 return 0; 490} 491 492bool kgd_gfx_v9_hqd_is_occupied(struct kgd_dev *kgd, uint64_t queue_address, 493 uint32_t pipe_id, uint32_t queue_id) 494{ 495 struct amdgpu_device *adev = get_amdgpu_device(kgd); 496 uint32_t act; 497 bool retval = false; 498 uint32_t low, high; 499 500 acquire_queue(kgd, pipe_id, queue_id); 501 act = RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_ACTIVE)); 502 if (act) { 503 low = lower_32_bits(queue_address >> 8); 504 high = upper_32_bits(queue_address >> 8); 505 506 if (low == RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_BASE)) && 507 high == RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_PQ_BASE_HI))) 508 retval = true; 509 } 510 release_queue(kgd); 511 return retval; 512} 513 514static bool kgd_hqd_sdma_is_occupied(struct kgd_dev *kgd, void *mqd) 515{ 516 struct amdgpu_device *adev = get_amdgpu_device(kgd); 517 struct v9_sdma_mqd *m; 518 uint32_t sdma_rlc_reg_offset; 519 uint32_t sdma_rlc_rb_cntl; 520 521 m = get_sdma_mqd(mqd); 522 sdma_rlc_reg_offset = get_sdma_rlc_reg_offset(adev, m->sdma_engine_id, 523 m->sdma_queue_id); 524 525 sdma_rlc_rb_cntl = RREG32(sdma_rlc_reg_offset + mmSDMA0_RLC0_RB_CNTL); 526 527 if (sdma_rlc_rb_cntl & SDMA0_RLC0_RB_CNTL__RB_ENABLE_MASK) 528 return true; 529 530 return false; 531} 532 533int kgd_gfx_v9_hqd_destroy(struct kgd_dev *kgd, void *mqd, 534 enum kfd_preempt_type reset_type, 535 unsigned int utimeout, uint32_t pipe_id, 536 uint32_t queue_id) 537{ 538 struct amdgpu_device *adev = get_amdgpu_device(kgd); 539 enum hqd_dequeue_request_type type; 540 unsigned long end_jiffies; 541 uint32_t temp; 542 struct v9_mqd *m = get_mqd(mqd); 543 544 if (adev->in_gpu_reset) 545 return -EIO; 546 547 acquire_queue(kgd, pipe_id, queue_id); 548 549 if (m->cp_hqd_vmid == 0) 550 WREG32_FIELD15_RLC(GC, 0, RLC_CP_SCHEDULERS, scheduler1, 0); 551 552 switch (reset_type) { 553 case KFD_PREEMPT_TYPE_WAVEFRONT_DRAIN: 554 type = DRAIN_PIPE; 555 break; 556 case KFD_PREEMPT_TYPE_WAVEFRONT_RESET: 557 type = RESET_WAVES; 558 break; 559 default: 560 type = DRAIN_PIPE; 561 break; 562 } 563 564 WREG32_RLC(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_DEQUEUE_REQUEST), type); 565 566 end_jiffies = (utimeout * HZ / 1000) + jiffies; 567 while (true) { 568 temp = RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_HQD_ACTIVE)); 569 if (!(temp & CP_HQD_ACTIVE__ACTIVE_MASK)) 570 break; 571 if (time_after(jiffies, end_jiffies)) { 572 pr_err("cp queue preemption time out.\n"); 573 release_queue(kgd); 574 return -ETIME; 575 } 576 usleep_range(500, 1000); 577 } 578 579 release_queue(kgd); 580 return 0; 581} 582 583static int kgd_hqd_sdma_destroy(struct kgd_dev *kgd, void *mqd, 584 unsigned int utimeout) 585{ 586 struct amdgpu_device *adev = get_amdgpu_device(kgd); 587 struct v9_sdma_mqd *m; 588 uint32_t sdma_rlc_reg_offset; 589 uint32_t temp; 590 unsigned long end_jiffies = (utimeout * HZ / 1000) + jiffies; 591 592 m = get_sdma_mqd(mqd); 593 sdma_rlc_reg_offset = get_sdma_rlc_reg_offset(adev, m->sdma_engine_id, 594 m->sdma_queue_id); 595 596 temp = RREG32(sdma_rlc_reg_offset + mmSDMA0_RLC0_RB_CNTL); 597 temp = temp & ~SDMA0_RLC0_RB_CNTL__RB_ENABLE_MASK; 598 WREG32(sdma_rlc_reg_offset + mmSDMA0_RLC0_RB_CNTL, temp); 599 600 while (true) { 601 temp = RREG32(sdma_rlc_reg_offset + mmSDMA0_RLC0_CONTEXT_STATUS); 602 if (temp & SDMA0_RLC0_CONTEXT_STATUS__IDLE_MASK) 603 break; 604 if (time_after(jiffies, end_jiffies)) { 605 pr_err("SDMA RLC not idle in %s\n", __func__); 606 return -ETIME; 607 } 608 usleep_range(500, 1000); 609 } 610 611 WREG32(sdma_rlc_reg_offset + mmSDMA0_RLC0_DOORBELL, 0); 612 WREG32(sdma_rlc_reg_offset + mmSDMA0_RLC0_RB_CNTL, 613 RREG32(sdma_rlc_reg_offset + mmSDMA0_RLC0_RB_CNTL) | 614 SDMA0_RLC0_RB_CNTL__RB_ENABLE_MASK); 615 616 m->sdmax_rlcx_rb_rptr = RREG32(sdma_rlc_reg_offset + mmSDMA0_RLC0_RB_RPTR); 617 m->sdmax_rlcx_rb_rptr_hi = 618 RREG32(sdma_rlc_reg_offset + mmSDMA0_RLC0_RB_RPTR_HI); 619 620 return 0; 621} 622 623bool kgd_gfx_v9_get_atc_vmid_pasid_mapping_info(struct kgd_dev *kgd, 624 uint8_t vmid, uint16_t *p_pasid) 625{ 626 uint32_t value; 627 struct amdgpu_device *adev = (struct amdgpu_device *) kgd; 628 629 value = RREG32(SOC15_REG_OFFSET(ATHUB, 0, mmATC_VMID0_PASID_MAPPING) 630 + vmid); 631 *p_pasid = value & ATC_VMID0_PASID_MAPPING__PASID_MASK; 632 633 return !!(value & ATC_VMID0_PASID_MAPPING__VALID_MASK); 634} 635 636int kgd_gfx_v9_address_watch_disable(struct kgd_dev *kgd) 637{ 638 return 0; 639} 640 641int kgd_gfx_v9_address_watch_execute(struct kgd_dev *kgd, 642 unsigned int watch_point_id, 643 uint32_t cntl_val, 644 uint32_t addr_hi, 645 uint32_t addr_lo) 646{ 647 return 0; 648} 649 650int kgd_gfx_v9_wave_control_execute(struct kgd_dev *kgd, 651 uint32_t gfx_index_val, 652 uint32_t sq_cmd) 653{ 654 struct amdgpu_device *adev = get_amdgpu_device(kgd); 655 uint32_t data = 0; 656 657 mutex_lock(&adev->grbm_idx_mutex); 658 659 WREG32_SOC15_RLC_SHADOW(GC, 0, mmGRBM_GFX_INDEX, gfx_index_val); 660 WREG32(SOC15_REG_OFFSET(GC, 0, mmSQ_CMD), sq_cmd); 661 662 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, 663 INSTANCE_BROADCAST_WRITES, 1); 664 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, 665 SH_BROADCAST_WRITES, 1); 666 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, 667 SE_BROADCAST_WRITES, 1); 668 669 WREG32_SOC15_RLC_SHADOW(GC, 0, mmGRBM_GFX_INDEX, data); 670 mutex_unlock(&adev->grbm_idx_mutex); 671 672 return 0; 673} 674 675uint32_t kgd_gfx_v9_address_watch_get_offset(struct kgd_dev *kgd, 676 unsigned int watch_point_id, 677 unsigned int reg_offset) 678{ 679 return 0; 680} 681 682static void kgd_gfx_v9_set_vm_context_page_table_base(struct kgd_dev *kgd, 683 uint32_t vmid, uint64_t page_table_base) 684{ 685 struct amdgpu_device *adev = get_amdgpu_device(kgd); 686 687 if (!amdgpu_amdkfd_is_kfd_vmid(adev, vmid)) { 688 pr_err("trying to set page table base for wrong VMID %u\n", 689 vmid); 690 return; 691 } 692 693 mmhub_v1_0_setup_vm_pt_regs(adev, vmid, page_table_base); 694 695 gfxhub_v1_0_setup_vm_pt_regs(adev, vmid, page_table_base); 696} 697 698const struct kfd2kgd_calls gfx_v9_kfd2kgd = { 699 .program_sh_mem_settings = kgd_gfx_v9_program_sh_mem_settings, 700 .set_pasid_vmid_mapping = kgd_gfx_v9_set_pasid_vmid_mapping, 701 .init_interrupts = kgd_gfx_v9_init_interrupts, 702 .hqd_load = kgd_gfx_v9_hqd_load, 703 .hiq_mqd_load = kgd_gfx_v9_hiq_mqd_load, 704 .hqd_sdma_load = kgd_hqd_sdma_load, 705 .hqd_dump = kgd_gfx_v9_hqd_dump, 706 .hqd_sdma_dump = kgd_hqd_sdma_dump, 707 .hqd_is_occupied = kgd_gfx_v9_hqd_is_occupied, 708 .hqd_sdma_is_occupied = kgd_hqd_sdma_is_occupied, 709 .hqd_destroy = kgd_gfx_v9_hqd_destroy, 710 .hqd_sdma_destroy = kgd_hqd_sdma_destroy, 711 .address_watch_disable = kgd_gfx_v9_address_watch_disable, 712 .address_watch_execute = kgd_gfx_v9_address_watch_execute, 713 .wave_control_execute = kgd_gfx_v9_wave_control_execute, 714 .address_watch_get_offset = kgd_gfx_v9_address_watch_get_offset, 715 .get_atc_vmid_pasid_mapping_info = 716 kgd_gfx_v9_get_atc_vmid_pasid_mapping_info, 717 .set_vm_context_page_table_base = kgd_gfx_v9_set_vm_context_page_table_base, 718 .get_hive_id = amdgpu_amdkfd_get_hive_id, 719 .get_unique_id = amdgpu_amdkfd_get_unique_id, 720}; 721