1254885Sdumbbell/* 2254885Sdumbbell * Copyright 2011 Advanced Micro Devices, Inc. 3254885Sdumbbell * 4254885Sdumbbell * Permission is hereby granted, free of charge, to any person obtaining a 5254885Sdumbbell * copy of this software and associated documentation files (the "Software"), 6254885Sdumbbell * to deal in the Software without restriction, including without limitation 7254885Sdumbbell * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8254885Sdumbbell * and/or sell copies of the Software, and to permit persons to whom the 9254885Sdumbbell * Software is furnished to do so, subject to the following conditions: 10254885Sdumbbell * 11254885Sdumbbell * The above copyright notice and this permission notice shall be included in 12254885Sdumbbell * all copies or substantial portions of the Software. 13254885Sdumbbell * 14254885Sdumbbell * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15254885Sdumbbell * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16254885Sdumbbell * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17254885Sdumbbell * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18254885Sdumbbell * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19254885Sdumbbell * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20254885Sdumbbell * OTHER DEALINGS IN THE SOFTWARE. 21254885Sdumbbell * 22254885Sdumbbell * Authors: Alex Deucher 23254885Sdumbbell */ 24254885Sdumbbell 25254885Sdumbbell#include <sys/cdefs.h> 26254885Sdumbbell__FBSDID("$FreeBSD$"); 27254885Sdumbbell 28254885Sdumbbell#include <dev/drm2/drmP.h> 29254885Sdumbbell#include "radeon.h" 30254885Sdumbbell#include "radeon_asic.h" 31254885Sdumbbell#include <dev/drm2/radeon/radeon_drm.h> 32254885Sdumbbell#include "sid.h" 33254885Sdumbbell#include "atom.h" 34254885Sdumbbell#include "si_blit_shaders.h" 35254885Sdumbbell 36254885Sdumbbell#define SI_PFP_UCODE_SIZE 2144 37254885Sdumbbell#define SI_PM4_UCODE_SIZE 2144 38254885Sdumbbell#define SI_CE_UCODE_SIZE 2144 39254885Sdumbbell#define SI_RLC_UCODE_SIZE 2048 40254885Sdumbbell#define SI_MC_UCODE_SIZE 7769 41254885Sdumbbell 42254885Sdumbbell/* get temperature in millidegrees */ 43254885Sdumbbellint si_get_temp(struct radeon_device *rdev) 44254885Sdumbbell{ 45254885Sdumbbell u32 temp; 46254885Sdumbbell int actual_temp = 0; 47254885Sdumbbell 48254885Sdumbbell temp = (RREG32(CG_MULT_THERMAL_STATUS) & CTF_TEMP_MASK) >> 49254885Sdumbbell CTF_TEMP_SHIFT; 50254885Sdumbbell 51254885Sdumbbell if (temp & 0x200) 52254885Sdumbbell actual_temp = 255; 53254885Sdumbbell else 54254885Sdumbbell actual_temp = temp & 0x1ff; 55254885Sdumbbell 56254885Sdumbbell actual_temp = (actual_temp * 1000); 57254885Sdumbbell 58254885Sdumbbell return actual_temp; 59254885Sdumbbell} 60254885Sdumbbell 61254885Sdumbbell#define TAHITI_IO_MC_REGS_SIZE 36 62254885Sdumbbell 63254885Sdumbbellstatic const u32 tahiti_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = { 64254885Sdumbbell {0x0000006f, 0x03044000}, 65254885Sdumbbell {0x00000070, 0x0480c018}, 66254885Sdumbbell {0x00000071, 0x00000040}, 67254885Sdumbbell {0x00000072, 0x01000000}, 68254885Sdumbbell {0x00000074, 0x000000ff}, 69254885Sdumbbell {0x00000075, 0x00143400}, 70254885Sdumbbell {0x00000076, 0x08ec0800}, 71254885Sdumbbell {0x00000077, 0x040000cc}, 72254885Sdumbbell {0x00000079, 0x00000000}, 73254885Sdumbbell {0x0000007a, 0x21000409}, 74254885Sdumbbell {0x0000007c, 0x00000000}, 75254885Sdumbbell {0x0000007d, 0xe8000000}, 76254885Sdumbbell {0x0000007e, 0x044408a8}, 77254885Sdumbbell {0x0000007f, 0x00000003}, 78254885Sdumbbell {0x00000080, 0x00000000}, 79254885Sdumbbell {0x00000081, 0x01000000}, 80254885Sdumbbell {0x00000082, 0x02000000}, 81254885Sdumbbell {0x00000083, 0x00000000}, 82254885Sdumbbell {0x00000084, 0xe3f3e4f4}, 83254885Sdumbbell {0x00000085, 0x00052024}, 84254885Sdumbbell {0x00000087, 0x00000000}, 85254885Sdumbbell {0x00000088, 0x66036603}, 86254885Sdumbbell {0x00000089, 0x01000000}, 87254885Sdumbbell {0x0000008b, 0x1c0a0000}, 88254885Sdumbbell {0x0000008c, 0xff010000}, 89254885Sdumbbell {0x0000008e, 0xffffefff}, 90254885Sdumbbell {0x0000008f, 0xfff3efff}, 91254885Sdumbbell {0x00000090, 0xfff3efbf}, 92254885Sdumbbell {0x00000094, 0x00101101}, 93254885Sdumbbell {0x00000095, 0x00000fff}, 94254885Sdumbbell {0x00000096, 0x00116fff}, 95254885Sdumbbell {0x00000097, 0x60010000}, 96254885Sdumbbell {0x00000098, 0x10010000}, 97254885Sdumbbell {0x00000099, 0x00006000}, 98254885Sdumbbell {0x0000009a, 0x00001000}, 99254885Sdumbbell {0x0000009f, 0x00a77400} 100254885Sdumbbell}; 101254885Sdumbbell 102254885Sdumbbellstatic const u32 pitcairn_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = { 103254885Sdumbbell {0x0000006f, 0x03044000}, 104254885Sdumbbell {0x00000070, 0x0480c018}, 105254885Sdumbbell {0x00000071, 0x00000040}, 106254885Sdumbbell {0x00000072, 0x01000000}, 107254885Sdumbbell {0x00000074, 0x000000ff}, 108254885Sdumbbell {0x00000075, 0x00143400}, 109254885Sdumbbell {0x00000076, 0x08ec0800}, 110254885Sdumbbell {0x00000077, 0x040000cc}, 111254885Sdumbbell {0x00000079, 0x00000000}, 112254885Sdumbbell {0x0000007a, 0x21000409}, 113254885Sdumbbell {0x0000007c, 0x00000000}, 114254885Sdumbbell {0x0000007d, 0xe8000000}, 115254885Sdumbbell {0x0000007e, 0x044408a8}, 116254885Sdumbbell {0x0000007f, 0x00000003}, 117254885Sdumbbell {0x00000080, 0x00000000}, 118254885Sdumbbell {0x00000081, 0x01000000}, 119254885Sdumbbell {0x00000082, 0x02000000}, 120254885Sdumbbell {0x00000083, 0x00000000}, 121254885Sdumbbell {0x00000084, 0xe3f3e4f4}, 122254885Sdumbbell {0x00000085, 0x00052024}, 123254885Sdumbbell {0x00000087, 0x00000000}, 124254885Sdumbbell {0x00000088, 0x66036603}, 125254885Sdumbbell {0x00000089, 0x01000000}, 126254885Sdumbbell {0x0000008b, 0x1c0a0000}, 127254885Sdumbbell {0x0000008c, 0xff010000}, 128254885Sdumbbell {0x0000008e, 0xffffefff}, 129254885Sdumbbell {0x0000008f, 0xfff3efff}, 130254885Sdumbbell {0x00000090, 0xfff3efbf}, 131254885Sdumbbell {0x00000094, 0x00101101}, 132254885Sdumbbell {0x00000095, 0x00000fff}, 133254885Sdumbbell {0x00000096, 0x00116fff}, 134254885Sdumbbell {0x00000097, 0x60010000}, 135254885Sdumbbell {0x00000098, 0x10010000}, 136254885Sdumbbell {0x00000099, 0x00006000}, 137254885Sdumbbell {0x0000009a, 0x00001000}, 138254885Sdumbbell {0x0000009f, 0x00a47400} 139254885Sdumbbell}; 140254885Sdumbbell 141254885Sdumbbellstatic const u32 verde_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = { 142254885Sdumbbell {0x0000006f, 0x03044000}, 143254885Sdumbbell {0x00000070, 0x0480c018}, 144254885Sdumbbell {0x00000071, 0x00000040}, 145254885Sdumbbell {0x00000072, 0x01000000}, 146254885Sdumbbell {0x00000074, 0x000000ff}, 147254885Sdumbbell {0x00000075, 0x00143400}, 148254885Sdumbbell {0x00000076, 0x08ec0800}, 149254885Sdumbbell {0x00000077, 0x040000cc}, 150254885Sdumbbell {0x00000079, 0x00000000}, 151254885Sdumbbell {0x0000007a, 0x21000409}, 152254885Sdumbbell {0x0000007c, 0x00000000}, 153254885Sdumbbell {0x0000007d, 0xe8000000}, 154254885Sdumbbell {0x0000007e, 0x044408a8}, 155254885Sdumbbell {0x0000007f, 0x00000003}, 156254885Sdumbbell {0x00000080, 0x00000000}, 157254885Sdumbbell {0x00000081, 0x01000000}, 158254885Sdumbbell {0x00000082, 0x02000000}, 159254885Sdumbbell {0x00000083, 0x00000000}, 160254885Sdumbbell {0x00000084, 0xe3f3e4f4}, 161254885Sdumbbell {0x00000085, 0x00052024}, 162254885Sdumbbell {0x00000087, 0x00000000}, 163254885Sdumbbell {0x00000088, 0x66036603}, 164254885Sdumbbell {0x00000089, 0x01000000}, 165254885Sdumbbell {0x0000008b, 0x1c0a0000}, 166254885Sdumbbell {0x0000008c, 0xff010000}, 167254885Sdumbbell {0x0000008e, 0xffffefff}, 168254885Sdumbbell {0x0000008f, 0xfff3efff}, 169254885Sdumbbell {0x00000090, 0xfff3efbf}, 170254885Sdumbbell {0x00000094, 0x00101101}, 171254885Sdumbbell {0x00000095, 0x00000fff}, 172254885Sdumbbell {0x00000096, 0x00116fff}, 173254885Sdumbbell {0x00000097, 0x60010000}, 174254885Sdumbbell {0x00000098, 0x10010000}, 175254885Sdumbbell {0x00000099, 0x00006000}, 176254885Sdumbbell {0x0000009a, 0x00001000}, 177254885Sdumbbell {0x0000009f, 0x00a37400} 178254885Sdumbbell}; 179254885Sdumbbell 180254885Sdumbbell/* ucode loading */ 181254885Sdumbbellstatic int si_mc_load_microcode(struct radeon_device *rdev) 182254885Sdumbbell{ 183254885Sdumbbell const __be32 *fw_data; 184254885Sdumbbell u32 running, blackout = 0; 185254885Sdumbbell u32 *io_mc_regs; 186254885Sdumbbell int i, ucode_size, regs_size; 187254885Sdumbbell 188254885Sdumbbell if (!rdev->mc_fw) 189254885Sdumbbell return -EINVAL; 190254885Sdumbbell 191254885Sdumbbell switch (rdev->family) { 192254885Sdumbbell case CHIP_TAHITI: 193254885Sdumbbell io_mc_regs = (u32 *)&tahiti_io_mc_regs; 194254885Sdumbbell ucode_size = SI_MC_UCODE_SIZE; 195254885Sdumbbell regs_size = TAHITI_IO_MC_REGS_SIZE; 196254885Sdumbbell break; 197254885Sdumbbell case CHIP_PITCAIRN: 198254885Sdumbbell io_mc_regs = (u32 *)&pitcairn_io_mc_regs; 199254885Sdumbbell ucode_size = SI_MC_UCODE_SIZE; 200254885Sdumbbell regs_size = TAHITI_IO_MC_REGS_SIZE; 201254885Sdumbbell break; 202254885Sdumbbell case CHIP_VERDE: 203254885Sdumbbell default: 204254885Sdumbbell io_mc_regs = (u32 *)&verde_io_mc_regs; 205254885Sdumbbell ucode_size = SI_MC_UCODE_SIZE; 206254885Sdumbbell regs_size = TAHITI_IO_MC_REGS_SIZE; 207254885Sdumbbell break; 208254885Sdumbbell } 209254885Sdumbbell 210254885Sdumbbell running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK; 211254885Sdumbbell 212254885Sdumbbell if (running == 0) { 213254885Sdumbbell if (running) { 214254885Sdumbbell blackout = RREG32(MC_SHARED_BLACKOUT_CNTL); 215254885Sdumbbell WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1); 216254885Sdumbbell } 217254885Sdumbbell 218254885Sdumbbell /* reset the engine and set to writable */ 219254885Sdumbbell WREG32(MC_SEQ_SUP_CNTL, 0x00000008); 220254885Sdumbbell WREG32(MC_SEQ_SUP_CNTL, 0x00000010); 221254885Sdumbbell 222254885Sdumbbell /* load mc io regs */ 223254885Sdumbbell for (i = 0; i < regs_size; i++) { 224254885Sdumbbell WREG32(MC_SEQ_IO_DEBUG_INDEX, io_mc_regs[(i << 1)]); 225254885Sdumbbell WREG32(MC_SEQ_IO_DEBUG_DATA, io_mc_regs[(i << 1) + 1]); 226254885Sdumbbell } 227254885Sdumbbell /* load the MC ucode */ 228254885Sdumbbell fw_data = (const __be32 *)rdev->mc_fw->data; 229254885Sdumbbell for (i = 0; i < ucode_size; i++) 230254885Sdumbbell WREG32(MC_SEQ_SUP_PGM, be32_to_cpup(fw_data++)); 231254885Sdumbbell 232254885Sdumbbell /* put the engine back into the active state */ 233254885Sdumbbell WREG32(MC_SEQ_SUP_CNTL, 0x00000008); 234254885Sdumbbell WREG32(MC_SEQ_SUP_CNTL, 0x00000004); 235254885Sdumbbell WREG32(MC_SEQ_SUP_CNTL, 0x00000001); 236254885Sdumbbell 237254885Sdumbbell /* wait for training to complete */ 238254885Sdumbbell for (i = 0; i < rdev->usec_timeout; i++) { 239254885Sdumbbell if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D0) 240254885Sdumbbell break; 241254885Sdumbbell DRM_UDELAY(1); 242254885Sdumbbell } 243254885Sdumbbell for (i = 0; i < rdev->usec_timeout; i++) { 244254885Sdumbbell if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D1) 245254885Sdumbbell break; 246254885Sdumbbell DRM_UDELAY(1); 247254885Sdumbbell } 248254885Sdumbbell 249254885Sdumbbell if (running) 250254885Sdumbbell WREG32(MC_SHARED_BLACKOUT_CNTL, blackout); 251254885Sdumbbell } 252254885Sdumbbell 253254885Sdumbbell return 0; 254254885Sdumbbell} 255254885Sdumbbell 256254885Sdumbbellstatic int si_init_microcode(struct radeon_device *rdev) 257254885Sdumbbell{ 258254885Sdumbbell const char *chip_name; 259254885Sdumbbell const char *rlc_chip_name; 260254885Sdumbbell size_t pfp_req_size, me_req_size, ce_req_size, rlc_req_size, mc_req_size; 261254885Sdumbbell char fw_name[30]; 262254885Sdumbbell int err; 263254885Sdumbbell 264254885Sdumbbell DRM_DEBUG("\n"); 265254885Sdumbbell 266254885Sdumbbell switch (rdev->family) { 267254885Sdumbbell case CHIP_TAHITI: 268254885Sdumbbell chip_name = "TAHITI"; 269254885Sdumbbell rlc_chip_name = "TAHITI"; 270254885Sdumbbell pfp_req_size = SI_PFP_UCODE_SIZE * 4; 271254885Sdumbbell me_req_size = SI_PM4_UCODE_SIZE * 4; 272254885Sdumbbell ce_req_size = SI_CE_UCODE_SIZE * 4; 273254885Sdumbbell rlc_req_size = SI_RLC_UCODE_SIZE * 4; 274254885Sdumbbell mc_req_size = SI_MC_UCODE_SIZE * 4; 275254885Sdumbbell break; 276254885Sdumbbell case CHIP_PITCAIRN: 277254885Sdumbbell chip_name = "PITCAIRN"; 278254885Sdumbbell rlc_chip_name = "PITCAIRN"; 279254885Sdumbbell pfp_req_size = SI_PFP_UCODE_SIZE * 4; 280254885Sdumbbell me_req_size = SI_PM4_UCODE_SIZE * 4; 281254885Sdumbbell ce_req_size = SI_CE_UCODE_SIZE * 4; 282254885Sdumbbell rlc_req_size = SI_RLC_UCODE_SIZE * 4; 283254885Sdumbbell mc_req_size = SI_MC_UCODE_SIZE * 4; 284254885Sdumbbell break; 285254885Sdumbbell case CHIP_VERDE: 286254885Sdumbbell chip_name = "VERDE"; 287254885Sdumbbell rlc_chip_name = "VERDE"; 288254885Sdumbbell pfp_req_size = SI_PFP_UCODE_SIZE * 4; 289254885Sdumbbell me_req_size = SI_PM4_UCODE_SIZE * 4; 290254885Sdumbbell ce_req_size = SI_CE_UCODE_SIZE * 4; 291254885Sdumbbell rlc_req_size = SI_RLC_UCODE_SIZE * 4; 292254885Sdumbbell mc_req_size = SI_MC_UCODE_SIZE * 4; 293254885Sdumbbell break; 294254885Sdumbbell default: panic("%s: Unsupported family %d", __func__, rdev->family); 295254885Sdumbbell } 296254885Sdumbbell 297254885Sdumbbell DRM_INFO("Loading %s Microcode\n", chip_name); 298254885Sdumbbell err = 0; 299254885Sdumbbell 300254885Sdumbbell snprintf(fw_name, sizeof(fw_name), "radeonkmsfw_%s_pfp", chip_name); 301254885Sdumbbell rdev->pfp_fw = firmware_get(fw_name); 302254885Sdumbbell if (rdev->pfp_fw == NULL) { 303254885Sdumbbell err = -ENOENT; 304254885Sdumbbell goto out; 305254885Sdumbbell } 306254885Sdumbbell if (rdev->pfp_fw->datasize != pfp_req_size) { 307254885Sdumbbell DRM_ERROR( 308254885Sdumbbell "si_cp: Bogus length %zu in firmware \"%s\"\n", 309254885Sdumbbell rdev->pfp_fw->datasize, fw_name); 310254885Sdumbbell err = -EINVAL; 311254885Sdumbbell goto out; 312254885Sdumbbell } 313254885Sdumbbell 314254885Sdumbbell snprintf(fw_name, sizeof(fw_name), "radeonkmsfw_%s_me", chip_name); 315254885Sdumbbell rdev->me_fw = firmware_get(fw_name); 316254885Sdumbbell if (rdev->me_fw == NULL) { 317254885Sdumbbell err = -ENOENT; 318254885Sdumbbell goto out; 319254885Sdumbbell } 320254885Sdumbbell if (rdev->me_fw->datasize != me_req_size) { 321254885Sdumbbell DRM_ERROR( 322254885Sdumbbell "si_cp: Bogus length %zu in firmware \"%s\"\n", 323254885Sdumbbell rdev->me_fw->datasize, fw_name); 324254885Sdumbbell err = -EINVAL; 325254885Sdumbbell } 326254885Sdumbbell 327254885Sdumbbell snprintf(fw_name, sizeof(fw_name), "radeonkmsfw_%s_ce", chip_name); 328254885Sdumbbell rdev->ce_fw = firmware_get(fw_name); 329254885Sdumbbell if (rdev->ce_fw == NULL) { 330254885Sdumbbell err = -ENOENT; 331254885Sdumbbell goto out; 332254885Sdumbbell } 333254885Sdumbbell if (rdev->ce_fw->datasize != ce_req_size) { 334254885Sdumbbell DRM_ERROR( 335254885Sdumbbell "si_cp: Bogus length %zu in firmware \"%s\"\n", 336254885Sdumbbell rdev->ce_fw->datasize, fw_name); 337254885Sdumbbell err = -EINVAL; 338254885Sdumbbell } 339254885Sdumbbell 340254885Sdumbbell snprintf(fw_name, sizeof(fw_name), "radeonkmsfw_%s_rlc", rlc_chip_name); 341254885Sdumbbell rdev->rlc_fw = firmware_get(fw_name); 342254885Sdumbbell if (rdev->rlc_fw == NULL) { 343254885Sdumbbell err = -ENOENT; 344254885Sdumbbell goto out; 345254885Sdumbbell } 346254885Sdumbbell if (rdev->rlc_fw->datasize != rlc_req_size) { 347254885Sdumbbell DRM_ERROR( 348254885Sdumbbell "si_rlc: Bogus length %zu in firmware \"%s\"\n", 349254885Sdumbbell rdev->rlc_fw->datasize, fw_name); 350254885Sdumbbell err = -EINVAL; 351254885Sdumbbell } 352254885Sdumbbell 353254885Sdumbbell snprintf(fw_name, sizeof(fw_name), "radeonkmsfw_%s_mc", chip_name); 354254885Sdumbbell rdev->mc_fw = firmware_get(fw_name); 355254885Sdumbbell if (rdev->mc_fw == NULL) { 356254885Sdumbbell err = -ENOENT; 357254885Sdumbbell goto out; 358254885Sdumbbell } 359254885Sdumbbell if (rdev->mc_fw->datasize != mc_req_size) { 360254885Sdumbbell DRM_ERROR( 361254885Sdumbbell "si_mc: Bogus length %zu in firmware \"%s\"\n", 362254885Sdumbbell rdev->mc_fw->datasize, fw_name); 363254885Sdumbbell err = -EINVAL; 364254885Sdumbbell } 365254885Sdumbbell 366254885Sdumbbellout: 367254885Sdumbbell if (err) { 368254885Sdumbbell if (err != -EINVAL) 369254885Sdumbbell DRM_ERROR( 370254885Sdumbbell "si_cp: Failed to load firmware \"%s\"\n", 371254885Sdumbbell fw_name); 372254885Sdumbbell if (rdev->pfp_fw != NULL) { 373254885Sdumbbell firmware_put(rdev->pfp_fw, FIRMWARE_UNLOAD); 374254885Sdumbbell rdev->pfp_fw = NULL; 375254885Sdumbbell } 376254885Sdumbbell if (rdev->me_fw != NULL) { 377254885Sdumbbell firmware_put(rdev->me_fw, FIRMWARE_UNLOAD); 378254885Sdumbbell rdev->me_fw = NULL; 379254885Sdumbbell } 380254885Sdumbbell if (rdev->ce_fw != NULL) { 381254885Sdumbbell firmware_put(rdev->ce_fw, FIRMWARE_UNLOAD); 382254885Sdumbbell rdev->ce_fw = NULL; 383254885Sdumbbell } 384254885Sdumbbell if (rdev->rlc_fw != NULL) { 385254885Sdumbbell firmware_put(rdev->rlc_fw, FIRMWARE_UNLOAD); 386254885Sdumbbell rdev->rlc_fw = NULL; 387254885Sdumbbell } 388254885Sdumbbell if (rdev->mc_fw != NULL) { 389254885Sdumbbell firmware_put(rdev->mc_fw, FIRMWARE_UNLOAD); 390254885Sdumbbell rdev->mc_fw = NULL; 391254885Sdumbbell } 392254885Sdumbbell } 393254885Sdumbbell return err; 394254885Sdumbbell} 395254885Sdumbbell 396254885Sdumbbell/** 397254885Sdumbbell * si_fini_microcode - drop the firmwares image references 398254885Sdumbbell * 399254885Sdumbbell * @rdev: radeon_device pointer 400254885Sdumbbell * 401254885Sdumbbell * Drop the pfp, me, rlc, mc and ce firmware image references. 402254885Sdumbbell * Called at driver shutdown. 403254885Sdumbbell */ 404254885Sdumbbellstatic void si_fini_microcode(struct radeon_device *rdev) 405254885Sdumbbell{ 406254885Sdumbbell 407254885Sdumbbell if (rdev->pfp_fw != NULL) { 408254885Sdumbbell firmware_put(rdev->pfp_fw, FIRMWARE_UNLOAD); 409254885Sdumbbell rdev->pfp_fw = NULL; 410254885Sdumbbell } 411254885Sdumbbell 412254885Sdumbbell if (rdev->me_fw != NULL) { 413254885Sdumbbell firmware_put(rdev->me_fw, FIRMWARE_UNLOAD); 414254885Sdumbbell rdev->me_fw = NULL; 415254885Sdumbbell } 416254885Sdumbbell 417254885Sdumbbell if (rdev->rlc_fw != NULL) { 418254885Sdumbbell firmware_put(rdev->rlc_fw, FIRMWARE_UNLOAD); 419254885Sdumbbell rdev->rlc_fw = NULL; 420254885Sdumbbell } 421254885Sdumbbell 422254885Sdumbbell if (rdev->mc_fw != NULL) { 423254885Sdumbbell firmware_put(rdev->mc_fw, FIRMWARE_UNLOAD); 424254885Sdumbbell rdev->mc_fw = NULL; 425254885Sdumbbell } 426254885Sdumbbell 427254885Sdumbbell if (rdev->ce_fw != NULL) { 428254885Sdumbbell firmware_put(rdev->ce_fw, FIRMWARE_UNLOAD); 429254885Sdumbbell rdev->ce_fw = NULL; 430254885Sdumbbell } 431254885Sdumbbell} 432254885Sdumbbell 433254885Sdumbbell/* watermark setup */ 434254885Sdumbbellstatic u32 dce6_line_buffer_adjust(struct radeon_device *rdev, 435254885Sdumbbell struct radeon_crtc *radeon_crtc, 436254885Sdumbbell struct drm_display_mode *mode, 437254885Sdumbbell struct drm_display_mode *other_mode) 438254885Sdumbbell{ 439254885Sdumbbell u32 tmp; 440254885Sdumbbell /* 441254885Sdumbbell * Line Buffer Setup 442254885Sdumbbell * There are 3 line buffers, each one shared by 2 display controllers. 443254885Sdumbbell * DC_LB_MEMORY_SPLIT controls how that line buffer is shared between 444254885Sdumbbell * the display controllers. The paritioning is done via one of four 445254885Sdumbbell * preset allocations specified in bits 21:20: 446254885Sdumbbell * 0 - half lb 447254885Sdumbbell * 2 - whole lb, other crtc must be disabled 448254885Sdumbbell */ 449254885Sdumbbell /* this can get tricky if we have two large displays on a paired group 450254885Sdumbbell * of crtcs. Ideally for multiple large displays we'd assign them to 451254885Sdumbbell * non-linked crtcs for maximum line buffer allocation. 452254885Sdumbbell */ 453254885Sdumbbell if (radeon_crtc->base.enabled && mode) { 454254885Sdumbbell if (other_mode) 455254885Sdumbbell tmp = 0; /* 1/2 */ 456254885Sdumbbell else 457254885Sdumbbell tmp = 2; /* whole */ 458254885Sdumbbell } else 459254885Sdumbbell tmp = 0; 460254885Sdumbbell 461254885Sdumbbell WREG32(DC_LB_MEMORY_SPLIT + radeon_crtc->crtc_offset, 462254885Sdumbbell DC_LB_MEMORY_CONFIG(tmp)); 463254885Sdumbbell 464254885Sdumbbell if (radeon_crtc->base.enabled && mode) { 465254885Sdumbbell switch (tmp) { 466254885Sdumbbell case 0: 467254885Sdumbbell default: 468254885Sdumbbell return 4096 * 2; 469254885Sdumbbell case 2: 470254885Sdumbbell return 8192 * 2; 471254885Sdumbbell } 472254885Sdumbbell } 473254885Sdumbbell 474254885Sdumbbell /* controller not enabled, so no lb used */ 475254885Sdumbbell return 0; 476254885Sdumbbell} 477254885Sdumbbell 478254885Sdumbbellstatic u32 si_get_number_of_dram_channels(struct radeon_device *rdev) 479254885Sdumbbell{ 480254885Sdumbbell u32 tmp = RREG32(MC_SHARED_CHMAP); 481254885Sdumbbell 482254885Sdumbbell switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) { 483254885Sdumbbell case 0: 484254885Sdumbbell default: 485254885Sdumbbell return 1; 486254885Sdumbbell case 1: 487254885Sdumbbell return 2; 488254885Sdumbbell case 2: 489254885Sdumbbell return 4; 490254885Sdumbbell case 3: 491254885Sdumbbell return 8; 492254885Sdumbbell case 4: 493254885Sdumbbell return 3; 494254885Sdumbbell case 5: 495254885Sdumbbell return 6; 496254885Sdumbbell case 6: 497254885Sdumbbell return 10; 498254885Sdumbbell case 7: 499254885Sdumbbell return 12; 500254885Sdumbbell case 8: 501254885Sdumbbell return 16; 502254885Sdumbbell } 503254885Sdumbbell} 504254885Sdumbbell 505254885Sdumbbellstruct dce6_wm_params { 506254885Sdumbbell u32 dram_channels; /* number of dram channels */ 507254885Sdumbbell u32 yclk; /* bandwidth per dram data pin in kHz */ 508254885Sdumbbell u32 sclk; /* engine clock in kHz */ 509254885Sdumbbell u32 disp_clk; /* display clock in kHz */ 510254885Sdumbbell u32 src_width; /* viewport width */ 511254885Sdumbbell u32 active_time; /* active display time in ns */ 512254885Sdumbbell u32 blank_time; /* blank time in ns */ 513254885Sdumbbell bool interlaced; /* mode is interlaced */ 514254885Sdumbbell fixed20_12 vsc; /* vertical scale ratio */ 515254885Sdumbbell u32 num_heads; /* number of active crtcs */ 516254885Sdumbbell u32 bytes_per_pixel; /* bytes per pixel display + overlay */ 517254885Sdumbbell u32 lb_size; /* line buffer allocated to pipe */ 518254885Sdumbbell u32 vtaps; /* vertical scaler taps */ 519254885Sdumbbell}; 520254885Sdumbbell 521254885Sdumbbellstatic u32 dce6_dram_bandwidth(struct dce6_wm_params *wm) 522254885Sdumbbell{ 523254885Sdumbbell /* Calculate raw DRAM Bandwidth */ 524254885Sdumbbell fixed20_12 dram_efficiency; /* 0.7 */ 525254885Sdumbbell fixed20_12 yclk, dram_channels, bandwidth; 526254885Sdumbbell fixed20_12 a; 527254885Sdumbbell 528254885Sdumbbell a.full = dfixed_const(1000); 529254885Sdumbbell yclk.full = dfixed_const(wm->yclk); 530254885Sdumbbell yclk.full = dfixed_div(yclk, a); 531254885Sdumbbell dram_channels.full = dfixed_const(wm->dram_channels * 4); 532254885Sdumbbell a.full = dfixed_const(10); 533254885Sdumbbell dram_efficiency.full = dfixed_const(7); 534254885Sdumbbell dram_efficiency.full = dfixed_div(dram_efficiency, a); 535254885Sdumbbell bandwidth.full = dfixed_mul(dram_channels, yclk); 536254885Sdumbbell bandwidth.full = dfixed_mul(bandwidth, dram_efficiency); 537254885Sdumbbell 538254885Sdumbbell return dfixed_trunc(bandwidth); 539254885Sdumbbell} 540254885Sdumbbell 541254885Sdumbbellstatic u32 dce6_dram_bandwidth_for_display(struct dce6_wm_params *wm) 542254885Sdumbbell{ 543254885Sdumbbell /* Calculate DRAM Bandwidth and the part allocated to display. */ 544254885Sdumbbell fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */ 545254885Sdumbbell fixed20_12 yclk, dram_channels, bandwidth; 546254885Sdumbbell fixed20_12 a; 547254885Sdumbbell 548254885Sdumbbell a.full = dfixed_const(1000); 549254885Sdumbbell yclk.full = dfixed_const(wm->yclk); 550254885Sdumbbell yclk.full = dfixed_div(yclk, a); 551254885Sdumbbell dram_channels.full = dfixed_const(wm->dram_channels * 4); 552254885Sdumbbell a.full = dfixed_const(10); 553254885Sdumbbell disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */ 554254885Sdumbbell disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a); 555254885Sdumbbell bandwidth.full = dfixed_mul(dram_channels, yclk); 556254885Sdumbbell bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation); 557254885Sdumbbell 558254885Sdumbbell return dfixed_trunc(bandwidth); 559254885Sdumbbell} 560254885Sdumbbell 561254885Sdumbbellstatic u32 dce6_data_return_bandwidth(struct dce6_wm_params *wm) 562254885Sdumbbell{ 563254885Sdumbbell /* Calculate the display Data return Bandwidth */ 564254885Sdumbbell fixed20_12 return_efficiency; /* 0.8 */ 565254885Sdumbbell fixed20_12 sclk, bandwidth; 566254885Sdumbbell fixed20_12 a; 567254885Sdumbbell 568254885Sdumbbell a.full = dfixed_const(1000); 569254885Sdumbbell sclk.full = dfixed_const(wm->sclk); 570254885Sdumbbell sclk.full = dfixed_div(sclk, a); 571254885Sdumbbell a.full = dfixed_const(10); 572254885Sdumbbell return_efficiency.full = dfixed_const(8); 573254885Sdumbbell return_efficiency.full = dfixed_div(return_efficiency, a); 574254885Sdumbbell a.full = dfixed_const(32); 575254885Sdumbbell bandwidth.full = dfixed_mul(a, sclk); 576254885Sdumbbell bandwidth.full = dfixed_mul(bandwidth, return_efficiency); 577254885Sdumbbell 578254885Sdumbbell return dfixed_trunc(bandwidth); 579254885Sdumbbell} 580254885Sdumbbell 581254885Sdumbbellstatic u32 dce6_get_dmif_bytes_per_request(struct dce6_wm_params *wm) 582254885Sdumbbell{ 583254885Sdumbbell return 32; 584254885Sdumbbell} 585254885Sdumbbell 586254885Sdumbbellstatic u32 dce6_dmif_request_bandwidth(struct dce6_wm_params *wm) 587254885Sdumbbell{ 588254885Sdumbbell /* Calculate the DMIF Request Bandwidth */ 589254885Sdumbbell fixed20_12 disp_clk_request_efficiency; /* 0.8 */ 590254885Sdumbbell fixed20_12 disp_clk, sclk, bandwidth; 591254885Sdumbbell fixed20_12 a, b1, b2; 592254885Sdumbbell u32 min_bandwidth; 593254885Sdumbbell 594254885Sdumbbell a.full = dfixed_const(1000); 595254885Sdumbbell disp_clk.full = dfixed_const(wm->disp_clk); 596254885Sdumbbell disp_clk.full = dfixed_div(disp_clk, a); 597254885Sdumbbell a.full = dfixed_const(dce6_get_dmif_bytes_per_request(wm) / 2); 598254885Sdumbbell b1.full = dfixed_mul(a, disp_clk); 599254885Sdumbbell 600254885Sdumbbell a.full = dfixed_const(1000); 601254885Sdumbbell sclk.full = dfixed_const(wm->sclk); 602254885Sdumbbell sclk.full = dfixed_div(sclk, a); 603254885Sdumbbell a.full = dfixed_const(dce6_get_dmif_bytes_per_request(wm)); 604254885Sdumbbell b2.full = dfixed_mul(a, sclk); 605254885Sdumbbell 606254885Sdumbbell a.full = dfixed_const(10); 607254885Sdumbbell disp_clk_request_efficiency.full = dfixed_const(8); 608254885Sdumbbell disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a); 609254885Sdumbbell 610254885Sdumbbell min_bandwidth = min(dfixed_trunc(b1), dfixed_trunc(b2)); 611254885Sdumbbell 612254885Sdumbbell a.full = dfixed_const(min_bandwidth); 613254885Sdumbbell bandwidth.full = dfixed_mul(a, disp_clk_request_efficiency); 614254885Sdumbbell 615254885Sdumbbell return dfixed_trunc(bandwidth); 616254885Sdumbbell} 617254885Sdumbbell 618254885Sdumbbellstatic u32 dce6_available_bandwidth(struct dce6_wm_params *wm) 619254885Sdumbbell{ 620254885Sdumbbell /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */ 621254885Sdumbbell u32 dram_bandwidth = dce6_dram_bandwidth(wm); 622254885Sdumbbell u32 data_return_bandwidth = dce6_data_return_bandwidth(wm); 623254885Sdumbbell u32 dmif_req_bandwidth = dce6_dmif_request_bandwidth(wm); 624254885Sdumbbell 625254885Sdumbbell return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth)); 626254885Sdumbbell} 627254885Sdumbbell 628254885Sdumbbellstatic u32 dce6_average_bandwidth(struct dce6_wm_params *wm) 629254885Sdumbbell{ 630254885Sdumbbell /* Calculate the display mode Average Bandwidth 631254885Sdumbbell * DisplayMode should contain the source and destination dimensions, 632254885Sdumbbell * timing, etc. 633254885Sdumbbell */ 634254885Sdumbbell fixed20_12 bpp; 635254885Sdumbbell fixed20_12 line_time; 636254885Sdumbbell fixed20_12 src_width; 637254885Sdumbbell fixed20_12 bandwidth; 638254885Sdumbbell fixed20_12 a; 639254885Sdumbbell 640254885Sdumbbell a.full = dfixed_const(1000); 641254885Sdumbbell line_time.full = dfixed_const(wm->active_time + wm->blank_time); 642254885Sdumbbell line_time.full = dfixed_div(line_time, a); 643254885Sdumbbell bpp.full = dfixed_const(wm->bytes_per_pixel); 644254885Sdumbbell src_width.full = dfixed_const(wm->src_width); 645254885Sdumbbell bandwidth.full = dfixed_mul(src_width, bpp); 646254885Sdumbbell bandwidth.full = dfixed_mul(bandwidth, wm->vsc); 647254885Sdumbbell bandwidth.full = dfixed_div(bandwidth, line_time); 648254885Sdumbbell 649254885Sdumbbell return dfixed_trunc(bandwidth); 650254885Sdumbbell} 651254885Sdumbbell 652254885Sdumbbellstatic u32 dce6_latency_watermark(struct dce6_wm_params *wm) 653254885Sdumbbell{ 654254885Sdumbbell /* First calcualte the latency in ns */ 655254885Sdumbbell u32 mc_latency = 2000; /* 2000 ns. */ 656254885Sdumbbell u32 available_bandwidth = dce6_available_bandwidth(wm); 657254885Sdumbbell u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth; 658254885Sdumbbell u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth; 659254885Sdumbbell u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */ 660254885Sdumbbell u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) + 661254885Sdumbbell (wm->num_heads * cursor_line_pair_return_time); 662254885Sdumbbell u32 latency = mc_latency + other_heads_data_return_time + dc_latency; 663254885Sdumbbell u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time; 664254885Sdumbbell u32 tmp, dmif_size = 12288; 665254885Sdumbbell fixed20_12 a, b, c; 666254885Sdumbbell 667254885Sdumbbell if (wm->num_heads == 0) 668254885Sdumbbell return 0; 669254885Sdumbbell 670254885Sdumbbell a.full = dfixed_const(2); 671254885Sdumbbell b.full = dfixed_const(1); 672254885Sdumbbell if ((wm->vsc.full > a.full) || 673254885Sdumbbell ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) || 674254885Sdumbbell (wm->vtaps >= 5) || 675254885Sdumbbell ((wm->vsc.full >= a.full) && wm->interlaced)) 676254885Sdumbbell max_src_lines_per_dst_line = 4; 677254885Sdumbbell else 678254885Sdumbbell max_src_lines_per_dst_line = 2; 679254885Sdumbbell 680254885Sdumbbell a.full = dfixed_const(available_bandwidth); 681254885Sdumbbell b.full = dfixed_const(wm->num_heads); 682254885Sdumbbell a.full = dfixed_div(a, b); 683254885Sdumbbell 684254885Sdumbbell b.full = dfixed_const(mc_latency + 512); 685254885Sdumbbell c.full = dfixed_const(wm->disp_clk); 686254885Sdumbbell b.full = dfixed_div(b, c); 687254885Sdumbbell 688254885Sdumbbell c.full = dfixed_const(dmif_size); 689254885Sdumbbell b.full = dfixed_div(c, b); 690254885Sdumbbell 691254885Sdumbbell tmp = min(dfixed_trunc(a), dfixed_trunc(b)); 692254885Sdumbbell 693254885Sdumbbell b.full = dfixed_const(1000); 694254885Sdumbbell c.full = dfixed_const(wm->disp_clk); 695254885Sdumbbell b.full = dfixed_div(c, b); 696254885Sdumbbell c.full = dfixed_const(wm->bytes_per_pixel); 697254885Sdumbbell b.full = dfixed_mul(b, c); 698254885Sdumbbell 699254885Sdumbbell lb_fill_bw = min(tmp, dfixed_trunc(b)); 700254885Sdumbbell 701254885Sdumbbell a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel); 702254885Sdumbbell b.full = dfixed_const(1000); 703254885Sdumbbell c.full = dfixed_const(lb_fill_bw); 704254885Sdumbbell b.full = dfixed_div(c, b); 705254885Sdumbbell a.full = dfixed_div(a, b); 706254885Sdumbbell line_fill_time = dfixed_trunc(a); 707254885Sdumbbell 708254885Sdumbbell if (line_fill_time < wm->active_time) 709254885Sdumbbell return latency; 710254885Sdumbbell else 711254885Sdumbbell return latency + (line_fill_time - wm->active_time); 712254885Sdumbbell 713254885Sdumbbell} 714254885Sdumbbell 715254885Sdumbbellstatic bool dce6_average_bandwidth_vs_dram_bandwidth_for_display(struct dce6_wm_params *wm) 716254885Sdumbbell{ 717254885Sdumbbell if (dce6_average_bandwidth(wm) <= 718254885Sdumbbell (dce6_dram_bandwidth_for_display(wm) / wm->num_heads)) 719254885Sdumbbell return true; 720254885Sdumbbell else 721254885Sdumbbell return false; 722254885Sdumbbell}; 723254885Sdumbbell 724254885Sdumbbellstatic bool dce6_average_bandwidth_vs_available_bandwidth(struct dce6_wm_params *wm) 725254885Sdumbbell{ 726254885Sdumbbell if (dce6_average_bandwidth(wm) <= 727254885Sdumbbell (dce6_available_bandwidth(wm) / wm->num_heads)) 728254885Sdumbbell return true; 729254885Sdumbbell else 730254885Sdumbbell return false; 731254885Sdumbbell}; 732254885Sdumbbell 733254885Sdumbbellstatic bool dce6_check_latency_hiding(struct dce6_wm_params *wm) 734254885Sdumbbell{ 735254885Sdumbbell u32 lb_partitions = wm->lb_size / wm->src_width; 736254885Sdumbbell u32 line_time = wm->active_time + wm->blank_time; 737254885Sdumbbell u32 latency_tolerant_lines; 738254885Sdumbbell u32 latency_hiding; 739254885Sdumbbell fixed20_12 a; 740254885Sdumbbell 741254885Sdumbbell a.full = dfixed_const(1); 742254885Sdumbbell if (wm->vsc.full > a.full) 743254885Sdumbbell latency_tolerant_lines = 1; 744254885Sdumbbell else { 745254885Sdumbbell if (lb_partitions <= (wm->vtaps + 1)) 746254885Sdumbbell latency_tolerant_lines = 1; 747254885Sdumbbell else 748254885Sdumbbell latency_tolerant_lines = 2; 749254885Sdumbbell } 750254885Sdumbbell 751254885Sdumbbell latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time); 752254885Sdumbbell 753254885Sdumbbell if (dce6_latency_watermark(wm) <= latency_hiding) 754254885Sdumbbell return true; 755254885Sdumbbell else 756254885Sdumbbell return false; 757254885Sdumbbell} 758254885Sdumbbell 759254885Sdumbbellstatic void dce6_program_watermarks(struct radeon_device *rdev, 760254885Sdumbbell struct radeon_crtc *radeon_crtc, 761254885Sdumbbell u32 lb_size, u32 num_heads) 762254885Sdumbbell{ 763254885Sdumbbell struct drm_display_mode *mode = &radeon_crtc->base.mode; 764254885Sdumbbell struct dce6_wm_params wm; 765254885Sdumbbell u32 pixel_period; 766254885Sdumbbell u32 line_time = 0; 767254885Sdumbbell u32 latency_watermark_a = 0, latency_watermark_b = 0; 768254885Sdumbbell u32 priority_a_mark = 0, priority_b_mark = 0; 769254885Sdumbbell u32 priority_a_cnt = PRIORITY_OFF; 770254885Sdumbbell u32 priority_b_cnt = PRIORITY_OFF; 771254885Sdumbbell u32 tmp, arb_control3; 772254885Sdumbbell fixed20_12 a, b, c; 773254885Sdumbbell 774254885Sdumbbell if (radeon_crtc->base.enabled && num_heads && mode) { 775254885Sdumbbell pixel_period = 1000000 / (u32)mode->clock; 776254885Sdumbbell line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535); 777254885Sdumbbell priority_a_cnt = 0; 778254885Sdumbbell priority_b_cnt = 0; 779254885Sdumbbell 780254885Sdumbbell wm.yclk = rdev->pm.current_mclk * 10; 781254885Sdumbbell wm.sclk = rdev->pm.current_sclk * 10; 782254885Sdumbbell wm.disp_clk = mode->clock; 783254885Sdumbbell wm.src_width = mode->crtc_hdisplay; 784254885Sdumbbell wm.active_time = mode->crtc_hdisplay * pixel_period; 785254885Sdumbbell wm.blank_time = line_time - wm.active_time; 786254885Sdumbbell wm.interlaced = false; 787254885Sdumbbell if (mode->flags & DRM_MODE_FLAG_INTERLACE) 788254885Sdumbbell wm.interlaced = true; 789254885Sdumbbell wm.vsc = radeon_crtc->vsc; 790254885Sdumbbell wm.vtaps = 1; 791254885Sdumbbell if (radeon_crtc->rmx_type != RMX_OFF) 792254885Sdumbbell wm.vtaps = 2; 793254885Sdumbbell wm.bytes_per_pixel = 4; /* XXX: get this from fb config */ 794254885Sdumbbell wm.lb_size = lb_size; 795254885Sdumbbell if (rdev->family == CHIP_ARUBA) 796254885Sdumbbell wm.dram_channels = evergreen_get_number_of_dram_channels(rdev); 797254885Sdumbbell else 798254885Sdumbbell wm.dram_channels = si_get_number_of_dram_channels(rdev); 799254885Sdumbbell wm.num_heads = num_heads; 800254885Sdumbbell 801254885Sdumbbell /* set for high clocks */ 802254885Sdumbbell latency_watermark_a = min(dce6_latency_watermark(&wm), (u32)65535); 803254885Sdumbbell /* set for low clocks */ 804254885Sdumbbell /* wm.yclk = low clk; wm.sclk = low clk */ 805254885Sdumbbell latency_watermark_b = min(dce6_latency_watermark(&wm), (u32)65535); 806254885Sdumbbell 807254885Sdumbbell /* possibly force display priority to high */ 808254885Sdumbbell /* should really do this at mode validation time... */ 809254885Sdumbbell if (!dce6_average_bandwidth_vs_dram_bandwidth_for_display(&wm) || 810254885Sdumbbell !dce6_average_bandwidth_vs_available_bandwidth(&wm) || 811254885Sdumbbell !dce6_check_latency_hiding(&wm) || 812254885Sdumbbell (rdev->disp_priority == 2)) { 813254885Sdumbbell DRM_DEBUG_KMS("force priority to high\n"); 814254885Sdumbbell priority_a_cnt |= PRIORITY_ALWAYS_ON; 815254885Sdumbbell priority_b_cnt |= PRIORITY_ALWAYS_ON; 816254885Sdumbbell } 817254885Sdumbbell 818254885Sdumbbell a.full = dfixed_const(1000); 819254885Sdumbbell b.full = dfixed_const(mode->clock); 820254885Sdumbbell b.full = dfixed_div(b, a); 821254885Sdumbbell c.full = dfixed_const(latency_watermark_a); 822254885Sdumbbell c.full = dfixed_mul(c, b); 823254885Sdumbbell c.full = dfixed_mul(c, radeon_crtc->hsc); 824254885Sdumbbell c.full = dfixed_div(c, a); 825254885Sdumbbell a.full = dfixed_const(16); 826254885Sdumbbell c.full = dfixed_div(c, a); 827254885Sdumbbell priority_a_mark = dfixed_trunc(c); 828254885Sdumbbell priority_a_cnt |= priority_a_mark & PRIORITY_MARK_MASK; 829254885Sdumbbell 830254885Sdumbbell a.full = dfixed_const(1000); 831254885Sdumbbell b.full = dfixed_const(mode->clock); 832254885Sdumbbell b.full = dfixed_div(b, a); 833254885Sdumbbell c.full = dfixed_const(latency_watermark_b); 834254885Sdumbbell c.full = dfixed_mul(c, b); 835254885Sdumbbell c.full = dfixed_mul(c, radeon_crtc->hsc); 836254885Sdumbbell c.full = dfixed_div(c, a); 837254885Sdumbbell a.full = dfixed_const(16); 838254885Sdumbbell c.full = dfixed_div(c, a); 839254885Sdumbbell priority_b_mark = dfixed_trunc(c); 840254885Sdumbbell priority_b_cnt |= priority_b_mark & PRIORITY_MARK_MASK; 841254885Sdumbbell } 842254885Sdumbbell 843254885Sdumbbell /* select wm A */ 844254885Sdumbbell arb_control3 = RREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset); 845254885Sdumbbell tmp = arb_control3; 846254885Sdumbbell tmp &= ~LATENCY_WATERMARK_MASK(3); 847254885Sdumbbell tmp |= LATENCY_WATERMARK_MASK(1); 848254885Sdumbbell WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, tmp); 849254885Sdumbbell WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset, 850254885Sdumbbell (LATENCY_LOW_WATERMARK(latency_watermark_a) | 851254885Sdumbbell LATENCY_HIGH_WATERMARK(line_time))); 852254885Sdumbbell /* select wm B */ 853254885Sdumbbell tmp = RREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset); 854254885Sdumbbell tmp &= ~LATENCY_WATERMARK_MASK(3); 855254885Sdumbbell tmp |= LATENCY_WATERMARK_MASK(2); 856254885Sdumbbell WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, tmp); 857254885Sdumbbell WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset, 858254885Sdumbbell (LATENCY_LOW_WATERMARK(latency_watermark_b) | 859254885Sdumbbell LATENCY_HIGH_WATERMARK(line_time))); 860254885Sdumbbell /* restore original selection */ 861254885Sdumbbell WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, arb_control3); 862254885Sdumbbell 863254885Sdumbbell /* write the priority marks */ 864254885Sdumbbell WREG32(PRIORITY_A_CNT + radeon_crtc->crtc_offset, priority_a_cnt); 865254885Sdumbbell WREG32(PRIORITY_B_CNT + radeon_crtc->crtc_offset, priority_b_cnt); 866254885Sdumbbell 867254885Sdumbbell} 868254885Sdumbbell 869254885Sdumbbellvoid dce6_bandwidth_update(struct radeon_device *rdev) 870254885Sdumbbell{ 871254885Sdumbbell struct drm_display_mode *mode0 = NULL; 872254885Sdumbbell struct drm_display_mode *mode1 = NULL; 873254885Sdumbbell u32 num_heads = 0, lb_size; 874254885Sdumbbell int i; 875254885Sdumbbell 876254885Sdumbbell radeon_update_display_priority(rdev); 877254885Sdumbbell 878254885Sdumbbell for (i = 0; i < rdev->num_crtc; i++) { 879254885Sdumbbell if (rdev->mode_info.crtcs[i]->base.enabled) 880254885Sdumbbell num_heads++; 881254885Sdumbbell } 882254885Sdumbbell for (i = 0; i < rdev->num_crtc; i += 2) { 883254885Sdumbbell mode0 = &rdev->mode_info.crtcs[i]->base.mode; 884254885Sdumbbell mode1 = &rdev->mode_info.crtcs[i+1]->base.mode; 885254885Sdumbbell lb_size = dce6_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode0, mode1); 886254885Sdumbbell dce6_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads); 887254885Sdumbbell lb_size = dce6_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i+1], mode1, mode0); 888254885Sdumbbell dce6_program_watermarks(rdev, rdev->mode_info.crtcs[i+1], lb_size, num_heads); 889254885Sdumbbell } 890254885Sdumbbell} 891254885Sdumbbell 892254885Sdumbbell/* 893254885Sdumbbell * Core functions 894254885Sdumbbell */ 895254885Sdumbbellstatic void si_tiling_mode_table_init(struct radeon_device *rdev) 896254885Sdumbbell{ 897254885Sdumbbell const u32 num_tile_mode_states = 32; 898254885Sdumbbell u32 reg_offset, gb_tile_moden, split_equal_to_row_size; 899254885Sdumbbell 900254885Sdumbbell switch (rdev->config.si.mem_row_size_in_kb) { 901254885Sdumbbell case 1: 902254885Sdumbbell split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_1KB; 903254885Sdumbbell break; 904254885Sdumbbell case 2: 905254885Sdumbbell default: 906254885Sdumbbell split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_2KB; 907254885Sdumbbell break; 908254885Sdumbbell case 4: 909254885Sdumbbell split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_4KB; 910254885Sdumbbell break; 911254885Sdumbbell } 912254885Sdumbbell 913254885Sdumbbell if ((rdev->family == CHIP_TAHITI) || 914254885Sdumbbell (rdev->family == CHIP_PITCAIRN)) { 915254885Sdumbbell for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) { 916254885Sdumbbell switch (reg_offset) { 917254885Sdumbbell case 0: /* non-AA compressed depth or any compressed stencil */ 918254885Sdumbbell gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 919254885Sdumbbell MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) | 920254885Sdumbbell PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | 921254885Sdumbbell TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) | 922254885Sdumbbell NUM_BANKS(ADDR_SURF_16_BANK) | 923254885Sdumbbell BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 924254885Sdumbbell BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 925254885Sdumbbell MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2)); 926254885Sdumbbell break; 927254885Sdumbbell case 1: /* 2xAA/4xAA compressed depth only */ 928254885Sdumbbell gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 929254885Sdumbbell MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) | 930254885Sdumbbell PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | 931254885Sdumbbell TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) | 932254885Sdumbbell NUM_BANKS(ADDR_SURF_16_BANK) | 933254885Sdumbbell BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 934254885Sdumbbell BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 935254885Sdumbbell MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2)); 936254885Sdumbbell break; 937254885Sdumbbell case 2: /* 8xAA compressed depth only */ 938254885Sdumbbell gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 939254885Sdumbbell MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) | 940254885Sdumbbell PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | 941254885Sdumbbell TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) | 942254885Sdumbbell NUM_BANKS(ADDR_SURF_16_BANK) | 943254885Sdumbbell BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 944254885Sdumbbell BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 945254885Sdumbbell MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2)); 946254885Sdumbbell break; 947254885Sdumbbell case 3: /* 2xAA/4xAA compressed depth with stencil (for depth buffer) */ 948254885Sdumbbell gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 949254885Sdumbbell MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) | 950254885Sdumbbell PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | 951254885Sdumbbell TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) | 952254885Sdumbbell NUM_BANKS(ADDR_SURF_16_BANK) | 953254885Sdumbbell BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 954254885Sdumbbell BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 955254885Sdumbbell MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2)); 956254885Sdumbbell break; 957254885Sdumbbell case 4: /* Maps w/ a dimension less than the 2D macro-tile dimensions (for mipmapped depth textures) */ 958254885Sdumbbell gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 959254885Sdumbbell MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) | 960254885Sdumbbell PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | 961254885Sdumbbell TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) | 962254885Sdumbbell NUM_BANKS(ADDR_SURF_16_BANK) | 963254885Sdumbbell BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 964254885Sdumbbell BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 965254885Sdumbbell MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2)); 966254885Sdumbbell break; 967254885Sdumbbell case 5: /* Uncompressed 16bpp depth - and stencil buffer allocated with it */ 968254885Sdumbbell gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 969254885Sdumbbell MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) | 970254885Sdumbbell PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | 971254885Sdumbbell TILE_SPLIT(split_equal_to_row_size) | 972254885Sdumbbell NUM_BANKS(ADDR_SURF_16_BANK) | 973254885Sdumbbell BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 974254885Sdumbbell BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 975254885Sdumbbell MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2)); 976254885Sdumbbell break; 977254885Sdumbbell case 6: /* Uncompressed 32bpp depth - and stencil buffer allocated with it */ 978254885Sdumbbell gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 979254885Sdumbbell MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) | 980254885Sdumbbell PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | 981254885Sdumbbell TILE_SPLIT(split_equal_to_row_size) | 982254885Sdumbbell NUM_BANKS(ADDR_SURF_16_BANK) | 983254885Sdumbbell BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 984254885Sdumbbell BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 985254885Sdumbbell MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1)); 986254885Sdumbbell break; 987254885Sdumbbell case 7: /* Uncompressed 8bpp stencil without depth (drivers typically do not use) */ 988254885Sdumbbell gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 989254885Sdumbbell MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) | 990254885Sdumbbell PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | 991254885Sdumbbell TILE_SPLIT(split_equal_to_row_size) | 992254885Sdumbbell NUM_BANKS(ADDR_SURF_16_BANK) | 993254885Sdumbbell BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 994254885Sdumbbell BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 995254885Sdumbbell MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2)); 996254885Sdumbbell break; 997254885Sdumbbell case 8: /* 1D and 1D Array Surfaces */ 998254885Sdumbbell gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) | 999254885Sdumbbell MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) | 1000254885Sdumbbell PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | 1001254885Sdumbbell TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) | 1002254885Sdumbbell NUM_BANKS(ADDR_SURF_16_BANK) | 1003254885Sdumbbell BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 1004254885Sdumbbell BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 1005254885Sdumbbell MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2)); 1006254885Sdumbbell break; 1007254885Sdumbbell case 9: /* Displayable maps. */ 1008254885Sdumbbell gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 1009254885Sdumbbell MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) | 1010254885Sdumbbell PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | 1011254885Sdumbbell TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) | 1012254885Sdumbbell NUM_BANKS(ADDR_SURF_16_BANK) | 1013254885Sdumbbell BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 1014254885Sdumbbell BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 1015254885Sdumbbell MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2)); 1016254885Sdumbbell break; 1017254885Sdumbbell case 10: /* Display 8bpp. */ 1018254885Sdumbbell gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 1019254885Sdumbbell MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) | 1020254885Sdumbbell PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | 1021254885Sdumbbell TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) | 1022254885Sdumbbell NUM_BANKS(ADDR_SURF_16_BANK) | 1023254885Sdumbbell BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 1024254885Sdumbbell BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 1025254885Sdumbbell MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2)); 1026254885Sdumbbell break; 1027254885Sdumbbell case 11: /* Display 16bpp. */ 1028254885Sdumbbell gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 1029254885Sdumbbell MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) | 1030254885Sdumbbell PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | 1031254885Sdumbbell TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) | 1032254885Sdumbbell NUM_BANKS(ADDR_SURF_16_BANK) | 1033254885Sdumbbell BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 1034254885Sdumbbell BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 1035254885Sdumbbell MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2)); 1036254885Sdumbbell break; 1037254885Sdumbbell case 12: /* Display 32bpp. */ 1038254885Sdumbbell gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 1039254885Sdumbbell MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) | 1040254885Sdumbbell PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | 1041254885Sdumbbell TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) | 1042254885Sdumbbell NUM_BANKS(ADDR_SURF_16_BANK) | 1043254885Sdumbbell BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 1044254885Sdumbbell BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 1045254885Sdumbbell MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1)); 1046254885Sdumbbell break; 1047254885Sdumbbell case 13: /* Thin. */ 1048254885Sdumbbell gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 1049254885Sdumbbell MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) | 1050254885Sdumbbell PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | 1051254885Sdumbbell TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) | 1052254885Sdumbbell NUM_BANKS(ADDR_SURF_16_BANK) | 1053254885Sdumbbell BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 1054254885Sdumbbell BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 1055254885Sdumbbell MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2)); 1056254885Sdumbbell break; 1057254885Sdumbbell case 14: /* Thin 8 bpp. */ 1058254885Sdumbbell gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 1059254885Sdumbbell MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) | 1060254885Sdumbbell PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | 1061254885Sdumbbell TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) | 1062254885Sdumbbell NUM_BANKS(ADDR_SURF_16_BANK) | 1063254885Sdumbbell BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 1064254885Sdumbbell BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 1065254885Sdumbbell MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1)); 1066254885Sdumbbell break; 1067254885Sdumbbell case 15: /* Thin 16 bpp. */ 1068254885Sdumbbell gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 1069254885Sdumbbell MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) | 1070254885Sdumbbell PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | 1071254885Sdumbbell TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) | 1072254885Sdumbbell NUM_BANKS(ADDR_SURF_16_BANK) | 1073254885Sdumbbell BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 1074254885Sdumbbell BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 1075254885Sdumbbell MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1)); 1076254885Sdumbbell break; 1077254885Sdumbbell case 16: /* Thin 32 bpp. */ 1078254885Sdumbbell gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 1079254885Sdumbbell MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) | 1080254885Sdumbbell PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | 1081254885Sdumbbell TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) | 1082254885Sdumbbell NUM_BANKS(ADDR_SURF_16_BANK) | 1083254885Sdumbbell BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 1084254885Sdumbbell BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 1085254885Sdumbbell MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1)); 1086254885Sdumbbell break; 1087254885Sdumbbell case 17: /* Thin 64 bpp. */ 1088254885Sdumbbell gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 1089254885Sdumbbell MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) | 1090254885Sdumbbell PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | 1091254885Sdumbbell TILE_SPLIT(split_equal_to_row_size) | 1092254885Sdumbbell NUM_BANKS(ADDR_SURF_16_BANK) | 1093254885Sdumbbell BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 1094254885Sdumbbell BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 1095254885Sdumbbell MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1)); 1096254885Sdumbbell break; 1097254885Sdumbbell case 21: /* 8 bpp PRT. */ 1098254885Sdumbbell gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 1099254885Sdumbbell MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) | 1100254885Sdumbbell PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | 1101254885Sdumbbell TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) | 1102254885Sdumbbell NUM_BANKS(ADDR_SURF_16_BANK) | 1103254885Sdumbbell BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) | 1104254885Sdumbbell BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 1105254885Sdumbbell MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2)); 1106254885Sdumbbell break; 1107254885Sdumbbell case 22: /* 16 bpp PRT */ 1108254885Sdumbbell gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 1109254885Sdumbbell MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) | 1110254885Sdumbbell PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | 1111254885Sdumbbell TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) | 1112254885Sdumbbell NUM_BANKS(ADDR_SURF_16_BANK) | 1113254885Sdumbbell BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 1114254885Sdumbbell BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 1115254885Sdumbbell MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4)); 1116254885Sdumbbell break; 1117254885Sdumbbell case 23: /* 32 bpp PRT */ 1118254885Sdumbbell gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 1119254885Sdumbbell MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) | 1120254885Sdumbbell PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | 1121254885Sdumbbell TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) | 1122254885Sdumbbell NUM_BANKS(ADDR_SURF_16_BANK) | 1123254885Sdumbbell BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 1124254885Sdumbbell BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 1125254885Sdumbbell MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2)); 1126254885Sdumbbell break; 1127254885Sdumbbell case 24: /* 64 bpp PRT */ 1128254885Sdumbbell gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 1129254885Sdumbbell MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) | 1130254885Sdumbbell PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | 1131254885Sdumbbell TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) | 1132254885Sdumbbell NUM_BANKS(ADDR_SURF_16_BANK) | 1133254885Sdumbbell BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 1134254885Sdumbbell BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 1135254885Sdumbbell MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2)); 1136254885Sdumbbell break; 1137254885Sdumbbell case 25: /* 128 bpp PRT */ 1138254885Sdumbbell gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 1139254885Sdumbbell MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) | 1140254885Sdumbbell PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | 1141254885Sdumbbell TILE_SPLIT(ADDR_SURF_TILE_SPLIT_1KB) | 1142254885Sdumbbell NUM_BANKS(ADDR_SURF_8_BANK) | 1143254885Sdumbbell BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 1144254885Sdumbbell BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 1145254885Sdumbbell MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1)); 1146254885Sdumbbell break; 1147254885Sdumbbell default: 1148254885Sdumbbell gb_tile_moden = 0; 1149254885Sdumbbell break; 1150254885Sdumbbell } 1151254885Sdumbbell WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden); 1152254885Sdumbbell } 1153254885Sdumbbell } else if (rdev->family == CHIP_VERDE) { 1154254885Sdumbbell for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) { 1155254885Sdumbbell switch (reg_offset) { 1156254885Sdumbbell case 0: /* non-AA compressed depth or any compressed stencil */ 1157254885Sdumbbell gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 1158254885Sdumbbell MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) | 1159254885Sdumbbell PIPE_CONFIG(ADDR_SURF_P4_8x16) | 1160254885Sdumbbell TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) | 1161254885Sdumbbell NUM_BANKS(ADDR_SURF_16_BANK) | 1162254885Sdumbbell BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 1163254885Sdumbbell BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 1164254885Sdumbbell MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4)); 1165254885Sdumbbell break; 1166254885Sdumbbell case 1: /* 2xAA/4xAA compressed depth only */ 1167254885Sdumbbell gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 1168254885Sdumbbell MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) | 1169254885Sdumbbell PIPE_CONFIG(ADDR_SURF_P4_8x16) | 1170254885Sdumbbell TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) | 1171254885Sdumbbell NUM_BANKS(ADDR_SURF_16_BANK) | 1172254885Sdumbbell BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 1173254885Sdumbbell BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 1174254885Sdumbbell MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4)); 1175254885Sdumbbell break; 1176254885Sdumbbell case 2: /* 8xAA compressed depth only */ 1177254885Sdumbbell gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 1178254885Sdumbbell MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) | 1179254885Sdumbbell PIPE_CONFIG(ADDR_SURF_P4_8x16) | 1180254885Sdumbbell TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) | 1181254885Sdumbbell NUM_BANKS(ADDR_SURF_16_BANK) | 1182254885Sdumbbell BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 1183254885Sdumbbell BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 1184254885Sdumbbell MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4)); 1185254885Sdumbbell break; 1186254885Sdumbbell case 3: /* 2xAA/4xAA compressed depth with stencil (for depth buffer) */ 1187254885Sdumbbell gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 1188254885Sdumbbell MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) | 1189254885Sdumbbell PIPE_CONFIG(ADDR_SURF_P4_8x16) | 1190254885Sdumbbell TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) | 1191254885Sdumbbell NUM_BANKS(ADDR_SURF_16_BANK) | 1192254885Sdumbbell BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 1193254885Sdumbbell BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 1194254885Sdumbbell MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4)); 1195254885Sdumbbell break; 1196254885Sdumbbell case 4: /* Maps w/ a dimension less than the 2D macro-tile dimensions (for mipmapped depth textures) */ 1197254885Sdumbbell gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 1198254885Sdumbbell MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) | 1199254885Sdumbbell PIPE_CONFIG(ADDR_SURF_P4_8x16) | 1200254885Sdumbbell TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) | 1201254885Sdumbbell NUM_BANKS(ADDR_SURF_16_BANK) | 1202254885Sdumbbell BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 1203254885Sdumbbell BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 1204254885Sdumbbell MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2)); 1205254885Sdumbbell break; 1206254885Sdumbbell case 5: /* Uncompressed 16bpp depth - and stencil buffer allocated with it */ 1207254885Sdumbbell gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 1208254885Sdumbbell MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) | 1209254885Sdumbbell PIPE_CONFIG(ADDR_SURF_P4_8x16) | 1210254885Sdumbbell TILE_SPLIT(split_equal_to_row_size) | 1211254885Sdumbbell NUM_BANKS(ADDR_SURF_16_BANK) | 1212254885Sdumbbell BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 1213254885Sdumbbell BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 1214254885Sdumbbell MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2)); 1215254885Sdumbbell break; 1216254885Sdumbbell case 6: /* Uncompressed 32bpp depth - and stencil buffer allocated with it */ 1217254885Sdumbbell gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 1218254885Sdumbbell MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) | 1219254885Sdumbbell PIPE_CONFIG(ADDR_SURF_P4_8x16) | 1220254885Sdumbbell TILE_SPLIT(split_equal_to_row_size) | 1221254885Sdumbbell NUM_BANKS(ADDR_SURF_16_BANK) | 1222254885Sdumbbell BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 1223254885Sdumbbell BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 1224254885Sdumbbell MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2)); 1225254885Sdumbbell break; 1226254885Sdumbbell case 7: /* Uncompressed 8bpp stencil without depth (drivers typically do not use) */ 1227254885Sdumbbell gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 1228254885Sdumbbell MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) | 1229254885Sdumbbell PIPE_CONFIG(ADDR_SURF_P4_8x16) | 1230254885Sdumbbell TILE_SPLIT(split_equal_to_row_size) | 1231254885Sdumbbell NUM_BANKS(ADDR_SURF_16_BANK) | 1232254885Sdumbbell BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 1233254885Sdumbbell BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 1234254885Sdumbbell MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4)); 1235254885Sdumbbell break; 1236254885Sdumbbell case 8: /* 1D and 1D Array Surfaces */ 1237254885Sdumbbell gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) | 1238254885Sdumbbell MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) | 1239254885Sdumbbell PIPE_CONFIG(ADDR_SURF_P4_8x16) | 1240254885Sdumbbell TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) | 1241254885Sdumbbell NUM_BANKS(ADDR_SURF_16_BANK) | 1242254885Sdumbbell BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 1243254885Sdumbbell BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 1244254885Sdumbbell MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2)); 1245254885Sdumbbell break; 1246254885Sdumbbell case 9: /* Displayable maps. */ 1247254885Sdumbbell gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 1248254885Sdumbbell MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) | 1249254885Sdumbbell PIPE_CONFIG(ADDR_SURF_P4_8x16) | 1250254885Sdumbbell TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) | 1251254885Sdumbbell NUM_BANKS(ADDR_SURF_16_BANK) | 1252254885Sdumbbell BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 1253254885Sdumbbell BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 1254254885Sdumbbell MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2)); 1255254885Sdumbbell break; 1256254885Sdumbbell case 10: /* Display 8bpp. */ 1257254885Sdumbbell gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 1258254885Sdumbbell MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) | 1259254885Sdumbbell PIPE_CONFIG(ADDR_SURF_P4_8x16) | 1260254885Sdumbbell TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) | 1261254885Sdumbbell NUM_BANKS(ADDR_SURF_16_BANK) | 1262254885Sdumbbell BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 1263254885Sdumbbell BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 1264254885Sdumbbell MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4)); 1265254885Sdumbbell break; 1266254885Sdumbbell case 11: /* Display 16bpp. */ 1267254885Sdumbbell gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 1268254885Sdumbbell MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) | 1269254885Sdumbbell PIPE_CONFIG(ADDR_SURF_P4_8x16) | 1270254885Sdumbbell TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) | 1271254885Sdumbbell NUM_BANKS(ADDR_SURF_16_BANK) | 1272254885Sdumbbell BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 1273254885Sdumbbell BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 1274254885Sdumbbell MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2)); 1275254885Sdumbbell break; 1276254885Sdumbbell case 12: /* Display 32bpp. */ 1277254885Sdumbbell gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 1278254885Sdumbbell MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) | 1279254885Sdumbbell PIPE_CONFIG(ADDR_SURF_P4_8x16) | 1280254885Sdumbbell TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) | 1281254885Sdumbbell NUM_BANKS(ADDR_SURF_16_BANK) | 1282254885Sdumbbell BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 1283254885Sdumbbell BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 1284254885Sdumbbell MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2)); 1285254885Sdumbbell break; 1286254885Sdumbbell case 13: /* Thin. */ 1287254885Sdumbbell gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 1288254885Sdumbbell MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) | 1289254885Sdumbbell PIPE_CONFIG(ADDR_SURF_P4_8x16) | 1290254885Sdumbbell TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) | 1291254885Sdumbbell NUM_BANKS(ADDR_SURF_16_BANK) | 1292254885Sdumbbell BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 1293254885Sdumbbell BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 1294254885Sdumbbell MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2)); 1295254885Sdumbbell break; 1296254885Sdumbbell case 14: /* Thin 8 bpp. */ 1297254885Sdumbbell gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 1298254885Sdumbbell MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) | 1299254885Sdumbbell PIPE_CONFIG(ADDR_SURF_P4_8x16) | 1300254885Sdumbbell TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) | 1301254885Sdumbbell NUM_BANKS(ADDR_SURF_16_BANK) | 1302254885Sdumbbell BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 1303254885Sdumbbell BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 1304254885Sdumbbell MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2)); 1305254885Sdumbbell break; 1306254885Sdumbbell case 15: /* Thin 16 bpp. */ 1307254885Sdumbbell gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 1308254885Sdumbbell MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) | 1309254885Sdumbbell PIPE_CONFIG(ADDR_SURF_P4_8x16) | 1310254885Sdumbbell TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) | 1311254885Sdumbbell NUM_BANKS(ADDR_SURF_16_BANK) | 1312254885Sdumbbell BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 1313254885Sdumbbell BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 1314254885Sdumbbell MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2)); 1315254885Sdumbbell break; 1316254885Sdumbbell case 16: /* Thin 32 bpp. */ 1317254885Sdumbbell gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 1318254885Sdumbbell MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) | 1319254885Sdumbbell PIPE_CONFIG(ADDR_SURF_P4_8x16) | 1320254885Sdumbbell TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) | 1321254885Sdumbbell NUM_BANKS(ADDR_SURF_16_BANK) | 1322254885Sdumbbell BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 1323254885Sdumbbell BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 1324254885Sdumbbell MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2)); 1325254885Sdumbbell break; 1326254885Sdumbbell case 17: /* Thin 64 bpp. */ 1327254885Sdumbbell gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 1328254885Sdumbbell MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) | 1329254885Sdumbbell PIPE_CONFIG(ADDR_SURF_P4_8x16) | 1330254885Sdumbbell TILE_SPLIT(split_equal_to_row_size) | 1331254885Sdumbbell NUM_BANKS(ADDR_SURF_16_BANK) | 1332254885Sdumbbell BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 1333254885Sdumbbell BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 1334254885Sdumbbell MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2)); 1335254885Sdumbbell break; 1336254885Sdumbbell case 21: /* 8 bpp PRT. */ 1337254885Sdumbbell gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 1338254885Sdumbbell MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) | 1339254885Sdumbbell PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | 1340254885Sdumbbell TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) | 1341254885Sdumbbell NUM_BANKS(ADDR_SURF_16_BANK) | 1342254885Sdumbbell BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) | 1343254885Sdumbbell BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 1344254885Sdumbbell MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2)); 1345254885Sdumbbell break; 1346254885Sdumbbell case 22: /* 16 bpp PRT */ 1347254885Sdumbbell gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 1348254885Sdumbbell MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) | 1349254885Sdumbbell PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | 1350254885Sdumbbell TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) | 1351254885Sdumbbell NUM_BANKS(ADDR_SURF_16_BANK) | 1352254885Sdumbbell BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 1353254885Sdumbbell BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 1354254885Sdumbbell MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4)); 1355254885Sdumbbell break; 1356254885Sdumbbell case 23: /* 32 bpp PRT */ 1357254885Sdumbbell gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 1358254885Sdumbbell MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) | 1359254885Sdumbbell PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | 1360254885Sdumbbell TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) | 1361254885Sdumbbell NUM_BANKS(ADDR_SURF_16_BANK) | 1362254885Sdumbbell BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 1363254885Sdumbbell BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 1364254885Sdumbbell MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2)); 1365254885Sdumbbell break; 1366254885Sdumbbell case 24: /* 64 bpp PRT */ 1367254885Sdumbbell gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 1368254885Sdumbbell MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) | 1369254885Sdumbbell PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | 1370254885Sdumbbell TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) | 1371254885Sdumbbell NUM_BANKS(ADDR_SURF_16_BANK) | 1372254885Sdumbbell BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 1373254885Sdumbbell BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 1374254885Sdumbbell MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2)); 1375254885Sdumbbell break; 1376254885Sdumbbell case 25: /* 128 bpp PRT */ 1377254885Sdumbbell gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 1378254885Sdumbbell MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) | 1379254885Sdumbbell PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | 1380254885Sdumbbell TILE_SPLIT(ADDR_SURF_TILE_SPLIT_1KB) | 1381254885Sdumbbell NUM_BANKS(ADDR_SURF_8_BANK) | 1382254885Sdumbbell BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 1383254885Sdumbbell BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 1384254885Sdumbbell MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1)); 1385254885Sdumbbell break; 1386254885Sdumbbell default: 1387254885Sdumbbell gb_tile_moden = 0; 1388254885Sdumbbell break; 1389254885Sdumbbell } 1390254885Sdumbbell WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden); 1391254885Sdumbbell } 1392254885Sdumbbell } else 1393254885Sdumbbell DRM_ERROR("unknown asic: 0x%x\n", rdev->family); 1394254885Sdumbbell} 1395254885Sdumbbell 1396254885Sdumbbellstatic void si_select_se_sh(struct radeon_device *rdev, 1397254885Sdumbbell u32 se_num, u32 sh_num) 1398254885Sdumbbell{ 1399254885Sdumbbell u32 data = INSTANCE_BROADCAST_WRITES; 1400254885Sdumbbell 1401254885Sdumbbell if ((se_num == 0xffffffff) && (sh_num == 0xffffffff)) 1402254885Sdumbbell data = SH_BROADCAST_WRITES | SE_BROADCAST_WRITES; 1403254885Sdumbbell else if (se_num == 0xffffffff) 1404254885Sdumbbell data |= SE_BROADCAST_WRITES | SH_INDEX(sh_num); 1405254885Sdumbbell else if (sh_num == 0xffffffff) 1406254885Sdumbbell data |= SH_BROADCAST_WRITES | SE_INDEX(se_num); 1407254885Sdumbbell else 1408254885Sdumbbell data |= SH_INDEX(sh_num) | SE_INDEX(se_num); 1409254885Sdumbbell WREG32(GRBM_GFX_INDEX, data); 1410254885Sdumbbell} 1411254885Sdumbbell 1412254885Sdumbbellstatic u32 si_create_bitmask(u32 bit_width) 1413254885Sdumbbell{ 1414254885Sdumbbell u32 i, mask = 0; 1415254885Sdumbbell 1416254885Sdumbbell for (i = 0; i < bit_width; i++) { 1417254885Sdumbbell mask <<= 1; 1418254885Sdumbbell mask |= 1; 1419254885Sdumbbell } 1420254885Sdumbbell return mask; 1421254885Sdumbbell} 1422254885Sdumbbell 1423254885Sdumbbellstatic u32 si_get_cu_enabled(struct radeon_device *rdev, u32 cu_per_sh) 1424254885Sdumbbell{ 1425254885Sdumbbell u32 data, mask; 1426254885Sdumbbell 1427254885Sdumbbell data = RREG32(CC_GC_SHADER_ARRAY_CONFIG); 1428254885Sdumbbell if (data & 1) 1429254885Sdumbbell data &= INACTIVE_CUS_MASK; 1430254885Sdumbbell else 1431254885Sdumbbell data = 0; 1432254885Sdumbbell data |= RREG32(GC_USER_SHADER_ARRAY_CONFIG); 1433254885Sdumbbell 1434254885Sdumbbell data >>= INACTIVE_CUS_SHIFT; 1435254885Sdumbbell 1436254885Sdumbbell mask = si_create_bitmask(cu_per_sh); 1437254885Sdumbbell 1438254885Sdumbbell return ~data & mask; 1439254885Sdumbbell} 1440254885Sdumbbell 1441254885Sdumbbellstatic void si_setup_spi(struct radeon_device *rdev, 1442254885Sdumbbell u32 se_num, u32 sh_per_se, 1443254885Sdumbbell u32 cu_per_sh) 1444254885Sdumbbell{ 1445254885Sdumbbell int i, j, k; 1446254885Sdumbbell u32 data, mask, active_cu; 1447254885Sdumbbell 1448254885Sdumbbell for (i = 0; i < se_num; i++) { 1449254885Sdumbbell for (j = 0; j < sh_per_se; j++) { 1450254885Sdumbbell si_select_se_sh(rdev, i, j); 1451254885Sdumbbell data = RREG32(SPI_STATIC_THREAD_MGMT_3); 1452254885Sdumbbell active_cu = si_get_cu_enabled(rdev, cu_per_sh); 1453254885Sdumbbell 1454254885Sdumbbell mask = 1; 1455254885Sdumbbell for (k = 0; k < 16; k++) { 1456254885Sdumbbell mask <<= k; 1457254885Sdumbbell if (active_cu & mask) { 1458254885Sdumbbell data &= ~mask; 1459254885Sdumbbell WREG32(SPI_STATIC_THREAD_MGMT_3, data); 1460254885Sdumbbell break; 1461254885Sdumbbell } 1462254885Sdumbbell } 1463254885Sdumbbell } 1464254885Sdumbbell } 1465254885Sdumbbell si_select_se_sh(rdev, 0xffffffff, 0xffffffff); 1466254885Sdumbbell} 1467254885Sdumbbell 1468254885Sdumbbellstatic u32 si_get_rb_disabled(struct radeon_device *rdev, 1469254885Sdumbbell u32 max_rb_num, u32 se_num, 1470254885Sdumbbell u32 sh_per_se) 1471254885Sdumbbell{ 1472254885Sdumbbell u32 data, mask; 1473254885Sdumbbell 1474254885Sdumbbell data = RREG32(CC_RB_BACKEND_DISABLE); 1475254885Sdumbbell if (data & 1) 1476254885Sdumbbell data &= BACKEND_DISABLE_MASK; 1477254885Sdumbbell else 1478254885Sdumbbell data = 0; 1479254885Sdumbbell data |= RREG32(GC_USER_RB_BACKEND_DISABLE); 1480254885Sdumbbell 1481254885Sdumbbell data >>= BACKEND_DISABLE_SHIFT; 1482254885Sdumbbell 1483254885Sdumbbell mask = si_create_bitmask(max_rb_num / se_num / sh_per_se); 1484254885Sdumbbell 1485254885Sdumbbell return data & mask; 1486254885Sdumbbell} 1487254885Sdumbbell 1488254885Sdumbbellstatic void si_setup_rb(struct radeon_device *rdev, 1489254885Sdumbbell u32 se_num, u32 sh_per_se, 1490254885Sdumbbell u32 max_rb_num) 1491254885Sdumbbell{ 1492254885Sdumbbell int i, j; 1493254885Sdumbbell u32 data, mask; 1494254885Sdumbbell u32 disabled_rbs = 0; 1495254885Sdumbbell u32 enabled_rbs = 0; 1496254885Sdumbbell 1497254885Sdumbbell for (i = 0; i < se_num; i++) { 1498254885Sdumbbell for (j = 0; j < sh_per_se; j++) { 1499254885Sdumbbell si_select_se_sh(rdev, i, j); 1500254885Sdumbbell data = si_get_rb_disabled(rdev, max_rb_num, se_num, sh_per_se); 1501254885Sdumbbell disabled_rbs |= data << ((i * sh_per_se + j) * TAHITI_RB_BITMAP_WIDTH_PER_SH); 1502254885Sdumbbell } 1503254885Sdumbbell } 1504254885Sdumbbell si_select_se_sh(rdev, 0xffffffff, 0xffffffff); 1505254885Sdumbbell 1506254885Sdumbbell mask = 1; 1507254885Sdumbbell for (i = 0; i < max_rb_num; i++) { 1508254885Sdumbbell if (!(disabled_rbs & mask)) 1509254885Sdumbbell enabled_rbs |= mask; 1510254885Sdumbbell mask <<= 1; 1511254885Sdumbbell } 1512254885Sdumbbell 1513254885Sdumbbell for (i = 0; i < se_num; i++) { 1514254885Sdumbbell si_select_se_sh(rdev, i, 0xffffffff); 1515254885Sdumbbell data = 0; 1516254885Sdumbbell for (j = 0; j < sh_per_se; j++) { 1517254885Sdumbbell switch (enabled_rbs & 3) { 1518254885Sdumbbell case 1: 1519254885Sdumbbell data |= (RASTER_CONFIG_RB_MAP_0 << (i * sh_per_se + j) * 2); 1520254885Sdumbbell break; 1521254885Sdumbbell case 2: 1522254885Sdumbbell data |= (RASTER_CONFIG_RB_MAP_3 << (i * sh_per_se + j) * 2); 1523254885Sdumbbell break; 1524254885Sdumbbell case 3: 1525254885Sdumbbell default: 1526254885Sdumbbell data |= (RASTER_CONFIG_RB_MAP_2 << (i * sh_per_se + j) * 2); 1527254885Sdumbbell break; 1528254885Sdumbbell } 1529254885Sdumbbell enabled_rbs >>= 2; 1530254885Sdumbbell } 1531254885Sdumbbell WREG32(PA_SC_RASTER_CONFIG, data); 1532254885Sdumbbell } 1533254885Sdumbbell si_select_se_sh(rdev, 0xffffffff, 0xffffffff); 1534254885Sdumbbell} 1535254885Sdumbbell 1536254885Sdumbbellstatic void si_gpu_init(struct radeon_device *rdev) 1537254885Sdumbbell{ 1538254885Sdumbbell u32 gb_addr_config = 0; 1539254885Sdumbbell u32 mc_shared_chmap, mc_arb_ramcfg; 1540254885Sdumbbell u32 sx_debug_1; 1541254885Sdumbbell u32 hdp_host_path_cntl; 1542254885Sdumbbell u32 tmp; 1543254885Sdumbbell int i, j; 1544254885Sdumbbell 1545254885Sdumbbell switch (rdev->family) { 1546254885Sdumbbell case CHIP_TAHITI: 1547254885Sdumbbell rdev->config.si.max_shader_engines = 2; 1548254885Sdumbbell rdev->config.si.max_tile_pipes = 12; 1549254885Sdumbbell rdev->config.si.max_cu_per_sh = 8; 1550254885Sdumbbell rdev->config.si.max_sh_per_se = 2; 1551254885Sdumbbell rdev->config.si.max_backends_per_se = 4; 1552254885Sdumbbell rdev->config.si.max_texture_channel_caches = 12; 1553254885Sdumbbell rdev->config.si.max_gprs = 256; 1554254885Sdumbbell rdev->config.si.max_gs_threads = 32; 1555254885Sdumbbell rdev->config.si.max_hw_contexts = 8; 1556254885Sdumbbell 1557254885Sdumbbell rdev->config.si.sc_prim_fifo_size_frontend = 0x20; 1558254885Sdumbbell rdev->config.si.sc_prim_fifo_size_backend = 0x100; 1559254885Sdumbbell rdev->config.si.sc_hiz_tile_fifo_size = 0x30; 1560254885Sdumbbell rdev->config.si.sc_earlyz_tile_fifo_size = 0x130; 1561254885Sdumbbell gb_addr_config = TAHITI_GB_ADDR_CONFIG_GOLDEN; 1562254885Sdumbbell break; 1563254885Sdumbbell case CHIP_PITCAIRN: 1564254885Sdumbbell rdev->config.si.max_shader_engines = 2; 1565254885Sdumbbell rdev->config.si.max_tile_pipes = 8; 1566254885Sdumbbell rdev->config.si.max_cu_per_sh = 5; 1567254885Sdumbbell rdev->config.si.max_sh_per_se = 2; 1568254885Sdumbbell rdev->config.si.max_backends_per_se = 4; 1569254885Sdumbbell rdev->config.si.max_texture_channel_caches = 8; 1570254885Sdumbbell rdev->config.si.max_gprs = 256; 1571254885Sdumbbell rdev->config.si.max_gs_threads = 32; 1572254885Sdumbbell rdev->config.si.max_hw_contexts = 8; 1573254885Sdumbbell 1574254885Sdumbbell rdev->config.si.sc_prim_fifo_size_frontend = 0x20; 1575254885Sdumbbell rdev->config.si.sc_prim_fifo_size_backend = 0x100; 1576254885Sdumbbell rdev->config.si.sc_hiz_tile_fifo_size = 0x30; 1577254885Sdumbbell rdev->config.si.sc_earlyz_tile_fifo_size = 0x130; 1578254885Sdumbbell gb_addr_config = TAHITI_GB_ADDR_CONFIG_GOLDEN; 1579254885Sdumbbell break; 1580254885Sdumbbell case CHIP_VERDE: 1581254885Sdumbbell default: 1582254885Sdumbbell rdev->config.si.max_shader_engines = 1; 1583254885Sdumbbell rdev->config.si.max_tile_pipes = 4; 1584254885Sdumbbell rdev->config.si.max_cu_per_sh = 2; 1585254885Sdumbbell rdev->config.si.max_sh_per_se = 2; 1586254885Sdumbbell rdev->config.si.max_backends_per_se = 4; 1587254885Sdumbbell rdev->config.si.max_texture_channel_caches = 4; 1588254885Sdumbbell rdev->config.si.max_gprs = 256; 1589254885Sdumbbell rdev->config.si.max_gs_threads = 32; 1590254885Sdumbbell rdev->config.si.max_hw_contexts = 8; 1591254885Sdumbbell 1592254885Sdumbbell rdev->config.si.sc_prim_fifo_size_frontend = 0x20; 1593254885Sdumbbell rdev->config.si.sc_prim_fifo_size_backend = 0x40; 1594254885Sdumbbell rdev->config.si.sc_hiz_tile_fifo_size = 0x30; 1595254885Sdumbbell rdev->config.si.sc_earlyz_tile_fifo_size = 0x130; 1596254885Sdumbbell gb_addr_config = VERDE_GB_ADDR_CONFIG_GOLDEN; 1597254885Sdumbbell break; 1598254885Sdumbbell } 1599254885Sdumbbell 1600254885Sdumbbell /* Initialize HDP */ 1601254885Sdumbbell for (i = 0, j = 0; i < 32; i++, j += 0x18) { 1602254885Sdumbbell WREG32((0x2c14 + j), 0x00000000); 1603254885Sdumbbell WREG32((0x2c18 + j), 0x00000000); 1604254885Sdumbbell WREG32((0x2c1c + j), 0x00000000); 1605254885Sdumbbell WREG32((0x2c20 + j), 0x00000000); 1606254885Sdumbbell WREG32((0x2c24 + j), 0x00000000); 1607254885Sdumbbell } 1608254885Sdumbbell 1609254885Sdumbbell WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff)); 1610254885Sdumbbell 1611254885Sdumbbell evergreen_fix_pci_max_read_req_size(rdev); 1612254885Sdumbbell 1613254885Sdumbbell WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN); 1614254885Sdumbbell 1615254885Sdumbbell mc_shared_chmap = RREG32(MC_SHARED_CHMAP); 1616254885Sdumbbell mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG); 1617254885Sdumbbell 1618254885Sdumbbell rdev->config.si.num_tile_pipes = rdev->config.si.max_tile_pipes; 1619254885Sdumbbell rdev->config.si.mem_max_burst_length_bytes = 256; 1620254885Sdumbbell tmp = (mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT; 1621254885Sdumbbell rdev->config.si.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024; 1622254885Sdumbbell if (rdev->config.si.mem_row_size_in_kb > 4) 1623254885Sdumbbell rdev->config.si.mem_row_size_in_kb = 4; 1624254885Sdumbbell /* XXX use MC settings? */ 1625254885Sdumbbell rdev->config.si.shader_engine_tile_size = 32; 1626254885Sdumbbell rdev->config.si.num_gpus = 1; 1627254885Sdumbbell rdev->config.si.multi_gpu_tile_size = 64; 1628254885Sdumbbell 1629254885Sdumbbell /* fix up row size */ 1630254885Sdumbbell gb_addr_config &= ~ROW_SIZE_MASK; 1631254885Sdumbbell switch (rdev->config.si.mem_row_size_in_kb) { 1632254885Sdumbbell case 1: 1633254885Sdumbbell default: 1634254885Sdumbbell gb_addr_config |= ROW_SIZE(0); 1635254885Sdumbbell break; 1636254885Sdumbbell case 2: 1637254885Sdumbbell gb_addr_config |= ROW_SIZE(1); 1638254885Sdumbbell break; 1639254885Sdumbbell case 4: 1640254885Sdumbbell gb_addr_config |= ROW_SIZE(2); 1641254885Sdumbbell break; 1642254885Sdumbbell } 1643254885Sdumbbell 1644254885Sdumbbell /* setup tiling info dword. gb_addr_config is not adequate since it does 1645254885Sdumbbell * not have bank info, so create a custom tiling dword. 1646254885Sdumbbell * bits 3:0 num_pipes 1647254885Sdumbbell * bits 7:4 num_banks 1648254885Sdumbbell * bits 11:8 group_size 1649254885Sdumbbell * bits 15:12 row_size 1650254885Sdumbbell */ 1651254885Sdumbbell rdev->config.si.tile_config = 0; 1652254885Sdumbbell switch (rdev->config.si.num_tile_pipes) { 1653254885Sdumbbell case 1: 1654254885Sdumbbell rdev->config.si.tile_config |= (0 << 0); 1655254885Sdumbbell break; 1656254885Sdumbbell case 2: 1657254885Sdumbbell rdev->config.si.tile_config |= (1 << 0); 1658254885Sdumbbell break; 1659254885Sdumbbell case 4: 1660254885Sdumbbell rdev->config.si.tile_config |= (2 << 0); 1661254885Sdumbbell break; 1662254885Sdumbbell case 8: 1663254885Sdumbbell default: 1664254885Sdumbbell /* XXX what about 12? */ 1665254885Sdumbbell rdev->config.si.tile_config |= (3 << 0); 1666254885Sdumbbell break; 1667254885Sdumbbell } 1668254885Sdumbbell switch ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) { 1669254885Sdumbbell case 0: /* four banks */ 1670254885Sdumbbell rdev->config.si.tile_config |= 0 << 4; 1671254885Sdumbbell break; 1672254885Sdumbbell case 1: /* eight banks */ 1673254885Sdumbbell rdev->config.si.tile_config |= 1 << 4; 1674254885Sdumbbell break; 1675254885Sdumbbell case 2: /* sixteen banks */ 1676254885Sdumbbell default: 1677254885Sdumbbell rdev->config.si.tile_config |= 2 << 4; 1678254885Sdumbbell break; 1679254885Sdumbbell } 1680254885Sdumbbell rdev->config.si.tile_config |= 1681254885Sdumbbell ((gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT) << 8; 1682254885Sdumbbell rdev->config.si.tile_config |= 1683254885Sdumbbell ((gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT) << 12; 1684254885Sdumbbell 1685254885Sdumbbell WREG32(GB_ADDR_CONFIG, gb_addr_config); 1686254885Sdumbbell WREG32(DMIF_ADDR_CONFIG, gb_addr_config); 1687254885Sdumbbell WREG32(HDP_ADDR_CONFIG, gb_addr_config); 1688254885Sdumbbell WREG32(DMA_TILING_CONFIG + DMA0_REGISTER_OFFSET, gb_addr_config); 1689254885Sdumbbell WREG32(DMA_TILING_CONFIG + DMA1_REGISTER_OFFSET, gb_addr_config); 1690254885Sdumbbell 1691254885Sdumbbell si_tiling_mode_table_init(rdev); 1692254885Sdumbbell 1693254885Sdumbbell si_setup_rb(rdev, rdev->config.si.max_shader_engines, 1694254885Sdumbbell rdev->config.si.max_sh_per_se, 1695254885Sdumbbell rdev->config.si.max_backends_per_se); 1696254885Sdumbbell 1697254885Sdumbbell si_setup_spi(rdev, rdev->config.si.max_shader_engines, 1698254885Sdumbbell rdev->config.si.max_sh_per_se, 1699254885Sdumbbell rdev->config.si.max_cu_per_sh); 1700254885Sdumbbell 1701254885Sdumbbell 1702254885Sdumbbell /* set HW defaults for 3D engine */ 1703254885Sdumbbell WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) | 1704254885Sdumbbell ROQ_IB2_START(0x2b))); 1705254885Sdumbbell WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) | MEQ2_START(0x60)); 1706254885Sdumbbell 1707254885Sdumbbell sx_debug_1 = RREG32(SX_DEBUG_1); 1708254885Sdumbbell WREG32(SX_DEBUG_1, sx_debug_1); 1709254885Sdumbbell 1710254885Sdumbbell WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4)); 1711254885Sdumbbell 1712254885Sdumbbell WREG32(PA_SC_FIFO_SIZE, (SC_FRONTEND_PRIM_FIFO_SIZE(rdev->config.si.sc_prim_fifo_size_frontend) | 1713254885Sdumbbell SC_BACKEND_PRIM_FIFO_SIZE(rdev->config.si.sc_prim_fifo_size_backend) | 1714254885Sdumbbell SC_HIZ_TILE_FIFO_SIZE(rdev->config.si.sc_hiz_tile_fifo_size) | 1715254885Sdumbbell SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.si.sc_earlyz_tile_fifo_size))); 1716254885Sdumbbell 1717254885Sdumbbell WREG32(VGT_NUM_INSTANCES, 1); 1718254885Sdumbbell 1719254885Sdumbbell WREG32(CP_PERFMON_CNTL, 0); 1720254885Sdumbbell 1721254885Sdumbbell WREG32(SQ_CONFIG, 0); 1722254885Sdumbbell 1723254885Sdumbbell WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) | 1724254885Sdumbbell FORCE_EOV_MAX_REZ_CNT(255))); 1725254885Sdumbbell 1726254885Sdumbbell WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC) | 1727254885Sdumbbell AUTO_INVLD_EN(ES_AND_GS_AUTO)); 1728254885Sdumbbell 1729254885Sdumbbell WREG32(VGT_GS_VERTEX_REUSE, 16); 1730254885Sdumbbell WREG32(PA_SC_LINE_STIPPLE_STATE, 0); 1731254885Sdumbbell 1732254885Sdumbbell WREG32(CB_PERFCOUNTER0_SELECT0, 0); 1733254885Sdumbbell WREG32(CB_PERFCOUNTER0_SELECT1, 0); 1734254885Sdumbbell WREG32(CB_PERFCOUNTER1_SELECT0, 0); 1735254885Sdumbbell WREG32(CB_PERFCOUNTER1_SELECT1, 0); 1736254885Sdumbbell WREG32(CB_PERFCOUNTER2_SELECT0, 0); 1737254885Sdumbbell WREG32(CB_PERFCOUNTER2_SELECT1, 0); 1738254885Sdumbbell WREG32(CB_PERFCOUNTER3_SELECT0, 0); 1739254885Sdumbbell WREG32(CB_PERFCOUNTER3_SELECT1, 0); 1740254885Sdumbbell 1741254885Sdumbbell tmp = RREG32(HDP_MISC_CNTL); 1742254885Sdumbbell tmp |= HDP_FLUSH_INVALIDATE_CACHE; 1743254885Sdumbbell WREG32(HDP_MISC_CNTL, tmp); 1744254885Sdumbbell 1745254885Sdumbbell hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL); 1746254885Sdumbbell WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl); 1747254885Sdumbbell 1748254885Sdumbbell WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3)); 1749254885Sdumbbell 1750254885Sdumbbell DRM_UDELAY(50); 1751254885Sdumbbell} 1752254885Sdumbbell 1753254885Sdumbbell/* 1754254885Sdumbbell * GPU scratch registers helpers function. 1755254885Sdumbbell */ 1756254885Sdumbbellstatic void si_scratch_init(struct radeon_device *rdev) 1757254885Sdumbbell{ 1758254885Sdumbbell int i; 1759254885Sdumbbell 1760254885Sdumbbell rdev->scratch.num_reg = 7; 1761254885Sdumbbell rdev->scratch.reg_base = SCRATCH_REG0; 1762254885Sdumbbell for (i = 0; i < rdev->scratch.num_reg; i++) { 1763254885Sdumbbell rdev->scratch.free[i] = true; 1764254885Sdumbbell rdev->scratch.reg[i] = rdev->scratch.reg_base + (i * 4); 1765254885Sdumbbell } 1766254885Sdumbbell} 1767254885Sdumbbell 1768254885Sdumbbellvoid si_fence_ring_emit(struct radeon_device *rdev, 1769254885Sdumbbell struct radeon_fence *fence) 1770254885Sdumbbell{ 1771254885Sdumbbell struct radeon_ring *ring = &rdev->ring[fence->ring]; 1772254885Sdumbbell u64 addr = rdev->fence_drv[fence->ring].gpu_addr; 1773254885Sdumbbell 1774254885Sdumbbell /* flush read cache over gart */ 1775254885Sdumbbell radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1)); 1776254885Sdumbbell radeon_ring_write(ring, (CP_COHER_CNTL2 - PACKET3_SET_CONFIG_REG_START) >> 2); 1777254885Sdumbbell radeon_ring_write(ring, 0); 1778254885Sdumbbell radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3)); 1779254885Sdumbbell radeon_ring_write(ring, PACKET3_TCL1_ACTION_ENA | 1780254885Sdumbbell PACKET3_TC_ACTION_ENA | 1781254885Sdumbbell PACKET3_SH_KCACHE_ACTION_ENA | 1782254885Sdumbbell PACKET3_SH_ICACHE_ACTION_ENA); 1783254885Sdumbbell radeon_ring_write(ring, 0xFFFFFFFF); 1784254885Sdumbbell radeon_ring_write(ring, 0); 1785254885Sdumbbell radeon_ring_write(ring, 10); /* poll interval */ 1786254885Sdumbbell /* EVENT_WRITE_EOP - flush caches, send int */ 1787254885Sdumbbell radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4)); 1788254885Sdumbbell radeon_ring_write(ring, EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) | EVENT_INDEX(5)); 1789254885Sdumbbell radeon_ring_write(ring, addr & 0xffffffff); 1790254885Sdumbbell radeon_ring_write(ring, (upper_32_bits(addr) & 0xff) | DATA_SEL(1) | INT_SEL(2)); 1791254885Sdumbbell radeon_ring_write(ring, fence->seq); 1792254885Sdumbbell radeon_ring_write(ring, 0); 1793254885Sdumbbell} 1794254885Sdumbbell 1795254885Sdumbbell/* 1796254885Sdumbbell * IB stuff 1797254885Sdumbbell */ 1798254885Sdumbbellvoid si_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib) 1799254885Sdumbbell{ 1800254885Sdumbbell struct radeon_ring *ring = &rdev->ring[ib->ring]; 1801254885Sdumbbell u32 header; 1802254885Sdumbbell 1803254885Sdumbbell if (ib->is_const_ib) { 1804254885Sdumbbell /* set switch buffer packet before const IB */ 1805254885Sdumbbell radeon_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0)); 1806254885Sdumbbell radeon_ring_write(ring, 0); 1807254885Sdumbbell 1808254885Sdumbbell header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2); 1809254885Sdumbbell } else { 1810254885Sdumbbell u32 next_rptr; 1811254885Sdumbbell if (ring->rptr_save_reg) { 1812254885Sdumbbell next_rptr = ring->wptr + 3 + 4 + 8; 1813254885Sdumbbell radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1)); 1814254885Sdumbbell radeon_ring_write(ring, ((ring->rptr_save_reg - 1815254885Sdumbbell PACKET3_SET_CONFIG_REG_START) >> 2)); 1816254885Sdumbbell radeon_ring_write(ring, next_rptr); 1817254885Sdumbbell } else if (rdev->wb.enabled) { 1818254885Sdumbbell next_rptr = ring->wptr + 5 + 4 + 8; 1819254885Sdumbbell radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 1820254885Sdumbbell radeon_ring_write(ring, (1 << 8)); 1821254885Sdumbbell radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc); 1822254885Sdumbbell radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr) & 0xffffffff); 1823254885Sdumbbell radeon_ring_write(ring, next_rptr); 1824254885Sdumbbell } 1825254885Sdumbbell 1826254885Sdumbbell header = PACKET3(PACKET3_INDIRECT_BUFFER, 2); 1827254885Sdumbbell } 1828254885Sdumbbell 1829254885Sdumbbell radeon_ring_write(ring, header); 1830254885Sdumbbell radeon_ring_write(ring, 1831254885Sdumbbell#ifdef __BIG_ENDIAN 1832254885Sdumbbell (2 << 0) | 1833254885Sdumbbell#endif 1834254885Sdumbbell (ib->gpu_addr & 0xFFFFFFFC)); 1835254885Sdumbbell radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF); 1836254885Sdumbbell radeon_ring_write(ring, ib->length_dw | 1837254885Sdumbbell (ib->vm ? (ib->vm->id << 24) : 0)); 1838254885Sdumbbell 1839254885Sdumbbell if (!ib->is_const_ib) { 1840254885Sdumbbell /* flush read cache over gart for this vmid */ 1841254885Sdumbbell radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1)); 1842254885Sdumbbell radeon_ring_write(ring, (CP_COHER_CNTL2 - PACKET3_SET_CONFIG_REG_START) >> 2); 1843254885Sdumbbell radeon_ring_write(ring, ib->vm ? ib->vm->id : 0); 1844254885Sdumbbell radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3)); 1845254885Sdumbbell radeon_ring_write(ring, PACKET3_TCL1_ACTION_ENA | 1846254885Sdumbbell PACKET3_TC_ACTION_ENA | 1847254885Sdumbbell PACKET3_SH_KCACHE_ACTION_ENA | 1848254885Sdumbbell PACKET3_SH_ICACHE_ACTION_ENA); 1849254885Sdumbbell radeon_ring_write(ring, 0xFFFFFFFF); 1850254885Sdumbbell radeon_ring_write(ring, 0); 1851254885Sdumbbell radeon_ring_write(ring, 10); /* poll interval */ 1852254885Sdumbbell } 1853254885Sdumbbell} 1854254885Sdumbbell 1855254885Sdumbbell/* 1856254885Sdumbbell * CP. 1857254885Sdumbbell */ 1858254885Sdumbbellstatic void si_cp_enable(struct radeon_device *rdev, bool enable) 1859254885Sdumbbell{ 1860254885Sdumbbell if (enable) 1861254885Sdumbbell WREG32(CP_ME_CNTL, 0); 1862254885Sdumbbell else { 1863254885Sdumbbell radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size); 1864254885Sdumbbell WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT)); 1865254885Sdumbbell WREG32(SCRATCH_UMSK, 0); 1866254885Sdumbbell rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false; 1867254885Sdumbbell rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false; 1868254885Sdumbbell rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false; 1869254885Sdumbbell } 1870254885Sdumbbell DRM_UDELAY(50); 1871254885Sdumbbell} 1872254885Sdumbbell 1873254885Sdumbbellstatic int si_cp_load_microcode(struct radeon_device *rdev) 1874254885Sdumbbell{ 1875254885Sdumbbell const __be32 *fw_data; 1876254885Sdumbbell int i; 1877254885Sdumbbell 1878254885Sdumbbell if (!rdev->me_fw || !rdev->pfp_fw) 1879254885Sdumbbell return -EINVAL; 1880254885Sdumbbell 1881254885Sdumbbell si_cp_enable(rdev, false); 1882254885Sdumbbell 1883254885Sdumbbell /* PFP */ 1884254885Sdumbbell fw_data = (const __be32 *)rdev->pfp_fw->data; 1885254885Sdumbbell WREG32(CP_PFP_UCODE_ADDR, 0); 1886254885Sdumbbell for (i = 0; i < SI_PFP_UCODE_SIZE; i++) 1887254885Sdumbbell WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++)); 1888254885Sdumbbell WREG32(CP_PFP_UCODE_ADDR, 0); 1889254885Sdumbbell 1890254885Sdumbbell /* CE */ 1891254885Sdumbbell fw_data = (const __be32 *)rdev->ce_fw->data; 1892254885Sdumbbell WREG32(CP_CE_UCODE_ADDR, 0); 1893254885Sdumbbell for (i = 0; i < SI_CE_UCODE_SIZE; i++) 1894254885Sdumbbell WREG32(CP_CE_UCODE_DATA, be32_to_cpup(fw_data++)); 1895254885Sdumbbell WREG32(CP_CE_UCODE_ADDR, 0); 1896254885Sdumbbell 1897254885Sdumbbell /* ME */ 1898254885Sdumbbell fw_data = (const __be32 *)rdev->me_fw->data; 1899254885Sdumbbell WREG32(CP_ME_RAM_WADDR, 0); 1900254885Sdumbbell for (i = 0; i < SI_PM4_UCODE_SIZE; i++) 1901254885Sdumbbell WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++)); 1902254885Sdumbbell WREG32(CP_ME_RAM_WADDR, 0); 1903254885Sdumbbell 1904254885Sdumbbell WREG32(CP_PFP_UCODE_ADDR, 0); 1905254885Sdumbbell WREG32(CP_CE_UCODE_ADDR, 0); 1906254885Sdumbbell WREG32(CP_ME_RAM_WADDR, 0); 1907254885Sdumbbell WREG32(CP_ME_RAM_RADDR, 0); 1908254885Sdumbbell return 0; 1909254885Sdumbbell} 1910254885Sdumbbell 1911254885Sdumbbellstatic int si_cp_start(struct radeon_device *rdev) 1912254885Sdumbbell{ 1913254885Sdumbbell struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]; 1914254885Sdumbbell int r, i; 1915254885Sdumbbell 1916254885Sdumbbell r = radeon_ring_lock(rdev, ring, 7 + 4); 1917254885Sdumbbell if (r) { 1918254885Sdumbbell DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r); 1919254885Sdumbbell return r; 1920254885Sdumbbell } 1921254885Sdumbbell /* init the CP */ 1922254885Sdumbbell radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5)); 1923254885Sdumbbell radeon_ring_write(ring, 0x1); 1924254885Sdumbbell radeon_ring_write(ring, 0x0); 1925254885Sdumbbell radeon_ring_write(ring, rdev->config.si.max_hw_contexts - 1); 1926254885Sdumbbell radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1)); 1927254885Sdumbbell radeon_ring_write(ring, 0); 1928254885Sdumbbell radeon_ring_write(ring, 0); 1929254885Sdumbbell 1930254885Sdumbbell /* init the CE partitions */ 1931254885Sdumbbell radeon_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2)); 1932254885Sdumbbell radeon_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE)); 1933254885Sdumbbell radeon_ring_write(ring, 0xc000); 1934254885Sdumbbell radeon_ring_write(ring, 0xe000); 1935254885Sdumbbell radeon_ring_unlock_commit(rdev, ring); 1936254885Sdumbbell 1937254885Sdumbbell si_cp_enable(rdev, true); 1938254885Sdumbbell 1939254885Sdumbbell r = radeon_ring_lock(rdev, ring, si_default_size + 10); 1940254885Sdumbbell if (r) { 1941254885Sdumbbell DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r); 1942254885Sdumbbell return r; 1943254885Sdumbbell } 1944254885Sdumbbell 1945254885Sdumbbell /* setup clear context state */ 1946254885Sdumbbell radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0)); 1947254885Sdumbbell radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE); 1948254885Sdumbbell 1949254885Sdumbbell for (i = 0; i < si_default_size; i++) 1950254885Sdumbbell radeon_ring_write(ring, si_default_state[i]); 1951254885Sdumbbell 1952254885Sdumbbell radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0)); 1953254885Sdumbbell radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE); 1954254885Sdumbbell 1955254885Sdumbbell /* set clear context state */ 1956254885Sdumbbell radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0)); 1957254885Sdumbbell radeon_ring_write(ring, 0); 1958254885Sdumbbell 1959254885Sdumbbell radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2)); 1960254885Sdumbbell radeon_ring_write(ring, 0x00000316); 1961254885Sdumbbell radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */ 1962254885Sdumbbell radeon_ring_write(ring, 0x00000010); /* VGT_OUT_DEALLOC_CNTL */ 1963254885Sdumbbell 1964254885Sdumbbell radeon_ring_unlock_commit(rdev, ring); 1965254885Sdumbbell 1966254885Sdumbbell for (i = RADEON_RING_TYPE_GFX_INDEX; i <= CAYMAN_RING_TYPE_CP2_INDEX; ++i) { 1967254885Sdumbbell ring = &rdev->ring[i]; 1968254885Sdumbbell r = radeon_ring_lock(rdev, ring, 2); 1969254885Sdumbbell 1970254885Sdumbbell /* clear the compute context state */ 1971254885Sdumbbell radeon_ring_write(ring, PACKET3_COMPUTE(PACKET3_CLEAR_STATE, 0)); 1972254885Sdumbbell radeon_ring_write(ring, 0); 1973254885Sdumbbell 1974254885Sdumbbell radeon_ring_unlock_commit(rdev, ring); 1975254885Sdumbbell } 1976254885Sdumbbell 1977254885Sdumbbell return 0; 1978254885Sdumbbell} 1979254885Sdumbbell 1980254885Sdumbbellstatic void si_cp_fini(struct radeon_device *rdev) 1981254885Sdumbbell{ 1982254885Sdumbbell struct radeon_ring *ring; 1983254885Sdumbbell si_cp_enable(rdev, false); 1984254885Sdumbbell 1985254885Sdumbbell ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]; 1986254885Sdumbbell radeon_ring_fini(rdev, ring); 1987254885Sdumbbell radeon_scratch_free(rdev, ring->rptr_save_reg); 1988254885Sdumbbell 1989254885Sdumbbell ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX]; 1990254885Sdumbbell radeon_ring_fini(rdev, ring); 1991254885Sdumbbell radeon_scratch_free(rdev, ring->rptr_save_reg); 1992254885Sdumbbell 1993254885Sdumbbell ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX]; 1994254885Sdumbbell radeon_ring_fini(rdev, ring); 1995254885Sdumbbell radeon_scratch_free(rdev, ring->rptr_save_reg); 1996254885Sdumbbell} 1997254885Sdumbbell 1998254885Sdumbbellstatic int si_cp_resume(struct radeon_device *rdev) 1999254885Sdumbbell{ 2000254885Sdumbbell struct radeon_ring *ring; 2001254885Sdumbbell u32 tmp; 2002254885Sdumbbell u32 rb_bufsz; 2003254885Sdumbbell int r; 2004254885Sdumbbell 2005254885Sdumbbell /* Reset cp; if cp is reset, then PA, SH, VGT also need to be reset */ 2006254885Sdumbbell WREG32(GRBM_SOFT_RESET, (SOFT_RESET_CP | 2007254885Sdumbbell SOFT_RESET_PA | 2008254885Sdumbbell SOFT_RESET_VGT | 2009254885Sdumbbell SOFT_RESET_SPI | 2010254885Sdumbbell SOFT_RESET_SX)); 2011254885Sdumbbell RREG32(GRBM_SOFT_RESET); 2012254885Sdumbbell DRM_MDELAY(15); 2013254885Sdumbbell WREG32(GRBM_SOFT_RESET, 0); 2014254885Sdumbbell RREG32(GRBM_SOFT_RESET); 2015254885Sdumbbell 2016254885Sdumbbell WREG32(CP_SEM_WAIT_TIMER, 0x0); 2017254885Sdumbbell WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0); 2018254885Sdumbbell 2019254885Sdumbbell /* Set the write pointer delay */ 2020254885Sdumbbell WREG32(CP_RB_WPTR_DELAY, 0); 2021254885Sdumbbell 2022254885Sdumbbell WREG32(CP_DEBUG, 0); 2023254885Sdumbbell WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF); 2024254885Sdumbbell 2025254885Sdumbbell /* ring 0 - compute and gfx */ 2026254885Sdumbbell /* Set ring buffer size */ 2027254885Sdumbbell ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]; 2028254885Sdumbbell rb_bufsz = drm_order(ring->ring_size / 8); 2029254885Sdumbbell tmp = (drm_order(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz; 2030254885Sdumbbell#ifdef __BIG_ENDIAN 2031254885Sdumbbell tmp |= BUF_SWAP_32BIT; 2032254885Sdumbbell#endif 2033254885Sdumbbell WREG32(CP_RB0_CNTL, tmp); 2034254885Sdumbbell 2035254885Sdumbbell /* Initialize the ring buffer's read and write pointers */ 2036254885Sdumbbell WREG32(CP_RB0_CNTL, tmp | RB_RPTR_WR_ENA); 2037254885Sdumbbell ring->wptr = 0; 2038254885Sdumbbell WREG32(CP_RB0_WPTR, ring->wptr); 2039254885Sdumbbell 2040254885Sdumbbell /* set the wb address whether it's enabled or not */ 2041254885Sdumbbell WREG32(CP_RB0_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC); 2042254885Sdumbbell WREG32(CP_RB0_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF); 2043254885Sdumbbell 2044254885Sdumbbell if (rdev->wb.enabled) 2045254885Sdumbbell WREG32(SCRATCH_UMSK, 0xff); 2046254885Sdumbbell else { 2047254885Sdumbbell tmp |= RB_NO_UPDATE; 2048254885Sdumbbell WREG32(SCRATCH_UMSK, 0); 2049254885Sdumbbell } 2050254885Sdumbbell 2051254885Sdumbbell DRM_MDELAY(1); 2052254885Sdumbbell WREG32(CP_RB0_CNTL, tmp); 2053254885Sdumbbell 2054254885Sdumbbell WREG32(CP_RB0_BASE, ring->gpu_addr >> 8); 2055254885Sdumbbell 2056254885Sdumbbell ring->rptr = RREG32(CP_RB0_RPTR); 2057254885Sdumbbell 2058254885Sdumbbell /* ring1 - compute only */ 2059254885Sdumbbell /* Set ring buffer size */ 2060254885Sdumbbell ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX]; 2061254885Sdumbbell rb_bufsz = drm_order(ring->ring_size / 8); 2062254885Sdumbbell tmp = (drm_order(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz; 2063254885Sdumbbell#ifdef __BIG_ENDIAN 2064254885Sdumbbell tmp |= BUF_SWAP_32BIT; 2065254885Sdumbbell#endif 2066254885Sdumbbell WREG32(CP_RB1_CNTL, tmp); 2067254885Sdumbbell 2068254885Sdumbbell /* Initialize the ring buffer's read and write pointers */ 2069254885Sdumbbell WREG32(CP_RB1_CNTL, tmp | RB_RPTR_WR_ENA); 2070254885Sdumbbell ring->wptr = 0; 2071254885Sdumbbell WREG32(CP_RB1_WPTR, ring->wptr); 2072254885Sdumbbell 2073254885Sdumbbell /* set the wb address whether it's enabled or not */ 2074254885Sdumbbell WREG32(CP_RB1_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET) & 0xFFFFFFFC); 2075254885Sdumbbell WREG32(CP_RB1_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET) & 0xFF); 2076254885Sdumbbell 2077254885Sdumbbell DRM_MDELAY(1); 2078254885Sdumbbell WREG32(CP_RB1_CNTL, tmp); 2079254885Sdumbbell 2080254885Sdumbbell WREG32(CP_RB1_BASE, ring->gpu_addr >> 8); 2081254885Sdumbbell 2082254885Sdumbbell ring->rptr = RREG32(CP_RB1_RPTR); 2083254885Sdumbbell 2084254885Sdumbbell /* ring2 - compute only */ 2085254885Sdumbbell /* Set ring buffer size */ 2086254885Sdumbbell ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX]; 2087254885Sdumbbell rb_bufsz = drm_order(ring->ring_size / 8); 2088254885Sdumbbell tmp = (drm_order(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz; 2089254885Sdumbbell#ifdef __BIG_ENDIAN 2090254885Sdumbbell tmp |= BUF_SWAP_32BIT; 2091254885Sdumbbell#endif 2092254885Sdumbbell WREG32(CP_RB2_CNTL, tmp); 2093254885Sdumbbell 2094254885Sdumbbell /* Initialize the ring buffer's read and write pointers */ 2095254885Sdumbbell WREG32(CP_RB2_CNTL, tmp | RB_RPTR_WR_ENA); 2096254885Sdumbbell ring->wptr = 0; 2097254885Sdumbbell WREG32(CP_RB2_WPTR, ring->wptr); 2098254885Sdumbbell 2099254885Sdumbbell /* set the wb address whether it's enabled or not */ 2100254885Sdumbbell WREG32(CP_RB2_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET) & 0xFFFFFFFC); 2101254885Sdumbbell WREG32(CP_RB2_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET) & 0xFF); 2102254885Sdumbbell 2103254885Sdumbbell DRM_MDELAY(1); 2104254885Sdumbbell WREG32(CP_RB2_CNTL, tmp); 2105254885Sdumbbell 2106254885Sdumbbell WREG32(CP_RB2_BASE, ring->gpu_addr >> 8); 2107254885Sdumbbell 2108254885Sdumbbell ring->rptr = RREG32(CP_RB2_RPTR); 2109254885Sdumbbell 2110254885Sdumbbell /* start the rings */ 2111254885Sdumbbell si_cp_start(rdev); 2112254885Sdumbbell rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true; 2113254885Sdumbbell rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = true; 2114254885Sdumbbell rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = true; 2115254885Sdumbbell r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]); 2116254885Sdumbbell if (r) { 2117254885Sdumbbell rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false; 2118254885Sdumbbell rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false; 2119254885Sdumbbell rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false; 2120254885Sdumbbell return r; 2121254885Sdumbbell } 2122254885Sdumbbell r = radeon_ring_test(rdev, CAYMAN_RING_TYPE_CP1_INDEX, &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX]); 2123254885Sdumbbell if (r) { 2124254885Sdumbbell rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false; 2125254885Sdumbbell } 2126254885Sdumbbell r = radeon_ring_test(rdev, CAYMAN_RING_TYPE_CP2_INDEX, &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX]); 2127254885Sdumbbell if (r) { 2128254885Sdumbbell rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false; 2129254885Sdumbbell } 2130254885Sdumbbell 2131254885Sdumbbell return 0; 2132254885Sdumbbell} 2133254885Sdumbbell 2134254885Sdumbbellbool si_gpu_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring) 2135254885Sdumbbell{ 2136254885Sdumbbell u32 srbm_status; 2137254885Sdumbbell u32 grbm_status, grbm_status2; 2138254885Sdumbbell u32 grbm_status_se0, grbm_status_se1; 2139254885Sdumbbell 2140254885Sdumbbell srbm_status = RREG32(SRBM_STATUS); 2141254885Sdumbbell grbm_status = RREG32(GRBM_STATUS); 2142254885Sdumbbell grbm_status2 = RREG32(GRBM_STATUS2); 2143254885Sdumbbell grbm_status_se0 = RREG32(GRBM_STATUS_SE0); 2144254885Sdumbbell grbm_status_se1 = RREG32(GRBM_STATUS_SE1); 2145254885Sdumbbell if (!(grbm_status & GUI_ACTIVE)) { 2146254885Sdumbbell radeon_ring_lockup_update(ring); 2147254885Sdumbbell return false; 2148254885Sdumbbell } 2149254885Sdumbbell /* force CP activities */ 2150254885Sdumbbell radeon_ring_force_activity(rdev, ring); 2151254885Sdumbbell return radeon_ring_test_lockup(rdev, ring); 2152254885Sdumbbell} 2153254885Sdumbbell 2154254885Sdumbbellstatic void si_gpu_soft_reset_gfx(struct radeon_device *rdev) 2155254885Sdumbbell{ 2156254885Sdumbbell u32 grbm_reset = 0; 2157254885Sdumbbell 2158254885Sdumbbell if (!(RREG32(GRBM_STATUS) & GUI_ACTIVE)) 2159254885Sdumbbell return; 2160254885Sdumbbell 2161254885Sdumbbell dev_info(rdev->dev, " GRBM_STATUS=0x%08X\n", 2162254885Sdumbbell RREG32(GRBM_STATUS)); 2163254885Sdumbbell dev_info(rdev->dev, " GRBM_STATUS2=0x%08X\n", 2164254885Sdumbbell RREG32(GRBM_STATUS2)); 2165254885Sdumbbell dev_info(rdev->dev, " GRBM_STATUS_SE0=0x%08X\n", 2166254885Sdumbbell RREG32(GRBM_STATUS_SE0)); 2167254885Sdumbbell dev_info(rdev->dev, " GRBM_STATUS_SE1=0x%08X\n", 2168254885Sdumbbell RREG32(GRBM_STATUS_SE1)); 2169254885Sdumbbell dev_info(rdev->dev, " SRBM_STATUS=0x%08X\n", 2170254885Sdumbbell RREG32(SRBM_STATUS)); 2171254885Sdumbbell 2172254885Sdumbbell /* Disable CP parsing/prefetching */ 2173254885Sdumbbell WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT); 2174254885Sdumbbell 2175254885Sdumbbell /* reset all the gfx blocks */ 2176254885Sdumbbell grbm_reset = (SOFT_RESET_CP | 2177254885Sdumbbell SOFT_RESET_CB | 2178254885Sdumbbell SOFT_RESET_DB | 2179254885Sdumbbell SOFT_RESET_GDS | 2180254885Sdumbbell SOFT_RESET_PA | 2181254885Sdumbbell SOFT_RESET_SC | 2182254885Sdumbbell SOFT_RESET_BCI | 2183254885Sdumbbell SOFT_RESET_SPI | 2184254885Sdumbbell SOFT_RESET_SX | 2185254885Sdumbbell SOFT_RESET_TC | 2186254885Sdumbbell SOFT_RESET_TA | 2187254885Sdumbbell SOFT_RESET_VGT | 2188254885Sdumbbell SOFT_RESET_IA); 2189254885Sdumbbell 2190254885Sdumbbell dev_info(rdev->dev, " GRBM_SOFT_RESET=0x%08X\n", grbm_reset); 2191254885Sdumbbell WREG32(GRBM_SOFT_RESET, grbm_reset); 2192254885Sdumbbell (void)RREG32(GRBM_SOFT_RESET); 2193254885Sdumbbell DRM_UDELAY(50); 2194254885Sdumbbell WREG32(GRBM_SOFT_RESET, 0); 2195254885Sdumbbell (void)RREG32(GRBM_SOFT_RESET); 2196254885Sdumbbell 2197254885Sdumbbell dev_info(rdev->dev, " GRBM_STATUS=0x%08X\n", 2198254885Sdumbbell RREG32(GRBM_STATUS)); 2199254885Sdumbbell dev_info(rdev->dev, " GRBM_STATUS2=0x%08X\n", 2200254885Sdumbbell RREG32(GRBM_STATUS2)); 2201254885Sdumbbell dev_info(rdev->dev, " GRBM_STATUS_SE0=0x%08X\n", 2202254885Sdumbbell RREG32(GRBM_STATUS_SE0)); 2203254885Sdumbbell dev_info(rdev->dev, " GRBM_STATUS_SE1=0x%08X\n", 2204254885Sdumbbell RREG32(GRBM_STATUS_SE1)); 2205254885Sdumbbell dev_info(rdev->dev, " SRBM_STATUS=0x%08X\n", 2206254885Sdumbbell RREG32(SRBM_STATUS)); 2207254885Sdumbbell} 2208254885Sdumbbell 2209254885Sdumbbellstatic void si_gpu_soft_reset_dma(struct radeon_device *rdev) 2210254885Sdumbbell{ 2211254885Sdumbbell u32 tmp; 2212254885Sdumbbell 2213254885Sdumbbell if (RREG32(DMA_STATUS_REG) & DMA_IDLE) 2214254885Sdumbbell return; 2215254885Sdumbbell 2216254885Sdumbbell dev_info(rdev->dev, " DMA_STATUS_REG = 0x%08X\n", 2217254885Sdumbbell RREG32(DMA_STATUS_REG)); 2218254885Sdumbbell 2219254885Sdumbbell /* dma0 */ 2220254885Sdumbbell tmp = RREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET); 2221254885Sdumbbell tmp &= ~DMA_RB_ENABLE; 2222254885Sdumbbell WREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET, tmp); 2223254885Sdumbbell 2224254885Sdumbbell /* dma1 */ 2225254885Sdumbbell tmp = RREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET); 2226254885Sdumbbell tmp &= ~DMA_RB_ENABLE; 2227254885Sdumbbell WREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET, tmp); 2228254885Sdumbbell 2229254885Sdumbbell /* Reset dma */ 2230254885Sdumbbell WREG32(SRBM_SOFT_RESET, SOFT_RESET_DMA | SOFT_RESET_DMA1); 2231254885Sdumbbell RREG32(SRBM_SOFT_RESET); 2232254885Sdumbbell DRM_UDELAY(50); 2233254885Sdumbbell WREG32(SRBM_SOFT_RESET, 0); 2234254885Sdumbbell 2235254885Sdumbbell dev_info(rdev->dev, " DMA_STATUS_REG = 0x%08X\n", 2236254885Sdumbbell RREG32(DMA_STATUS_REG)); 2237254885Sdumbbell} 2238254885Sdumbbell 2239254885Sdumbbellstatic int si_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask) 2240254885Sdumbbell{ 2241254885Sdumbbell struct evergreen_mc_save save; 2242254885Sdumbbell 2243254885Sdumbbell if (!(RREG32(GRBM_STATUS) & GUI_ACTIVE)) 2244254885Sdumbbell reset_mask &= ~(RADEON_RESET_GFX | RADEON_RESET_COMPUTE); 2245254885Sdumbbell 2246254885Sdumbbell if (RREG32(DMA_STATUS_REG) & DMA_IDLE) 2247254885Sdumbbell reset_mask &= ~RADEON_RESET_DMA; 2248254885Sdumbbell 2249254885Sdumbbell if (reset_mask == 0) 2250254885Sdumbbell return 0; 2251254885Sdumbbell 2252254885Sdumbbell dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask); 2253254885Sdumbbell 2254254885Sdumbbell dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n", 2255254885Sdumbbell RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR)); 2256254885Sdumbbell dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n", 2257254885Sdumbbell RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS)); 2258254885Sdumbbell 2259254885Sdumbbell evergreen_mc_stop(rdev, &save); 2260254885Sdumbbell if (radeon_mc_wait_for_idle(rdev)) { 2261254885Sdumbbell dev_warn(rdev->dev, "Wait for MC idle timedout !\n"); 2262254885Sdumbbell } 2263254885Sdumbbell 2264254885Sdumbbell if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE)) 2265254885Sdumbbell si_gpu_soft_reset_gfx(rdev); 2266254885Sdumbbell 2267254885Sdumbbell if (reset_mask & RADEON_RESET_DMA) 2268254885Sdumbbell si_gpu_soft_reset_dma(rdev); 2269254885Sdumbbell 2270254885Sdumbbell /* Wait a little for things to settle down */ 2271254885Sdumbbell DRM_UDELAY(50); 2272254885Sdumbbell 2273254885Sdumbbell evergreen_mc_resume(rdev, &save); 2274254885Sdumbbell return 0; 2275254885Sdumbbell} 2276254885Sdumbbell 2277254885Sdumbbellint si_asic_reset(struct radeon_device *rdev) 2278254885Sdumbbell{ 2279254885Sdumbbell return si_gpu_soft_reset(rdev, (RADEON_RESET_GFX | 2280254885Sdumbbell RADEON_RESET_COMPUTE | 2281254885Sdumbbell RADEON_RESET_DMA)); 2282254885Sdumbbell} 2283254885Sdumbbell 2284254885Sdumbbell/* MC */ 2285254885Sdumbbellstatic void si_mc_program(struct radeon_device *rdev) 2286254885Sdumbbell{ 2287254885Sdumbbell struct evergreen_mc_save save; 2288254885Sdumbbell u32 tmp; 2289254885Sdumbbell int i, j; 2290254885Sdumbbell 2291254885Sdumbbell /* Initialize HDP */ 2292254885Sdumbbell for (i = 0, j = 0; i < 32; i++, j += 0x18) { 2293254885Sdumbbell WREG32((0x2c14 + j), 0x00000000); 2294254885Sdumbbell WREG32((0x2c18 + j), 0x00000000); 2295254885Sdumbbell WREG32((0x2c1c + j), 0x00000000); 2296254885Sdumbbell WREG32((0x2c20 + j), 0x00000000); 2297254885Sdumbbell WREG32((0x2c24 + j), 0x00000000); 2298254885Sdumbbell } 2299254885Sdumbbell WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0); 2300254885Sdumbbell 2301254885Sdumbbell evergreen_mc_stop(rdev, &save); 2302254885Sdumbbell if (radeon_mc_wait_for_idle(rdev)) { 2303254885Sdumbbell dev_warn(rdev->dev, "Wait for MC idle timedout !\n"); 2304254885Sdumbbell } 2305254885Sdumbbell /* Lockout access through VGA aperture*/ 2306254885Sdumbbell WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE); 2307254885Sdumbbell /* Update configuration */ 2308254885Sdumbbell WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR, 2309254885Sdumbbell rdev->mc.vram_start >> 12); 2310254885Sdumbbell WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR, 2311254885Sdumbbell rdev->mc.vram_end >> 12); 2312254885Sdumbbell WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR, 2313254885Sdumbbell rdev->vram_scratch.gpu_addr >> 12); 2314254885Sdumbbell tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16; 2315254885Sdumbbell tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF); 2316254885Sdumbbell WREG32(MC_VM_FB_LOCATION, tmp); 2317254885Sdumbbell /* XXX double check these! */ 2318254885Sdumbbell WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8)); 2319254885Sdumbbell WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30)); 2320254885Sdumbbell WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF); 2321254885Sdumbbell WREG32(MC_VM_AGP_BASE, 0); 2322254885Sdumbbell WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF); 2323254885Sdumbbell WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF); 2324254885Sdumbbell if (radeon_mc_wait_for_idle(rdev)) { 2325254885Sdumbbell dev_warn(rdev->dev, "Wait for MC idle timedout !\n"); 2326254885Sdumbbell } 2327254885Sdumbbell evergreen_mc_resume(rdev, &save); 2328254885Sdumbbell /* we need to own VRAM, so turn off the VGA renderer here 2329254885Sdumbbell * to stop it overwriting our objects */ 2330254885Sdumbbell rv515_vga_render_disable(rdev); 2331254885Sdumbbell} 2332254885Sdumbbell 2333254885Sdumbbell/* SI MC address space is 40 bits */ 2334254885Sdumbbellstatic void si_vram_location(struct radeon_device *rdev, 2335254885Sdumbbell struct radeon_mc *mc, u64 base) 2336254885Sdumbbell{ 2337254885Sdumbbell mc->vram_start = base; 2338254885Sdumbbell if (mc->mc_vram_size > (0xFFFFFFFFFFULL - base + 1)) { 2339254885Sdumbbell dev_warn(rdev->dev, "limiting VRAM to PCI aperture size\n"); 2340254885Sdumbbell mc->real_vram_size = mc->aper_size; 2341254885Sdumbbell mc->mc_vram_size = mc->aper_size; 2342254885Sdumbbell } 2343254885Sdumbbell mc->vram_end = mc->vram_start + mc->mc_vram_size - 1; 2344254885Sdumbbell dev_info(rdev->dev, "VRAM: %juM 0x%016jX - 0x%016jX (%juM used)\n", 2345254885Sdumbbell (uintmax_t)mc->mc_vram_size >> 20, (uintmax_t)mc->vram_start, 2346254885Sdumbbell (uintmax_t)mc->vram_end, (uintmax_t)mc->real_vram_size >> 20); 2347254885Sdumbbell} 2348254885Sdumbbell 2349254885Sdumbbellstatic void si_gtt_location(struct radeon_device *rdev, struct radeon_mc *mc) 2350254885Sdumbbell{ 2351254885Sdumbbell u64 size_af, size_bf; 2352254885Sdumbbell 2353254885Sdumbbell size_af = ((0xFFFFFFFFFFULL - mc->vram_end) + mc->gtt_base_align) & ~mc->gtt_base_align; 2354254885Sdumbbell size_bf = mc->vram_start & ~mc->gtt_base_align; 2355254885Sdumbbell if (size_bf > size_af) { 2356254885Sdumbbell if (mc->gtt_size > size_bf) { 2357254885Sdumbbell dev_warn(rdev->dev, "limiting GTT\n"); 2358254885Sdumbbell mc->gtt_size = size_bf; 2359254885Sdumbbell } 2360254885Sdumbbell mc->gtt_start = (mc->vram_start & ~mc->gtt_base_align) - mc->gtt_size; 2361254885Sdumbbell } else { 2362254885Sdumbbell if (mc->gtt_size > size_af) { 2363254885Sdumbbell dev_warn(rdev->dev, "limiting GTT\n"); 2364254885Sdumbbell mc->gtt_size = size_af; 2365254885Sdumbbell } 2366254885Sdumbbell mc->gtt_start = (mc->vram_end + 1 + mc->gtt_base_align) & ~mc->gtt_base_align; 2367254885Sdumbbell } 2368254885Sdumbbell mc->gtt_end = mc->gtt_start + mc->gtt_size - 1; 2369254885Sdumbbell dev_info(rdev->dev, "GTT: %juM 0x%016jX - 0x%016jX\n", 2370254885Sdumbbell (uintmax_t)mc->gtt_size >> 20, (uintmax_t)mc->gtt_start, (uintmax_t)mc->gtt_end); 2371254885Sdumbbell} 2372254885Sdumbbell 2373254885Sdumbbellstatic void si_vram_gtt_location(struct radeon_device *rdev, 2374254885Sdumbbell struct radeon_mc *mc) 2375254885Sdumbbell{ 2376254885Sdumbbell if (mc->mc_vram_size > 0xFFC0000000ULL) { 2377254885Sdumbbell /* leave room for at least 1024M GTT */ 2378254885Sdumbbell dev_warn(rdev->dev, "limiting VRAM\n"); 2379254885Sdumbbell mc->real_vram_size = 0xFFC0000000ULL; 2380254885Sdumbbell mc->mc_vram_size = 0xFFC0000000ULL; 2381254885Sdumbbell } 2382254885Sdumbbell si_vram_location(rdev, &rdev->mc, 0); 2383254885Sdumbbell rdev->mc.gtt_base_align = 0; 2384254885Sdumbbell si_gtt_location(rdev, mc); 2385254885Sdumbbell} 2386254885Sdumbbell 2387254885Sdumbbellstatic int si_mc_init(struct radeon_device *rdev) 2388254885Sdumbbell{ 2389254885Sdumbbell u32 tmp; 2390254885Sdumbbell int chansize, numchan; 2391254885Sdumbbell 2392254885Sdumbbell /* Get VRAM informations */ 2393254885Sdumbbell rdev->mc.vram_is_ddr = true; 2394254885Sdumbbell tmp = RREG32(MC_ARB_RAMCFG); 2395254885Sdumbbell if (tmp & CHANSIZE_OVERRIDE) { 2396254885Sdumbbell chansize = 16; 2397254885Sdumbbell } else if (tmp & CHANSIZE_MASK) { 2398254885Sdumbbell chansize = 64; 2399254885Sdumbbell } else { 2400254885Sdumbbell chansize = 32; 2401254885Sdumbbell } 2402254885Sdumbbell tmp = RREG32(MC_SHARED_CHMAP); 2403254885Sdumbbell switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) { 2404254885Sdumbbell case 0: 2405254885Sdumbbell default: 2406254885Sdumbbell numchan = 1; 2407254885Sdumbbell break; 2408254885Sdumbbell case 1: 2409254885Sdumbbell numchan = 2; 2410254885Sdumbbell break; 2411254885Sdumbbell case 2: 2412254885Sdumbbell numchan = 4; 2413254885Sdumbbell break; 2414254885Sdumbbell case 3: 2415254885Sdumbbell numchan = 8; 2416254885Sdumbbell break; 2417254885Sdumbbell case 4: 2418254885Sdumbbell numchan = 3; 2419254885Sdumbbell break; 2420254885Sdumbbell case 5: 2421254885Sdumbbell numchan = 6; 2422254885Sdumbbell break; 2423254885Sdumbbell case 6: 2424254885Sdumbbell numchan = 10; 2425254885Sdumbbell break; 2426254885Sdumbbell case 7: 2427254885Sdumbbell numchan = 12; 2428254885Sdumbbell break; 2429254885Sdumbbell case 8: 2430254885Sdumbbell numchan = 16; 2431254885Sdumbbell break; 2432254885Sdumbbell } 2433254885Sdumbbell rdev->mc.vram_width = numchan * chansize; 2434254885Sdumbbell /* Could aper size report 0 ? */ 2435254885Sdumbbell rdev->mc.aper_base = drm_get_resource_start(rdev->ddev, 0); 2436254885Sdumbbell rdev->mc.aper_size = drm_get_resource_len(rdev->ddev, 0); 2437254885Sdumbbell /* size in MB on si */ 2438254885Sdumbbell rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024 * 1024; 2439254885Sdumbbell rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024 * 1024; 2440254885Sdumbbell rdev->mc.visible_vram_size = rdev->mc.aper_size; 2441254885Sdumbbell si_vram_gtt_location(rdev, &rdev->mc); 2442254885Sdumbbell radeon_update_bandwidth_info(rdev); 2443254885Sdumbbell 2444254885Sdumbbell return 0; 2445254885Sdumbbell} 2446254885Sdumbbell 2447254885Sdumbbell/* 2448254885Sdumbbell * GART 2449254885Sdumbbell */ 2450254885Sdumbbellvoid si_pcie_gart_tlb_flush(struct radeon_device *rdev) 2451254885Sdumbbell{ 2452254885Sdumbbell /* flush hdp cache */ 2453254885Sdumbbell WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1); 2454254885Sdumbbell 2455254885Sdumbbell /* bits 0-15 are the VM contexts0-15 */ 2456254885Sdumbbell WREG32(VM_INVALIDATE_REQUEST, 1); 2457254885Sdumbbell} 2458254885Sdumbbell 2459254885Sdumbbellstatic int si_pcie_gart_enable(struct radeon_device *rdev) 2460254885Sdumbbell{ 2461254885Sdumbbell int r, i; 2462254885Sdumbbell 2463254885Sdumbbell if (rdev->gart.robj == NULL) { 2464254885Sdumbbell dev_err(rdev->dev, "No VRAM object for PCIE GART.\n"); 2465254885Sdumbbell return -EINVAL; 2466254885Sdumbbell } 2467254885Sdumbbell r = radeon_gart_table_vram_pin(rdev); 2468254885Sdumbbell if (r) 2469254885Sdumbbell return r; 2470254885Sdumbbell radeon_gart_restore(rdev); 2471254885Sdumbbell /* Setup TLB control */ 2472254885Sdumbbell WREG32(MC_VM_MX_L1_TLB_CNTL, 2473254885Sdumbbell (0xA << 7) | 2474254885Sdumbbell ENABLE_L1_TLB | 2475254885Sdumbbell SYSTEM_ACCESS_MODE_NOT_IN_SYS | 2476254885Sdumbbell ENABLE_ADVANCED_DRIVER_MODEL | 2477254885Sdumbbell SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU); 2478254885Sdumbbell /* Setup L2 cache */ 2479254885Sdumbbell WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | 2480254885Sdumbbell ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE | 2481254885Sdumbbell ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE | 2482254885Sdumbbell EFFECTIVE_L2_QUEUE_SIZE(7) | 2483254885Sdumbbell CONTEXT1_IDENTITY_ACCESS_MODE(1)); 2484254885Sdumbbell WREG32(VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS | INVALIDATE_L2_CACHE); 2485254885Sdumbbell WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY | 2486254885Sdumbbell L2_CACHE_BIGK_FRAGMENT_SIZE(0)); 2487254885Sdumbbell /* setup context0 */ 2488254885Sdumbbell WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12); 2489254885Sdumbbell WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12); 2490254885Sdumbbell WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12); 2491254885Sdumbbell WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR, 2492254885Sdumbbell (u32)(rdev->dummy_page.addr >> 12)); 2493254885Sdumbbell WREG32(VM_CONTEXT0_CNTL2, 0); 2494254885Sdumbbell WREG32(VM_CONTEXT0_CNTL, (ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) | 2495254885Sdumbbell RANGE_PROTECTION_FAULT_ENABLE_DEFAULT)); 2496254885Sdumbbell 2497254885Sdumbbell WREG32(0x15D4, 0); 2498254885Sdumbbell WREG32(0x15D8, 0); 2499254885Sdumbbell WREG32(0x15DC, 0); 2500254885Sdumbbell 2501254885Sdumbbell /* empty context1-15 */ 2502254885Sdumbbell /* set vm size, must be a multiple of 4 */ 2503254885Sdumbbell WREG32(VM_CONTEXT1_PAGE_TABLE_START_ADDR, 0); 2504254885Sdumbbell WREG32(VM_CONTEXT1_PAGE_TABLE_END_ADDR, rdev->vm_manager.max_pfn); 2505254885Sdumbbell /* Assign the pt base to something valid for now; the pts used for 2506254885Sdumbbell * the VMs are determined by the application and setup and assigned 2507254885Sdumbbell * on the fly in the vm part of radeon_gart.c 2508254885Sdumbbell */ 2509254885Sdumbbell for (i = 1; i < 16; i++) { 2510254885Sdumbbell if (i < 8) 2511254885Sdumbbell WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2), 2512254885Sdumbbell rdev->gart.table_addr >> 12); 2513254885Sdumbbell else 2514254885Sdumbbell WREG32(VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2), 2515254885Sdumbbell rdev->gart.table_addr >> 12); 2516254885Sdumbbell } 2517254885Sdumbbell 2518254885Sdumbbell /* enable context1-15 */ 2519254885Sdumbbell WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR, 2520254885Sdumbbell (u32)(rdev->dummy_page.addr >> 12)); 2521254885Sdumbbell WREG32(VM_CONTEXT1_CNTL2, 4); 2522254885Sdumbbell WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(1) | 2523254885Sdumbbell RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT | 2524254885Sdumbbell RANGE_PROTECTION_FAULT_ENABLE_DEFAULT | 2525254885Sdumbbell DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT | 2526254885Sdumbbell DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT | 2527254885Sdumbbell PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT | 2528254885Sdumbbell PDE0_PROTECTION_FAULT_ENABLE_DEFAULT | 2529254885Sdumbbell VALID_PROTECTION_FAULT_ENABLE_INTERRUPT | 2530254885Sdumbbell VALID_PROTECTION_FAULT_ENABLE_DEFAULT | 2531254885Sdumbbell READ_PROTECTION_FAULT_ENABLE_INTERRUPT | 2532254885Sdumbbell READ_PROTECTION_FAULT_ENABLE_DEFAULT | 2533254885Sdumbbell WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT | 2534254885Sdumbbell WRITE_PROTECTION_FAULT_ENABLE_DEFAULT); 2535254885Sdumbbell 2536254885Sdumbbell si_pcie_gart_tlb_flush(rdev); 2537254885Sdumbbell DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n", 2538254885Sdumbbell (unsigned)(rdev->mc.gtt_size >> 20), 2539254885Sdumbbell (unsigned long long)rdev->gart.table_addr); 2540254885Sdumbbell rdev->gart.ready = true; 2541254885Sdumbbell return 0; 2542254885Sdumbbell} 2543254885Sdumbbell 2544254885Sdumbbellstatic void si_pcie_gart_disable(struct radeon_device *rdev) 2545254885Sdumbbell{ 2546254885Sdumbbell /* Disable all tables */ 2547254885Sdumbbell WREG32(VM_CONTEXT0_CNTL, 0); 2548254885Sdumbbell WREG32(VM_CONTEXT1_CNTL, 0); 2549254885Sdumbbell /* Setup TLB control */ 2550254885Sdumbbell WREG32(MC_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE_NOT_IN_SYS | 2551254885Sdumbbell SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU); 2552254885Sdumbbell /* Setup L2 cache */ 2553254885Sdumbbell WREG32(VM_L2_CNTL, ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE | 2554254885Sdumbbell ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE | 2555254885Sdumbbell EFFECTIVE_L2_QUEUE_SIZE(7) | 2556254885Sdumbbell CONTEXT1_IDENTITY_ACCESS_MODE(1)); 2557254885Sdumbbell WREG32(VM_L2_CNTL2, 0); 2558254885Sdumbbell WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY | 2559254885Sdumbbell L2_CACHE_BIGK_FRAGMENT_SIZE(0)); 2560254885Sdumbbell radeon_gart_table_vram_unpin(rdev); 2561254885Sdumbbell} 2562254885Sdumbbell 2563254885Sdumbbellstatic void si_pcie_gart_fini(struct radeon_device *rdev) 2564254885Sdumbbell{ 2565254885Sdumbbell si_pcie_gart_disable(rdev); 2566254885Sdumbbell radeon_gart_table_vram_free(rdev); 2567254885Sdumbbell radeon_gart_fini(rdev); 2568254885Sdumbbell} 2569254885Sdumbbell 2570254885Sdumbbell/* vm parser */ 2571254885Sdumbbellstatic bool si_vm_reg_valid(u32 reg) 2572254885Sdumbbell{ 2573254885Sdumbbell /* context regs are fine */ 2574254885Sdumbbell if (reg >= 0x28000) 2575254885Sdumbbell return true; 2576254885Sdumbbell 2577254885Sdumbbell /* check config regs */ 2578254885Sdumbbell switch (reg) { 2579254885Sdumbbell case GRBM_GFX_INDEX: 2580254885Sdumbbell case CP_STRMOUT_CNTL: 2581254885Sdumbbell case VGT_VTX_VECT_EJECT_REG: 2582254885Sdumbbell case VGT_CACHE_INVALIDATION: 2583254885Sdumbbell case VGT_ESGS_RING_SIZE: 2584254885Sdumbbell case VGT_GSVS_RING_SIZE: 2585254885Sdumbbell case VGT_GS_VERTEX_REUSE: 2586254885Sdumbbell case VGT_PRIMITIVE_TYPE: 2587254885Sdumbbell case VGT_INDEX_TYPE: 2588254885Sdumbbell case VGT_NUM_INDICES: 2589254885Sdumbbell case VGT_NUM_INSTANCES: 2590254885Sdumbbell case VGT_TF_RING_SIZE: 2591254885Sdumbbell case VGT_HS_OFFCHIP_PARAM: 2592254885Sdumbbell case VGT_TF_MEMORY_BASE: 2593254885Sdumbbell case PA_CL_ENHANCE: 2594254885Sdumbbell case PA_SU_LINE_STIPPLE_VALUE: 2595254885Sdumbbell case PA_SC_LINE_STIPPLE_STATE: 2596254885Sdumbbell case PA_SC_ENHANCE: 2597254885Sdumbbell case SQC_CACHES: 2598254885Sdumbbell case SPI_STATIC_THREAD_MGMT_1: 2599254885Sdumbbell case SPI_STATIC_THREAD_MGMT_2: 2600254885Sdumbbell case SPI_STATIC_THREAD_MGMT_3: 2601254885Sdumbbell case SPI_PS_MAX_WAVE_ID: 2602254885Sdumbbell case SPI_CONFIG_CNTL: 2603254885Sdumbbell case SPI_CONFIG_CNTL_1: 2604254885Sdumbbell case TA_CNTL_AUX: 2605254885Sdumbbell return true; 2606254885Sdumbbell default: 2607254885Sdumbbell DRM_ERROR("Invalid register 0x%x in CS\n", reg); 2608254885Sdumbbell return false; 2609254885Sdumbbell } 2610254885Sdumbbell} 2611254885Sdumbbell 2612254885Sdumbbellstatic int si_vm_packet3_ce_check(struct radeon_device *rdev, 2613254885Sdumbbell u32 *ib, struct radeon_cs_packet *pkt) 2614254885Sdumbbell{ 2615254885Sdumbbell switch (pkt->opcode) { 2616254885Sdumbbell case PACKET3_NOP: 2617254885Sdumbbell case PACKET3_SET_BASE: 2618254885Sdumbbell case PACKET3_SET_CE_DE_COUNTERS: 2619254885Sdumbbell case PACKET3_LOAD_CONST_RAM: 2620254885Sdumbbell case PACKET3_WRITE_CONST_RAM: 2621254885Sdumbbell case PACKET3_WRITE_CONST_RAM_OFFSET: 2622254885Sdumbbell case PACKET3_DUMP_CONST_RAM: 2623254885Sdumbbell case PACKET3_INCREMENT_CE_COUNTER: 2624254885Sdumbbell case PACKET3_WAIT_ON_DE_COUNTER: 2625254885Sdumbbell case PACKET3_CE_WRITE: 2626254885Sdumbbell break; 2627254885Sdumbbell default: 2628254885Sdumbbell DRM_ERROR("Invalid CE packet3: 0x%x\n", pkt->opcode); 2629254885Sdumbbell return -EINVAL; 2630254885Sdumbbell } 2631254885Sdumbbell return 0; 2632254885Sdumbbell} 2633254885Sdumbbell 2634254885Sdumbbellstatic int si_vm_packet3_gfx_check(struct radeon_device *rdev, 2635254885Sdumbbell u32 *ib, struct radeon_cs_packet *pkt) 2636254885Sdumbbell{ 2637254885Sdumbbell u32 idx = pkt->idx + 1; 2638254885Sdumbbell u32 idx_value = ib[idx]; 2639254885Sdumbbell u32 start_reg, end_reg, reg, i; 2640254885Sdumbbell u32 command, info; 2641254885Sdumbbell 2642254885Sdumbbell switch (pkt->opcode) { 2643254885Sdumbbell case PACKET3_NOP: 2644254885Sdumbbell case PACKET3_SET_BASE: 2645254885Sdumbbell case PACKET3_CLEAR_STATE: 2646254885Sdumbbell case PACKET3_INDEX_BUFFER_SIZE: 2647254885Sdumbbell case PACKET3_DISPATCH_DIRECT: 2648254885Sdumbbell case PACKET3_DISPATCH_INDIRECT: 2649254885Sdumbbell case PACKET3_ALLOC_GDS: 2650254885Sdumbbell case PACKET3_WRITE_GDS_RAM: 2651254885Sdumbbell case PACKET3_ATOMIC_GDS: 2652254885Sdumbbell case PACKET3_ATOMIC: 2653254885Sdumbbell case PACKET3_OCCLUSION_QUERY: 2654254885Sdumbbell case PACKET3_SET_PREDICATION: 2655254885Sdumbbell case PACKET3_COND_EXEC: 2656254885Sdumbbell case PACKET3_PRED_EXEC: 2657254885Sdumbbell case PACKET3_DRAW_INDIRECT: 2658254885Sdumbbell case PACKET3_DRAW_INDEX_INDIRECT: 2659254885Sdumbbell case PACKET3_INDEX_BASE: 2660254885Sdumbbell case PACKET3_DRAW_INDEX_2: 2661254885Sdumbbell case PACKET3_CONTEXT_CONTROL: 2662254885Sdumbbell case PACKET3_INDEX_TYPE: 2663254885Sdumbbell case PACKET3_DRAW_INDIRECT_MULTI: 2664254885Sdumbbell case PACKET3_DRAW_INDEX_AUTO: 2665254885Sdumbbell case PACKET3_DRAW_INDEX_IMMD: 2666254885Sdumbbell case PACKET3_NUM_INSTANCES: 2667254885Sdumbbell case PACKET3_DRAW_INDEX_MULTI_AUTO: 2668254885Sdumbbell case PACKET3_STRMOUT_BUFFER_UPDATE: 2669254885Sdumbbell case PACKET3_DRAW_INDEX_OFFSET_2: 2670254885Sdumbbell case PACKET3_DRAW_INDEX_MULTI_ELEMENT: 2671254885Sdumbbell case PACKET3_DRAW_INDEX_INDIRECT_MULTI: 2672254885Sdumbbell case PACKET3_MPEG_INDEX: 2673254885Sdumbbell case PACKET3_WAIT_REG_MEM: 2674254885Sdumbbell case PACKET3_MEM_WRITE: 2675254885Sdumbbell case PACKET3_PFP_SYNC_ME: 2676254885Sdumbbell case PACKET3_SURFACE_SYNC: 2677254885Sdumbbell case PACKET3_EVENT_WRITE: 2678254885Sdumbbell case PACKET3_EVENT_WRITE_EOP: 2679254885Sdumbbell case PACKET3_EVENT_WRITE_EOS: 2680254885Sdumbbell case PACKET3_SET_CONTEXT_REG: 2681254885Sdumbbell case PACKET3_SET_CONTEXT_REG_INDIRECT: 2682254885Sdumbbell case PACKET3_SET_SH_REG: 2683254885Sdumbbell case PACKET3_SET_SH_REG_OFFSET: 2684254885Sdumbbell case PACKET3_INCREMENT_DE_COUNTER: 2685254885Sdumbbell case PACKET3_WAIT_ON_CE_COUNTER: 2686254885Sdumbbell case PACKET3_WAIT_ON_AVAIL_BUFFER: 2687254885Sdumbbell case PACKET3_ME_WRITE: 2688254885Sdumbbell break; 2689254885Sdumbbell case PACKET3_COPY_DATA: 2690254885Sdumbbell if ((idx_value & 0xf00) == 0) { 2691254885Sdumbbell reg = ib[idx + 3] * 4; 2692254885Sdumbbell if (!si_vm_reg_valid(reg)) 2693254885Sdumbbell return -EINVAL; 2694254885Sdumbbell } 2695254885Sdumbbell break; 2696254885Sdumbbell case PACKET3_WRITE_DATA: 2697254885Sdumbbell if ((idx_value & 0xf00) == 0) { 2698254885Sdumbbell start_reg = ib[idx + 1] * 4; 2699254885Sdumbbell if (idx_value & 0x10000) { 2700254885Sdumbbell if (!si_vm_reg_valid(start_reg)) 2701254885Sdumbbell return -EINVAL; 2702254885Sdumbbell } else { 2703254885Sdumbbell for (i = 0; i < (pkt->count - 2); i++) { 2704254885Sdumbbell reg = start_reg + (4 * i); 2705254885Sdumbbell if (!si_vm_reg_valid(reg)) 2706254885Sdumbbell return -EINVAL; 2707254885Sdumbbell } 2708254885Sdumbbell } 2709254885Sdumbbell } 2710254885Sdumbbell break; 2711254885Sdumbbell case PACKET3_COND_WRITE: 2712254885Sdumbbell if (idx_value & 0x100) { 2713254885Sdumbbell reg = ib[idx + 5] * 4; 2714254885Sdumbbell if (!si_vm_reg_valid(reg)) 2715254885Sdumbbell return -EINVAL; 2716254885Sdumbbell } 2717254885Sdumbbell break; 2718254885Sdumbbell case PACKET3_COPY_DW: 2719254885Sdumbbell if (idx_value & 0x2) { 2720254885Sdumbbell reg = ib[idx + 3] * 4; 2721254885Sdumbbell if (!si_vm_reg_valid(reg)) 2722254885Sdumbbell return -EINVAL; 2723254885Sdumbbell } 2724254885Sdumbbell break; 2725254885Sdumbbell case PACKET3_SET_CONFIG_REG: 2726254885Sdumbbell start_reg = (idx_value << 2) + PACKET3_SET_CONFIG_REG_START; 2727254885Sdumbbell end_reg = 4 * pkt->count + start_reg - 4; 2728254885Sdumbbell if ((start_reg < PACKET3_SET_CONFIG_REG_START) || 2729254885Sdumbbell (start_reg >= PACKET3_SET_CONFIG_REG_END) || 2730254885Sdumbbell (end_reg >= PACKET3_SET_CONFIG_REG_END)) { 2731254885Sdumbbell DRM_ERROR("bad PACKET3_SET_CONFIG_REG\n"); 2732254885Sdumbbell return -EINVAL; 2733254885Sdumbbell } 2734254885Sdumbbell for (i = 0; i < pkt->count; i++) { 2735254885Sdumbbell reg = start_reg + (4 * i); 2736254885Sdumbbell if (!si_vm_reg_valid(reg)) 2737254885Sdumbbell return -EINVAL; 2738254885Sdumbbell } 2739254885Sdumbbell break; 2740254885Sdumbbell case PACKET3_CP_DMA: 2741254885Sdumbbell command = ib[idx + 4]; 2742254885Sdumbbell info = ib[idx + 1]; 2743254885Sdumbbell if (command & PACKET3_CP_DMA_CMD_SAS) { 2744254885Sdumbbell /* src address space is register */ 2745254885Sdumbbell if (((info & 0x60000000) >> 29) == 0) { 2746254885Sdumbbell start_reg = idx_value << 2; 2747254885Sdumbbell if (command & PACKET3_CP_DMA_CMD_SAIC) { 2748254885Sdumbbell reg = start_reg; 2749254885Sdumbbell if (!si_vm_reg_valid(reg)) { 2750254885Sdumbbell DRM_ERROR("CP DMA Bad SRC register\n"); 2751254885Sdumbbell return -EINVAL; 2752254885Sdumbbell } 2753254885Sdumbbell } else { 2754254885Sdumbbell for (i = 0; i < (command & 0x1fffff); i++) { 2755254885Sdumbbell reg = start_reg + (4 * i); 2756254885Sdumbbell if (!si_vm_reg_valid(reg)) { 2757254885Sdumbbell DRM_ERROR("CP DMA Bad SRC register\n"); 2758254885Sdumbbell return -EINVAL; 2759254885Sdumbbell } 2760254885Sdumbbell } 2761254885Sdumbbell } 2762254885Sdumbbell } 2763254885Sdumbbell } 2764254885Sdumbbell if (command & PACKET3_CP_DMA_CMD_DAS) { 2765254885Sdumbbell /* dst address space is register */ 2766254885Sdumbbell if (((info & 0x00300000) >> 20) == 0) { 2767254885Sdumbbell start_reg = ib[idx + 2]; 2768254885Sdumbbell if (command & PACKET3_CP_DMA_CMD_DAIC) { 2769254885Sdumbbell reg = start_reg; 2770254885Sdumbbell if (!si_vm_reg_valid(reg)) { 2771254885Sdumbbell DRM_ERROR("CP DMA Bad DST register\n"); 2772254885Sdumbbell return -EINVAL; 2773254885Sdumbbell } 2774254885Sdumbbell } else { 2775254885Sdumbbell for (i = 0; i < (command & 0x1fffff); i++) { 2776254885Sdumbbell reg = start_reg + (4 * i); 2777254885Sdumbbell if (!si_vm_reg_valid(reg)) { 2778254885Sdumbbell DRM_ERROR("CP DMA Bad DST register\n"); 2779254885Sdumbbell return -EINVAL; 2780254885Sdumbbell } 2781254885Sdumbbell } 2782254885Sdumbbell } 2783254885Sdumbbell } 2784254885Sdumbbell } 2785254885Sdumbbell break; 2786254885Sdumbbell default: 2787254885Sdumbbell DRM_ERROR("Invalid GFX packet3: 0x%x\n", pkt->opcode); 2788254885Sdumbbell return -EINVAL; 2789254885Sdumbbell } 2790254885Sdumbbell return 0; 2791254885Sdumbbell} 2792254885Sdumbbell 2793254885Sdumbbellstatic int si_vm_packet3_compute_check(struct radeon_device *rdev, 2794254885Sdumbbell u32 *ib, struct radeon_cs_packet *pkt) 2795254885Sdumbbell{ 2796254885Sdumbbell u32 idx = pkt->idx + 1; 2797254885Sdumbbell u32 idx_value = ib[idx]; 2798254885Sdumbbell u32 start_reg, reg, i; 2799254885Sdumbbell 2800254885Sdumbbell switch (pkt->opcode) { 2801254885Sdumbbell case PACKET3_NOP: 2802254885Sdumbbell case PACKET3_SET_BASE: 2803254885Sdumbbell case PACKET3_CLEAR_STATE: 2804254885Sdumbbell case PACKET3_DISPATCH_DIRECT: 2805254885Sdumbbell case PACKET3_DISPATCH_INDIRECT: 2806254885Sdumbbell case PACKET3_ALLOC_GDS: 2807254885Sdumbbell case PACKET3_WRITE_GDS_RAM: 2808254885Sdumbbell case PACKET3_ATOMIC_GDS: 2809254885Sdumbbell case PACKET3_ATOMIC: 2810254885Sdumbbell case PACKET3_OCCLUSION_QUERY: 2811254885Sdumbbell case PACKET3_SET_PREDICATION: 2812254885Sdumbbell case PACKET3_COND_EXEC: 2813254885Sdumbbell case PACKET3_PRED_EXEC: 2814254885Sdumbbell case PACKET3_CONTEXT_CONTROL: 2815254885Sdumbbell case PACKET3_STRMOUT_BUFFER_UPDATE: 2816254885Sdumbbell case PACKET3_WAIT_REG_MEM: 2817254885Sdumbbell case PACKET3_MEM_WRITE: 2818254885Sdumbbell case PACKET3_PFP_SYNC_ME: 2819254885Sdumbbell case PACKET3_SURFACE_SYNC: 2820254885Sdumbbell case PACKET3_EVENT_WRITE: 2821254885Sdumbbell case PACKET3_EVENT_WRITE_EOP: 2822254885Sdumbbell case PACKET3_EVENT_WRITE_EOS: 2823254885Sdumbbell case PACKET3_SET_CONTEXT_REG: 2824254885Sdumbbell case PACKET3_SET_CONTEXT_REG_INDIRECT: 2825254885Sdumbbell case PACKET3_SET_SH_REG: 2826254885Sdumbbell case PACKET3_SET_SH_REG_OFFSET: 2827254885Sdumbbell case PACKET3_INCREMENT_DE_COUNTER: 2828254885Sdumbbell case PACKET3_WAIT_ON_CE_COUNTER: 2829254885Sdumbbell case PACKET3_WAIT_ON_AVAIL_BUFFER: 2830254885Sdumbbell case PACKET3_ME_WRITE: 2831254885Sdumbbell break; 2832254885Sdumbbell case PACKET3_COPY_DATA: 2833254885Sdumbbell if ((idx_value & 0xf00) == 0) { 2834254885Sdumbbell reg = ib[idx + 3] * 4; 2835254885Sdumbbell if (!si_vm_reg_valid(reg)) 2836254885Sdumbbell return -EINVAL; 2837254885Sdumbbell } 2838254885Sdumbbell break; 2839254885Sdumbbell case PACKET3_WRITE_DATA: 2840254885Sdumbbell if ((idx_value & 0xf00) == 0) { 2841254885Sdumbbell start_reg = ib[idx + 1] * 4; 2842254885Sdumbbell if (idx_value & 0x10000) { 2843254885Sdumbbell if (!si_vm_reg_valid(start_reg)) 2844254885Sdumbbell return -EINVAL; 2845254885Sdumbbell } else { 2846254885Sdumbbell for (i = 0; i < (pkt->count - 2); i++) { 2847254885Sdumbbell reg = start_reg + (4 * i); 2848254885Sdumbbell if (!si_vm_reg_valid(reg)) 2849254885Sdumbbell return -EINVAL; 2850254885Sdumbbell } 2851254885Sdumbbell } 2852254885Sdumbbell } 2853254885Sdumbbell break; 2854254885Sdumbbell case PACKET3_COND_WRITE: 2855254885Sdumbbell if (idx_value & 0x100) { 2856254885Sdumbbell reg = ib[idx + 5] * 4; 2857254885Sdumbbell if (!si_vm_reg_valid(reg)) 2858254885Sdumbbell return -EINVAL; 2859254885Sdumbbell } 2860254885Sdumbbell break; 2861254885Sdumbbell case PACKET3_COPY_DW: 2862254885Sdumbbell if (idx_value & 0x2) { 2863254885Sdumbbell reg = ib[idx + 3] * 4; 2864254885Sdumbbell if (!si_vm_reg_valid(reg)) 2865254885Sdumbbell return -EINVAL; 2866254885Sdumbbell } 2867254885Sdumbbell break; 2868254885Sdumbbell default: 2869254885Sdumbbell DRM_ERROR("Invalid Compute packet3: 0x%x\n", pkt->opcode); 2870254885Sdumbbell return -EINVAL; 2871254885Sdumbbell } 2872254885Sdumbbell return 0; 2873254885Sdumbbell} 2874254885Sdumbbell 2875254885Sdumbbellint si_ib_parse(struct radeon_device *rdev, struct radeon_ib *ib) 2876254885Sdumbbell{ 2877254885Sdumbbell int ret = 0; 2878254885Sdumbbell u32 idx = 0; 2879254885Sdumbbell struct radeon_cs_packet pkt; 2880254885Sdumbbell 2881254885Sdumbbell do { 2882254885Sdumbbell pkt.idx = idx; 2883254885Sdumbbell pkt.type = CP_PACKET_GET_TYPE(ib->ptr[idx]); 2884254885Sdumbbell pkt.count = CP_PACKET_GET_COUNT(ib->ptr[idx]); 2885254885Sdumbbell pkt.one_reg_wr = 0; 2886254885Sdumbbell switch (pkt.type) { 2887254885Sdumbbell case PACKET_TYPE0: 2888254885Sdumbbell dev_err(rdev->dev, "Packet0 not allowed!\n"); 2889254885Sdumbbell ret = -EINVAL; 2890254885Sdumbbell break; 2891254885Sdumbbell case PACKET_TYPE2: 2892254885Sdumbbell idx += 1; 2893254885Sdumbbell break; 2894254885Sdumbbell case PACKET_TYPE3: 2895254885Sdumbbell pkt.opcode = CP_PACKET3_GET_OPCODE(ib->ptr[idx]); 2896254885Sdumbbell if (ib->is_const_ib) 2897254885Sdumbbell ret = si_vm_packet3_ce_check(rdev, ib->ptr, &pkt); 2898254885Sdumbbell else { 2899254885Sdumbbell switch (ib->ring) { 2900254885Sdumbbell case RADEON_RING_TYPE_GFX_INDEX: 2901254885Sdumbbell ret = si_vm_packet3_gfx_check(rdev, ib->ptr, &pkt); 2902254885Sdumbbell break; 2903254885Sdumbbell case CAYMAN_RING_TYPE_CP1_INDEX: 2904254885Sdumbbell case CAYMAN_RING_TYPE_CP2_INDEX: 2905254885Sdumbbell ret = si_vm_packet3_compute_check(rdev, ib->ptr, &pkt); 2906254885Sdumbbell break; 2907254885Sdumbbell default: 2908254885Sdumbbell dev_err(rdev->dev, "Non-PM4 ring %d !\n", ib->ring); 2909254885Sdumbbell ret = -EINVAL; 2910254885Sdumbbell break; 2911254885Sdumbbell } 2912254885Sdumbbell } 2913254885Sdumbbell idx += pkt.count + 2; 2914254885Sdumbbell break; 2915254885Sdumbbell default: 2916254885Sdumbbell dev_err(rdev->dev, "Unknown packet type %d !\n", pkt.type); 2917254885Sdumbbell ret = -EINVAL; 2918254885Sdumbbell break; 2919254885Sdumbbell } 2920254885Sdumbbell if (ret) 2921254885Sdumbbell break; 2922254885Sdumbbell } while (idx < ib->length_dw); 2923254885Sdumbbell 2924254885Sdumbbell return ret; 2925254885Sdumbbell} 2926254885Sdumbbell 2927254885Sdumbbell/* 2928254885Sdumbbell * vm 2929254885Sdumbbell */ 2930254885Sdumbbellint si_vm_init(struct radeon_device *rdev) 2931254885Sdumbbell{ 2932254885Sdumbbell /* number of VMs */ 2933254885Sdumbbell rdev->vm_manager.nvm = 16; 2934254885Sdumbbell /* base offset of vram pages */ 2935254885Sdumbbell rdev->vm_manager.vram_base_offset = 0; 2936254885Sdumbbell 2937254885Sdumbbell return 0; 2938254885Sdumbbell} 2939254885Sdumbbell 2940254885Sdumbbellvoid si_vm_fini(struct radeon_device *rdev) 2941254885Sdumbbell{ 2942254885Sdumbbell} 2943254885Sdumbbell 2944254885Sdumbbell/** 2945254885Sdumbbell * si_vm_set_page - update the page tables using the CP 2946254885Sdumbbell * 2947254885Sdumbbell * @rdev: radeon_device pointer 2948254885Sdumbbell * @pe: addr of the page entry 2949254885Sdumbbell * @addr: dst addr to write into pe 2950254885Sdumbbell * @count: number of page entries to update 2951254885Sdumbbell * @incr: increase next addr by incr bytes 2952254885Sdumbbell * @flags: access flags 2953254885Sdumbbell * 2954254885Sdumbbell * Update the page tables using the CP (cayman-si). 2955254885Sdumbbell */ 2956254885Sdumbbellvoid si_vm_set_page(struct radeon_device *rdev, uint64_t pe, 2957254885Sdumbbell uint64_t addr, unsigned count, 2958254885Sdumbbell uint32_t incr, uint32_t flags) 2959254885Sdumbbell{ 2960254885Sdumbbell struct radeon_ring *ring = &rdev->ring[rdev->asic->vm.pt_ring_index]; 2961254885Sdumbbell uint32_t r600_flags = cayman_vm_page_flags(rdev, flags); 2962254885Sdumbbell uint64_t value; 2963254885Sdumbbell unsigned ndw; 2964254885Sdumbbell 2965254885Sdumbbell if (rdev->asic->vm.pt_ring_index == RADEON_RING_TYPE_GFX_INDEX) { 2966254885Sdumbbell while (count) { 2967254885Sdumbbell ndw = 2 + count * 2; 2968254885Sdumbbell if (ndw > 0x3FFE) 2969254885Sdumbbell ndw = 0x3FFE; 2970254885Sdumbbell 2971254885Sdumbbell radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, ndw)); 2972254885Sdumbbell radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) | 2973254885Sdumbbell WRITE_DATA_DST_SEL(1))); 2974254885Sdumbbell radeon_ring_write(ring, pe); 2975254885Sdumbbell radeon_ring_write(ring, upper_32_bits(pe)); 2976254885Sdumbbell for (; ndw > 2; ndw -= 2, --count, pe += 8) { 2977254885Sdumbbell if (flags & RADEON_VM_PAGE_SYSTEM) { 2978254885Sdumbbell value = radeon_vm_map_gart(rdev, addr); 2979254885Sdumbbell value &= 0xFFFFFFFFFFFFF000ULL; 2980254885Sdumbbell } else if (flags & RADEON_VM_PAGE_VALID) { 2981254885Sdumbbell value = addr; 2982254885Sdumbbell } else { 2983254885Sdumbbell value = 0; 2984254885Sdumbbell } 2985254885Sdumbbell addr += incr; 2986254885Sdumbbell value |= r600_flags; 2987254885Sdumbbell radeon_ring_write(ring, value); 2988254885Sdumbbell radeon_ring_write(ring, upper_32_bits(value)); 2989254885Sdumbbell } 2990254885Sdumbbell } 2991254885Sdumbbell } else { 2992254885Sdumbbell /* DMA */ 2993254885Sdumbbell if (flags & RADEON_VM_PAGE_SYSTEM) { 2994254885Sdumbbell while (count) { 2995254885Sdumbbell ndw = count * 2; 2996254885Sdumbbell if (ndw > 0xFFFFE) 2997254885Sdumbbell ndw = 0xFFFFE; 2998254885Sdumbbell 2999254885Sdumbbell /* for non-physically contiguous pages (system) */ 3000254885Sdumbbell radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_WRITE, 0, 0, 0, ndw)); 3001254885Sdumbbell radeon_ring_write(ring, pe); 3002254885Sdumbbell radeon_ring_write(ring, upper_32_bits(pe) & 0xff); 3003254885Sdumbbell for (; ndw > 0; ndw -= 2, --count, pe += 8) { 3004254885Sdumbbell if (flags & RADEON_VM_PAGE_SYSTEM) { 3005254885Sdumbbell value = radeon_vm_map_gart(rdev, addr); 3006254885Sdumbbell value &= 0xFFFFFFFFFFFFF000ULL; 3007254885Sdumbbell } else if (flags & RADEON_VM_PAGE_VALID) { 3008254885Sdumbbell value = addr; 3009254885Sdumbbell } else { 3010254885Sdumbbell value = 0; 3011254885Sdumbbell } 3012254885Sdumbbell addr += incr; 3013254885Sdumbbell value |= r600_flags; 3014254885Sdumbbell radeon_ring_write(ring, value); 3015254885Sdumbbell radeon_ring_write(ring, upper_32_bits(value)); 3016254885Sdumbbell } 3017254885Sdumbbell } 3018254885Sdumbbell } else { 3019254885Sdumbbell while (count) { 3020254885Sdumbbell ndw = count * 2; 3021254885Sdumbbell if (ndw > 0xFFFFE) 3022254885Sdumbbell ndw = 0xFFFFE; 3023254885Sdumbbell 3024254885Sdumbbell if (flags & RADEON_VM_PAGE_VALID) 3025254885Sdumbbell value = addr; 3026254885Sdumbbell else 3027254885Sdumbbell value = 0; 3028254885Sdumbbell /* for physically contiguous pages (vram) */ 3029254885Sdumbbell radeon_ring_write(ring, DMA_PTE_PDE_PACKET(ndw)); 3030254885Sdumbbell radeon_ring_write(ring, pe); /* dst addr */ 3031254885Sdumbbell radeon_ring_write(ring, upper_32_bits(pe) & 0xff); 3032254885Sdumbbell radeon_ring_write(ring, r600_flags); /* mask */ 3033254885Sdumbbell radeon_ring_write(ring, 0); 3034254885Sdumbbell radeon_ring_write(ring, value); /* value */ 3035254885Sdumbbell radeon_ring_write(ring, upper_32_bits(value)); 3036254885Sdumbbell radeon_ring_write(ring, incr); /* increment size */ 3037254885Sdumbbell radeon_ring_write(ring, 0); 3038254885Sdumbbell pe += ndw * 4; 3039254885Sdumbbell addr += (ndw / 2) * incr; 3040254885Sdumbbell count -= ndw / 2; 3041254885Sdumbbell } 3042254885Sdumbbell } 3043254885Sdumbbell } 3044254885Sdumbbell} 3045254885Sdumbbell 3046254885Sdumbbellvoid si_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm) 3047254885Sdumbbell{ 3048254885Sdumbbell struct radeon_ring *ring = &rdev->ring[ridx]; 3049254885Sdumbbell 3050254885Sdumbbell if (vm == NULL) 3051254885Sdumbbell return; 3052254885Sdumbbell 3053254885Sdumbbell /* write new base address */ 3054254885Sdumbbell radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 3055254885Sdumbbell radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) | 3056254885Sdumbbell WRITE_DATA_DST_SEL(0))); 3057254885Sdumbbell 3058254885Sdumbbell if (vm->id < 8) { 3059254885Sdumbbell radeon_ring_write(ring, 3060254885Sdumbbell (VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm->id << 2)) >> 2); 3061254885Sdumbbell } else { 3062254885Sdumbbell radeon_ring_write(ring, 3063254885Sdumbbell (VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm->id - 8) << 2)) >> 2); 3064254885Sdumbbell } 3065254885Sdumbbell radeon_ring_write(ring, 0); 3066254885Sdumbbell radeon_ring_write(ring, vm->pd_gpu_addr >> 12); 3067254885Sdumbbell 3068254885Sdumbbell /* flush hdp cache */ 3069254885Sdumbbell radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 3070254885Sdumbbell radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) | 3071254885Sdumbbell WRITE_DATA_DST_SEL(0))); 3072254885Sdumbbell radeon_ring_write(ring, HDP_MEM_COHERENCY_FLUSH_CNTL >> 2); 3073254885Sdumbbell radeon_ring_write(ring, 0); 3074254885Sdumbbell radeon_ring_write(ring, 0x1); 3075254885Sdumbbell 3076254885Sdumbbell /* bits 0-15 are the VM contexts0-15 */ 3077254885Sdumbbell radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 3078254885Sdumbbell radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) | 3079254885Sdumbbell WRITE_DATA_DST_SEL(0))); 3080254885Sdumbbell radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2); 3081254885Sdumbbell radeon_ring_write(ring, 0); 3082254885Sdumbbell radeon_ring_write(ring, 1 << vm->id); 3083254885Sdumbbell 3084254885Sdumbbell /* sync PFP to ME, otherwise we might get invalid PFP reads */ 3085254885Sdumbbell radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0)); 3086254885Sdumbbell radeon_ring_write(ring, 0x0); 3087254885Sdumbbell} 3088254885Sdumbbell 3089254885Sdumbbellvoid si_dma_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm) 3090254885Sdumbbell{ 3091254885Sdumbbell struct radeon_ring *ring = &rdev->ring[ridx]; 3092254885Sdumbbell 3093254885Sdumbbell if (vm == NULL) 3094254885Sdumbbell return; 3095254885Sdumbbell 3096254885Sdumbbell radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_SRBM_WRITE, 0, 0, 0, 0)); 3097254885Sdumbbell if (vm->id < 8) { 3098254885Sdumbbell radeon_ring_write(ring, (0xf << 16) | ((VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm->id << 2)) >> 2)); 3099254885Sdumbbell } else { 3100254885Sdumbbell radeon_ring_write(ring, (0xf << 16) | ((VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm->id - 8) << 2)) >> 2)); 3101254885Sdumbbell } 3102254885Sdumbbell radeon_ring_write(ring, vm->pd_gpu_addr >> 12); 3103254885Sdumbbell 3104254885Sdumbbell /* flush hdp cache */ 3105254885Sdumbbell radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_SRBM_WRITE, 0, 0, 0, 0)); 3106254885Sdumbbell radeon_ring_write(ring, (0xf << 16) | (HDP_MEM_COHERENCY_FLUSH_CNTL >> 2)); 3107254885Sdumbbell radeon_ring_write(ring, 1); 3108254885Sdumbbell 3109254885Sdumbbell /* bits 0-7 are the VM contexts0-7 */ 3110254885Sdumbbell radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_SRBM_WRITE, 0, 0, 0, 0)); 3111254885Sdumbbell radeon_ring_write(ring, (0xf << 16) | (VM_INVALIDATE_REQUEST >> 2)); 3112254885Sdumbbell radeon_ring_write(ring, 1 << vm->id); 3113254885Sdumbbell} 3114254885Sdumbbell 3115254885Sdumbbell/* 3116254885Sdumbbell * RLC 3117254885Sdumbbell */ 3118254885Sdumbbellvoid si_rlc_fini(struct radeon_device *rdev) 3119254885Sdumbbell{ 3120254885Sdumbbell int r; 3121254885Sdumbbell 3122254885Sdumbbell /* save restore block */ 3123254885Sdumbbell if (rdev->rlc.save_restore_obj) { 3124254885Sdumbbell r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false); 3125254885Sdumbbell if (unlikely(r != 0)) 3126254885Sdumbbell dev_warn(rdev->dev, "(%d) reserve RLC sr bo failed\n", r); 3127254885Sdumbbell radeon_bo_unpin(rdev->rlc.save_restore_obj); 3128254885Sdumbbell radeon_bo_unreserve(rdev->rlc.save_restore_obj); 3129254885Sdumbbell 3130254885Sdumbbell radeon_bo_unref(&rdev->rlc.save_restore_obj); 3131254885Sdumbbell rdev->rlc.save_restore_obj = NULL; 3132254885Sdumbbell } 3133254885Sdumbbell 3134254885Sdumbbell /* clear state block */ 3135254885Sdumbbell if (rdev->rlc.clear_state_obj) { 3136254885Sdumbbell r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false); 3137254885Sdumbbell if (unlikely(r != 0)) 3138254885Sdumbbell dev_warn(rdev->dev, "(%d) reserve RLC c bo failed\n", r); 3139254885Sdumbbell radeon_bo_unpin(rdev->rlc.clear_state_obj); 3140254885Sdumbbell radeon_bo_unreserve(rdev->rlc.clear_state_obj); 3141254885Sdumbbell 3142254885Sdumbbell radeon_bo_unref(&rdev->rlc.clear_state_obj); 3143254885Sdumbbell rdev->rlc.clear_state_obj = NULL; 3144254885Sdumbbell } 3145254885Sdumbbell} 3146254885Sdumbbell 3147254885Sdumbbellint si_rlc_init(struct radeon_device *rdev) 3148254885Sdumbbell{ 3149254885Sdumbbell int r; 3150254885Sdumbbell 3151254885Sdumbbell /* save restore block */ 3152254885Sdumbbell if (rdev->rlc.save_restore_obj == NULL) { 3153254885Sdumbbell r = radeon_bo_create(rdev, RADEON_GPU_PAGE_SIZE, PAGE_SIZE, true, 3154254885Sdumbbell RADEON_GEM_DOMAIN_VRAM, NULL, 3155254885Sdumbbell &rdev->rlc.save_restore_obj); 3156254885Sdumbbell if (r) { 3157254885Sdumbbell dev_warn(rdev->dev, "(%d) create RLC sr bo failed\n", r); 3158254885Sdumbbell return r; 3159254885Sdumbbell } 3160254885Sdumbbell } 3161254885Sdumbbell 3162254885Sdumbbell r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false); 3163254885Sdumbbell if (unlikely(r != 0)) { 3164254885Sdumbbell si_rlc_fini(rdev); 3165254885Sdumbbell return r; 3166254885Sdumbbell } 3167254885Sdumbbell r = radeon_bo_pin(rdev->rlc.save_restore_obj, RADEON_GEM_DOMAIN_VRAM, 3168254885Sdumbbell &rdev->rlc.save_restore_gpu_addr); 3169254885Sdumbbell radeon_bo_unreserve(rdev->rlc.save_restore_obj); 3170254885Sdumbbell if (r) { 3171254885Sdumbbell dev_warn(rdev->dev, "(%d) pin RLC sr bo failed\n", r); 3172254885Sdumbbell si_rlc_fini(rdev); 3173254885Sdumbbell return r; 3174254885Sdumbbell } 3175254885Sdumbbell 3176254885Sdumbbell /* clear state block */ 3177254885Sdumbbell if (rdev->rlc.clear_state_obj == NULL) { 3178254885Sdumbbell r = radeon_bo_create(rdev, RADEON_GPU_PAGE_SIZE, PAGE_SIZE, true, 3179254885Sdumbbell RADEON_GEM_DOMAIN_VRAM, NULL, 3180254885Sdumbbell &rdev->rlc.clear_state_obj); 3181254885Sdumbbell if (r) { 3182254885Sdumbbell dev_warn(rdev->dev, "(%d) create RLC c bo failed\n", r); 3183254885Sdumbbell si_rlc_fini(rdev); 3184254885Sdumbbell return r; 3185254885Sdumbbell } 3186254885Sdumbbell } 3187254885Sdumbbell r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false); 3188254885Sdumbbell if (unlikely(r != 0)) { 3189254885Sdumbbell si_rlc_fini(rdev); 3190254885Sdumbbell return r; 3191254885Sdumbbell } 3192254885Sdumbbell r = radeon_bo_pin(rdev->rlc.clear_state_obj, RADEON_GEM_DOMAIN_VRAM, 3193254885Sdumbbell &rdev->rlc.clear_state_gpu_addr); 3194254885Sdumbbell radeon_bo_unreserve(rdev->rlc.clear_state_obj); 3195254885Sdumbbell if (r) { 3196254885Sdumbbell dev_warn(rdev->dev, "(%d) pin RLC c bo failed\n", r); 3197254885Sdumbbell si_rlc_fini(rdev); 3198254885Sdumbbell return r; 3199254885Sdumbbell } 3200254885Sdumbbell 3201254885Sdumbbell return 0; 3202254885Sdumbbell} 3203254885Sdumbbell 3204254885Sdumbbellstatic void si_rlc_stop(struct radeon_device *rdev) 3205254885Sdumbbell{ 3206254885Sdumbbell WREG32(RLC_CNTL, 0); 3207254885Sdumbbell} 3208254885Sdumbbell 3209254885Sdumbbellstatic void si_rlc_start(struct radeon_device *rdev) 3210254885Sdumbbell{ 3211254885Sdumbbell WREG32(RLC_CNTL, RLC_ENABLE); 3212254885Sdumbbell} 3213254885Sdumbbell 3214254885Sdumbbellstatic int si_rlc_resume(struct radeon_device *rdev) 3215254885Sdumbbell{ 3216254885Sdumbbell u32 i; 3217254885Sdumbbell const __be32 *fw_data; 3218254885Sdumbbell 3219254885Sdumbbell if (!rdev->rlc_fw) 3220254885Sdumbbell return -EINVAL; 3221254885Sdumbbell 3222254885Sdumbbell si_rlc_stop(rdev); 3223254885Sdumbbell 3224254885Sdumbbell WREG32(RLC_RL_BASE, 0); 3225254885Sdumbbell WREG32(RLC_RL_SIZE, 0); 3226254885Sdumbbell WREG32(RLC_LB_CNTL, 0); 3227254885Sdumbbell WREG32(RLC_LB_CNTR_MAX, 0xffffffff); 3228254885Sdumbbell WREG32(RLC_LB_CNTR_INIT, 0); 3229254885Sdumbbell 3230254885Sdumbbell WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8); 3231254885Sdumbbell WREG32(RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8); 3232254885Sdumbbell 3233254885Sdumbbell WREG32(RLC_MC_CNTL, 0); 3234254885Sdumbbell WREG32(RLC_UCODE_CNTL, 0); 3235254885Sdumbbell 3236254885Sdumbbell fw_data = (const __be32 *)rdev->rlc_fw->data; 3237254885Sdumbbell for (i = 0; i < SI_RLC_UCODE_SIZE; i++) { 3238254885Sdumbbell WREG32(RLC_UCODE_ADDR, i); 3239254885Sdumbbell WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++)); 3240254885Sdumbbell } 3241254885Sdumbbell WREG32(RLC_UCODE_ADDR, 0); 3242254885Sdumbbell 3243254885Sdumbbell si_rlc_start(rdev); 3244254885Sdumbbell 3245254885Sdumbbell return 0; 3246254885Sdumbbell} 3247254885Sdumbbell 3248254885Sdumbbellstatic void si_enable_interrupts(struct radeon_device *rdev) 3249254885Sdumbbell{ 3250254885Sdumbbell u32 ih_cntl = RREG32(IH_CNTL); 3251254885Sdumbbell u32 ih_rb_cntl = RREG32(IH_RB_CNTL); 3252254885Sdumbbell 3253254885Sdumbbell ih_cntl |= ENABLE_INTR; 3254254885Sdumbbell ih_rb_cntl |= IH_RB_ENABLE; 3255254885Sdumbbell WREG32(IH_CNTL, ih_cntl); 3256254885Sdumbbell WREG32(IH_RB_CNTL, ih_rb_cntl); 3257254885Sdumbbell rdev->ih.enabled = true; 3258254885Sdumbbell} 3259254885Sdumbbell 3260254885Sdumbbellstatic void si_disable_interrupts(struct radeon_device *rdev) 3261254885Sdumbbell{ 3262254885Sdumbbell u32 ih_rb_cntl = RREG32(IH_RB_CNTL); 3263254885Sdumbbell u32 ih_cntl = RREG32(IH_CNTL); 3264254885Sdumbbell 3265254885Sdumbbell ih_rb_cntl &= ~IH_RB_ENABLE; 3266254885Sdumbbell ih_cntl &= ~ENABLE_INTR; 3267254885Sdumbbell WREG32(IH_RB_CNTL, ih_rb_cntl); 3268254885Sdumbbell WREG32(IH_CNTL, ih_cntl); 3269254885Sdumbbell /* set rptr, wptr to 0 */ 3270254885Sdumbbell WREG32(IH_RB_RPTR, 0); 3271254885Sdumbbell WREG32(IH_RB_WPTR, 0); 3272254885Sdumbbell rdev->ih.enabled = false; 3273254885Sdumbbell rdev->ih.rptr = 0; 3274254885Sdumbbell} 3275254885Sdumbbell 3276254885Sdumbbellstatic void si_disable_interrupt_state(struct radeon_device *rdev) 3277254885Sdumbbell{ 3278254885Sdumbbell u32 tmp; 3279254885Sdumbbell 3280254885Sdumbbell WREG32(CP_INT_CNTL_RING0, CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE); 3281254885Sdumbbell WREG32(CP_INT_CNTL_RING1, 0); 3282254885Sdumbbell WREG32(CP_INT_CNTL_RING2, 0); 3283254885Sdumbbell tmp = RREG32(DMA_CNTL + DMA0_REGISTER_OFFSET) & ~TRAP_ENABLE; 3284254885Sdumbbell WREG32(DMA_CNTL + DMA0_REGISTER_OFFSET, tmp); 3285254885Sdumbbell tmp = RREG32(DMA_CNTL + DMA1_REGISTER_OFFSET) & ~TRAP_ENABLE; 3286254885Sdumbbell WREG32(DMA_CNTL + DMA1_REGISTER_OFFSET, tmp); 3287254885Sdumbbell WREG32(GRBM_INT_CNTL, 0); 3288254885Sdumbbell WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0); 3289254885Sdumbbell WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0); 3290254885Sdumbbell if (rdev->num_crtc >= 4) { 3291254885Sdumbbell WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0); 3292254885Sdumbbell WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0); 3293254885Sdumbbell } 3294254885Sdumbbell if (rdev->num_crtc >= 6) { 3295254885Sdumbbell WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0); 3296254885Sdumbbell WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0); 3297254885Sdumbbell } 3298254885Sdumbbell 3299254885Sdumbbell WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0); 3300254885Sdumbbell WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0); 3301254885Sdumbbell if (rdev->num_crtc >= 4) { 3302254885Sdumbbell WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0); 3303254885Sdumbbell WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0); 3304254885Sdumbbell } 3305254885Sdumbbell if (rdev->num_crtc >= 6) { 3306254885Sdumbbell WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0); 3307254885Sdumbbell WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0); 3308254885Sdumbbell } 3309254885Sdumbbell 3310254885Sdumbbell WREG32(DACA_AUTODETECT_INT_CONTROL, 0); 3311254885Sdumbbell 3312254885Sdumbbell tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY; 3313254885Sdumbbell WREG32(DC_HPD1_INT_CONTROL, tmp); 3314254885Sdumbbell tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY; 3315254885Sdumbbell WREG32(DC_HPD2_INT_CONTROL, tmp); 3316254885Sdumbbell tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY; 3317254885Sdumbbell WREG32(DC_HPD3_INT_CONTROL, tmp); 3318254885Sdumbbell tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY; 3319254885Sdumbbell WREG32(DC_HPD4_INT_CONTROL, tmp); 3320254885Sdumbbell tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY; 3321254885Sdumbbell WREG32(DC_HPD5_INT_CONTROL, tmp); 3322254885Sdumbbell tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY; 3323254885Sdumbbell WREG32(DC_HPD6_INT_CONTROL, tmp); 3324254885Sdumbbell 3325254885Sdumbbell} 3326254885Sdumbbell 3327254885Sdumbbellstatic int si_irq_init(struct radeon_device *rdev) 3328254885Sdumbbell{ 3329254885Sdumbbell int ret = 0; 3330254885Sdumbbell int rb_bufsz; 3331254885Sdumbbell u32 interrupt_cntl, ih_cntl, ih_rb_cntl; 3332254885Sdumbbell 3333254885Sdumbbell /* allocate ring */ 3334254885Sdumbbell ret = r600_ih_ring_alloc(rdev); 3335254885Sdumbbell if (ret) 3336254885Sdumbbell return ret; 3337254885Sdumbbell 3338254885Sdumbbell /* disable irqs */ 3339254885Sdumbbell si_disable_interrupts(rdev); 3340254885Sdumbbell 3341254885Sdumbbell /* init rlc */ 3342254885Sdumbbell ret = si_rlc_resume(rdev); 3343254885Sdumbbell if (ret) { 3344254885Sdumbbell r600_ih_ring_fini(rdev); 3345254885Sdumbbell return ret; 3346254885Sdumbbell } 3347254885Sdumbbell 3348254885Sdumbbell /* setup interrupt control */ 3349254885Sdumbbell /* set dummy read address to ring address */ 3350254885Sdumbbell WREG32(INTERRUPT_CNTL2, rdev->ih.gpu_addr >> 8); 3351254885Sdumbbell interrupt_cntl = RREG32(INTERRUPT_CNTL); 3352254885Sdumbbell /* IH_DUMMY_RD_OVERRIDE=0 - dummy read disabled with msi, enabled without msi 3353254885Sdumbbell * IH_DUMMY_RD_OVERRIDE=1 - dummy read controlled by IH_DUMMY_RD_EN 3354254885Sdumbbell */ 3355254885Sdumbbell interrupt_cntl &= ~IH_DUMMY_RD_OVERRIDE; 3356254885Sdumbbell /* IH_REQ_NONSNOOP_EN=1 if ring is in non-cacheable memory, e.g., vram */ 3357254885Sdumbbell interrupt_cntl &= ~IH_REQ_NONSNOOP_EN; 3358254885Sdumbbell WREG32(INTERRUPT_CNTL, interrupt_cntl); 3359254885Sdumbbell 3360254885Sdumbbell WREG32(IH_RB_BASE, rdev->ih.gpu_addr >> 8); 3361254885Sdumbbell rb_bufsz = drm_order(rdev->ih.ring_size / 4); 3362254885Sdumbbell 3363254885Sdumbbell ih_rb_cntl = (IH_WPTR_OVERFLOW_ENABLE | 3364254885Sdumbbell IH_WPTR_OVERFLOW_CLEAR | 3365254885Sdumbbell (rb_bufsz << 1)); 3366254885Sdumbbell 3367254885Sdumbbell if (rdev->wb.enabled) 3368254885Sdumbbell ih_rb_cntl |= IH_WPTR_WRITEBACK_ENABLE; 3369254885Sdumbbell 3370254885Sdumbbell /* set the writeback address whether it's enabled or not */ 3371254885Sdumbbell WREG32(IH_RB_WPTR_ADDR_LO, (rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFFFFFFFC); 3372254885Sdumbbell WREG32(IH_RB_WPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFF); 3373254885Sdumbbell 3374254885Sdumbbell WREG32(IH_RB_CNTL, ih_rb_cntl); 3375254885Sdumbbell 3376254885Sdumbbell /* set rptr, wptr to 0 */ 3377254885Sdumbbell WREG32(IH_RB_RPTR, 0); 3378254885Sdumbbell WREG32(IH_RB_WPTR, 0); 3379254885Sdumbbell 3380254885Sdumbbell /* Default settings for IH_CNTL (disabled at first) */ 3381254885Sdumbbell ih_cntl = MC_WRREQ_CREDIT(0x10) | MC_WR_CLEAN_CNT(0x10) | MC_VMID(0); 3382254885Sdumbbell /* RPTR_REARM only works if msi's are enabled */ 3383254885Sdumbbell if (rdev->msi_enabled) 3384254885Sdumbbell ih_cntl |= RPTR_REARM; 3385254885Sdumbbell WREG32(IH_CNTL, ih_cntl); 3386254885Sdumbbell 3387254885Sdumbbell /* force the active interrupt state to all disabled */ 3388254885Sdumbbell si_disable_interrupt_state(rdev); 3389254885Sdumbbell 3390254885Sdumbbell pci_enable_busmaster(rdev->dev); 3391254885Sdumbbell 3392254885Sdumbbell /* enable irqs */ 3393254885Sdumbbell si_enable_interrupts(rdev); 3394254885Sdumbbell 3395254885Sdumbbell return ret; 3396254885Sdumbbell} 3397254885Sdumbbell 3398254885Sdumbbellint si_irq_set(struct radeon_device *rdev) 3399254885Sdumbbell{ 3400254885Sdumbbell u32 cp_int_cntl = CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE; 3401254885Sdumbbell u32 cp_int_cntl1 = 0, cp_int_cntl2 = 0; 3402254885Sdumbbell u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0; 3403254885Sdumbbell u32 hpd1, hpd2, hpd3, hpd4, hpd5, hpd6; 3404254885Sdumbbell u32 grbm_int_cntl = 0; 3405254885Sdumbbell u32 grph1 = 0, grph2 = 0, grph3 = 0, grph4 = 0, grph5 = 0, grph6 = 0; 3406254885Sdumbbell u32 dma_cntl, dma_cntl1; 3407254885Sdumbbell 3408254885Sdumbbell if (!rdev->irq.installed) { 3409254885Sdumbbell DRM_ERROR("Can't enable IRQ/MSI because no handler is installed\n"); 3410254885Sdumbbell return -EINVAL; 3411254885Sdumbbell } 3412254885Sdumbbell /* don't enable anything if the ih is disabled */ 3413254885Sdumbbell if (!rdev->ih.enabled) { 3414254885Sdumbbell si_disable_interrupts(rdev); 3415254885Sdumbbell /* force the active interrupt state to all disabled */ 3416254885Sdumbbell si_disable_interrupt_state(rdev); 3417254885Sdumbbell return 0; 3418254885Sdumbbell } 3419254885Sdumbbell 3420254885Sdumbbell hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~DC_HPDx_INT_EN; 3421254885Sdumbbell hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~DC_HPDx_INT_EN; 3422254885Sdumbbell hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~DC_HPDx_INT_EN; 3423254885Sdumbbell hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~DC_HPDx_INT_EN; 3424254885Sdumbbell hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN; 3425254885Sdumbbell hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN; 3426254885Sdumbbell 3427254885Sdumbbell dma_cntl = RREG32(DMA_CNTL + DMA0_REGISTER_OFFSET) & ~TRAP_ENABLE; 3428254885Sdumbbell dma_cntl1 = RREG32(DMA_CNTL + DMA1_REGISTER_OFFSET) & ~TRAP_ENABLE; 3429254885Sdumbbell 3430254885Sdumbbell /* enable CP interrupts on all rings */ 3431254885Sdumbbell if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) { 3432254885Sdumbbell DRM_DEBUG("si_irq_set: sw int gfx\n"); 3433254885Sdumbbell cp_int_cntl |= TIME_STAMP_INT_ENABLE; 3434254885Sdumbbell } 3435254885Sdumbbell if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) { 3436254885Sdumbbell DRM_DEBUG("si_irq_set: sw int cp1\n"); 3437254885Sdumbbell cp_int_cntl1 |= TIME_STAMP_INT_ENABLE; 3438254885Sdumbbell } 3439254885Sdumbbell if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) { 3440254885Sdumbbell DRM_DEBUG("si_irq_set: sw int cp2\n"); 3441254885Sdumbbell cp_int_cntl2 |= TIME_STAMP_INT_ENABLE; 3442254885Sdumbbell } 3443254885Sdumbbell if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) { 3444254885Sdumbbell DRM_DEBUG("si_irq_set: sw int dma\n"); 3445254885Sdumbbell dma_cntl |= TRAP_ENABLE; 3446254885Sdumbbell } 3447254885Sdumbbell 3448254885Sdumbbell if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) { 3449254885Sdumbbell DRM_DEBUG("si_irq_set: sw int dma1\n"); 3450254885Sdumbbell dma_cntl1 |= TRAP_ENABLE; 3451254885Sdumbbell } 3452254885Sdumbbell if (rdev->irq.crtc_vblank_int[0] || 3453254885Sdumbbell atomic_read(&rdev->irq.pflip[0])) { 3454254885Sdumbbell DRM_DEBUG("si_irq_set: vblank 0\n"); 3455254885Sdumbbell crtc1 |= VBLANK_INT_MASK; 3456254885Sdumbbell } 3457254885Sdumbbell if (rdev->irq.crtc_vblank_int[1] || 3458254885Sdumbbell atomic_read(&rdev->irq.pflip[1])) { 3459254885Sdumbbell DRM_DEBUG("si_irq_set: vblank 1\n"); 3460254885Sdumbbell crtc2 |= VBLANK_INT_MASK; 3461254885Sdumbbell } 3462254885Sdumbbell if (rdev->irq.crtc_vblank_int[2] || 3463254885Sdumbbell atomic_read(&rdev->irq.pflip[2])) { 3464254885Sdumbbell DRM_DEBUG("si_irq_set: vblank 2\n"); 3465254885Sdumbbell crtc3 |= VBLANK_INT_MASK; 3466254885Sdumbbell } 3467254885Sdumbbell if (rdev->irq.crtc_vblank_int[3] || 3468254885Sdumbbell atomic_read(&rdev->irq.pflip[3])) { 3469254885Sdumbbell DRM_DEBUG("si_irq_set: vblank 3\n"); 3470254885Sdumbbell crtc4 |= VBLANK_INT_MASK; 3471254885Sdumbbell } 3472254885Sdumbbell if (rdev->irq.crtc_vblank_int[4] || 3473254885Sdumbbell atomic_read(&rdev->irq.pflip[4])) { 3474254885Sdumbbell DRM_DEBUG("si_irq_set: vblank 4\n"); 3475254885Sdumbbell crtc5 |= VBLANK_INT_MASK; 3476254885Sdumbbell } 3477254885Sdumbbell if (rdev->irq.crtc_vblank_int[5] || 3478254885Sdumbbell atomic_read(&rdev->irq.pflip[5])) { 3479254885Sdumbbell DRM_DEBUG("si_irq_set: vblank 5\n"); 3480254885Sdumbbell crtc6 |= VBLANK_INT_MASK; 3481254885Sdumbbell } 3482254885Sdumbbell if (rdev->irq.hpd[0]) { 3483254885Sdumbbell DRM_DEBUG("si_irq_set: hpd 1\n"); 3484254885Sdumbbell hpd1 |= DC_HPDx_INT_EN; 3485254885Sdumbbell } 3486254885Sdumbbell if (rdev->irq.hpd[1]) { 3487254885Sdumbbell DRM_DEBUG("si_irq_set: hpd 2\n"); 3488254885Sdumbbell hpd2 |= DC_HPDx_INT_EN; 3489254885Sdumbbell } 3490254885Sdumbbell if (rdev->irq.hpd[2]) { 3491254885Sdumbbell DRM_DEBUG("si_irq_set: hpd 3\n"); 3492254885Sdumbbell hpd3 |= DC_HPDx_INT_EN; 3493254885Sdumbbell } 3494254885Sdumbbell if (rdev->irq.hpd[3]) { 3495254885Sdumbbell DRM_DEBUG("si_irq_set: hpd 4\n"); 3496254885Sdumbbell hpd4 |= DC_HPDx_INT_EN; 3497254885Sdumbbell } 3498254885Sdumbbell if (rdev->irq.hpd[4]) { 3499254885Sdumbbell DRM_DEBUG("si_irq_set: hpd 5\n"); 3500254885Sdumbbell hpd5 |= DC_HPDx_INT_EN; 3501254885Sdumbbell } 3502254885Sdumbbell if (rdev->irq.hpd[5]) { 3503254885Sdumbbell DRM_DEBUG("si_irq_set: hpd 6\n"); 3504254885Sdumbbell hpd6 |= DC_HPDx_INT_EN; 3505254885Sdumbbell } 3506254885Sdumbbell 3507254885Sdumbbell WREG32(CP_INT_CNTL_RING0, cp_int_cntl); 3508254885Sdumbbell WREG32(CP_INT_CNTL_RING1, cp_int_cntl1); 3509254885Sdumbbell WREG32(CP_INT_CNTL_RING2, cp_int_cntl2); 3510254885Sdumbbell 3511254885Sdumbbell WREG32(DMA_CNTL + DMA0_REGISTER_OFFSET, dma_cntl); 3512254885Sdumbbell WREG32(DMA_CNTL + DMA1_REGISTER_OFFSET, dma_cntl1); 3513254885Sdumbbell 3514254885Sdumbbell WREG32(GRBM_INT_CNTL, grbm_int_cntl); 3515254885Sdumbbell 3516254885Sdumbbell WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1); 3517254885Sdumbbell WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2); 3518254885Sdumbbell if (rdev->num_crtc >= 4) { 3519254885Sdumbbell WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3); 3520254885Sdumbbell WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4); 3521254885Sdumbbell } 3522254885Sdumbbell if (rdev->num_crtc >= 6) { 3523254885Sdumbbell WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5); 3524254885Sdumbbell WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6); 3525254885Sdumbbell } 3526254885Sdumbbell 3527254885Sdumbbell WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, grph1); 3528254885Sdumbbell WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, grph2); 3529254885Sdumbbell if (rdev->num_crtc >= 4) { 3530254885Sdumbbell WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, grph3); 3531254885Sdumbbell WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, grph4); 3532254885Sdumbbell } 3533254885Sdumbbell if (rdev->num_crtc >= 6) { 3534254885Sdumbbell WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, grph5); 3535254885Sdumbbell WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, grph6); 3536254885Sdumbbell } 3537254885Sdumbbell 3538254885Sdumbbell WREG32(DC_HPD1_INT_CONTROL, hpd1); 3539254885Sdumbbell WREG32(DC_HPD2_INT_CONTROL, hpd2); 3540254885Sdumbbell WREG32(DC_HPD3_INT_CONTROL, hpd3); 3541254885Sdumbbell WREG32(DC_HPD4_INT_CONTROL, hpd4); 3542254885Sdumbbell WREG32(DC_HPD5_INT_CONTROL, hpd5); 3543254885Sdumbbell WREG32(DC_HPD6_INT_CONTROL, hpd6); 3544254885Sdumbbell 3545254885Sdumbbell return 0; 3546254885Sdumbbell} 3547254885Sdumbbell 3548254885Sdumbbellstatic inline void si_irq_ack(struct radeon_device *rdev) 3549254885Sdumbbell{ 3550254885Sdumbbell u32 tmp; 3551254885Sdumbbell 3552254885Sdumbbell rdev->irq.stat_regs.evergreen.disp_int = RREG32(DISP_INTERRUPT_STATUS); 3553254885Sdumbbell rdev->irq.stat_regs.evergreen.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE); 3554254885Sdumbbell rdev->irq.stat_regs.evergreen.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2); 3555254885Sdumbbell rdev->irq.stat_regs.evergreen.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3); 3556254885Sdumbbell rdev->irq.stat_regs.evergreen.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4); 3557254885Sdumbbell rdev->irq.stat_regs.evergreen.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5); 3558254885Sdumbbell rdev->irq.stat_regs.evergreen.d1grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET); 3559254885Sdumbbell rdev->irq.stat_regs.evergreen.d2grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET); 3560254885Sdumbbell if (rdev->num_crtc >= 4) { 3561254885Sdumbbell rdev->irq.stat_regs.evergreen.d3grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET); 3562254885Sdumbbell rdev->irq.stat_regs.evergreen.d4grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET); 3563254885Sdumbbell } 3564254885Sdumbbell if (rdev->num_crtc >= 6) { 3565254885Sdumbbell rdev->irq.stat_regs.evergreen.d5grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET); 3566254885Sdumbbell rdev->irq.stat_regs.evergreen.d6grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET); 3567254885Sdumbbell } 3568254885Sdumbbell 3569254885Sdumbbell if (rdev->irq.stat_regs.evergreen.d1grph_int & GRPH_PFLIP_INT_OCCURRED) 3570254885Sdumbbell WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR); 3571254885Sdumbbell if (rdev->irq.stat_regs.evergreen.d2grph_int & GRPH_PFLIP_INT_OCCURRED) 3572254885Sdumbbell WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR); 3573254885Sdumbbell if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT) 3574254885Sdumbbell WREG32(VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK); 3575254885Sdumbbell if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT) 3576254885Sdumbbell WREG32(VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK); 3577254885Sdumbbell if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT) 3578254885Sdumbbell WREG32(VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK); 3579254885Sdumbbell if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT) 3580254885Sdumbbell WREG32(VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK); 3581254885Sdumbbell 3582254885Sdumbbell if (rdev->num_crtc >= 4) { 3583254885Sdumbbell if (rdev->irq.stat_regs.evergreen.d3grph_int & GRPH_PFLIP_INT_OCCURRED) 3584254885Sdumbbell WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR); 3585254885Sdumbbell if (rdev->irq.stat_regs.evergreen.d4grph_int & GRPH_PFLIP_INT_OCCURRED) 3586254885Sdumbbell WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR); 3587254885Sdumbbell if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) 3588254885Sdumbbell WREG32(VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK); 3589254885Sdumbbell if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) 3590254885Sdumbbell WREG32(VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK); 3591254885Sdumbbell if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) 3592254885Sdumbbell WREG32(VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK); 3593254885Sdumbbell if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) 3594254885Sdumbbell WREG32(VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK); 3595254885Sdumbbell } 3596254885Sdumbbell 3597254885Sdumbbell if (rdev->num_crtc >= 6) { 3598254885Sdumbbell if (rdev->irq.stat_regs.evergreen.d5grph_int & GRPH_PFLIP_INT_OCCURRED) 3599254885Sdumbbell WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR); 3600254885Sdumbbell if (rdev->irq.stat_regs.evergreen.d6grph_int & GRPH_PFLIP_INT_OCCURRED) 3601254885Sdumbbell WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR); 3602254885Sdumbbell if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) 3603254885Sdumbbell WREG32(VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK); 3604254885Sdumbbell if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) 3605254885Sdumbbell WREG32(VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK); 3606254885Sdumbbell if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) 3607254885Sdumbbell WREG32(VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK); 3608254885Sdumbbell if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) 3609254885Sdumbbell WREG32(VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK); 3610254885Sdumbbell } 3611254885Sdumbbell 3612254885Sdumbbell if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) { 3613254885Sdumbbell tmp = RREG32(DC_HPD1_INT_CONTROL); 3614254885Sdumbbell tmp |= DC_HPDx_INT_ACK; 3615254885Sdumbbell WREG32(DC_HPD1_INT_CONTROL, tmp); 3616254885Sdumbbell } 3617254885Sdumbbell if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) { 3618254885Sdumbbell tmp = RREG32(DC_HPD2_INT_CONTROL); 3619254885Sdumbbell tmp |= DC_HPDx_INT_ACK; 3620254885Sdumbbell WREG32(DC_HPD2_INT_CONTROL, tmp); 3621254885Sdumbbell } 3622254885Sdumbbell if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) { 3623254885Sdumbbell tmp = RREG32(DC_HPD3_INT_CONTROL); 3624254885Sdumbbell tmp |= DC_HPDx_INT_ACK; 3625254885Sdumbbell WREG32(DC_HPD3_INT_CONTROL, tmp); 3626254885Sdumbbell } 3627254885Sdumbbell if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) { 3628254885Sdumbbell tmp = RREG32(DC_HPD4_INT_CONTROL); 3629254885Sdumbbell tmp |= DC_HPDx_INT_ACK; 3630254885Sdumbbell WREG32(DC_HPD4_INT_CONTROL, tmp); 3631254885Sdumbbell } 3632254885Sdumbbell if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) { 3633254885Sdumbbell tmp = RREG32(DC_HPD5_INT_CONTROL); 3634254885Sdumbbell tmp |= DC_HPDx_INT_ACK; 3635254885Sdumbbell WREG32(DC_HPD5_INT_CONTROL, tmp); 3636254885Sdumbbell } 3637254885Sdumbbell if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) { 3638254885Sdumbbell tmp = RREG32(DC_HPD5_INT_CONTROL); 3639254885Sdumbbell tmp |= DC_HPDx_INT_ACK; 3640254885Sdumbbell WREG32(DC_HPD6_INT_CONTROL, tmp); 3641254885Sdumbbell } 3642254885Sdumbbell} 3643254885Sdumbbell 3644254885Sdumbbellstatic void si_irq_disable(struct radeon_device *rdev) 3645254885Sdumbbell{ 3646254885Sdumbbell si_disable_interrupts(rdev); 3647254885Sdumbbell /* Wait and acknowledge irq */ 3648254885Sdumbbell DRM_MDELAY(1); 3649254885Sdumbbell si_irq_ack(rdev); 3650254885Sdumbbell si_disable_interrupt_state(rdev); 3651254885Sdumbbell} 3652254885Sdumbbell 3653254885Sdumbbellstatic void si_irq_suspend(struct radeon_device *rdev) 3654254885Sdumbbell{ 3655254885Sdumbbell si_irq_disable(rdev); 3656254885Sdumbbell si_rlc_stop(rdev); 3657254885Sdumbbell} 3658254885Sdumbbell 3659254885Sdumbbellstatic void si_irq_fini(struct radeon_device *rdev) 3660254885Sdumbbell{ 3661254885Sdumbbell si_irq_suspend(rdev); 3662254885Sdumbbell r600_ih_ring_fini(rdev); 3663254885Sdumbbell} 3664254885Sdumbbell 3665254885Sdumbbellstatic inline u32 si_get_ih_wptr(struct radeon_device *rdev) 3666254885Sdumbbell{ 3667254885Sdumbbell u32 wptr, tmp; 3668254885Sdumbbell 3669254885Sdumbbell if (rdev->wb.enabled) 3670254885Sdumbbell wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]); 3671254885Sdumbbell else 3672254885Sdumbbell wptr = RREG32(IH_RB_WPTR); 3673254885Sdumbbell 3674254885Sdumbbell if (wptr & RB_OVERFLOW) { 3675254885Sdumbbell /* When a ring buffer overflow happen start parsing interrupt 3676254885Sdumbbell * from the last not overwritten vector (wptr + 16). Hopefully 3677254885Sdumbbell * this should allow us to catchup. 3678254885Sdumbbell */ 3679254885Sdumbbell dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, %d, %d)\n", 3680254885Sdumbbell wptr, rdev->ih.rptr, (wptr + 16) + rdev->ih.ptr_mask); 3681254885Sdumbbell rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask; 3682254885Sdumbbell tmp = RREG32(IH_RB_CNTL); 3683254885Sdumbbell tmp |= IH_WPTR_OVERFLOW_CLEAR; 3684254885Sdumbbell WREG32(IH_RB_CNTL, tmp); 3685254885Sdumbbell } 3686254885Sdumbbell return (wptr & rdev->ih.ptr_mask); 3687254885Sdumbbell} 3688254885Sdumbbell 3689254885Sdumbbell/* SI IV Ring 3690254885Sdumbbell * Each IV ring entry is 128 bits: 3691254885Sdumbbell * [7:0] - interrupt source id 3692254885Sdumbbell * [31:8] - reserved 3693254885Sdumbbell * [59:32] - interrupt source data 3694254885Sdumbbell * [63:60] - reserved 3695254885Sdumbbell * [71:64] - RINGID 3696254885Sdumbbell * [79:72] - VMID 3697254885Sdumbbell * [127:80] - reserved 3698254885Sdumbbell */ 3699254885Sdumbbellirqreturn_t si_irq_process(struct radeon_device *rdev) 3700254885Sdumbbell{ 3701254885Sdumbbell u32 wptr; 3702254885Sdumbbell u32 rptr; 3703254885Sdumbbell u32 src_id, src_data, ring_id; 3704254885Sdumbbell u32 ring_index; 3705254885Sdumbbell bool queue_hotplug = false; 3706254885Sdumbbell 3707254885Sdumbbell if (!rdev->ih.enabled || rdev->shutdown) 3708254885Sdumbbell return IRQ_NONE; 3709254885Sdumbbell 3710254885Sdumbbell wptr = si_get_ih_wptr(rdev); 3711254885Sdumbbell 3712254885Sdumbbellrestart_ih: 3713254885Sdumbbell /* is somebody else already processing irqs? */ 3714254885Sdumbbell if (atomic_xchg(&rdev->ih.lock, 1)) 3715254885Sdumbbell return IRQ_NONE; 3716254885Sdumbbell 3717254885Sdumbbell rptr = rdev->ih.rptr; 3718254885Sdumbbell DRM_DEBUG("si_irq_process start: rptr %d, wptr %d\n", rptr, wptr); 3719254885Sdumbbell 3720254885Sdumbbell /* Order reading of wptr vs. reading of IH ring data */ 3721254885Sdumbbell rmb(); 3722254885Sdumbbell 3723254885Sdumbbell /* display interrupts */ 3724254885Sdumbbell si_irq_ack(rdev); 3725254885Sdumbbell 3726254885Sdumbbell while (rptr != wptr) { 3727254885Sdumbbell /* wptr/rptr are in bytes! */ 3728254885Sdumbbell ring_index = rptr / 4; 3729254885Sdumbbell src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff; 3730254885Sdumbbell src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff; 3731254885Sdumbbell ring_id = le32_to_cpu(rdev->ih.ring[ring_index + 2]) & 0xff; 3732254885Sdumbbell 3733254885Sdumbbell switch (src_id) { 3734254885Sdumbbell case 1: /* D1 vblank/vline */ 3735254885Sdumbbell switch (src_data) { 3736254885Sdumbbell case 0: /* D1 vblank */ 3737254885Sdumbbell if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT) { 3738254885Sdumbbell if (rdev->irq.crtc_vblank_int[0]) { 3739254885Sdumbbell drm_handle_vblank(rdev->ddev, 0); 3740254885Sdumbbell rdev->pm.vblank_sync = true; 3741254885Sdumbbell DRM_WAKEUP(&rdev->irq.vblank_queue); 3742254885Sdumbbell } 3743254885Sdumbbell if (atomic_read(&rdev->irq.pflip[0])) 3744254885Sdumbbell radeon_crtc_handle_flip(rdev, 0); 3745254885Sdumbbell rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VBLANK_INTERRUPT; 3746254885Sdumbbell DRM_DEBUG("IH: D1 vblank\n"); 3747254885Sdumbbell } 3748254885Sdumbbell break; 3749254885Sdumbbell case 1: /* D1 vline */ 3750254885Sdumbbell if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT) { 3751254885Sdumbbell rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VLINE_INTERRUPT; 3752254885Sdumbbell DRM_DEBUG("IH: D1 vline\n"); 3753254885Sdumbbell } 3754254885Sdumbbell break; 3755254885Sdumbbell default: 3756254885Sdumbbell DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); 3757254885Sdumbbell break; 3758254885Sdumbbell } 3759254885Sdumbbell break; 3760254885Sdumbbell case 2: /* D2 vblank/vline */ 3761254885Sdumbbell switch (src_data) { 3762254885Sdumbbell case 0: /* D2 vblank */ 3763254885Sdumbbell if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT) { 3764254885Sdumbbell if (rdev->irq.crtc_vblank_int[1]) { 3765254885Sdumbbell drm_handle_vblank(rdev->ddev, 1); 3766254885Sdumbbell rdev->pm.vblank_sync = true; 3767254885Sdumbbell DRM_WAKEUP(&rdev->irq.vblank_queue); 3768254885Sdumbbell } 3769254885Sdumbbell if (atomic_read(&rdev->irq.pflip[1])) 3770254885Sdumbbell radeon_crtc_handle_flip(rdev, 1); 3771254885Sdumbbell rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT; 3772254885Sdumbbell DRM_DEBUG("IH: D2 vblank\n"); 3773254885Sdumbbell } 3774254885Sdumbbell break; 3775254885Sdumbbell case 1: /* D2 vline */ 3776254885Sdumbbell if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT) { 3777254885Sdumbbell rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT; 3778254885Sdumbbell DRM_DEBUG("IH: D2 vline\n"); 3779254885Sdumbbell } 3780254885Sdumbbell break; 3781254885Sdumbbell default: 3782254885Sdumbbell DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); 3783254885Sdumbbell break; 3784254885Sdumbbell } 3785254885Sdumbbell break; 3786254885Sdumbbell case 3: /* D3 vblank/vline */ 3787254885Sdumbbell switch (src_data) { 3788254885Sdumbbell case 0: /* D3 vblank */ 3789254885Sdumbbell if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) { 3790254885Sdumbbell if (rdev->irq.crtc_vblank_int[2]) { 3791254885Sdumbbell drm_handle_vblank(rdev->ddev, 2); 3792254885Sdumbbell rdev->pm.vblank_sync = true; 3793254885Sdumbbell DRM_WAKEUP(&rdev->irq.vblank_queue); 3794254885Sdumbbell } 3795254885Sdumbbell if (atomic_read(&rdev->irq.pflip[2])) 3796254885Sdumbbell radeon_crtc_handle_flip(rdev, 2); 3797254885Sdumbbell rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT; 3798254885Sdumbbell DRM_DEBUG("IH: D3 vblank\n"); 3799254885Sdumbbell } 3800254885Sdumbbell break; 3801254885Sdumbbell case 1: /* D3 vline */ 3802254885Sdumbbell if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) { 3803254885Sdumbbell rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT; 3804254885Sdumbbell DRM_DEBUG("IH: D3 vline\n"); 3805254885Sdumbbell } 3806254885Sdumbbell break; 3807254885Sdumbbell default: 3808254885Sdumbbell DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); 3809254885Sdumbbell break; 3810254885Sdumbbell } 3811254885Sdumbbell break; 3812254885Sdumbbell case 4: /* D4 vblank/vline */ 3813254885Sdumbbell switch (src_data) { 3814254885Sdumbbell case 0: /* D4 vblank */ 3815254885Sdumbbell if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) { 3816254885Sdumbbell if (rdev->irq.crtc_vblank_int[3]) { 3817254885Sdumbbell drm_handle_vblank(rdev->ddev, 3); 3818254885Sdumbbell rdev->pm.vblank_sync = true; 3819254885Sdumbbell DRM_WAKEUP(&rdev->irq.vblank_queue); 3820254885Sdumbbell } 3821254885Sdumbbell if (atomic_read(&rdev->irq.pflip[3])) 3822254885Sdumbbell radeon_crtc_handle_flip(rdev, 3); 3823254885Sdumbbell rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT; 3824254885Sdumbbell DRM_DEBUG("IH: D4 vblank\n"); 3825254885Sdumbbell } 3826254885Sdumbbell break; 3827254885Sdumbbell case 1: /* D4 vline */ 3828254885Sdumbbell if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) { 3829254885Sdumbbell rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT; 3830254885Sdumbbell DRM_DEBUG("IH: D4 vline\n"); 3831254885Sdumbbell } 3832254885Sdumbbell break; 3833254885Sdumbbell default: 3834254885Sdumbbell DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); 3835254885Sdumbbell break; 3836254885Sdumbbell } 3837254885Sdumbbell break; 3838254885Sdumbbell case 5: /* D5 vblank/vline */ 3839254885Sdumbbell switch (src_data) { 3840254885Sdumbbell case 0: /* D5 vblank */ 3841254885Sdumbbell if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) { 3842254885Sdumbbell if (rdev->irq.crtc_vblank_int[4]) { 3843254885Sdumbbell drm_handle_vblank(rdev->ddev, 4); 3844254885Sdumbbell rdev->pm.vblank_sync = true; 3845254885Sdumbbell DRM_WAKEUP(&rdev->irq.vblank_queue); 3846254885Sdumbbell } 3847254885Sdumbbell if (atomic_read(&rdev->irq.pflip[4])) 3848254885Sdumbbell radeon_crtc_handle_flip(rdev, 4); 3849254885Sdumbbell rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT; 3850254885Sdumbbell DRM_DEBUG("IH: D5 vblank\n"); 3851254885Sdumbbell } 3852254885Sdumbbell break; 3853254885Sdumbbell case 1: /* D5 vline */ 3854254885Sdumbbell if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) { 3855254885Sdumbbell rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT; 3856254885Sdumbbell DRM_DEBUG("IH: D5 vline\n"); 3857254885Sdumbbell } 3858254885Sdumbbell break; 3859254885Sdumbbell default: 3860254885Sdumbbell DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); 3861254885Sdumbbell break; 3862254885Sdumbbell } 3863254885Sdumbbell break; 3864254885Sdumbbell case 6: /* D6 vblank/vline */ 3865254885Sdumbbell switch (src_data) { 3866254885Sdumbbell case 0: /* D6 vblank */ 3867254885Sdumbbell if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) { 3868254885Sdumbbell if (rdev->irq.crtc_vblank_int[5]) { 3869254885Sdumbbell drm_handle_vblank(rdev->ddev, 5); 3870254885Sdumbbell rdev->pm.vblank_sync = true; 3871254885Sdumbbell DRM_WAKEUP(&rdev->irq.vblank_queue); 3872254885Sdumbbell } 3873254885Sdumbbell if (atomic_read(&rdev->irq.pflip[5])) 3874254885Sdumbbell radeon_crtc_handle_flip(rdev, 5); 3875254885Sdumbbell rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT; 3876254885Sdumbbell DRM_DEBUG("IH: D6 vblank\n"); 3877254885Sdumbbell } 3878254885Sdumbbell break; 3879254885Sdumbbell case 1: /* D6 vline */ 3880254885Sdumbbell if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) { 3881254885Sdumbbell rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT; 3882254885Sdumbbell DRM_DEBUG("IH: D6 vline\n"); 3883254885Sdumbbell } 3884254885Sdumbbell break; 3885254885Sdumbbell default: 3886254885Sdumbbell DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); 3887254885Sdumbbell break; 3888254885Sdumbbell } 3889254885Sdumbbell break; 3890254885Sdumbbell case 42: /* HPD hotplug */ 3891254885Sdumbbell switch (src_data) { 3892254885Sdumbbell case 0: 3893254885Sdumbbell if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) { 3894254885Sdumbbell rdev->irq.stat_regs.evergreen.disp_int &= ~DC_HPD1_INTERRUPT; 3895254885Sdumbbell queue_hotplug = true; 3896254885Sdumbbell DRM_DEBUG("IH: HPD1\n"); 3897254885Sdumbbell } 3898254885Sdumbbell break; 3899254885Sdumbbell case 1: 3900254885Sdumbbell if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) { 3901254885Sdumbbell rdev->irq.stat_regs.evergreen.disp_int_cont &= ~DC_HPD2_INTERRUPT; 3902254885Sdumbbell queue_hotplug = true; 3903254885Sdumbbell DRM_DEBUG("IH: HPD2\n"); 3904254885Sdumbbell } 3905254885Sdumbbell break; 3906254885Sdumbbell case 2: 3907254885Sdumbbell if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) { 3908254885Sdumbbell rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~DC_HPD3_INTERRUPT; 3909254885Sdumbbell queue_hotplug = true; 3910254885Sdumbbell DRM_DEBUG("IH: HPD3\n"); 3911254885Sdumbbell } 3912254885Sdumbbell break; 3913254885Sdumbbell case 3: 3914254885Sdumbbell if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) { 3915254885Sdumbbell rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~DC_HPD4_INTERRUPT; 3916254885Sdumbbell queue_hotplug = true; 3917254885Sdumbbell DRM_DEBUG("IH: HPD4\n"); 3918254885Sdumbbell } 3919254885Sdumbbell break; 3920254885Sdumbbell case 4: 3921254885Sdumbbell if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) { 3922254885Sdumbbell rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~DC_HPD5_INTERRUPT; 3923254885Sdumbbell queue_hotplug = true; 3924254885Sdumbbell DRM_DEBUG("IH: HPD5\n"); 3925254885Sdumbbell } 3926254885Sdumbbell break; 3927254885Sdumbbell case 5: 3928254885Sdumbbell if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) { 3929254885Sdumbbell rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~DC_HPD6_INTERRUPT; 3930254885Sdumbbell queue_hotplug = true; 3931254885Sdumbbell DRM_DEBUG("IH: HPD6\n"); 3932254885Sdumbbell } 3933254885Sdumbbell break; 3934254885Sdumbbell default: 3935254885Sdumbbell DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); 3936254885Sdumbbell break; 3937254885Sdumbbell } 3938254885Sdumbbell break; 3939254885Sdumbbell case 146: 3940254885Sdumbbell case 147: 3941254885Sdumbbell dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data); 3942254885Sdumbbell dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n", 3943254885Sdumbbell RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR)); 3944254885Sdumbbell dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n", 3945254885Sdumbbell RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS)); 3946254885Sdumbbell /* reset addr and status */ 3947254885Sdumbbell WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1); 3948254885Sdumbbell break; 3949254885Sdumbbell case 176: /* RINGID0 CP_INT */ 3950254885Sdumbbell radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX); 3951254885Sdumbbell break; 3952254885Sdumbbell case 177: /* RINGID1 CP_INT */ 3953254885Sdumbbell radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX); 3954254885Sdumbbell break; 3955254885Sdumbbell case 178: /* RINGID2 CP_INT */ 3956254885Sdumbbell radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX); 3957254885Sdumbbell break; 3958254885Sdumbbell case 181: /* CP EOP event */ 3959254885Sdumbbell DRM_DEBUG("IH: CP EOP\n"); 3960254885Sdumbbell switch (ring_id) { 3961254885Sdumbbell case 0: 3962254885Sdumbbell radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX); 3963254885Sdumbbell break; 3964254885Sdumbbell case 1: 3965254885Sdumbbell radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX); 3966254885Sdumbbell break; 3967254885Sdumbbell case 2: 3968254885Sdumbbell radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX); 3969254885Sdumbbell break; 3970254885Sdumbbell } 3971254885Sdumbbell break; 3972254885Sdumbbell case 224: /* DMA trap event */ 3973254885Sdumbbell DRM_DEBUG("IH: DMA trap\n"); 3974254885Sdumbbell radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX); 3975254885Sdumbbell break; 3976254885Sdumbbell case 233: /* GUI IDLE */ 3977254885Sdumbbell DRM_DEBUG("IH: GUI idle\n"); 3978254885Sdumbbell break; 3979254885Sdumbbell case 244: /* DMA trap event */ 3980254885Sdumbbell DRM_DEBUG("IH: DMA1 trap\n"); 3981254885Sdumbbell radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX); 3982254885Sdumbbell break; 3983254885Sdumbbell default: 3984254885Sdumbbell DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); 3985254885Sdumbbell break; 3986254885Sdumbbell } 3987254885Sdumbbell 3988254885Sdumbbell /* wptr/rptr are in bytes! */ 3989254885Sdumbbell rptr += 16; 3990254885Sdumbbell rptr &= rdev->ih.ptr_mask; 3991254885Sdumbbell } 3992254885Sdumbbell if (queue_hotplug) 3993254885Sdumbbell taskqueue_enqueue(rdev->tq, &rdev->hotplug_work); 3994254885Sdumbbell rdev->ih.rptr = rptr; 3995254885Sdumbbell WREG32(IH_RB_RPTR, rdev->ih.rptr); 3996254885Sdumbbell atomic_set(&rdev->ih.lock, 0); 3997254885Sdumbbell 3998254885Sdumbbell /* make sure wptr hasn't changed while processing */ 3999254885Sdumbbell wptr = si_get_ih_wptr(rdev); 4000254885Sdumbbell if (wptr != rptr) 4001254885Sdumbbell goto restart_ih; 4002254885Sdumbbell 4003254885Sdumbbell return IRQ_HANDLED; 4004254885Sdumbbell} 4005254885Sdumbbell 4006254885Sdumbbell/** 4007254885Sdumbbell * si_copy_dma - copy pages using the DMA engine 4008254885Sdumbbell * 4009254885Sdumbbell * @rdev: radeon_device pointer 4010254885Sdumbbell * @src_offset: src GPU address 4011254885Sdumbbell * @dst_offset: dst GPU address 4012254885Sdumbbell * @num_gpu_pages: number of GPU pages to xfer 4013254885Sdumbbell * @fence: radeon fence object 4014254885Sdumbbell * 4015254885Sdumbbell * Copy GPU paging using the DMA engine (SI). 4016254885Sdumbbell * Used by the radeon ttm implementation to move pages if 4017254885Sdumbbell * registered as the asic copy callback. 4018254885Sdumbbell */ 4019254885Sdumbbellint si_copy_dma(struct radeon_device *rdev, 4020254885Sdumbbell uint64_t src_offset, uint64_t dst_offset, 4021254885Sdumbbell unsigned num_gpu_pages, 4022254885Sdumbbell struct radeon_fence **fence) 4023254885Sdumbbell{ 4024254885Sdumbbell struct radeon_semaphore *sem = NULL; 4025254885Sdumbbell int ring_index = rdev->asic->copy.dma_ring_index; 4026254885Sdumbbell struct radeon_ring *ring = &rdev->ring[ring_index]; 4027254885Sdumbbell u32 size_in_bytes, cur_size_in_bytes; 4028254885Sdumbbell int i, num_loops; 4029254885Sdumbbell int r = 0; 4030254885Sdumbbell 4031254885Sdumbbell r = radeon_semaphore_create(rdev, &sem); 4032254885Sdumbbell if (r) { 4033254885Sdumbbell DRM_ERROR("radeon: moving bo (%d).\n", r); 4034254885Sdumbbell return r; 4035254885Sdumbbell } 4036254885Sdumbbell 4037254885Sdumbbell size_in_bytes = (num_gpu_pages << RADEON_GPU_PAGE_SHIFT); 4038254885Sdumbbell num_loops = DIV_ROUND_UP(size_in_bytes, 0xfffff); 4039254885Sdumbbell r = radeon_ring_lock(rdev, ring, num_loops * 5 + 11); 4040254885Sdumbbell if (r) { 4041254885Sdumbbell DRM_ERROR("radeon: moving bo (%d).\n", r); 4042254885Sdumbbell radeon_semaphore_free(rdev, &sem, NULL); 4043254885Sdumbbell return r; 4044254885Sdumbbell } 4045254885Sdumbbell 4046254885Sdumbbell if (radeon_fence_need_sync(*fence, ring->idx)) { 4047254885Sdumbbell radeon_semaphore_sync_rings(rdev, sem, (*fence)->ring, 4048254885Sdumbbell ring->idx); 4049254885Sdumbbell radeon_fence_note_sync(*fence, ring->idx); 4050254885Sdumbbell } else { 4051254885Sdumbbell radeon_semaphore_free(rdev, &sem, NULL); 4052254885Sdumbbell } 4053254885Sdumbbell 4054254885Sdumbbell for (i = 0; i < num_loops; i++) { 4055254885Sdumbbell cur_size_in_bytes = size_in_bytes; 4056254885Sdumbbell if (cur_size_in_bytes > 0xFFFFF) 4057254885Sdumbbell cur_size_in_bytes = 0xFFFFF; 4058254885Sdumbbell size_in_bytes -= cur_size_in_bytes; 4059254885Sdumbbell radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_COPY, 1, 0, 0, cur_size_in_bytes)); 4060254885Sdumbbell radeon_ring_write(ring, dst_offset & 0xffffffff); 4061254885Sdumbbell radeon_ring_write(ring, src_offset & 0xffffffff); 4062254885Sdumbbell radeon_ring_write(ring, upper_32_bits(dst_offset) & 0xff); 4063254885Sdumbbell radeon_ring_write(ring, upper_32_bits(src_offset) & 0xff); 4064254885Sdumbbell src_offset += cur_size_in_bytes; 4065254885Sdumbbell dst_offset += cur_size_in_bytes; 4066254885Sdumbbell } 4067254885Sdumbbell 4068254885Sdumbbell r = radeon_fence_emit(rdev, fence, ring->idx); 4069254885Sdumbbell if (r) { 4070254885Sdumbbell radeon_ring_unlock_undo(rdev, ring); 4071254885Sdumbbell return r; 4072254885Sdumbbell } 4073254885Sdumbbell 4074254885Sdumbbell radeon_ring_unlock_commit(rdev, ring); 4075254885Sdumbbell radeon_semaphore_free(rdev, &sem, *fence); 4076254885Sdumbbell 4077254885Sdumbbell return r; 4078254885Sdumbbell} 4079254885Sdumbbell 4080254885Sdumbbell/* 4081254885Sdumbbell * startup/shutdown callbacks 4082254885Sdumbbell */ 4083254885Sdumbbellstatic int si_startup(struct radeon_device *rdev) 4084254885Sdumbbell{ 4085254885Sdumbbell struct radeon_ring *ring; 4086254885Sdumbbell int r; 4087254885Sdumbbell 4088254885Sdumbbell if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw || 4089254885Sdumbbell !rdev->rlc_fw || !rdev->mc_fw) { 4090254885Sdumbbell r = si_init_microcode(rdev); 4091254885Sdumbbell if (r) { 4092254885Sdumbbell DRM_ERROR("Failed to load firmware!\n"); 4093254885Sdumbbell return r; 4094254885Sdumbbell } 4095254885Sdumbbell } 4096254885Sdumbbell 4097254885Sdumbbell r = si_mc_load_microcode(rdev); 4098254885Sdumbbell if (r) { 4099254885Sdumbbell DRM_ERROR("Failed to load MC firmware!\n"); 4100254885Sdumbbell return r; 4101254885Sdumbbell } 4102254885Sdumbbell 4103254885Sdumbbell r = r600_vram_scratch_init(rdev); 4104254885Sdumbbell if (r) 4105254885Sdumbbell return r; 4106254885Sdumbbell 4107254885Sdumbbell si_mc_program(rdev); 4108254885Sdumbbell r = si_pcie_gart_enable(rdev); 4109254885Sdumbbell if (r) 4110254885Sdumbbell return r; 4111254885Sdumbbell si_gpu_init(rdev); 4112254885Sdumbbell 4113254885Sdumbbell#if 0 4114254885Sdumbbell r = evergreen_blit_init(rdev); 4115254885Sdumbbell if (r) { 4116254885Sdumbbell r600_blit_fini(rdev); 4117254885Sdumbbell rdev->asic->copy = NULL; 4118254885Sdumbbell dev_warn(rdev->dev, "failed blitter (%d) falling back to memcpy\n", r); 4119254885Sdumbbell } 4120254885Sdumbbell#endif 4121254885Sdumbbell /* allocate rlc buffers */ 4122254885Sdumbbell r = si_rlc_init(rdev); 4123254885Sdumbbell if (r) { 4124254885Sdumbbell DRM_ERROR("Failed to init rlc BOs!\n"); 4125254885Sdumbbell return r; 4126254885Sdumbbell } 4127254885Sdumbbell 4128254885Sdumbbell /* allocate wb buffer */ 4129254885Sdumbbell r = radeon_wb_init(rdev); 4130254885Sdumbbell if (r) 4131254885Sdumbbell return r; 4132254885Sdumbbell 4133254885Sdumbbell r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX); 4134254885Sdumbbell if (r) { 4135254885Sdumbbell dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r); 4136254885Sdumbbell return r; 4137254885Sdumbbell } 4138254885Sdumbbell 4139254885Sdumbbell r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP1_INDEX); 4140254885Sdumbbell if (r) { 4141254885Sdumbbell dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r); 4142254885Sdumbbell return r; 4143254885Sdumbbell } 4144254885Sdumbbell 4145254885Sdumbbell r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP2_INDEX); 4146254885Sdumbbell if (r) { 4147254885Sdumbbell dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r); 4148254885Sdumbbell return r; 4149254885Sdumbbell } 4150254885Sdumbbell 4151254885Sdumbbell r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX); 4152254885Sdumbbell if (r) { 4153254885Sdumbbell dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r); 4154254885Sdumbbell return r; 4155254885Sdumbbell } 4156254885Sdumbbell 4157254885Sdumbbell r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_DMA1_INDEX); 4158254885Sdumbbell if (r) { 4159254885Sdumbbell dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r); 4160254885Sdumbbell return r; 4161254885Sdumbbell } 4162254885Sdumbbell 4163254885Sdumbbell /* Enable IRQ */ 4164254885Sdumbbell r = si_irq_init(rdev); 4165254885Sdumbbell if (r) { 4166254885Sdumbbell DRM_ERROR("radeon: IH init failed (%d).\n", r); 4167254885Sdumbbell radeon_irq_kms_fini(rdev); 4168254885Sdumbbell return r; 4169254885Sdumbbell } 4170254885Sdumbbell si_irq_set(rdev); 4171254885Sdumbbell 4172254885Sdumbbell ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]; 4173254885Sdumbbell r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET, 4174254885Sdumbbell CP_RB0_RPTR, CP_RB0_WPTR, 4175254885Sdumbbell 0, 0xfffff, RADEON_CP_PACKET2); 4176254885Sdumbbell if (r) 4177254885Sdumbbell return r; 4178254885Sdumbbell 4179254885Sdumbbell ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX]; 4180254885Sdumbbell r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP1_RPTR_OFFSET, 4181254885Sdumbbell CP_RB1_RPTR, CP_RB1_WPTR, 4182254885Sdumbbell 0, 0xfffff, RADEON_CP_PACKET2); 4183254885Sdumbbell if (r) 4184254885Sdumbbell return r; 4185254885Sdumbbell 4186254885Sdumbbell ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX]; 4187254885Sdumbbell r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP2_RPTR_OFFSET, 4188254885Sdumbbell CP_RB2_RPTR, CP_RB2_WPTR, 4189254885Sdumbbell 0, 0xfffff, RADEON_CP_PACKET2); 4190254885Sdumbbell if (r) 4191254885Sdumbbell return r; 4192254885Sdumbbell 4193254885Sdumbbell ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX]; 4194254885Sdumbbell r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET, 4195254885Sdumbbell DMA_RB_RPTR + DMA0_REGISTER_OFFSET, 4196254885Sdumbbell DMA_RB_WPTR + DMA0_REGISTER_OFFSET, 4197254885Sdumbbell 2, 0x3fffc, DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0, 0)); 4198254885Sdumbbell if (r) 4199254885Sdumbbell return r; 4200254885Sdumbbell 4201254885Sdumbbell ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX]; 4202254885Sdumbbell r = radeon_ring_init(rdev, ring, ring->ring_size, CAYMAN_WB_DMA1_RPTR_OFFSET, 4203254885Sdumbbell DMA_RB_RPTR + DMA1_REGISTER_OFFSET, 4204254885Sdumbbell DMA_RB_WPTR + DMA1_REGISTER_OFFSET, 4205254885Sdumbbell 2, 0x3fffc, DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0, 0)); 4206254885Sdumbbell if (r) 4207254885Sdumbbell return r; 4208254885Sdumbbell 4209254885Sdumbbell r = si_cp_load_microcode(rdev); 4210254885Sdumbbell if (r) 4211254885Sdumbbell return r; 4212254885Sdumbbell r = si_cp_resume(rdev); 4213254885Sdumbbell if (r) 4214254885Sdumbbell return r; 4215254885Sdumbbell 4216254885Sdumbbell r = cayman_dma_resume(rdev); 4217254885Sdumbbell if (r) 4218254885Sdumbbell return r; 4219254885Sdumbbell 4220254885Sdumbbell r = radeon_ib_pool_init(rdev); 4221254885Sdumbbell if (r) { 4222254885Sdumbbell dev_err(rdev->dev, "IB initialization failed (%d).\n", r); 4223254885Sdumbbell return r; 4224254885Sdumbbell } 4225254885Sdumbbell 4226254885Sdumbbell r = radeon_vm_manager_init(rdev); 4227254885Sdumbbell if (r) { 4228254885Sdumbbell dev_err(rdev->dev, "vm manager initialization failed (%d).\n", r); 4229254885Sdumbbell return r; 4230254885Sdumbbell } 4231254885Sdumbbell 4232254885Sdumbbell return 0; 4233254885Sdumbbell} 4234254885Sdumbbell 4235254885Sdumbbellint si_resume(struct radeon_device *rdev) 4236254885Sdumbbell{ 4237254885Sdumbbell int r; 4238254885Sdumbbell 4239254885Sdumbbell /* Do not reset GPU before posting, on rv770 hw unlike on r500 hw, 4240254885Sdumbbell * posting will perform necessary task to bring back GPU into good 4241254885Sdumbbell * shape. 4242254885Sdumbbell */ 4243254885Sdumbbell /* post card */ 4244254885Sdumbbell atom_asic_init(rdev->mode_info.atom_context); 4245254885Sdumbbell 4246254885Sdumbbell rdev->accel_working = true; 4247254885Sdumbbell r = si_startup(rdev); 4248254885Sdumbbell if (r) { 4249254885Sdumbbell DRM_ERROR("si startup failed on resume\n"); 4250254885Sdumbbell rdev->accel_working = false; 4251254885Sdumbbell return r; 4252254885Sdumbbell } 4253254885Sdumbbell 4254254885Sdumbbell return r; 4255254885Sdumbbell 4256254885Sdumbbell} 4257254885Sdumbbell 4258254885Sdumbbellint si_suspend(struct radeon_device *rdev) 4259254885Sdumbbell{ 4260254885Sdumbbell si_cp_enable(rdev, false); 4261254885Sdumbbell cayman_dma_stop(rdev); 4262254885Sdumbbell si_irq_suspend(rdev); 4263254885Sdumbbell radeon_wb_disable(rdev); 4264254885Sdumbbell si_pcie_gart_disable(rdev); 4265254885Sdumbbell return 0; 4266254885Sdumbbell} 4267254885Sdumbbell 4268254885Sdumbbell/* Plan is to move initialization in that function and use 4269254885Sdumbbell * helper function so that radeon_device_init pretty much 4270254885Sdumbbell * do nothing more than calling asic specific function. This 4271254885Sdumbbell * should also allow to remove a bunch of callback function 4272254885Sdumbbell * like vram_info. 4273254885Sdumbbell */ 4274254885Sdumbbellint si_init(struct radeon_device *rdev) 4275254885Sdumbbell{ 4276254885Sdumbbell struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]; 4277254885Sdumbbell int r; 4278254885Sdumbbell 4279254885Sdumbbell /* Read BIOS */ 4280254885Sdumbbell if (!radeon_get_bios(rdev)) { 4281254885Sdumbbell if (ASIC_IS_AVIVO(rdev)) 4282254885Sdumbbell return -EINVAL; 4283254885Sdumbbell } 4284254885Sdumbbell /* Must be an ATOMBIOS */ 4285254885Sdumbbell if (!rdev->is_atom_bios) { 4286254885Sdumbbell dev_err(rdev->dev, "Expecting atombios for cayman GPU\n"); 4287254885Sdumbbell return -EINVAL; 4288254885Sdumbbell } 4289254885Sdumbbell r = radeon_atombios_init(rdev); 4290254885Sdumbbell if (r) 4291254885Sdumbbell return r; 4292254885Sdumbbell 4293254885Sdumbbell /* Post card if necessary */ 4294254885Sdumbbell if (!radeon_card_posted(rdev)) { 4295254885Sdumbbell if (!rdev->bios) { 4296254885Sdumbbell dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n"); 4297254885Sdumbbell return -EINVAL; 4298254885Sdumbbell } 4299254885Sdumbbell DRM_INFO("GPU not posted. posting now...\n"); 4300254885Sdumbbell atom_asic_init(rdev->mode_info.atom_context); 4301254885Sdumbbell } 4302254885Sdumbbell /* Initialize scratch registers */ 4303254885Sdumbbell si_scratch_init(rdev); 4304254885Sdumbbell /* Initialize surface registers */ 4305254885Sdumbbell radeon_surface_init(rdev); 4306254885Sdumbbell /* Initialize clocks */ 4307254885Sdumbbell radeon_get_clock_info(rdev->ddev); 4308254885Sdumbbell 4309254885Sdumbbell /* Fence driver */ 4310254885Sdumbbell r = radeon_fence_driver_init(rdev); 4311254885Sdumbbell if (r) 4312254885Sdumbbell return r; 4313254885Sdumbbell 4314254885Sdumbbell /* initialize memory controller */ 4315254885Sdumbbell r = si_mc_init(rdev); 4316254885Sdumbbell if (r) 4317254885Sdumbbell return r; 4318254885Sdumbbell /* Memory manager */ 4319254885Sdumbbell r = radeon_bo_init(rdev); 4320254885Sdumbbell if (r) 4321254885Sdumbbell return r; 4322254885Sdumbbell 4323254885Sdumbbell r = radeon_irq_kms_init(rdev); 4324254885Sdumbbell if (r) 4325254885Sdumbbell return r; 4326254885Sdumbbell 4327254885Sdumbbell ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]; 4328254885Sdumbbell ring->ring_obj = NULL; 4329254885Sdumbbell r600_ring_init(rdev, ring, 1024 * 1024); 4330254885Sdumbbell 4331254885Sdumbbell ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX]; 4332254885Sdumbbell ring->ring_obj = NULL; 4333254885Sdumbbell r600_ring_init(rdev, ring, 1024 * 1024); 4334254885Sdumbbell 4335254885Sdumbbell ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX]; 4336254885Sdumbbell ring->ring_obj = NULL; 4337254885Sdumbbell r600_ring_init(rdev, ring, 1024 * 1024); 4338254885Sdumbbell 4339254885Sdumbbell ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX]; 4340254885Sdumbbell ring->ring_obj = NULL; 4341254885Sdumbbell r600_ring_init(rdev, ring, 64 * 1024); 4342254885Sdumbbell 4343254885Sdumbbell ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX]; 4344254885Sdumbbell ring->ring_obj = NULL; 4345254885Sdumbbell r600_ring_init(rdev, ring, 64 * 1024); 4346254885Sdumbbell 4347254885Sdumbbell rdev->ih.ring_obj = NULL; 4348254885Sdumbbell r600_ih_ring_init(rdev, 64 * 1024); 4349254885Sdumbbell 4350254885Sdumbbell r = r600_pcie_gart_init(rdev); 4351254885Sdumbbell if (r) 4352254885Sdumbbell return r; 4353254885Sdumbbell 4354254885Sdumbbell rdev->accel_working = true; 4355254885Sdumbbell r = si_startup(rdev); 4356254885Sdumbbell if (r) { 4357254885Sdumbbell dev_err(rdev->dev, "disabling GPU acceleration\n"); 4358254885Sdumbbell si_cp_fini(rdev); 4359254885Sdumbbell cayman_dma_fini(rdev); 4360254885Sdumbbell si_irq_fini(rdev); 4361254885Sdumbbell si_rlc_fini(rdev); 4362254885Sdumbbell radeon_wb_fini(rdev); 4363254885Sdumbbell radeon_ib_pool_fini(rdev); 4364254885Sdumbbell radeon_vm_manager_fini(rdev); 4365254885Sdumbbell radeon_irq_kms_fini(rdev); 4366254885Sdumbbell si_pcie_gart_fini(rdev); 4367254885Sdumbbell rdev->accel_working = false; 4368254885Sdumbbell } 4369254885Sdumbbell 4370254885Sdumbbell /* Don't start up if the MC ucode is missing. 4371254885Sdumbbell * The default clocks and voltages before the MC ucode 4372254885Sdumbbell * is loaded are not suffient for advanced operations. 4373254885Sdumbbell */ 4374254885Sdumbbell if (!rdev->mc_fw) { 4375254885Sdumbbell DRM_ERROR("radeon: MC ucode required for NI+.\n"); 4376254885Sdumbbell return -EINVAL; 4377254885Sdumbbell } 4378254885Sdumbbell 4379254885Sdumbbell return 0; 4380254885Sdumbbell} 4381254885Sdumbbell 4382254885Sdumbbellvoid si_fini(struct radeon_device *rdev) 4383254885Sdumbbell{ 4384254885Sdumbbell#if 0 4385254885Sdumbbell r600_blit_fini(rdev); 4386254885Sdumbbell#endif 4387254885Sdumbbell si_cp_fini(rdev); 4388254885Sdumbbell cayman_dma_fini(rdev); 4389254885Sdumbbell si_irq_fini(rdev); 4390254885Sdumbbell si_rlc_fini(rdev); 4391254885Sdumbbell radeon_wb_fini(rdev); 4392254885Sdumbbell radeon_vm_manager_fini(rdev); 4393254885Sdumbbell radeon_ib_pool_fini(rdev); 4394254885Sdumbbell radeon_irq_kms_fini(rdev); 4395254885Sdumbbell si_pcie_gart_fini(rdev); 4396254885Sdumbbell r600_vram_scratch_fini(rdev); 4397254885Sdumbbell radeon_gem_fini(rdev); 4398254885Sdumbbell radeon_fence_driver_fini(rdev); 4399254885Sdumbbell radeon_bo_fini(rdev); 4400254885Sdumbbell radeon_atombios_fini(rdev); 4401254885Sdumbbell si_fini_microcode(rdev); 4402254885Sdumbbell free(rdev->bios, DRM_MEM_DRIVER); 4403254885Sdumbbell rdev->bios = NULL; 4404254885Sdumbbell} 4405254885Sdumbbell 4406254885Sdumbbell/** 4407254885Sdumbbell * si_get_gpu_clock - return GPU clock counter snapshot 4408254885Sdumbbell * 4409254885Sdumbbell * @rdev: radeon_device pointer 4410254885Sdumbbell * 4411254885Sdumbbell * Fetches a GPU clock counter snapshot (SI). 4412254885Sdumbbell * Returns the 64 bit clock counter snapshot. 4413254885Sdumbbell */ 4414254885Sdumbbelluint64_t si_get_gpu_clock(struct radeon_device *rdev) 4415254885Sdumbbell{ 4416254885Sdumbbell uint64_t clock; 4417254885Sdumbbell 4418254885Sdumbbell sx_xlock(&rdev->gpu_clock_mutex); 4419254885Sdumbbell WREG32(RLC_CAPTURE_GPU_CLOCK_COUNT, 1); 4420254885Sdumbbell clock = (uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_LSB) | 4421254885Sdumbbell ((uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_MSB) << 32ULL); 4422254885Sdumbbell sx_xunlock(&rdev->gpu_clock_mutex); 4423254885Sdumbbell return clock; 4424254885Sdumbbell} 4425