1/* $NetBSD: radeon_evergreen.c,v 1.6 2023/09/30 10:46:45 mrg Exp $ */ 2 3/* 4 * Copyright 2010 Advanced Micro Devices, Inc. 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining a 7 * copy of this software and associated documentation files (the "Software"), 8 * to deal in the Software without restriction, including without limitation 9 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 10 * and/or sell copies of the Software, and to permit persons to whom the 11 * Software is furnished to do so, subject to the following conditions: 12 * 13 * The above copyright notice and this permission notice shall be included in 14 * all copies or substantial portions of the Software. 15 * 16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 19 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 22 * OTHER DEALINGS IN THE SOFTWARE. 23 * 24 * Authors: Alex Deucher 25 */ 26 27#include <sys/cdefs.h> 28__KERNEL_RCSID(0, "$NetBSD: radeon_evergreen.c,v 1.6 2023/09/30 10:46:45 mrg Exp $"); 29 30#include <linux/firmware.h> 31#include <linux/pci.h> 32#include <linux/slab.h> 33 34#include <drm/drm_vblank.h> 35#include <drm/radeon_drm.h> 36 37#include "atom.h" 38#include "avivod.h" 39#include "evergreen_blit_shaders.h" 40#include "evergreen_reg.h" 41#include "evergreend.h" 42#include "radeon.h" 43#include "radeon_asic.h" 44#include "radeon_audio.h" 45#include "radeon_ucode.h" 46 47#define DC_HPDx_CONTROL(x) (DC_HPD1_CONTROL + (x * 0xc)) 48#define DC_HPDx_INT_CONTROL(x) (DC_HPD1_INT_CONTROL + (x * 0xc)) 49#define DC_HPDx_INT_STATUS_REG(x) (DC_HPD1_INT_STATUS + (x * 0xc)) 50 51/* 52 * Indirect registers accessor 53 */ 54u32 eg_cg_rreg(struct radeon_device *rdev, u32 reg) 55{ 56 unsigned long flags; 57 u32 r; 58 59 spin_lock_irqsave(&rdev->cg_idx_lock, flags); 60 WREG32(EVERGREEN_CG_IND_ADDR, ((reg) & 0xffff)); 61 r = RREG32(EVERGREEN_CG_IND_DATA); 62 spin_unlock_irqrestore(&rdev->cg_idx_lock, flags); 63 return r; 64} 65 66void eg_cg_wreg(struct radeon_device *rdev, u32 reg, u32 v) 67{ 68 unsigned long flags; 69 70 spin_lock_irqsave(&rdev->cg_idx_lock, flags); 71 WREG32(EVERGREEN_CG_IND_ADDR, ((reg) & 0xffff)); 72 WREG32(EVERGREEN_CG_IND_DATA, (v)); 73 spin_unlock_irqrestore(&rdev->cg_idx_lock, flags); 74} 75 76u32 eg_pif_phy0_rreg(struct radeon_device *rdev, u32 reg) 77{ 78 unsigned long flags; 79 u32 r; 80 81 spin_lock_irqsave(&rdev->pif_idx_lock, flags); 82 WREG32(EVERGREEN_PIF_PHY0_INDEX, ((reg) & 0xffff)); 83 r = RREG32(EVERGREEN_PIF_PHY0_DATA); 84 spin_unlock_irqrestore(&rdev->pif_idx_lock, flags); 85 return r; 86} 87 88void eg_pif_phy0_wreg(struct radeon_device *rdev, u32 reg, u32 v) 89{ 90 unsigned long flags; 91 92 spin_lock_irqsave(&rdev->pif_idx_lock, flags); 93 WREG32(EVERGREEN_PIF_PHY0_INDEX, ((reg) & 0xffff)); 94 WREG32(EVERGREEN_PIF_PHY0_DATA, (v)); 95 spin_unlock_irqrestore(&rdev->pif_idx_lock, flags); 96} 97 98u32 eg_pif_phy1_rreg(struct radeon_device *rdev, u32 reg) 99{ 100 unsigned long flags; 101 u32 r; 102 103 spin_lock_irqsave(&rdev->pif_idx_lock, flags); 104 WREG32(EVERGREEN_PIF_PHY1_INDEX, ((reg) & 0xffff)); 105 r = RREG32(EVERGREEN_PIF_PHY1_DATA); 106 spin_unlock_irqrestore(&rdev->pif_idx_lock, flags); 107 return r; 108} 109 110void eg_pif_phy1_wreg(struct radeon_device *rdev, u32 reg, u32 v) 111{ 112 unsigned long flags; 113 114 spin_lock_irqsave(&rdev->pif_idx_lock, flags); 115 WREG32(EVERGREEN_PIF_PHY1_INDEX, ((reg) & 0xffff)); 116 WREG32(EVERGREEN_PIF_PHY1_DATA, (v)); 117 spin_unlock_irqrestore(&rdev->pif_idx_lock, flags); 118} 119 120static const u32 crtc_offsets[6] = 121{ 122 EVERGREEN_CRTC0_REGISTER_OFFSET, 123 EVERGREEN_CRTC1_REGISTER_OFFSET, 124 EVERGREEN_CRTC2_REGISTER_OFFSET, 125 EVERGREEN_CRTC3_REGISTER_OFFSET, 126 EVERGREEN_CRTC4_REGISTER_OFFSET, 127 EVERGREEN_CRTC5_REGISTER_OFFSET 128}; 129 130#include "clearstate_evergreen.h" 131 132static const u32 sumo_rlc_save_restore_register_list[] = 133{ 134 0x98fc, 135 0x9830, 136 0x9834, 137 0x9838, 138 0x9870, 139 0x9874, 140 0x8a14, 141 0x8b24, 142 0x8bcc, 143 0x8b10, 144 0x8d00, 145 0x8d04, 146 0x8c00, 147 0x8c04, 148 0x8c08, 149 0x8c0c, 150 0x8d8c, 151 0x8c20, 152 0x8c24, 153 0x8c28, 154 0x8c18, 155 0x8c1c, 156 0x8cf0, 157 0x8e2c, 158 0x8e38, 159 0x8c30, 160 0x9508, 161 0x9688, 162 0x9608, 163 0x960c, 164 0x9610, 165 0x9614, 166 0x88c4, 167 0x88d4, 168 0xa008, 169 0x900c, 170 0x9100, 171 0x913c, 172 0x98f8, 173 0x98f4, 174 0x9b7c, 175 0x3f8c, 176 0x8950, 177 0x8954, 178 0x8a18, 179 0x8b28, 180 0x9144, 181 0x9148, 182 0x914c, 183 0x3f90, 184 0x3f94, 185 0x915c, 186 0x9160, 187 0x9178, 188 0x917c, 189 0x9180, 190 0x918c, 191 0x9190, 192 0x9194, 193 0x9198, 194 0x919c, 195 0x91a8, 196 0x91ac, 197 0x91b0, 198 0x91b4, 199 0x91b8, 200 0x91c4, 201 0x91c8, 202 0x91cc, 203 0x91d0, 204 0x91d4, 205 0x91e0, 206 0x91e4, 207 0x91ec, 208 0x91f0, 209 0x91f4, 210 0x9200, 211 0x9204, 212 0x929c, 213 0x9150, 214 0x802c, 215}; 216 217static void evergreen_gpu_init(struct radeon_device *rdev); 218void evergreen_fini(struct radeon_device *rdev); 219void evergreen_pcie_gen2_enable(struct radeon_device *rdev); 220void evergreen_program_aspm(struct radeon_device *rdev); 221extern void cayman_cp_int_cntl_setup(struct radeon_device *rdev, 222 int ring, u32 cp_int_cntl); 223extern void cayman_vm_decode_fault(struct radeon_device *rdev, 224 u32 status, u32 addr); 225void cik_init_cp_pg_table(struct radeon_device *rdev); 226 227extern u32 si_get_csb_size(struct radeon_device *rdev); 228extern void si_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer); 229extern u32 cik_get_csb_size(struct radeon_device *rdev); 230extern void cik_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer); 231extern void rv770_set_clk_bypass_mode(struct radeon_device *rdev); 232 233static const u32 evergreen_golden_registers[] = 234{ 235 0x3f90, 0xffff0000, 0xff000000, 236 0x9148, 0xffff0000, 0xff000000, 237 0x3f94, 0xffff0000, 0xff000000, 238 0x914c, 0xffff0000, 0xff000000, 239 0x9b7c, 0xffffffff, 0x00000000, 240 0x8a14, 0xffffffff, 0x00000007, 241 0x8b10, 0xffffffff, 0x00000000, 242 0x960c, 0xffffffff, 0x54763210, 243 0x88c4, 0xffffffff, 0x000000c2, 244 0x88d4, 0xffffffff, 0x00000010, 245 0x8974, 0xffffffff, 0x00000000, 246 0xc78, 0x00000080, 0x00000080, 247 0x5eb4, 0xffffffff, 0x00000002, 248 0x5e78, 0xffffffff, 0x001000f0, 249 0x6104, 0x01000300, 0x00000000, 250 0x5bc0, 0x00300000, 0x00000000, 251 0x7030, 0xffffffff, 0x00000011, 252 0x7c30, 0xffffffff, 0x00000011, 253 0x10830, 0xffffffff, 0x00000011, 254 0x11430, 0xffffffff, 0x00000011, 255 0x12030, 0xffffffff, 0x00000011, 256 0x12c30, 0xffffffff, 0x00000011, 257 0xd02c, 0xffffffff, 0x08421000, 258 0x240c, 0xffffffff, 0x00000380, 259 0x8b24, 0xffffffff, 0x00ff0fff, 260 0x28a4c, 0x06000000, 0x06000000, 261 0x10c, 0x00000001, 0x00000001, 262 0x8d00, 0xffffffff, 0x100e4848, 263 0x8d04, 0xffffffff, 0x00164745, 264 0x8c00, 0xffffffff, 0xe4000003, 265 0x8c04, 0xffffffff, 0x40600060, 266 0x8c08, 0xffffffff, 0x001c001c, 267 0x8cf0, 0xffffffff, 0x08e00620, 268 0x8c20, 0xffffffff, 0x00800080, 269 0x8c24, 0xffffffff, 0x00800080, 270 0x8c18, 0xffffffff, 0x20202078, 271 0x8c1c, 0xffffffff, 0x00001010, 272 0x28350, 0xffffffff, 0x00000000, 273 0xa008, 0xffffffff, 0x00010000, 274 0x5c4, 0xffffffff, 0x00000001, 275 0x9508, 0xffffffff, 0x00000002, 276 0x913c, 0x0000000f, 0x0000000a 277}; 278 279static const u32 evergreen_golden_registers2[] = 280{ 281 0x2f4c, 0xffffffff, 0x00000000, 282 0x54f4, 0xffffffff, 0x00000000, 283 0x54f0, 0xffffffff, 0x00000000, 284 0x5498, 0xffffffff, 0x00000000, 285 0x549c, 0xffffffff, 0x00000000, 286 0x5494, 0xffffffff, 0x00000000, 287 0x53cc, 0xffffffff, 0x00000000, 288 0x53c8, 0xffffffff, 0x00000000, 289 0x53c4, 0xffffffff, 0x00000000, 290 0x53c0, 0xffffffff, 0x00000000, 291 0x53bc, 0xffffffff, 0x00000000, 292 0x53b8, 0xffffffff, 0x00000000, 293 0x53b4, 0xffffffff, 0x00000000, 294 0x53b0, 0xffffffff, 0x00000000 295}; 296 297static const u32 cypress_mgcg_init[] = 298{ 299 0x802c, 0xffffffff, 0xc0000000, 300 0x5448, 0xffffffff, 0x00000100, 301 0x55e4, 0xffffffff, 0x00000100, 302 0x160c, 0xffffffff, 0x00000100, 303 0x5644, 0xffffffff, 0x00000100, 304 0xc164, 0xffffffff, 0x00000100, 305 0x8a18, 0xffffffff, 0x00000100, 306 0x897c, 0xffffffff, 0x06000100, 307 0x8b28, 0xffffffff, 0x00000100, 308 0x9144, 0xffffffff, 0x00000100, 309 0x9a60, 0xffffffff, 0x00000100, 310 0x9868, 0xffffffff, 0x00000100, 311 0x8d58, 0xffffffff, 0x00000100, 312 0x9510, 0xffffffff, 0x00000100, 313 0x949c, 0xffffffff, 0x00000100, 314 0x9654, 0xffffffff, 0x00000100, 315 0x9030, 0xffffffff, 0x00000100, 316 0x9034, 0xffffffff, 0x00000100, 317 0x9038, 0xffffffff, 0x00000100, 318 0x903c, 0xffffffff, 0x00000100, 319 0x9040, 0xffffffff, 0x00000100, 320 0xa200, 0xffffffff, 0x00000100, 321 0xa204, 0xffffffff, 0x00000100, 322 0xa208, 0xffffffff, 0x00000100, 323 0xa20c, 0xffffffff, 0x00000100, 324 0x971c, 0xffffffff, 0x00000100, 325 0x977c, 0xffffffff, 0x00000100, 326 0x3f80, 0xffffffff, 0x00000100, 327 0xa210, 0xffffffff, 0x00000100, 328 0xa214, 0xffffffff, 0x00000100, 329 0x4d8, 0xffffffff, 0x00000100, 330 0x9784, 0xffffffff, 0x00000100, 331 0x9698, 0xffffffff, 0x00000100, 332 0x4d4, 0xffffffff, 0x00000200, 333 0x30cc, 0xffffffff, 0x00000100, 334 0xd0c0, 0xffffffff, 0xff000100, 335 0x802c, 0xffffffff, 0x40000000, 336 0x915c, 0xffffffff, 0x00010000, 337 0x9160, 0xffffffff, 0x00030002, 338 0x9178, 0xffffffff, 0x00070000, 339 0x917c, 0xffffffff, 0x00030002, 340 0x9180, 0xffffffff, 0x00050004, 341 0x918c, 0xffffffff, 0x00010006, 342 0x9190, 0xffffffff, 0x00090008, 343 0x9194, 0xffffffff, 0x00070000, 344 0x9198, 0xffffffff, 0x00030002, 345 0x919c, 0xffffffff, 0x00050004, 346 0x91a8, 0xffffffff, 0x00010006, 347 0x91ac, 0xffffffff, 0x00090008, 348 0x91b0, 0xffffffff, 0x00070000, 349 0x91b4, 0xffffffff, 0x00030002, 350 0x91b8, 0xffffffff, 0x00050004, 351 0x91c4, 0xffffffff, 0x00010006, 352 0x91c8, 0xffffffff, 0x00090008, 353 0x91cc, 0xffffffff, 0x00070000, 354 0x91d0, 0xffffffff, 0x00030002, 355 0x91d4, 0xffffffff, 0x00050004, 356 0x91e0, 0xffffffff, 0x00010006, 357 0x91e4, 0xffffffff, 0x00090008, 358 0x91e8, 0xffffffff, 0x00000000, 359 0x91ec, 0xffffffff, 0x00070000, 360 0x91f0, 0xffffffff, 0x00030002, 361 0x91f4, 0xffffffff, 0x00050004, 362 0x9200, 0xffffffff, 0x00010006, 363 0x9204, 0xffffffff, 0x00090008, 364 0x9208, 0xffffffff, 0x00070000, 365 0x920c, 0xffffffff, 0x00030002, 366 0x9210, 0xffffffff, 0x00050004, 367 0x921c, 0xffffffff, 0x00010006, 368 0x9220, 0xffffffff, 0x00090008, 369 0x9224, 0xffffffff, 0x00070000, 370 0x9228, 0xffffffff, 0x00030002, 371 0x922c, 0xffffffff, 0x00050004, 372 0x9238, 0xffffffff, 0x00010006, 373 0x923c, 0xffffffff, 0x00090008, 374 0x9240, 0xffffffff, 0x00070000, 375 0x9244, 0xffffffff, 0x00030002, 376 0x9248, 0xffffffff, 0x00050004, 377 0x9254, 0xffffffff, 0x00010006, 378 0x9258, 0xffffffff, 0x00090008, 379 0x925c, 0xffffffff, 0x00070000, 380 0x9260, 0xffffffff, 0x00030002, 381 0x9264, 0xffffffff, 0x00050004, 382 0x9270, 0xffffffff, 0x00010006, 383 0x9274, 0xffffffff, 0x00090008, 384 0x9278, 0xffffffff, 0x00070000, 385 0x927c, 0xffffffff, 0x00030002, 386 0x9280, 0xffffffff, 0x00050004, 387 0x928c, 0xffffffff, 0x00010006, 388 0x9290, 0xffffffff, 0x00090008, 389 0x9294, 0xffffffff, 0x00000000, 390 0x929c, 0xffffffff, 0x00000001, 391 0x802c, 0xffffffff, 0x40010000, 392 0x915c, 0xffffffff, 0x00010000, 393 0x9160, 0xffffffff, 0x00030002, 394 0x9178, 0xffffffff, 0x00070000, 395 0x917c, 0xffffffff, 0x00030002, 396 0x9180, 0xffffffff, 0x00050004, 397 0x918c, 0xffffffff, 0x00010006, 398 0x9190, 0xffffffff, 0x00090008, 399 0x9194, 0xffffffff, 0x00070000, 400 0x9198, 0xffffffff, 0x00030002, 401 0x919c, 0xffffffff, 0x00050004, 402 0x91a8, 0xffffffff, 0x00010006, 403 0x91ac, 0xffffffff, 0x00090008, 404 0x91b0, 0xffffffff, 0x00070000, 405 0x91b4, 0xffffffff, 0x00030002, 406 0x91b8, 0xffffffff, 0x00050004, 407 0x91c4, 0xffffffff, 0x00010006, 408 0x91c8, 0xffffffff, 0x00090008, 409 0x91cc, 0xffffffff, 0x00070000, 410 0x91d0, 0xffffffff, 0x00030002, 411 0x91d4, 0xffffffff, 0x00050004, 412 0x91e0, 0xffffffff, 0x00010006, 413 0x91e4, 0xffffffff, 0x00090008, 414 0x91e8, 0xffffffff, 0x00000000, 415 0x91ec, 0xffffffff, 0x00070000, 416 0x91f0, 0xffffffff, 0x00030002, 417 0x91f4, 0xffffffff, 0x00050004, 418 0x9200, 0xffffffff, 0x00010006, 419 0x9204, 0xffffffff, 0x00090008, 420 0x9208, 0xffffffff, 0x00070000, 421 0x920c, 0xffffffff, 0x00030002, 422 0x9210, 0xffffffff, 0x00050004, 423 0x921c, 0xffffffff, 0x00010006, 424 0x9220, 0xffffffff, 0x00090008, 425 0x9224, 0xffffffff, 0x00070000, 426 0x9228, 0xffffffff, 0x00030002, 427 0x922c, 0xffffffff, 0x00050004, 428 0x9238, 0xffffffff, 0x00010006, 429 0x923c, 0xffffffff, 0x00090008, 430 0x9240, 0xffffffff, 0x00070000, 431 0x9244, 0xffffffff, 0x00030002, 432 0x9248, 0xffffffff, 0x00050004, 433 0x9254, 0xffffffff, 0x00010006, 434 0x9258, 0xffffffff, 0x00090008, 435 0x925c, 0xffffffff, 0x00070000, 436 0x9260, 0xffffffff, 0x00030002, 437 0x9264, 0xffffffff, 0x00050004, 438 0x9270, 0xffffffff, 0x00010006, 439 0x9274, 0xffffffff, 0x00090008, 440 0x9278, 0xffffffff, 0x00070000, 441 0x927c, 0xffffffff, 0x00030002, 442 0x9280, 0xffffffff, 0x00050004, 443 0x928c, 0xffffffff, 0x00010006, 444 0x9290, 0xffffffff, 0x00090008, 445 0x9294, 0xffffffff, 0x00000000, 446 0x929c, 0xffffffff, 0x00000001, 447 0x802c, 0xffffffff, 0xc0000000 448}; 449 450static const u32 redwood_mgcg_init[] = 451{ 452 0x802c, 0xffffffff, 0xc0000000, 453 0x5448, 0xffffffff, 0x00000100, 454 0x55e4, 0xffffffff, 0x00000100, 455 0x160c, 0xffffffff, 0x00000100, 456 0x5644, 0xffffffff, 0x00000100, 457 0xc164, 0xffffffff, 0x00000100, 458 0x8a18, 0xffffffff, 0x00000100, 459 0x897c, 0xffffffff, 0x06000100, 460 0x8b28, 0xffffffff, 0x00000100, 461 0x9144, 0xffffffff, 0x00000100, 462 0x9a60, 0xffffffff, 0x00000100, 463 0x9868, 0xffffffff, 0x00000100, 464 0x8d58, 0xffffffff, 0x00000100, 465 0x9510, 0xffffffff, 0x00000100, 466 0x949c, 0xffffffff, 0x00000100, 467 0x9654, 0xffffffff, 0x00000100, 468 0x9030, 0xffffffff, 0x00000100, 469 0x9034, 0xffffffff, 0x00000100, 470 0x9038, 0xffffffff, 0x00000100, 471 0x903c, 0xffffffff, 0x00000100, 472 0x9040, 0xffffffff, 0x00000100, 473 0xa200, 0xffffffff, 0x00000100, 474 0xa204, 0xffffffff, 0x00000100, 475 0xa208, 0xffffffff, 0x00000100, 476 0xa20c, 0xffffffff, 0x00000100, 477 0x971c, 0xffffffff, 0x00000100, 478 0x977c, 0xffffffff, 0x00000100, 479 0x3f80, 0xffffffff, 0x00000100, 480 0xa210, 0xffffffff, 0x00000100, 481 0xa214, 0xffffffff, 0x00000100, 482 0x4d8, 0xffffffff, 0x00000100, 483 0x9784, 0xffffffff, 0x00000100, 484 0x9698, 0xffffffff, 0x00000100, 485 0x4d4, 0xffffffff, 0x00000200, 486 0x30cc, 0xffffffff, 0x00000100, 487 0xd0c0, 0xffffffff, 0xff000100, 488 0x802c, 0xffffffff, 0x40000000, 489 0x915c, 0xffffffff, 0x00010000, 490 0x9160, 0xffffffff, 0x00030002, 491 0x9178, 0xffffffff, 0x00070000, 492 0x917c, 0xffffffff, 0x00030002, 493 0x9180, 0xffffffff, 0x00050004, 494 0x918c, 0xffffffff, 0x00010006, 495 0x9190, 0xffffffff, 0x00090008, 496 0x9194, 0xffffffff, 0x00070000, 497 0x9198, 0xffffffff, 0x00030002, 498 0x919c, 0xffffffff, 0x00050004, 499 0x91a8, 0xffffffff, 0x00010006, 500 0x91ac, 0xffffffff, 0x00090008, 501 0x91b0, 0xffffffff, 0x00070000, 502 0x91b4, 0xffffffff, 0x00030002, 503 0x91b8, 0xffffffff, 0x00050004, 504 0x91c4, 0xffffffff, 0x00010006, 505 0x91c8, 0xffffffff, 0x00090008, 506 0x91cc, 0xffffffff, 0x00070000, 507 0x91d0, 0xffffffff, 0x00030002, 508 0x91d4, 0xffffffff, 0x00050004, 509 0x91e0, 0xffffffff, 0x00010006, 510 0x91e4, 0xffffffff, 0x00090008, 511 0x91e8, 0xffffffff, 0x00000000, 512 0x91ec, 0xffffffff, 0x00070000, 513 0x91f0, 0xffffffff, 0x00030002, 514 0x91f4, 0xffffffff, 0x00050004, 515 0x9200, 0xffffffff, 0x00010006, 516 0x9204, 0xffffffff, 0x00090008, 517 0x9294, 0xffffffff, 0x00000000, 518 0x929c, 0xffffffff, 0x00000001, 519 0x802c, 0xffffffff, 0xc0000000 520}; 521 522static const u32 cedar_golden_registers[] = 523{ 524 0x3f90, 0xffff0000, 0xff000000, 525 0x9148, 0xffff0000, 0xff000000, 526 0x3f94, 0xffff0000, 0xff000000, 527 0x914c, 0xffff0000, 0xff000000, 528 0x9b7c, 0xffffffff, 0x00000000, 529 0x8a14, 0xffffffff, 0x00000007, 530 0x8b10, 0xffffffff, 0x00000000, 531 0x960c, 0xffffffff, 0x54763210, 532 0x88c4, 0xffffffff, 0x000000c2, 533 0x88d4, 0xffffffff, 0x00000000, 534 0x8974, 0xffffffff, 0x00000000, 535 0xc78, 0x00000080, 0x00000080, 536 0x5eb4, 0xffffffff, 0x00000002, 537 0x5e78, 0xffffffff, 0x001000f0, 538 0x6104, 0x01000300, 0x00000000, 539 0x5bc0, 0x00300000, 0x00000000, 540 0x7030, 0xffffffff, 0x00000011, 541 0x7c30, 0xffffffff, 0x00000011, 542 0x10830, 0xffffffff, 0x00000011, 543 0x11430, 0xffffffff, 0x00000011, 544 0xd02c, 0xffffffff, 0x08421000, 545 0x240c, 0xffffffff, 0x00000380, 546 0x8b24, 0xffffffff, 0x00ff0fff, 547 0x28a4c, 0x06000000, 0x06000000, 548 0x10c, 0x00000001, 0x00000001, 549 0x8d00, 0xffffffff, 0x100e4848, 550 0x8d04, 0xffffffff, 0x00164745, 551 0x8c00, 0xffffffff, 0xe4000003, 552 0x8c04, 0xffffffff, 0x40600060, 553 0x8c08, 0xffffffff, 0x001c001c, 554 0x8cf0, 0xffffffff, 0x08e00410, 555 0x8c20, 0xffffffff, 0x00800080, 556 0x8c24, 0xffffffff, 0x00800080, 557 0x8c18, 0xffffffff, 0x20202078, 558 0x8c1c, 0xffffffff, 0x00001010, 559 0x28350, 0xffffffff, 0x00000000, 560 0xa008, 0xffffffff, 0x00010000, 561 0x5c4, 0xffffffff, 0x00000001, 562 0x9508, 0xffffffff, 0x00000002 563}; 564 565static const u32 cedar_mgcg_init[] = 566{ 567 0x802c, 0xffffffff, 0xc0000000, 568 0x5448, 0xffffffff, 0x00000100, 569 0x55e4, 0xffffffff, 0x00000100, 570 0x160c, 0xffffffff, 0x00000100, 571 0x5644, 0xffffffff, 0x00000100, 572 0xc164, 0xffffffff, 0x00000100, 573 0x8a18, 0xffffffff, 0x00000100, 574 0x897c, 0xffffffff, 0x06000100, 575 0x8b28, 0xffffffff, 0x00000100, 576 0x9144, 0xffffffff, 0x00000100, 577 0x9a60, 0xffffffff, 0x00000100, 578 0x9868, 0xffffffff, 0x00000100, 579 0x8d58, 0xffffffff, 0x00000100, 580 0x9510, 0xffffffff, 0x00000100, 581 0x949c, 0xffffffff, 0x00000100, 582 0x9654, 0xffffffff, 0x00000100, 583 0x9030, 0xffffffff, 0x00000100, 584 0x9034, 0xffffffff, 0x00000100, 585 0x9038, 0xffffffff, 0x00000100, 586 0x903c, 0xffffffff, 0x00000100, 587 0x9040, 0xffffffff, 0x00000100, 588 0xa200, 0xffffffff, 0x00000100, 589 0xa204, 0xffffffff, 0x00000100, 590 0xa208, 0xffffffff, 0x00000100, 591 0xa20c, 0xffffffff, 0x00000100, 592 0x971c, 0xffffffff, 0x00000100, 593 0x977c, 0xffffffff, 0x00000100, 594 0x3f80, 0xffffffff, 0x00000100, 595 0xa210, 0xffffffff, 0x00000100, 596 0xa214, 0xffffffff, 0x00000100, 597 0x4d8, 0xffffffff, 0x00000100, 598 0x9784, 0xffffffff, 0x00000100, 599 0x9698, 0xffffffff, 0x00000100, 600 0x4d4, 0xffffffff, 0x00000200, 601 0x30cc, 0xffffffff, 0x00000100, 602 0xd0c0, 0xffffffff, 0xff000100, 603 0x802c, 0xffffffff, 0x40000000, 604 0x915c, 0xffffffff, 0x00010000, 605 0x9178, 0xffffffff, 0x00050000, 606 0x917c, 0xffffffff, 0x00030002, 607 0x918c, 0xffffffff, 0x00010004, 608 0x9190, 0xffffffff, 0x00070006, 609 0x9194, 0xffffffff, 0x00050000, 610 0x9198, 0xffffffff, 0x00030002, 611 0x91a8, 0xffffffff, 0x00010004, 612 0x91ac, 0xffffffff, 0x00070006, 613 0x91e8, 0xffffffff, 0x00000000, 614 0x9294, 0xffffffff, 0x00000000, 615 0x929c, 0xffffffff, 0x00000001, 616 0x802c, 0xffffffff, 0xc0000000 617}; 618 619static const u32 juniper_mgcg_init[] = 620{ 621 0x802c, 0xffffffff, 0xc0000000, 622 0x5448, 0xffffffff, 0x00000100, 623 0x55e4, 0xffffffff, 0x00000100, 624 0x160c, 0xffffffff, 0x00000100, 625 0x5644, 0xffffffff, 0x00000100, 626 0xc164, 0xffffffff, 0x00000100, 627 0x8a18, 0xffffffff, 0x00000100, 628 0x897c, 0xffffffff, 0x06000100, 629 0x8b28, 0xffffffff, 0x00000100, 630 0x9144, 0xffffffff, 0x00000100, 631 0x9a60, 0xffffffff, 0x00000100, 632 0x9868, 0xffffffff, 0x00000100, 633 0x8d58, 0xffffffff, 0x00000100, 634 0x9510, 0xffffffff, 0x00000100, 635 0x949c, 0xffffffff, 0x00000100, 636 0x9654, 0xffffffff, 0x00000100, 637 0x9030, 0xffffffff, 0x00000100, 638 0x9034, 0xffffffff, 0x00000100, 639 0x9038, 0xffffffff, 0x00000100, 640 0x903c, 0xffffffff, 0x00000100, 641 0x9040, 0xffffffff, 0x00000100, 642 0xa200, 0xffffffff, 0x00000100, 643 0xa204, 0xffffffff, 0x00000100, 644 0xa208, 0xffffffff, 0x00000100, 645 0xa20c, 0xffffffff, 0x00000100, 646 0x971c, 0xffffffff, 0x00000100, 647 0xd0c0, 0xffffffff, 0xff000100, 648 0x802c, 0xffffffff, 0x40000000, 649 0x915c, 0xffffffff, 0x00010000, 650 0x9160, 0xffffffff, 0x00030002, 651 0x9178, 0xffffffff, 0x00070000, 652 0x917c, 0xffffffff, 0x00030002, 653 0x9180, 0xffffffff, 0x00050004, 654 0x918c, 0xffffffff, 0x00010006, 655 0x9190, 0xffffffff, 0x00090008, 656 0x9194, 0xffffffff, 0x00070000, 657 0x9198, 0xffffffff, 0x00030002, 658 0x919c, 0xffffffff, 0x00050004, 659 0x91a8, 0xffffffff, 0x00010006, 660 0x91ac, 0xffffffff, 0x00090008, 661 0x91b0, 0xffffffff, 0x00070000, 662 0x91b4, 0xffffffff, 0x00030002, 663 0x91b8, 0xffffffff, 0x00050004, 664 0x91c4, 0xffffffff, 0x00010006, 665 0x91c8, 0xffffffff, 0x00090008, 666 0x91cc, 0xffffffff, 0x00070000, 667 0x91d0, 0xffffffff, 0x00030002, 668 0x91d4, 0xffffffff, 0x00050004, 669 0x91e0, 0xffffffff, 0x00010006, 670 0x91e4, 0xffffffff, 0x00090008, 671 0x91e8, 0xffffffff, 0x00000000, 672 0x91ec, 0xffffffff, 0x00070000, 673 0x91f0, 0xffffffff, 0x00030002, 674 0x91f4, 0xffffffff, 0x00050004, 675 0x9200, 0xffffffff, 0x00010006, 676 0x9204, 0xffffffff, 0x00090008, 677 0x9208, 0xffffffff, 0x00070000, 678 0x920c, 0xffffffff, 0x00030002, 679 0x9210, 0xffffffff, 0x00050004, 680 0x921c, 0xffffffff, 0x00010006, 681 0x9220, 0xffffffff, 0x00090008, 682 0x9224, 0xffffffff, 0x00070000, 683 0x9228, 0xffffffff, 0x00030002, 684 0x922c, 0xffffffff, 0x00050004, 685 0x9238, 0xffffffff, 0x00010006, 686 0x923c, 0xffffffff, 0x00090008, 687 0x9240, 0xffffffff, 0x00070000, 688 0x9244, 0xffffffff, 0x00030002, 689 0x9248, 0xffffffff, 0x00050004, 690 0x9254, 0xffffffff, 0x00010006, 691 0x9258, 0xffffffff, 0x00090008, 692 0x925c, 0xffffffff, 0x00070000, 693 0x9260, 0xffffffff, 0x00030002, 694 0x9264, 0xffffffff, 0x00050004, 695 0x9270, 0xffffffff, 0x00010006, 696 0x9274, 0xffffffff, 0x00090008, 697 0x9278, 0xffffffff, 0x00070000, 698 0x927c, 0xffffffff, 0x00030002, 699 0x9280, 0xffffffff, 0x00050004, 700 0x928c, 0xffffffff, 0x00010006, 701 0x9290, 0xffffffff, 0x00090008, 702 0x9294, 0xffffffff, 0x00000000, 703 0x929c, 0xffffffff, 0x00000001, 704 0x802c, 0xffffffff, 0xc0000000, 705 0x977c, 0xffffffff, 0x00000100, 706 0x3f80, 0xffffffff, 0x00000100, 707 0xa210, 0xffffffff, 0x00000100, 708 0xa214, 0xffffffff, 0x00000100, 709 0x4d8, 0xffffffff, 0x00000100, 710 0x9784, 0xffffffff, 0x00000100, 711 0x9698, 0xffffffff, 0x00000100, 712 0x4d4, 0xffffffff, 0x00000200, 713 0x30cc, 0xffffffff, 0x00000100, 714 0x802c, 0xffffffff, 0xc0000000 715}; 716 717static const u32 supersumo_golden_registers[] = 718{ 719 0x5eb4, 0xffffffff, 0x00000002, 720 0x5c4, 0xffffffff, 0x00000001, 721 0x7030, 0xffffffff, 0x00000011, 722 0x7c30, 0xffffffff, 0x00000011, 723 0x6104, 0x01000300, 0x00000000, 724 0x5bc0, 0x00300000, 0x00000000, 725 0x8c04, 0xffffffff, 0x40600060, 726 0x8c08, 0xffffffff, 0x001c001c, 727 0x8c20, 0xffffffff, 0x00800080, 728 0x8c24, 0xffffffff, 0x00800080, 729 0x8c18, 0xffffffff, 0x20202078, 730 0x8c1c, 0xffffffff, 0x00001010, 731 0x918c, 0xffffffff, 0x00010006, 732 0x91a8, 0xffffffff, 0x00010006, 733 0x91c4, 0xffffffff, 0x00010006, 734 0x91e0, 0xffffffff, 0x00010006, 735 0x9200, 0xffffffff, 0x00010006, 736 0x9150, 0xffffffff, 0x6e944040, 737 0x917c, 0xffffffff, 0x00030002, 738 0x9180, 0xffffffff, 0x00050004, 739 0x9198, 0xffffffff, 0x00030002, 740 0x919c, 0xffffffff, 0x00050004, 741 0x91b4, 0xffffffff, 0x00030002, 742 0x91b8, 0xffffffff, 0x00050004, 743 0x91d0, 0xffffffff, 0x00030002, 744 0x91d4, 0xffffffff, 0x00050004, 745 0x91f0, 0xffffffff, 0x00030002, 746 0x91f4, 0xffffffff, 0x00050004, 747 0x915c, 0xffffffff, 0x00010000, 748 0x9160, 0xffffffff, 0x00030002, 749 0x3f90, 0xffff0000, 0xff000000, 750 0x9178, 0xffffffff, 0x00070000, 751 0x9194, 0xffffffff, 0x00070000, 752 0x91b0, 0xffffffff, 0x00070000, 753 0x91cc, 0xffffffff, 0x00070000, 754 0x91ec, 0xffffffff, 0x00070000, 755 0x9148, 0xffff0000, 0xff000000, 756 0x9190, 0xffffffff, 0x00090008, 757 0x91ac, 0xffffffff, 0x00090008, 758 0x91c8, 0xffffffff, 0x00090008, 759 0x91e4, 0xffffffff, 0x00090008, 760 0x9204, 0xffffffff, 0x00090008, 761 0x3f94, 0xffff0000, 0xff000000, 762 0x914c, 0xffff0000, 0xff000000, 763 0x929c, 0xffffffff, 0x00000001, 764 0x8a18, 0xffffffff, 0x00000100, 765 0x8b28, 0xffffffff, 0x00000100, 766 0x9144, 0xffffffff, 0x00000100, 767 0x5644, 0xffffffff, 0x00000100, 768 0x9b7c, 0xffffffff, 0x00000000, 769 0x8030, 0xffffffff, 0x0000100a, 770 0x8a14, 0xffffffff, 0x00000007, 771 0x8b24, 0xffffffff, 0x00ff0fff, 772 0x8b10, 0xffffffff, 0x00000000, 773 0x28a4c, 0x06000000, 0x06000000, 774 0x4d8, 0xffffffff, 0x00000100, 775 0x913c, 0xffff000f, 0x0100000a, 776 0x960c, 0xffffffff, 0x54763210, 777 0x88c4, 0xffffffff, 0x000000c2, 778 0x88d4, 0xffffffff, 0x00000010, 779 0x8974, 0xffffffff, 0x00000000, 780 0xc78, 0x00000080, 0x00000080, 781 0x5e78, 0xffffffff, 0x001000f0, 782 0xd02c, 0xffffffff, 0x08421000, 783 0xa008, 0xffffffff, 0x00010000, 784 0x8d00, 0xffffffff, 0x100e4848, 785 0x8d04, 0xffffffff, 0x00164745, 786 0x8c00, 0xffffffff, 0xe4000003, 787 0x8cf0, 0x1fffffff, 0x08e00620, 788 0x28350, 0xffffffff, 0x00000000, 789 0x9508, 0xffffffff, 0x00000002 790}; 791 792static const u32 sumo_golden_registers[] = 793{ 794 0x900c, 0x00ffffff, 0x0017071f, 795 0x8c18, 0xffffffff, 0x10101060, 796 0x8c1c, 0xffffffff, 0x00001010, 797 0x8c30, 0x0000000f, 0x00000005, 798 0x9688, 0x0000000f, 0x00000007 799}; 800 801static const u32 wrestler_golden_registers[] = 802{ 803 0x5eb4, 0xffffffff, 0x00000002, 804 0x5c4, 0xffffffff, 0x00000001, 805 0x7030, 0xffffffff, 0x00000011, 806 0x7c30, 0xffffffff, 0x00000011, 807 0x6104, 0x01000300, 0x00000000, 808 0x5bc0, 0x00300000, 0x00000000, 809 0x918c, 0xffffffff, 0x00010006, 810 0x91a8, 0xffffffff, 0x00010006, 811 0x9150, 0xffffffff, 0x6e944040, 812 0x917c, 0xffffffff, 0x00030002, 813 0x9198, 0xffffffff, 0x00030002, 814 0x915c, 0xffffffff, 0x00010000, 815 0x3f90, 0xffff0000, 0xff000000, 816 0x9178, 0xffffffff, 0x00070000, 817 0x9194, 0xffffffff, 0x00070000, 818 0x9148, 0xffff0000, 0xff000000, 819 0x9190, 0xffffffff, 0x00090008, 820 0x91ac, 0xffffffff, 0x00090008, 821 0x3f94, 0xffff0000, 0xff000000, 822 0x914c, 0xffff0000, 0xff000000, 823 0x929c, 0xffffffff, 0x00000001, 824 0x8a18, 0xffffffff, 0x00000100, 825 0x8b28, 0xffffffff, 0x00000100, 826 0x9144, 0xffffffff, 0x00000100, 827 0x9b7c, 0xffffffff, 0x00000000, 828 0x8030, 0xffffffff, 0x0000100a, 829 0x8a14, 0xffffffff, 0x00000001, 830 0x8b24, 0xffffffff, 0x00ff0fff, 831 0x8b10, 0xffffffff, 0x00000000, 832 0x28a4c, 0x06000000, 0x06000000, 833 0x4d8, 0xffffffff, 0x00000100, 834 0x913c, 0xffff000f, 0x0100000a, 835 0x960c, 0xffffffff, 0x54763210, 836 0x88c4, 0xffffffff, 0x000000c2, 837 0x88d4, 0xffffffff, 0x00000010, 838 0x8974, 0xffffffff, 0x00000000, 839 0xc78, 0x00000080, 0x00000080, 840 0x5e78, 0xffffffff, 0x001000f0, 841 0xd02c, 0xffffffff, 0x08421000, 842 0xa008, 0xffffffff, 0x00010000, 843 0x8d00, 0xffffffff, 0x100e4848, 844 0x8d04, 0xffffffff, 0x00164745, 845 0x8c00, 0xffffffff, 0xe4000003, 846 0x8cf0, 0x1fffffff, 0x08e00410, 847 0x28350, 0xffffffff, 0x00000000, 848 0x9508, 0xffffffff, 0x00000002, 849 0x900c, 0xffffffff, 0x0017071f, 850 0x8c18, 0xffffffff, 0x10101060, 851 0x8c1c, 0xffffffff, 0x00001010 852}; 853 854static const u32 barts_golden_registers[] = 855{ 856 0x5eb4, 0xffffffff, 0x00000002, 857 0x5e78, 0x8f311ff1, 0x001000f0, 858 0x3f90, 0xffff0000, 0xff000000, 859 0x9148, 0xffff0000, 0xff000000, 860 0x3f94, 0xffff0000, 0xff000000, 861 0x914c, 0xffff0000, 0xff000000, 862 0xc78, 0x00000080, 0x00000080, 863 0xbd4, 0x70073777, 0x00010001, 864 0xd02c, 0xbfffff1f, 0x08421000, 865 0xd0b8, 0x03773777, 0x02011003, 866 0x5bc0, 0x00200000, 0x50100000, 867 0x98f8, 0x33773777, 0x02011003, 868 0x98fc, 0xffffffff, 0x76543210, 869 0x7030, 0x31000311, 0x00000011, 870 0x2f48, 0x00000007, 0x02011003, 871 0x6b28, 0x00000010, 0x00000012, 872 0x7728, 0x00000010, 0x00000012, 873 0x10328, 0x00000010, 0x00000012, 874 0x10f28, 0x00000010, 0x00000012, 875 0x11b28, 0x00000010, 0x00000012, 876 0x12728, 0x00000010, 0x00000012, 877 0x240c, 0x000007ff, 0x00000380, 878 0x8a14, 0xf000001f, 0x00000007, 879 0x8b24, 0x3fff3fff, 0x00ff0fff, 880 0x8b10, 0x0000ff0f, 0x00000000, 881 0x28a4c, 0x07ffffff, 0x06000000, 882 0x10c, 0x00000001, 0x00010003, 883 0xa02c, 0xffffffff, 0x0000009b, 884 0x913c, 0x0000000f, 0x0100000a, 885 0x8d00, 0xffff7f7f, 0x100e4848, 886 0x8d04, 0x00ffffff, 0x00164745, 887 0x8c00, 0xfffc0003, 0xe4000003, 888 0x8c04, 0xf8ff00ff, 0x40600060, 889 0x8c08, 0x00ff00ff, 0x001c001c, 890 0x8cf0, 0x1fff1fff, 0x08e00620, 891 0x8c20, 0x0fff0fff, 0x00800080, 892 0x8c24, 0x0fff0fff, 0x00800080, 893 0x8c18, 0xffffffff, 0x20202078, 894 0x8c1c, 0x0000ffff, 0x00001010, 895 0x28350, 0x00000f01, 0x00000000, 896 0x9508, 0x3700001f, 0x00000002, 897 0x960c, 0xffffffff, 0x54763210, 898 0x88c4, 0x001f3ae3, 0x000000c2, 899 0x88d4, 0x0000001f, 0x00000010, 900 0x8974, 0xffffffff, 0x00000000 901}; 902 903static const u32 turks_golden_registers[] = 904{ 905 0x5eb4, 0xffffffff, 0x00000002, 906 0x5e78, 0x8f311ff1, 0x001000f0, 907 0x8c8, 0x00003000, 0x00001070, 908 0x8cc, 0x000fffff, 0x00040035, 909 0x3f90, 0xffff0000, 0xfff00000, 910 0x9148, 0xffff0000, 0xfff00000, 911 0x3f94, 0xffff0000, 0xfff00000, 912 0x914c, 0xffff0000, 0xfff00000, 913 0xc78, 0x00000080, 0x00000080, 914 0xbd4, 0x00073007, 0x00010002, 915 0xd02c, 0xbfffff1f, 0x08421000, 916 0xd0b8, 0x03773777, 0x02010002, 917 0x5bc0, 0x00200000, 0x50100000, 918 0x98f8, 0x33773777, 0x00010002, 919 0x98fc, 0xffffffff, 0x33221100, 920 0x7030, 0x31000311, 0x00000011, 921 0x2f48, 0x33773777, 0x00010002, 922 0x6b28, 0x00000010, 0x00000012, 923 0x7728, 0x00000010, 0x00000012, 924 0x10328, 0x00000010, 0x00000012, 925 0x10f28, 0x00000010, 0x00000012, 926 0x11b28, 0x00000010, 0x00000012, 927 0x12728, 0x00000010, 0x00000012, 928 0x240c, 0x000007ff, 0x00000380, 929 0x8a14, 0xf000001f, 0x00000007, 930 0x8b24, 0x3fff3fff, 0x00ff0fff, 931 0x8b10, 0x0000ff0f, 0x00000000, 932 0x28a4c, 0x07ffffff, 0x06000000, 933 0x10c, 0x00000001, 0x00010003, 934 0xa02c, 0xffffffff, 0x0000009b, 935 0x913c, 0x0000000f, 0x0100000a, 936 0x8d00, 0xffff7f7f, 0x100e4848, 937 0x8d04, 0x00ffffff, 0x00164745, 938 0x8c00, 0xfffc0003, 0xe4000003, 939 0x8c04, 0xf8ff00ff, 0x40600060, 940 0x8c08, 0x00ff00ff, 0x001c001c, 941 0x8cf0, 0x1fff1fff, 0x08e00410, 942 0x8c20, 0x0fff0fff, 0x00800080, 943 0x8c24, 0x0fff0fff, 0x00800080, 944 0x8c18, 0xffffffff, 0x20202078, 945 0x8c1c, 0x0000ffff, 0x00001010, 946 0x28350, 0x00000f01, 0x00000000, 947 0x9508, 0x3700001f, 0x00000002, 948 0x960c, 0xffffffff, 0x54763210, 949 0x88c4, 0x001f3ae3, 0x000000c2, 950 0x88d4, 0x0000001f, 0x00000010, 951 0x8974, 0xffffffff, 0x00000000 952}; 953 954static const u32 caicos_golden_registers[] = 955{ 956 0x5eb4, 0xffffffff, 0x00000002, 957 0x5e78, 0x8f311ff1, 0x001000f0, 958 0x8c8, 0x00003420, 0x00001450, 959 0x8cc, 0x000fffff, 0x00040035, 960 0x3f90, 0xffff0000, 0xfffc0000, 961 0x9148, 0xffff0000, 0xfffc0000, 962 0x3f94, 0xffff0000, 0xfffc0000, 963 0x914c, 0xffff0000, 0xfffc0000, 964 0xc78, 0x00000080, 0x00000080, 965 0xbd4, 0x00073007, 0x00010001, 966 0xd02c, 0xbfffff1f, 0x08421000, 967 0xd0b8, 0x03773777, 0x02010001, 968 0x5bc0, 0x00200000, 0x50100000, 969 0x98f8, 0x33773777, 0x02010001, 970 0x98fc, 0xffffffff, 0x33221100, 971 0x7030, 0x31000311, 0x00000011, 972 0x2f48, 0x33773777, 0x02010001, 973 0x6b28, 0x00000010, 0x00000012, 974 0x7728, 0x00000010, 0x00000012, 975 0x10328, 0x00000010, 0x00000012, 976 0x10f28, 0x00000010, 0x00000012, 977 0x11b28, 0x00000010, 0x00000012, 978 0x12728, 0x00000010, 0x00000012, 979 0x240c, 0x000007ff, 0x00000380, 980 0x8a14, 0xf000001f, 0x00000001, 981 0x8b24, 0x3fff3fff, 0x00ff0fff, 982 0x8b10, 0x0000ff0f, 0x00000000, 983 0x28a4c, 0x07ffffff, 0x06000000, 984 0x10c, 0x00000001, 0x00010003, 985 0xa02c, 0xffffffff, 0x0000009b, 986 0x913c, 0x0000000f, 0x0100000a, 987 0x8d00, 0xffff7f7f, 0x100e4848, 988 0x8d04, 0x00ffffff, 0x00164745, 989 0x8c00, 0xfffc0003, 0xe4000003, 990 0x8c04, 0xf8ff00ff, 0x40600060, 991 0x8c08, 0x00ff00ff, 0x001c001c, 992 0x8cf0, 0x1fff1fff, 0x08e00410, 993 0x8c20, 0x0fff0fff, 0x00800080, 994 0x8c24, 0x0fff0fff, 0x00800080, 995 0x8c18, 0xffffffff, 0x20202078, 996 0x8c1c, 0x0000ffff, 0x00001010, 997 0x28350, 0x00000f01, 0x00000000, 998 0x9508, 0x3700001f, 0x00000002, 999 0x960c, 0xffffffff, 0x54763210, 1000 0x88c4, 0x001f3ae3, 0x000000c2, 1001 0x88d4, 0x0000001f, 0x00000010, 1002 0x8974, 0xffffffff, 0x00000000 1003}; 1004 1005static void evergreen_init_golden_registers(struct radeon_device *rdev) 1006{ 1007 switch (rdev->family) { 1008 case CHIP_CYPRESS: 1009 case CHIP_HEMLOCK: 1010 radeon_program_register_sequence(rdev, 1011 evergreen_golden_registers, 1012 (const u32)ARRAY_SIZE(evergreen_golden_registers)); 1013 radeon_program_register_sequence(rdev, 1014 evergreen_golden_registers2, 1015 (const u32)ARRAY_SIZE(evergreen_golden_registers2)); 1016 radeon_program_register_sequence(rdev, 1017 cypress_mgcg_init, 1018 (const u32)ARRAY_SIZE(cypress_mgcg_init)); 1019 break; 1020 case CHIP_JUNIPER: 1021 radeon_program_register_sequence(rdev, 1022 evergreen_golden_registers, 1023 (const u32)ARRAY_SIZE(evergreen_golden_registers)); 1024 radeon_program_register_sequence(rdev, 1025 evergreen_golden_registers2, 1026 (const u32)ARRAY_SIZE(evergreen_golden_registers2)); 1027 radeon_program_register_sequence(rdev, 1028 juniper_mgcg_init, 1029 (const u32)ARRAY_SIZE(juniper_mgcg_init)); 1030 break; 1031 case CHIP_REDWOOD: 1032 radeon_program_register_sequence(rdev, 1033 evergreen_golden_registers, 1034 (const u32)ARRAY_SIZE(evergreen_golden_registers)); 1035 radeon_program_register_sequence(rdev, 1036 evergreen_golden_registers2, 1037 (const u32)ARRAY_SIZE(evergreen_golden_registers2)); 1038 radeon_program_register_sequence(rdev, 1039 redwood_mgcg_init, 1040 (const u32)ARRAY_SIZE(redwood_mgcg_init)); 1041 break; 1042 case CHIP_CEDAR: 1043 radeon_program_register_sequence(rdev, 1044 cedar_golden_registers, 1045 (const u32)ARRAY_SIZE(cedar_golden_registers)); 1046 radeon_program_register_sequence(rdev, 1047 evergreen_golden_registers2, 1048 (const u32)ARRAY_SIZE(evergreen_golden_registers2)); 1049 radeon_program_register_sequence(rdev, 1050 cedar_mgcg_init, 1051 (const u32)ARRAY_SIZE(cedar_mgcg_init)); 1052 break; 1053 case CHIP_PALM: 1054 radeon_program_register_sequence(rdev, 1055 wrestler_golden_registers, 1056 (const u32)ARRAY_SIZE(wrestler_golden_registers)); 1057 break; 1058 case CHIP_SUMO: 1059 radeon_program_register_sequence(rdev, 1060 supersumo_golden_registers, 1061 (const u32)ARRAY_SIZE(supersumo_golden_registers)); 1062 break; 1063 case CHIP_SUMO2: 1064 radeon_program_register_sequence(rdev, 1065 supersumo_golden_registers, 1066 (const u32)ARRAY_SIZE(supersumo_golden_registers)); 1067 radeon_program_register_sequence(rdev, 1068 sumo_golden_registers, 1069 (const u32)ARRAY_SIZE(sumo_golden_registers)); 1070 break; 1071 case CHIP_BARTS: 1072 radeon_program_register_sequence(rdev, 1073 barts_golden_registers, 1074 (const u32)ARRAY_SIZE(barts_golden_registers)); 1075 break; 1076 case CHIP_TURKS: 1077 radeon_program_register_sequence(rdev, 1078 turks_golden_registers, 1079 (const u32)ARRAY_SIZE(turks_golden_registers)); 1080 break; 1081 case CHIP_CAICOS: 1082 radeon_program_register_sequence(rdev, 1083 caicos_golden_registers, 1084 (const u32)ARRAY_SIZE(caicos_golden_registers)); 1085 break; 1086 default: 1087 break; 1088 } 1089} 1090 1091/** 1092 * evergreen_get_allowed_info_register - fetch the register for the info ioctl 1093 * 1094 * @rdev: radeon_device pointer 1095 * @reg: register offset in bytes 1096 * @val: register value 1097 * 1098 * Returns 0 for success or -EINVAL for an invalid register 1099 * 1100 */ 1101int evergreen_get_allowed_info_register(struct radeon_device *rdev, 1102 u32 reg, u32 *val) 1103{ 1104 switch (reg) { 1105 case GRBM_STATUS: 1106 case GRBM_STATUS_SE0: 1107 case GRBM_STATUS_SE1: 1108 case SRBM_STATUS: 1109 case SRBM_STATUS2: 1110 case DMA_STATUS_REG: 1111 case UVD_STATUS: 1112 *val = RREG32(reg); 1113 return 0; 1114 default: 1115 return -EINVAL; 1116 } 1117} 1118 1119void evergreen_tiling_fields(unsigned tiling_flags, unsigned *bankw, 1120 unsigned *bankh, unsigned *mtaspect, 1121 unsigned *tile_split) 1122{ 1123 *bankw = (tiling_flags >> RADEON_TILING_EG_BANKW_SHIFT) & RADEON_TILING_EG_BANKW_MASK; 1124 *bankh = (tiling_flags >> RADEON_TILING_EG_BANKH_SHIFT) & RADEON_TILING_EG_BANKH_MASK; 1125 *mtaspect = (tiling_flags >> RADEON_TILING_EG_MACRO_TILE_ASPECT_SHIFT) & RADEON_TILING_EG_MACRO_TILE_ASPECT_MASK; 1126 *tile_split = (tiling_flags >> RADEON_TILING_EG_TILE_SPLIT_SHIFT) & RADEON_TILING_EG_TILE_SPLIT_MASK; 1127 switch (*bankw) { 1128 default: 1129 case 1: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_1; break; 1130 case 2: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_2; break; 1131 case 4: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_4; break; 1132 case 8: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_8; break; 1133 } 1134 switch (*bankh) { 1135 default: 1136 case 1: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_1; break; 1137 case 2: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_2; break; 1138 case 4: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_4; break; 1139 case 8: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_8; break; 1140 } 1141 switch (*mtaspect) { 1142 default: 1143 case 1: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_1; break; 1144 case 2: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_2; break; 1145 case 4: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_4; break; 1146 case 8: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_8; break; 1147 } 1148} 1149 1150static int sumo_set_uvd_clock(struct radeon_device *rdev, u32 clock, 1151 u32 cntl_reg, u32 status_reg) 1152{ 1153 int r, i; 1154 struct atom_clock_dividers dividers; 1155 1156 r = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM, 1157 clock, false, ÷rs); 1158 if (r) 1159 return r; 1160 1161 WREG32_P(cntl_reg, dividers.post_div, ~(DCLK_DIR_CNTL_EN|DCLK_DIVIDER_MASK)); 1162 1163 for (i = 0; i < 100; i++) { 1164 if (RREG32(status_reg) & DCLK_STATUS) 1165 break; 1166 mdelay(10); 1167 } 1168 if (i == 100) 1169 return -ETIMEDOUT; 1170 1171 return 0; 1172} 1173 1174int sumo_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk) 1175{ 1176 int r = 0; 1177 u32 cg_scratch = RREG32(CG_SCRATCH1); 1178 1179 r = sumo_set_uvd_clock(rdev, vclk, CG_VCLK_CNTL, CG_VCLK_STATUS); 1180 if (r) 1181 goto done; 1182 cg_scratch &= 0xffff0000; 1183 cg_scratch |= vclk / 100; /* Mhz */ 1184 1185 r = sumo_set_uvd_clock(rdev, dclk, CG_DCLK_CNTL, CG_DCLK_STATUS); 1186 if (r) 1187 goto done; 1188 cg_scratch &= 0x0000ffff; 1189 cg_scratch |= (dclk / 100) << 16; /* Mhz */ 1190 1191done: 1192 WREG32(CG_SCRATCH1, cg_scratch); 1193 1194 return r; 1195} 1196 1197int evergreen_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk) 1198{ 1199 /* start off with something large */ 1200 unsigned fb_div = 0, vclk_div = 0, dclk_div = 0; 1201 int r; 1202 1203 /* bypass vclk and dclk with bclk */ 1204 WREG32_P(CG_UPLL_FUNC_CNTL_2, 1205 VCLK_SRC_SEL(1) | DCLK_SRC_SEL(1), 1206 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK)); 1207 1208 /* put PLL in bypass mode */ 1209 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_BYPASS_EN_MASK, ~UPLL_BYPASS_EN_MASK); 1210 1211 if (!vclk || !dclk) { 1212 /* keep the Bypass mode, put PLL to sleep */ 1213 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK); 1214 return 0; 1215 } 1216 1217 r = radeon_uvd_calc_upll_dividers(rdev, vclk, dclk, 125000, 250000, 1218 16384, 0x03FFFFFF, 0, 128, 5, 1219 &fb_div, &vclk_div, &dclk_div); 1220 if (r) 1221 return r; 1222 1223 /* set VCO_MODE to 1 */ 1224 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_VCO_MODE_MASK, ~UPLL_VCO_MODE_MASK); 1225 1226 /* toggle UPLL_SLEEP to 1 then back to 0 */ 1227 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK); 1228 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_SLEEP_MASK); 1229 1230 /* deassert UPLL_RESET */ 1231 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK); 1232 1233 mdelay(1); 1234 1235 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL); 1236 if (r) 1237 return r; 1238 1239 /* assert UPLL_RESET again */ 1240 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_RESET_MASK, ~UPLL_RESET_MASK); 1241 1242 /* disable spread spectrum. */ 1243 WREG32_P(CG_UPLL_SPREAD_SPECTRUM, 0, ~SSEN_MASK); 1244 1245 /* set feedback divider */ 1246 WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(fb_div), ~UPLL_FB_DIV_MASK); 1247 1248 /* set ref divider to 0 */ 1249 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_REF_DIV_MASK); 1250 1251 if (fb_div < 307200) 1252 WREG32_P(CG_UPLL_FUNC_CNTL_4, 0, ~UPLL_SPARE_ISPARE9); 1253 else 1254 WREG32_P(CG_UPLL_FUNC_CNTL_4, UPLL_SPARE_ISPARE9, ~UPLL_SPARE_ISPARE9); 1255 1256 /* set PDIV_A and PDIV_B */ 1257 WREG32_P(CG_UPLL_FUNC_CNTL_2, 1258 UPLL_PDIV_A(vclk_div) | UPLL_PDIV_B(dclk_div), 1259 ~(UPLL_PDIV_A_MASK | UPLL_PDIV_B_MASK)); 1260 1261 /* give the PLL some time to settle */ 1262 mdelay(15); 1263 1264 /* deassert PLL_RESET */ 1265 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK); 1266 1267 mdelay(15); 1268 1269 /* switch from bypass mode to normal mode */ 1270 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_BYPASS_EN_MASK); 1271 1272 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL); 1273 if (r) 1274 return r; 1275 1276 /* switch VCLK and DCLK selection */ 1277 WREG32_P(CG_UPLL_FUNC_CNTL_2, 1278 VCLK_SRC_SEL(2) | DCLK_SRC_SEL(2), 1279 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK)); 1280 1281 mdelay(100); 1282 1283 return 0; 1284} 1285 1286void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev) 1287{ 1288 int readrq; 1289 u16 v; 1290 1291 readrq = pcie_get_readrq(rdev->pdev); 1292 v = ffs(readrq) - 8; 1293 /* if bios or OS sets MAX_READ_REQUEST_SIZE to an invalid value, fix it 1294 * to avoid hangs or perfomance issues 1295 */ 1296 if ((v == 0) || (v == 6) || (v == 7)) 1297 pcie_set_readrq(rdev->pdev, 512); 1298} 1299 1300void dce4_program_fmt(struct drm_encoder *encoder) 1301{ 1302 struct drm_device *dev = encoder->dev; 1303 struct radeon_device *rdev = dev->dev_private; 1304 struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder); 1305 struct radeon_crtc *radeon_crtc = to_radeon_crtc(encoder->crtc); 1306 struct drm_connector *connector = radeon_get_connector_for_encoder(encoder); 1307 int bpc = 0; 1308 u32 tmp = 0; 1309 enum radeon_connector_dither dither = RADEON_FMT_DITHER_DISABLE; 1310 1311 if (connector) { 1312 struct radeon_connector *radeon_connector = to_radeon_connector(connector); 1313 bpc = radeon_get_monitor_bpc(connector); 1314 dither = radeon_connector->dither; 1315 } 1316 1317 /* LVDS/eDP FMT is set up by atom */ 1318 if (radeon_encoder->devices & ATOM_DEVICE_LCD_SUPPORT) 1319 return; 1320 1321 /* not needed for analog */ 1322 if ((radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC1) || 1323 (radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC2)) 1324 return; 1325 1326 if (bpc == 0) 1327 return; 1328 1329 switch (bpc) { 1330 case 6: 1331 if (dither == RADEON_FMT_DITHER_ENABLE) 1332 /* XXX sort out optimal dither settings */ 1333 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE | 1334 FMT_SPATIAL_DITHER_EN); 1335 else 1336 tmp |= FMT_TRUNCATE_EN; 1337 break; 1338 case 8: 1339 if (dither == RADEON_FMT_DITHER_ENABLE) 1340 /* XXX sort out optimal dither settings */ 1341 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE | 1342 FMT_RGB_RANDOM_ENABLE | 1343 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH); 1344 else 1345 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH); 1346 break; 1347 case 10: 1348 default: 1349 /* not needed */ 1350 break; 1351 } 1352 1353 WREG32(FMT_BIT_DEPTH_CONTROL + radeon_crtc->crtc_offset, tmp); 1354} 1355 1356static bool dce4_is_in_vblank(struct radeon_device *rdev, int crtc) 1357{ 1358 if (RREG32(EVERGREEN_CRTC_STATUS + crtc_offsets[crtc]) & EVERGREEN_CRTC_V_BLANK) 1359 return true; 1360 else 1361 return false; 1362} 1363 1364static bool dce4_is_counter_moving(struct radeon_device *rdev, int crtc) 1365{ 1366 u32 pos1, pos2; 1367 1368 pos1 = RREG32(EVERGREEN_CRTC_STATUS_POSITION + crtc_offsets[crtc]); 1369 pos2 = RREG32(EVERGREEN_CRTC_STATUS_POSITION + crtc_offsets[crtc]); 1370 1371 if (pos1 != pos2) 1372 return true; 1373 else 1374 return false; 1375} 1376 1377/** 1378 * dce4_wait_for_vblank - vblank wait asic callback. 1379 * 1380 * @rdev: radeon_device pointer 1381 * @crtc: crtc to wait for vblank on 1382 * 1383 * Wait for vblank on the requested crtc (evergreen+). 1384 */ 1385void dce4_wait_for_vblank(struct radeon_device *rdev, int crtc) 1386{ 1387 unsigned i = 0; 1388 1389 if (crtc >= rdev->num_crtc) 1390 return; 1391 1392 if (!(RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[crtc]) & EVERGREEN_CRTC_MASTER_EN)) 1393 return; 1394 1395 /* depending on when we hit vblank, we may be close to active; if so, 1396 * wait for another frame. 1397 */ 1398 while (dce4_is_in_vblank(rdev, crtc)) { 1399 if (i++ % 100 == 0) { 1400 if (!dce4_is_counter_moving(rdev, crtc)) 1401 break; 1402 } 1403 } 1404 1405 while (!dce4_is_in_vblank(rdev, crtc)) { 1406 if (i++ % 100 == 0) { 1407 if (!dce4_is_counter_moving(rdev, crtc)) 1408 break; 1409 } 1410 } 1411} 1412 1413/** 1414 * evergreen_page_flip - pageflip callback. 1415 * 1416 * @rdev: radeon_device pointer 1417 * @crtc_id: crtc to cleanup pageflip on 1418 * @crtc_base: new address of the crtc (GPU MC address) 1419 * 1420 * Triggers the actual pageflip by updating the primary 1421 * surface base address (evergreen+). 1422 */ 1423void evergreen_page_flip(struct radeon_device *rdev, int crtc_id, u64 crtc_base, 1424 bool async) 1425{ 1426 struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id]; 1427 1428 /* update the scanout addresses */ 1429 WREG32(EVERGREEN_GRPH_FLIP_CONTROL + radeon_crtc->crtc_offset, 1430 async ? EVERGREEN_GRPH_SURFACE_UPDATE_H_RETRACE_EN : 0); 1431 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + radeon_crtc->crtc_offset, 1432 upper_32_bits(crtc_base)); 1433 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset, 1434 (u32)crtc_base); 1435 /* post the write */ 1436 RREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset); 1437} 1438 1439/** 1440 * evergreen_page_flip_pending - check if page flip is still pending 1441 * 1442 * @rdev: radeon_device pointer 1443 * @crtc_id: crtc to check 1444 * 1445 * Returns the current update pending status. 1446 */ 1447bool evergreen_page_flip_pending(struct radeon_device *rdev, int crtc_id) 1448{ 1449 struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id]; 1450 1451 /* Return current update_pending status: */ 1452 return !!(RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset) & 1453 EVERGREEN_GRPH_SURFACE_UPDATE_PENDING); 1454} 1455 1456/* get temperature in millidegrees */ 1457int evergreen_get_temp(struct radeon_device *rdev) 1458{ 1459 u32 temp, toffset; 1460 int actual_temp = 0; 1461 1462 if (rdev->family == CHIP_JUNIPER) { 1463 toffset = (RREG32(CG_THERMAL_CTRL) & TOFFSET_MASK) >> 1464 TOFFSET_SHIFT; 1465 temp = (RREG32(CG_TS0_STATUS) & TS0_ADC_DOUT_MASK) >> 1466 TS0_ADC_DOUT_SHIFT; 1467 1468 if (toffset & 0x100) 1469 actual_temp = temp / 2 - (0x200 - toffset); 1470 else 1471 actual_temp = temp / 2 + toffset; 1472 1473 actual_temp = actual_temp * 1000; 1474 1475 } else { 1476 temp = (RREG32(CG_MULT_THERMAL_STATUS) & ASIC_T_MASK) >> 1477 ASIC_T_SHIFT; 1478 1479 if (temp & 0x400) 1480 actual_temp = -256; 1481 else if (temp & 0x200) 1482 actual_temp = 255; 1483 else if (temp & 0x100) { 1484 actual_temp = temp & 0x1ff; 1485 actual_temp |= ~0x1ff; 1486 } else 1487 actual_temp = temp & 0xff; 1488 1489 actual_temp = (actual_temp * 1000) / 2; 1490 } 1491 1492 return actual_temp; 1493} 1494 1495int sumo_get_temp(struct radeon_device *rdev) 1496{ 1497 u32 temp = RREG32(CG_THERMAL_STATUS) & 0xff; 1498 int actual_temp = temp - 49; 1499 1500 return actual_temp * 1000; 1501} 1502 1503/** 1504 * sumo_pm_init_profile - Initialize power profiles callback. 1505 * 1506 * @rdev: radeon_device pointer 1507 * 1508 * Initialize the power states used in profile mode 1509 * (sumo, trinity, SI). 1510 * Used for profile mode only. 1511 */ 1512void sumo_pm_init_profile(struct radeon_device *rdev) 1513{ 1514 int idx; 1515 1516 /* default */ 1517 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index; 1518 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index; 1519 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0; 1520 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 0; 1521 1522 /* low,mid sh/mh */ 1523 if (rdev->flags & RADEON_IS_MOBILITY) 1524 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 0); 1525 else 1526 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0); 1527 1528 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = idx; 1529 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = idx; 1530 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0; 1531 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0; 1532 1533 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = idx; 1534 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = idx; 1535 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0; 1536 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0; 1537 1538 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = idx; 1539 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = idx; 1540 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0; 1541 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 0; 1542 1543 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = idx; 1544 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = idx; 1545 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0; 1546 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 0; 1547 1548 /* high sh/mh */ 1549 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0); 1550 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = idx; 1551 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = idx; 1552 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0; 1553 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx = 1554 rdev->pm.power_state[idx].num_clock_modes - 1; 1555 1556 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = idx; 1557 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = idx; 1558 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0; 1559 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx = 1560 rdev->pm.power_state[idx].num_clock_modes - 1; 1561} 1562 1563/** 1564 * btc_pm_init_profile - Initialize power profiles callback. 1565 * 1566 * @rdev: radeon_device pointer 1567 * 1568 * Initialize the power states used in profile mode 1569 * (BTC, cayman). 1570 * Used for profile mode only. 1571 */ 1572void btc_pm_init_profile(struct radeon_device *rdev) 1573{ 1574 int idx; 1575 1576 /* default */ 1577 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index; 1578 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index; 1579 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0; 1580 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 2; 1581 /* starting with BTC, there is one state that is used for both 1582 * MH and SH. Difference is that we always use the high clock index for 1583 * mclk. 1584 */ 1585 if (rdev->flags & RADEON_IS_MOBILITY) 1586 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 0); 1587 else 1588 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0); 1589 /* low sh */ 1590 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = idx; 1591 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = idx; 1592 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0; 1593 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0; 1594 /* mid sh */ 1595 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = idx; 1596 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = idx; 1597 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0; 1598 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 1; 1599 /* high sh */ 1600 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = idx; 1601 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = idx; 1602 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0; 1603 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx = 2; 1604 /* low mh */ 1605 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = idx; 1606 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = idx; 1607 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0; 1608 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0; 1609 /* mid mh */ 1610 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = idx; 1611 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = idx; 1612 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0; 1613 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 1; 1614 /* high mh */ 1615 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = idx; 1616 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = idx; 1617 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0; 1618 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx = 2; 1619} 1620 1621/** 1622 * evergreen_pm_misc - set additional pm hw parameters callback. 1623 * 1624 * @rdev: radeon_device pointer 1625 * 1626 * Set non-clock parameters associated with a power state 1627 * (voltage, etc.) (evergreen+). 1628 */ 1629void evergreen_pm_misc(struct radeon_device *rdev) 1630{ 1631 int req_ps_idx = rdev->pm.requested_power_state_index; 1632 int req_cm_idx = rdev->pm.requested_clock_mode_index; 1633 struct radeon_power_state *ps = &rdev->pm.power_state[req_ps_idx]; 1634 struct radeon_voltage *voltage = &ps->clock_info[req_cm_idx].voltage; 1635 1636 if (voltage->type == VOLTAGE_SW) { 1637 /* 0xff0x are flags rather then an actual voltage */ 1638 if ((voltage->voltage & 0xff00) == 0xff00) 1639 return; 1640 if (voltage->voltage && (voltage->voltage != rdev->pm.current_vddc)) { 1641 radeon_atom_set_voltage(rdev, voltage->voltage, SET_VOLTAGE_TYPE_ASIC_VDDC); 1642 rdev->pm.current_vddc = voltage->voltage; 1643 DRM_DEBUG("Setting: vddc: %d\n", voltage->voltage); 1644 } 1645 1646 /* starting with BTC, there is one state that is used for both 1647 * MH and SH. Difference is that we always use the high clock index for 1648 * mclk and vddci. 1649 */ 1650 if ((rdev->pm.pm_method == PM_METHOD_PROFILE) && 1651 (rdev->family >= CHIP_BARTS) && 1652 rdev->pm.active_crtc_count && 1653 ((rdev->pm.profile_index == PM_PROFILE_MID_MH_IDX) || 1654 (rdev->pm.profile_index == PM_PROFILE_LOW_MH_IDX))) 1655 voltage = &rdev->pm.power_state[req_ps_idx]. 1656 clock_info[rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx].voltage; 1657 1658 /* 0xff0x are flags rather then an actual voltage */ 1659 if ((voltage->vddci & 0xff00) == 0xff00) 1660 return; 1661 if (voltage->vddci && (voltage->vddci != rdev->pm.current_vddci)) { 1662 radeon_atom_set_voltage(rdev, voltage->vddci, SET_VOLTAGE_TYPE_ASIC_VDDCI); 1663 rdev->pm.current_vddci = voltage->vddci; 1664 DRM_DEBUG("Setting: vddci: %d\n", voltage->vddci); 1665 } 1666 } 1667} 1668 1669/** 1670 * evergreen_pm_prepare - pre-power state change callback. 1671 * 1672 * @rdev: radeon_device pointer 1673 * 1674 * Prepare for a power state change (evergreen+). 1675 */ 1676void evergreen_pm_prepare(struct radeon_device *rdev) 1677{ 1678 struct drm_device *ddev = rdev->ddev; 1679 struct drm_crtc *crtc; 1680 struct radeon_crtc *radeon_crtc; 1681 u32 tmp; 1682 1683 /* disable any active CRTCs */ 1684 list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) { 1685 radeon_crtc = to_radeon_crtc(crtc); 1686 if (radeon_crtc->enabled) { 1687 tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset); 1688 tmp |= EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE; 1689 WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp); 1690 } 1691 } 1692} 1693 1694/** 1695 * evergreen_pm_finish - post-power state change callback. 1696 * 1697 * @rdev: radeon_device pointer 1698 * 1699 * Clean up after a power state change (evergreen+). 1700 */ 1701void evergreen_pm_finish(struct radeon_device *rdev) 1702{ 1703 struct drm_device *ddev = rdev->ddev; 1704 struct drm_crtc *crtc; 1705 struct radeon_crtc *radeon_crtc; 1706 u32 tmp; 1707 1708 /* enable any active CRTCs */ 1709 list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) { 1710 radeon_crtc = to_radeon_crtc(crtc); 1711 if (radeon_crtc->enabled) { 1712 tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset); 1713 tmp &= ~EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE; 1714 WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp); 1715 } 1716 } 1717} 1718 1719/** 1720 * evergreen_hpd_sense - hpd sense callback. 1721 * 1722 * @rdev: radeon_device pointer 1723 * @hpd: hpd (hotplug detect) pin 1724 * 1725 * Checks if a digital monitor is connected (evergreen+). 1726 * Returns true if connected, false if not connected. 1727 */ 1728bool evergreen_hpd_sense(struct radeon_device *rdev, enum radeon_hpd_id hpd) 1729{ 1730 if (hpd == RADEON_HPD_NONE) 1731 return false; 1732 1733 return !!(RREG32(DC_HPDx_INT_STATUS_REG(hpd)) & DC_HPDx_SENSE); 1734} 1735 1736/** 1737 * evergreen_hpd_set_polarity - hpd set polarity callback. 1738 * 1739 * @rdev: radeon_device pointer 1740 * @hpd: hpd (hotplug detect) pin 1741 * 1742 * Set the polarity of the hpd pin (evergreen+). 1743 */ 1744void evergreen_hpd_set_polarity(struct radeon_device *rdev, 1745 enum radeon_hpd_id hpd) 1746{ 1747 bool connected = evergreen_hpd_sense(rdev, hpd); 1748 1749 if (hpd == RADEON_HPD_NONE) 1750 return; 1751 1752 if (connected) 1753 WREG32_AND(DC_HPDx_INT_CONTROL(hpd), ~DC_HPDx_INT_POLARITY); 1754 else 1755 WREG32_OR(DC_HPDx_INT_CONTROL(hpd), DC_HPDx_INT_POLARITY); 1756} 1757 1758/** 1759 * evergreen_hpd_init - hpd setup callback. 1760 * 1761 * @rdev: radeon_device pointer 1762 * 1763 * Setup the hpd pins used by the card (evergreen+). 1764 * Enable the pin, set the polarity, and enable the hpd interrupts. 1765 */ 1766void evergreen_hpd_init(struct radeon_device *rdev) 1767{ 1768 struct drm_device *dev = rdev->ddev; 1769 struct drm_connector *connector; 1770 unsigned enabled = 0; 1771 u32 tmp = DC_HPDx_CONNECTION_TIMER(0x9c4) | 1772 DC_HPDx_RX_INT_TIMER(0xfa) | DC_HPDx_EN; 1773 1774 list_for_each_entry(connector, &dev->mode_config.connector_list, head) { 1775 enum radeon_hpd_id hpd = 1776 to_radeon_connector(connector)->hpd.hpd; 1777 1778 if (connector->connector_type == DRM_MODE_CONNECTOR_eDP || 1779 connector->connector_type == DRM_MODE_CONNECTOR_LVDS) { 1780 /* don't try to enable hpd on eDP or LVDS avoid breaking the 1781 * aux dp channel on imac and help (but not completely fix) 1782 * https://bugzilla.redhat.com/show_bug.cgi?id=726143 1783 * also avoid interrupt storms during dpms. 1784 */ 1785 continue; 1786 } 1787 1788 if (hpd == RADEON_HPD_NONE) 1789 continue; 1790 1791 WREG32(DC_HPDx_CONTROL(hpd), tmp); 1792 enabled |= 1 << hpd; 1793 1794 radeon_hpd_set_polarity(rdev, hpd); 1795 } 1796 radeon_irq_kms_enable_hpd(rdev, enabled); 1797} 1798 1799/** 1800 * evergreen_hpd_fini - hpd tear down callback. 1801 * 1802 * @rdev: radeon_device pointer 1803 * 1804 * Tear down the hpd pins used by the card (evergreen+). 1805 * Disable the hpd interrupts. 1806 */ 1807void evergreen_hpd_fini(struct radeon_device *rdev) 1808{ 1809 struct drm_device *dev = rdev->ddev; 1810 struct drm_connector *connector; 1811 unsigned disabled = 0; 1812 1813 list_for_each_entry(connector, &dev->mode_config.connector_list, head) { 1814 enum radeon_hpd_id hpd = 1815 to_radeon_connector(connector)->hpd.hpd; 1816 1817 if (hpd == RADEON_HPD_NONE) 1818 continue; 1819 1820 WREG32(DC_HPDx_CONTROL(hpd), 0); 1821 disabled |= 1 << hpd; 1822 } 1823 radeon_irq_kms_disable_hpd(rdev, disabled); 1824} 1825 1826/* watermark setup */ 1827 1828static u32 evergreen_line_buffer_adjust(struct radeon_device *rdev, 1829 struct radeon_crtc *radeon_crtc, 1830 struct drm_display_mode *mode, 1831 struct drm_display_mode *other_mode) 1832{ 1833 u32 tmp, buffer_alloc, i; 1834 u32 pipe_offset = radeon_crtc->crtc_id * 0x20; 1835 /* 1836 * Line Buffer Setup 1837 * There are 3 line buffers, each one shared by 2 display controllers. 1838 * DC_LB_MEMORY_SPLIT controls how that line buffer is shared between 1839 * the display controllers. The paritioning is done via one of four 1840 * preset allocations specified in bits 2:0: 1841 * first display controller 1842 * 0 - first half of lb (3840 * 2) 1843 * 1 - first 3/4 of lb (5760 * 2) 1844 * 2 - whole lb (7680 * 2), other crtc must be disabled 1845 * 3 - first 1/4 of lb (1920 * 2) 1846 * second display controller 1847 * 4 - second half of lb (3840 * 2) 1848 * 5 - second 3/4 of lb (5760 * 2) 1849 * 6 - whole lb (7680 * 2), other crtc must be disabled 1850 * 7 - last 1/4 of lb (1920 * 2) 1851 */ 1852 /* this can get tricky if we have two large displays on a paired group 1853 * of crtcs. Ideally for multiple large displays we'd assign them to 1854 * non-linked crtcs for maximum line buffer allocation. 1855 */ 1856 if (radeon_crtc->base.enabled && mode) { 1857 if (other_mode) { 1858 tmp = 0; /* 1/2 */ 1859 buffer_alloc = 1; 1860 } else { 1861 tmp = 2; /* whole */ 1862 buffer_alloc = 2; 1863 } 1864 } else { 1865 tmp = 0; 1866 buffer_alloc = 0; 1867 } 1868 1869 /* second controller of the pair uses second half of the lb */ 1870 if (radeon_crtc->crtc_id % 2) 1871 tmp += 4; 1872 WREG32(DC_LB_MEMORY_SPLIT + radeon_crtc->crtc_offset, tmp); 1873 1874 if (ASIC_IS_DCE41(rdev) || ASIC_IS_DCE5(rdev)) { 1875 WREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset, 1876 DMIF_BUFFERS_ALLOCATED(buffer_alloc)); 1877 for (i = 0; i < rdev->usec_timeout; i++) { 1878 if (RREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset) & 1879 DMIF_BUFFERS_ALLOCATED_COMPLETED) 1880 break; 1881 udelay(1); 1882 } 1883 } 1884 1885 if (radeon_crtc->base.enabled && mode) { 1886 switch (tmp) { 1887 case 0: 1888 case 4: 1889 default: 1890 if (ASIC_IS_DCE5(rdev)) 1891 return 4096 * 2; 1892 else 1893 return 3840 * 2; 1894 case 1: 1895 case 5: 1896 if (ASIC_IS_DCE5(rdev)) 1897 return 6144 * 2; 1898 else 1899 return 5760 * 2; 1900 case 2: 1901 case 6: 1902 if (ASIC_IS_DCE5(rdev)) 1903 return 8192 * 2; 1904 else 1905 return 7680 * 2; 1906 case 3: 1907 case 7: 1908 if (ASIC_IS_DCE5(rdev)) 1909 return 2048 * 2; 1910 else 1911 return 1920 * 2; 1912 } 1913 } 1914 1915 /* controller not enabled, so no lb used */ 1916 return 0; 1917} 1918 1919u32 evergreen_get_number_of_dram_channels(struct radeon_device *rdev) 1920{ 1921 u32 tmp = RREG32(MC_SHARED_CHMAP); 1922 1923 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) { 1924 case 0: 1925 default: 1926 return 1; 1927 case 1: 1928 return 2; 1929 case 2: 1930 return 4; 1931 case 3: 1932 return 8; 1933 } 1934} 1935 1936struct evergreen_wm_params { 1937 u32 dram_channels; /* number of dram channels */ 1938 u32 yclk; /* bandwidth per dram data pin in kHz */ 1939 u32 sclk; /* engine clock in kHz */ 1940 u32 disp_clk; /* display clock in kHz */ 1941 u32 src_width; /* viewport width */ 1942 u32 active_time; /* active display time in ns */ 1943 u32 blank_time; /* blank time in ns */ 1944 bool interlaced; /* mode is interlaced */ 1945 fixed20_12 vsc; /* vertical scale ratio */ 1946 u32 num_heads; /* number of active crtcs */ 1947 u32 bytes_per_pixel; /* bytes per pixel display + overlay */ 1948 u32 lb_size; /* line buffer allocated to pipe */ 1949 u32 vtaps; /* vertical scaler taps */ 1950}; 1951 1952static u32 evergreen_dram_bandwidth(struct evergreen_wm_params *wm) 1953{ 1954 /* Calculate DRAM Bandwidth and the part allocated to display. */ 1955 fixed20_12 dram_efficiency; /* 0.7 */ 1956 fixed20_12 yclk, dram_channels, bandwidth; 1957 fixed20_12 a; 1958 1959 a.full = dfixed_const(1000); 1960 yclk.full = dfixed_const(wm->yclk); 1961 yclk.full = dfixed_div(yclk, a); 1962 dram_channels.full = dfixed_const(wm->dram_channels * 4); 1963 a.full = dfixed_const(10); 1964 dram_efficiency.full = dfixed_const(7); 1965 dram_efficiency.full = dfixed_div(dram_efficiency, a); 1966 bandwidth.full = dfixed_mul(dram_channels, yclk); 1967 bandwidth.full = dfixed_mul(bandwidth, dram_efficiency); 1968 1969 return dfixed_trunc(bandwidth); 1970} 1971 1972static u32 evergreen_dram_bandwidth_for_display(struct evergreen_wm_params *wm) 1973{ 1974 /* Calculate DRAM Bandwidth and the part allocated to display. */ 1975 fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */ 1976 fixed20_12 yclk, dram_channels, bandwidth; 1977 fixed20_12 a; 1978 1979 a.full = dfixed_const(1000); 1980 yclk.full = dfixed_const(wm->yclk); 1981 yclk.full = dfixed_div(yclk, a); 1982 dram_channels.full = dfixed_const(wm->dram_channels * 4); 1983 a.full = dfixed_const(10); 1984 disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */ 1985 disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a); 1986 bandwidth.full = dfixed_mul(dram_channels, yclk); 1987 bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation); 1988 1989 return dfixed_trunc(bandwidth); 1990} 1991 1992static u32 evergreen_data_return_bandwidth(struct evergreen_wm_params *wm) 1993{ 1994 /* Calculate the display Data return Bandwidth */ 1995 fixed20_12 return_efficiency; /* 0.8 */ 1996 fixed20_12 sclk, bandwidth; 1997 fixed20_12 a; 1998 1999 a.full = dfixed_const(1000); 2000 sclk.full = dfixed_const(wm->sclk); 2001 sclk.full = dfixed_div(sclk, a); 2002 a.full = dfixed_const(10); 2003 return_efficiency.full = dfixed_const(8); 2004 return_efficiency.full = dfixed_div(return_efficiency, a); 2005 a.full = dfixed_const(32); 2006 bandwidth.full = dfixed_mul(a, sclk); 2007 bandwidth.full = dfixed_mul(bandwidth, return_efficiency); 2008 2009 return dfixed_trunc(bandwidth); 2010} 2011 2012static u32 evergreen_dmif_request_bandwidth(struct evergreen_wm_params *wm) 2013{ 2014 /* Calculate the DMIF Request Bandwidth */ 2015 fixed20_12 disp_clk_request_efficiency; /* 0.8 */ 2016 fixed20_12 disp_clk, bandwidth; 2017 fixed20_12 a; 2018 2019 a.full = dfixed_const(1000); 2020 disp_clk.full = dfixed_const(wm->disp_clk); 2021 disp_clk.full = dfixed_div(disp_clk, a); 2022 a.full = dfixed_const(10); 2023 disp_clk_request_efficiency.full = dfixed_const(8); 2024 disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a); 2025 a.full = dfixed_const(32); 2026 bandwidth.full = dfixed_mul(a, disp_clk); 2027 bandwidth.full = dfixed_mul(bandwidth, disp_clk_request_efficiency); 2028 2029 return dfixed_trunc(bandwidth); 2030} 2031 2032static u32 evergreen_available_bandwidth(struct evergreen_wm_params *wm) 2033{ 2034 /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */ 2035 u32 dram_bandwidth = evergreen_dram_bandwidth(wm); 2036 u32 data_return_bandwidth = evergreen_data_return_bandwidth(wm); 2037 u32 dmif_req_bandwidth = evergreen_dmif_request_bandwidth(wm); 2038 2039 return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth)); 2040} 2041 2042static u32 evergreen_average_bandwidth(struct evergreen_wm_params *wm) 2043{ 2044 /* Calculate the display mode Average Bandwidth 2045 * DisplayMode should contain the source and destination dimensions, 2046 * timing, etc. 2047 */ 2048 fixed20_12 bpp; 2049 fixed20_12 line_time; 2050 fixed20_12 src_width; 2051 fixed20_12 bandwidth; 2052 fixed20_12 a; 2053 2054 a.full = dfixed_const(1000); 2055 line_time.full = dfixed_const(wm->active_time + wm->blank_time); 2056 line_time.full = dfixed_div(line_time, a); 2057 bpp.full = dfixed_const(wm->bytes_per_pixel); 2058 src_width.full = dfixed_const(wm->src_width); 2059 bandwidth.full = dfixed_mul(src_width, bpp); 2060 bandwidth.full = dfixed_mul(bandwidth, wm->vsc); 2061 bandwidth.full = dfixed_div(bandwidth, line_time); 2062 2063 return dfixed_trunc(bandwidth); 2064} 2065 2066static u32 evergreen_latency_watermark(struct evergreen_wm_params *wm) 2067{ 2068 /* First calcualte the latency in ns */ 2069 u32 mc_latency = 2000; /* 2000 ns. */ 2070 u32 available_bandwidth = evergreen_available_bandwidth(wm); 2071 u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth; 2072 u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth; 2073 u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */ 2074 u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) + 2075 (wm->num_heads * cursor_line_pair_return_time); 2076 u32 latency = mc_latency + other_heads_data_return_time + dc_latency; 2077 u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time; 2078 fixed20_12 a, b, c; 2079 2080 if (wm->num_heads == 0) 2081 return 0; 2082 2083 a.full = dfixed_const(2); 2084 b.full = dfixed_const(1); 2085 if ((wm->vsc.full > a.full) || 2086 ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) || 2087 (wm->vtaps >= 5) || 2088 ((wm->vsc.full >= a.full) && wm->interlaced)) 2089 max_src_lines_per_dst_line = 4; 2090 else 2091 max_src_lines_per_dst_line = 2; 2092 2093 a.full = dfixed_const(available_bandwidth); 2094 b.full = dfixed_const(wm->num_heads); 2095 a.full = dfixed_div(a, b); 2096 2097 lb_fill_bw = min(dfixed_trunc(a), wm->disp_clk * wm->bytes_per_pixel / 1000); 2098 2099 a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel); 2100 b.full = dfixed_const(1000); 2101 c.full = dfixed_const(lb_fill_bw); 2102 b.full = dfixed_div(c, b); 2103 a.full = dfixed_div(a, b); 2104 line_fill_time = dfixed_trunc(a); 2105 2106 if (line_fill_time < wm->active_time) 2107 return latency; 2108 else 2109 return latency + (line_fill_time - wm->active_time); 2110 2111} 2112 2113static bool evergreen_average_bandwidth_vs_dram_bandwidth_for_display(struct evergreen_wm_params *wm) 2114{ 2115 if (evergreen_average_bandwidth(wm) <= 2116 (evergreen_dram_bandwidth_for_display(wm) / wm->num_heads)) 2117 return true; 2118 else 2119 return false; 2120}; 2121 2122static bool evergreen_average_bandwidth_vs_available_bandwidth(struct evergreen_wm_params *wm) 2123{ 2124 if (evergreen_average_bandwidth(wm) <= 2125 (evergreen_available_bandwidth(wm) / wm->num_heads)) 2126 return true; 2127 else 2128 return false; 2129}; 2130 2131static bool evergreen_check_latency_hiding(struct evergreen_wm_params *wm) 2132{ 2133 u32 lb_partitions = wm->lb_size / wm->src_width; 2134 u32 line_time = wm->active_time + wm->blank_time; 2135 u32 latency_tolerant_lines; 2136 u32 latency_hiding; 2137 fixed20_12 a; 2138 2139 a.full = dfixed_const(1); 2140 if (wm->vsc.full > a.full) 2141 latency_tolerant_lines = 1; 2142 else { 2143 if (lb_partitions <= (wm->vtaps + 1)) 2144 latency_tolerant_lines = 1; 2145 else 2146 latency_tolerant_lines = 2; 2147 } 2148 2149 latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time); 2150 2151 if (evergreen_latency_watermark(wm) <= latency_hiding) 2152 return true; 2153 else 2154 return false; 2155} 2156 2157static void evergreen_program_watermarks(struct radeon_device *rdev, 2158 struct radeon_crtc *radeon_crtc, 2159 u32 lb_size, u32 num_heads) 2160{ 2161 struct drm_display_mode *mode = &radeon_crtc->base.mode; 2162 struct evergreen_wm_params wm_low, wm_high; 2163 u32 dram_channels; 2164 u32 active_time; 2165 u32 line_time = 0; 2166 u32 latency_watermark_a = 0, latency_watermark_b = 0; 2167 u32 priority_a_mark = 0, priority_b_mark = 0; 2168 u32 priority_a_cnt = PRIORITY_OFF; 2169 u32 priority_b_cnt = PRIORITY_OFF; 2170 u32 pipe_offset = radeon_crtc->crtc_id * 16; 2171 u32 tmp, arb_control3; 2172 fixed20_12 a, b, c; 2173 2174 if (radeon_crtc->base.enabled && num_heads && mode) { 2175 active_time = (u32) div_u64((u64)mode->crtc_hdisplay * 1000000, 2176 (u32)mode->clock); 2177 line_time = (u32) div_u64((u64)mode->crtc_htotal * 1000000, 2178 (u32)mode->clock); 2179 line_time = min(line_time, (u32)65535); 2180 priority_a_cnt = 0; 2181 priority_b_cnt = 0; 2182 dram_channels = evergreen_get_number_of_dram_channels(rdev); 2183 2184 /* watermark for high clocks */ 2185 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) { 2186 wm_high.yclk = 2187 radeon_dpm_get_mclk(rdev, false) * 10; 2188 wm_high.sclk = 2189 radeon_dpm_get_sclk(rdev, false) * 10; 2190 } else { 2191 wm_high.yclk = rdev->pm.current_mclk * 10; 2192 wm_high.sclk = rdev->pm.current_sclk * 10; 2193 } 2194 2195 wm_high.disp_clk = mode->clock; 2196 wm_high.src_width = mode->crtc_hdisplay; 2197 wm_high.active_time = active_time; 2198 wm_high.blank_time = line_time - wm_high.active_time; 2199 wm_high.interlaced = false; 2200 if (mode->flags & DRM_MODE_FLAG_INTERLACE) 2201 wm_high.interlaced = true; 2202 wm_high.vsc = radeon_crtc->vsc; 2203 wm_high.vtaps = 1; 2204 if (radeon_crtc->rmx_type != RMX_OFF) 2205 wm_high.vtaps = 2; 2206 wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */ 2207 wm_high.lb_size = lb_size; 2208 wm_high.dram_channels = dram_channels; 2209 wm_high.num_heads = num_heads; 2210 2211 /* watermark for low clocks */ 2212 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) { 2213 wm_low.yclk = 2214 radeon_dpm_get_mclk(rdev, true) * 10; 2215 wm_low.sclk = 2216 radeon_dpm_get_sclk(rdev, true) * 10; 2217 } else { 2218 wm_low.yclk = rdev->pm.current_mclk * 10; 2219 wm_low.sclk = rdev->pm.current_sclk * 10; 2220 } 2221 2222 wm_low.disp_clk = mode->clock; 2223 wm_low.src_width = mode->crtc_hdisplay; 2224 wm_low.active_time = active_time; 2225 wm_low.blank_time = line_time - wm_low.active_time; 2226 wm_low.interlaced = false; 2227 if (mode->flags & DRM_MODE_FLAG_INTERLACE) 2228 wm_low.interlaced = true; 2229 wm_low.vsc = radeon_crtc->vsc; 2230 wm_low.vtaps = 1; 2231 if (radeon_crtc->rmx_type != RMX_OFF) 2232 wm_low.vtaps = 2; 2233 wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */ 2234 wm_low.lb_size = lb_size; 2235 wm_low.dram_channels = dram_channels; 2236 wm_low.num_heads = num_heads; 2237 2238 /* set for high clocks */ 2239 latency_watermark_a = min(evergreen_latency_watermark(&wm_high), (u32)65535); 2240 /* set for low clocks */ 2241 latency_watermark_b = min(evergreen_latency_watermark(&wm_low), (u32)65535); 2242 2243 /* possibly force display priority to high */ 2244 /* should really do this at mode validation time... */ 2245 if (!evergreen_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) || 2246 !evergreen_average_bandwidth_vs_available_bandwidth(&wm_high) || 2247 !evergreen_check_latency_hiding(&wm_high) || 2248 (rdev->disp_priority == 2)) { 2249 DRM_DEBUG_KMS("force priority a to high\n"); 2250 priority_a_cnt |= PRIORITY_ALWAYS_ON; 2251 } 2252 if (!evergreen_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) || 2253 !evergreen_average_bandwidth_vs_available_bandwidth(&wm_low) || 2254 !evergreen_check_latency_hiding(&wm_low) || 2255 (rdev->disp_priority == 2)) { 2256 DRM_DEBUG_KMS("force priority b to high\n"); 2257 priority_b_cnt |= PRIORITY_ALWAYS_ON; 2258 } 2259 2260 a.full = dfixed_const(1000); 2261 b.full = dfixed_const(mode->clock); 2262 b.full = dfixed_div(b, a); 2263 c.full = dfixed_const(latency_watermark_a); 2264 c.full = dfixed_mul(c, b); 2265 c.full = dfixed_mul(c, radeon_crtc->hsc); 2266 c.full = dfixed_div(c, a); 2267 a.full = dfixed_const(16); 2268 c.full = dfixed_div(c, a); 2269 priority_a_mark = dfixed_trunc(c); 2270 priority_a_cnt |= priority_a_mark & PRIORITY_MARK_MASK; 2271 2272 a.full = dfixed_const(1000); 2273 b.full = dfixed_const(mode->clock); 2274 b.full = dfixed_div(b, a); 2275 c.full = dfixed_const(latency_watermark_b); 2276 c.full = dfixed_mul(c, b); 2277 c.full = dfixed_mul(c, radeon_crtc->hsc); 2278 c.full = dfixed_div(c, a); 2279 a.full = dfixed_const(16); 2280 c.full = dfixed_div(c, a); 2281 priority_b_mark = dfixed_trunc(c); 2282 priority_b_cnt |= priority_b_mark & PRIORITY_MARK_MASK; 2283 2284 /* Save number of lines the linebuffer leads before the scanout */ 2285 radeon_crtc->lb_vblank_lead_lines = DIV_ROUND_UP(lb_size, mode->crtc_hdisplay); 2286 } 2287 2288 /* select wm A */ 2289 arb_control3 = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset); 2290 tmp = arb_control3; 2291 tmp &= ~LATENCY_WATERMARK_MASK(3); 2292 tmp |= LATENCY_WATERMARK_MASK(1); 2293 WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp); 2294 WREG32(PIPE0_LATENCY_CONTROL + pipe_offset, 2295 (LATENCY_LOW_WATERMARK(latency_watermark_a) | 2296 LATENCY_HIGH_WATERMARK(line_time))); 2297 /* select wm B */ 2298 tmp = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset); 2299 tmp &= ~LATENCY_WATERMARK_MASK(3); 2300 tmp |= LATENCY_WATERMARK_MASK(2); 2301 WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp); 2302 WREG32(PIPE0_LATENCY_CONTROL + pipe_offset, 2303 (LATENCY_LOW_WATERMARK(latency_watermark_b) | 2304 LATENCY_HIGH_WATERMARK(line_time))); 2305 /* restore original selection */ 2306 WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, arb_control3); 2307 2308 /* write the priority marks */ 2309 WREG32(PRIORITY_A_CNT + radeon_crtc->crtc_offset, priority_a_cnt); 2310 WREG32(PRIORITY_B_CNT + radeon_crtc->crtc_offset, priority_b_cnt); 2311 2312 /* save values for DPM */ 2313 radeon_crtc->line_time = line_time; 2314 radeon_crtc->wm_high = latency_watermark_a; 2315 radeon_crtc->wm_low = latency_watermark_b; 2316} 2317 2318/** 2319 * evergreen_bandwidth_update - update display watermarks callback. 2320 * 2321 * @rdev: radeon_device pointer 2322 * 2323 * Update the display watermarks based on the requested mode(s) 2324 * (evergreen+). 2325 */ 2326void evergreen_bandwidth_update(struct radeon_device *rdev) 2327{ 2328 struct drm_display_mode *mode0 = NULL; 2329 struct drm_display_mode *mode1 = NULL; 2330 u32 num_heads = 0, lb_size; 2331 int i; 2332 2333 if (!rdev->mode_info.mode_config_initialized) 2334 return; 2335 2336 radeon_update_display_priority(rdev); 2337 2338 for (i = 0; i < rdev->num_crtc; i++) { 2339 if (rdev->mode_info.crtcs[i]->base.enabled) 2340 num_heads++; 2341 } 2342 for (i = 0; i < rdev->num_crtc; i += 2) { 2343 mode0 = &rdev->mode_info.crtcs[i]->base.mode; 2344 mode1 = &rdev->mode_info.crtcs[i+1]->base.mode; 2345 lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode0, mode1); 2346 evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads); 2347 lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i+1], mode1, mode0); 2348 evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i+1], lb_size, num_heads); 2349 } 2350} 2351 2352/** 2353 * evergreen_mc_wait_for_idle - wait for MC idle callback. 2354 * 2355 * @rdev: radeon_device pointer 2356 * 2357 * Wait for the MC (memory controller) to be idle. 2358 * (evergreen+). 2359 * Returns 0 if the MC is idle, -1 if not. 2360 */ 2361int evergreen_mc_wait_for_idle(struct radeon_device *rdev) 2362{ 2363 unsigned i; 2364 u32 tmp; 2365 2366 for (i = 0; i < rdev->usec_timeout; i++) { 2367 /* read MC_STATUS */ 2368 tmp = RREG32(SRBM_STATUS) & 0x1F00; 2369 if (!tmp) 2370 return 0; 2371 udelay(1); 2372 } 2373 return -1; 2374} 2375 2376/* 2377 * GART 2378 */ 2379void evergreen_pcie_gart_tlb_flush(struct radeon_device *rdev) 2380{ 2381 unsigned i; 2382 u32 tmp; 2383 2384 WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1); 2385 2386 WREG32(VM_CONTEXT0_REQUEST_RESPONSE, REQUEST_TYPE(1)); 2387 for (i = 0; i < rdev->usec_timeout; i++) { 2388 /* read MC_STATUS */ 2389 tmp = RREG32(VM_CONTEXT0_REQUEST_RESPONSE); 2390 tmp = (tmp & RESPONSE_TYPE_MASK) >> RESPONSE_TYPE_SHIFT; 2391 if (tmp == 2) { 2392 pr_warn("[drm] r600 flush TLB failed\n"); 2393 return; 2394 } 2395 if (tmp) { 2396 return; 2397 } 2398 udelay(1); 2399 } 2400} 2401 2402static int evergreen_pcie_gart_enable(struct radeon_device *rdev) 2403{ 2404 u32 tmp; 2405 int r; 2406 2407 if (rdev->gart.robj == NULL) { 2408 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n"); 2409 return -EINVAL; 2410 } 2411 r = radeon_gart_table_vram_pin(rdev); 2412 if (r) 2413 return r; 2414 /* Setup L2 cache */ 2415 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING | 2416 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE | 2417 EFFECTIVE_L2_QUEUE_SIZE(7)); 2418 WREG32(VM_L2_CNTL2, 0); 2419 WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2)); 2420 /* Setup TLB control */ 2421 tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING | 2422 SYSTEM_ACCESS_MODE_NOT_IN_SYS | 2423 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU | 2424 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5); 2425 if (rdev->flags & RADEON_IS_IGP) { 2426 WREG32(FUS_MC_VM_MD_L1_TLB0_CNTL, tmp); 2427 WREG32(FUS_MC_VM_MD_L1_TLB1_CNTL, tmp); 2428 WREG32(FUS_MC_VM_MD_L1_TLB2_CNTL, tmp); 2429 } else { 2430 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp); 2431 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp); 2432 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp); 2433 if ((rdev->family == CHIP_JUNIPER) || 2434 (rdev->family == CHIP_CYPRESS) || 2435 (rdev->family == CHIP_HEMLOCK) || 2436 (rdev->family == CHIP_BARTS)) 2437 WREG32(MC_VM_MD_L1_TLB3_CNTL, tmp); 2438 } 2439 WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp); 2440 WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp); 2441 WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp); 2442 WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp); 2443 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12); 2444 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12); 2445 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12); 2446 WREG32(VM_CONTEXT0_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) | 2447 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT); 2448 WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR, 2449 (u32)(rdev->dummy_page.addr >> 12)); 2450 WREG32(VM_CONTEXT1_CNTL, 0); 2451 2452 evergreen_pcie_gart_tlb_flush(rdev); 2453 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n", 2454 (unsigned)(rdev->mc.gtt_size >> 20), 2455 (unsigned long long)rdev->gart.table_addr); 2456 rdev->gart.ready = true; 2457 return 0; 2458} 2459 2460static void evergreen_pcie_gart_disable(struct radeon_device *rdev) 2461{ 2462 u32 tmp; 2463 2464 /* Disable all tables */ 2465 WREG32(VM_CONTEXT0_CNTL, 0); 2466 WREG32(VM_CONTEXT1_CNTL, 0); 2467 2468 /* Setup L2 cache */ 2469 WREG32(VM_L2_CNTL, ENABLE_L2_FRAGMENT_PROCESSING | 2470 EFFECTIVE_L2_QUEUE_SIZE(7)); 2471 WREG32(VM_L2_CNTL2, 0); 2472 WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2)); 2473 /* Setup TLB control */ 2474 tmp = EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5); 2475 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp); 2476 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp); 2477 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp); 2478 WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp); 2479 WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp); 2480 WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp); 2481 WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp); 2482 radeon_gart_table_vram_unpin(rdev); 2483} 2484 2485static void evergreen_pcie_gart_fini(struct radeon_device *rdev) 2486{ 2487 evergreen_pcie_gart_disable(rdev); 2488 radeon_gart_table_vram_free(rdev); 2489 radeon_gart_fini(rdev); 2490} 2491 2492 2493static void evergreen_agp_enable(struct radeon_device *rdev) 2494{ 2495 u32 tmp; 2496 2497 /* Setup L2 cache */ 2498 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING | 2499 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE | 2500 EFFECTIVE_L2_QUEUE_SIZE(7)); 2501 WREG32(VM_L2_CNTL2, 0); 2502 WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2)); 2503 /* Setup TLB control */ 2504 tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING | 2505 SYSTEM_ACCESS_MODE_NOT_IN_SYS | 2506 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU | 2507 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5); 2508 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp); 2509 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp); 2510 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp); 2511 WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp); 2512 WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp); 2513 WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp); 2514 WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp); 2515 WREG32(VM_CONTEXT0_CNTL, 0); 2516 WREG32(VM_CONTEXT1_CNTL, 0); 2517} 2518 2519static const unsigned ni_dig_offsets[] = 2520{ 2521 NI_DIG0_REGISTER_OFFSET, 2522 NI_DIG1_REGISTER_OFFSET, 2523 NI_DIG2_REGISTER_OFFSET, 2524 NI_DIG3_REGISTER_OFFSET, 2525 NI_DIG4_REGISTER_OFFSET, 2526 NI_DIG5_REGISTER_OFFSET 2527}; 2528 2529static const unsigned ni_tx_offsets[] = 2530{ 2531 NI_DCIO_UNIPHY0_UNIPHY_TX_CONTROL1, 2532 NI_DCIO_UNIPHY1_UNIPHY_TX_CONTROL1, 2533 NI_DCIO_UNIPHY2_UNIPHY_TX_CONTROL1, 2534 NI_DCIO_UNIPHY3_UNIPHY_TX_CONTROL1, 2535 NI_DCIO_UNIPHY4_UNIPHY_TX_CONTROL1, 2536 NI_DCIO_UNIPHY5_UNIPHY_TX_CONTROL1 2537}; 2538 2539static const unsigned evergreen_dp_offsets[] = 2540{ 2541 EVERGREEN_DP0_REGISTER_OFFSET, 2542 EVERGREEN_DP1_REGISTER_OFFSET, 2543 EVERGREEN_DP2_REGISTER_OFFSET, 2544 EVERGREEN_DP3_REGISTER_OFFSET, 2545 EVERGREEN_DP4_REGISTER_OFFSET, 2546 EVERGREEN_DP5_REGISTER_OFFSET 2547}; 2548 2549static const unsigned evergreen_disp_int_status[] = 2550{ 2551 DISP_INTERRUPT_STATUS, 2552 DISP_INTERRUPT_STATUS_CONTINUE, 2553 DISP_INTERRUPT_STATUS_CONTINUE2, 2554 DISP_INTERRUPT_STATUS_CONTINUE3, 2555 DISP_INTERRUPT_STATUS_CONTINUE4, 2556 DISP_INTERRUPT_STATUS_CONTINUE5 2557}; 2558 2559/* 2560 * Assumption is that EVERGREEN_CRTC_MASTER_EN enable for requested crtc 2561 * We go from crtc to connector and it is not relible since it 2562 * should be an opposite direction .If crtc is enable then 2563 * find the dig_fe which selects this crtc and insure that it enable. 2564 * if such dig_fe is found then find dig_be which selects found dig_be and 2565 * insure that it enable and in DP_SST mode. 2566 * if UNIPHY_PLL_CONTROL1.enable then we should disconnect timing 2567 * from dp symbols clocks . 2568 */ 2569static bool evergreen_is_dp_sst_stream_enabled(struct radeon_device *rdev, 2570 unsigned crtc_id, unsigned *ret_dig_fe) 2571{ 2572 unsigned i; 2573 unsigned dig_fe; 2574 unsigned dig_be; 2575 unsigned dig_en_be; 2576 unsigned uniphy_pll; 2577 unsigned digs_fe_selected; 2578 unsigned dig_be_mode; 2579 unsigned dig_fe_mask; 2580 bool is_enabled = false; 2581 bool found_crtc = false; 2582 2583 /* loop through all running dig_fe to find selected crtc */ 2584 for (i = 0; i < ARRAY_SIZE(ni_dig_offsets); i++) { 2585 dig_fe = RREG32(NI_DIG_FE_CNTL + ni_dig_offsets[i]); 2586 if (dig_fe & NI_DIG_FE_CNTL_SYMCLK_FE_ON && 2587 crtc_id == NI_DIG_FE_CNTL_SOURCE_SELECT(dig_fe)) { 2588 /* found running pipe */ 2589 found_crtc = true; 2590 dig_fe_mask = 1 << i; 2591 dig_fe = i; 2592 break; 2593 } 2594 } 2595 2596 if (found_crtc) { 2597 /* loop through all running dig_be to find selected dig_fe */ 2598 for (i = 0; i < ARRAY_SIZE(ni_dig_offsets); i++) { 2599 dig_be = RREG32(NI_DIG_BE_CNTL + ni_dig_offsets[i]); 2600 /* if dig_fe_selected by dig_be? */ 2601 digs_fe_selected = NI_DIG_BE_CNTL_FE_SOURCE_SELECT(dig_be); 2602 dig_be_mode = NI_DIG_FE_CNTL_MODE(dig_be); 2603 if (dig_fe_mask & digs_fe_selected && 2604 /* if dig_be in sst mode? */ 2605 dig_be_mode == NI_DIG_BE_DPSST) { 2606 dig_en_be = RREG32(NI_DIG_BE_EN_CNTL + 2607 ni_dig_offsets[i]); 2608 uniphy_pll = RREG32(NI_DCIO_UNIPHY0_PLL_CONTROL1 + 2609 ni_tx_offsets[i]); 2610 /* dig_be enable and tx is running */ 2611 if (dig_en_be & NI_DIG_BE_EN_CNTL_ENABLE && 2612 dig_en_be & NI_DIG_BE_EN_CNTL_SYMBCLK_ON && 2613 uniphy_pll & NI_DCIO_UNIPHY0_PLL_CONTROL1_ENABLE) { 2614 is_enabled = true; 2615 *ret_dig_fe = dig_fe; 2616 break; 2617 } 2618 } 2619 } 2620 } 2621 2622 return is_enabled; 2623} 2624 2625/* 2626 * Blank dig when in dp sst mode 2627 * Dig ignores crtc timing 2628 */ 2629static void evergreen_blank_dp_output(struct radeon_device *rdev, 2630 unsigned dig_fe) 2631{ 2632 unsigned stream_ctrl; 2633 unsigned fifo_ctrl; 2634 unsigned counter = 0; 2635 2636 if (dig_fe >= ARRAY_SIZE(evergreen_dp_offsets)) { 2637 DRM_ERROR("invalid dig_fe %d\n", dig_fe); 2638 return; 2639 } 2640 2641 stream_ctrl = RREG32(EVERGREEN_DP_VID_STREAM_CNTL + 2642 evergreen_dp_offsets[dig_fe]); 2643 if (!(stream_ctrl & EVERGREEN_DP_VID_STREAM_CNTL_ENABLE)) { 2644 DRM_ERROR("dig %d , should be enable\n", dig_fe); 2645 return; 2646 } 2647 2648 stream_ctrl &=~EVERGREEN_DP_VID_STREAM_CNTL_ENABLE; 2649 WREG32(EVERGREEN_DP_VID_STREAM_CNTL + 2650 evergreen_dp_offsets[dig_fe], stream_ctrl); 2651 2652 stream_ctrl = RREG32(EVERGREEN_DP_VID_STREAM_CNTL + 2653 evergreen_dp_offsets[dig_fe]); 2654 while (counter < 32 && stream_ctrl & EVERGREEN_DP_VID_STREAM_STATUS) { 2655 msleep(1); 2656 counter++; 2657 stream_ctrl = RREG32(EVERGREEN_DP_VID_STREAM_CNTL + 2658 evergreen_dp_offsets[dig_fe]); 2659 } 2660 if (counter >= 32 ) 2661 DRM_ERROR("counter exceeds %d\n", counter); 2662 2663 fifo_ctrl = RREG32(EVERGREEN_DP_STEER_FIFO + evergreen_dp_offsets[dig_fe]); 2664 fifo_ctrl |= EVERGREEN_DP_STEER_FIFO_RESET; 2665 WREG32(EVERGREEN_DP_STEER_FIFO + evergreen_dp_offsets[dig_fe], fifo_ctrl); 2666 2667} 2668 2669void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save) 2670{ 2671 u32 crtc_enabled, tmp, frame_count, blackout; 2672 int i, j; 2673 unsigned dig_fe; 2674 2675 if (!ASIC_IS_NODCE(rdev)) { 2676 save->vga_render_control = RREG32(VGA_RENDER_CONTROL); 2677 save->vga_hdp_control = RREG32(VGA_HDP_CONTROL); 2678 2679 /* disable VGA render */ 2680 WREG32(VGA_RENDER_CONTROL, 0); 2681 } 2682 /* blank the display controllers */ 2683 for (i = 0; i < rdev->num_crtc; i++) { 2684 crtc_enabled = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]) & EVERGREEN_CRTC_MASTER_EN; 2685 if (crtc_enabled) { 2686 save->crtc_enabled[i] = true; 2687 if (ASIC_IS_DCE6(rdev)) { 2688 tmp = RREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i]); 2689 if (!(tmp & EVERGREEN_CRTC_BLANK_DATA_EN)) { 2690 radeon_wait_for_vblank(rdev, i); 2691 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1); 2692 tmp |= EVERGREEN_CRTC_BLANK_DATA_EN; 2693 WREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i], tmp); 2694 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0); 2695 } 2696 } else { 2697 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]); 2698 if (!(tmp & EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE)) { 2699 radeon_wait_for_vblank(rdev, i); 2700 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1); 2701 tmp |= EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE; 2702 WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp); 2703 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0); 2704 } 2705 } 2706 /* wait for the next frame */ 2707 frame_count = radeon_get_vblank_counter(rdev, i); 2708 for (j = 0; j < rdev->usec_timeout; j++) { 2709 if (radeon_get_vblank_counter(rdev, i) != frame_count) 2710 break; 2711 udelay(1); 2712 } 2713 /*we should disable dig if it drives dp sst*/ 2714 /*but we are in radeon_device_init and the topology is unknown*/ 2715 /*and it is available after radeon_modeset_init*/ 2716 /*the following method radeon_atom_encoder_dpms_dig*/ 2717 /*does the job if we initialize it properly*/ 2718 /*for now we do it this manually*/ 2719 /**/ 2720 if (ASIC_IS_DCE5(rdev) && 2721 evergreen_is_dp_sst_stream_enabled(rdev, i ,&dig_fe)) 2722 evergreen_blank_dp_output(rdev, dig_fe); 2723 /*we could remove 6 lines below*/ 2724 /* XXX this is a hack to avoid strange behavior with EFI on certain systems */ 2725 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1); 2726 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]); 2727 tmp &= ~EVERGREEN_CRTC_MASTER_EN; 2728 WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp); 2729 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0); 2730 save->crtc_enabled[i] = false; 2731 /* ***** */ 2732 } else { 2733 save->crtc_enabled[i] = false; 2734 } 2735 } 2736 2737 radeon_mc_wait_for_idle(rdev); 2738 2739 blackout = RREG32(MC_SHARED_BLACKOUT_CNTL); 2740 if ((blackout & BLACKOUT_MODE_MASK) != 1) { 2741 /* Block CPU access */ 2742 WREG32(BIF_FB_EN, 0); 2743 /* blackout the MC */ 2744 blackout &= ~BLACKOUT_MODE_MASK; 2745 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1); 2746 } 2747 /* wait for the MC to settle */ 2748 udelay(100); 2749 2750 /* lock double buffered regs */ 2751 for (i = 0; i < rdev->num_crtc; i++) { 2752 if (save->crtc_enabled[i]) { 2753 tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]); 2754 if (!(tmp & EVERGREEN_GRPH_UPDATE_LOCK)) { 2755 tmp |= EVERGREEN_GRPH_UPDATE_LOCK; 2756 WREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i], tmp); 2757 } 2758 tmp = RREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i]); 2759 if (!(tmp & 1)) { 2760 tmp |= 1; 2761 WREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i], tmp); 2762 } 2763 } 2764 } 2765} 2766 2767void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save) 2768{ 2769 u32 tmp, frame_count; 2770 int i, j; 2771 2772 /* update crtc base addresses */ 2773 for (i = 0; i < rdev->num_crtc; i++) { 2774 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i], 2775 upper_32_bits(rdev->mc.vram_start)); 2776 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i], 2777 upper_32_bits(rdev->mc.vram_start)); 2778 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + crtc_offsets[i], 2779 (u32)rdev->mc.vram_start); 2780 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + crtc_offsets[i], 2781 (u32)rdev->mc.vram_start); 2782 } 2783 2784 if (!ASIC_IS_NODCE(rdev)) { 2785 WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS_HIGH, upper_32_bits(rdev->mc.vram_start)); 2786 WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS, (u32)rdev->mc.vram_start); 2787 } 2788 2789 /* unlock regs and wait for update */ 2790 for (i = 0; i < rdev->num_crtc; i++) { 2791 if (save->crtc_enabled[i]) { 2792 tmp = RREG32(EVERGREEN_MASTER_UPDATE_MODE + crtc_offsets[i]); 2793 if ((tmp & 0x7) != 0) { 2794 tmp &= ~0x7; 2795 WREG32(EVERGREEN_MASTER_UPDATE_MODE + crtc_offsets[i], tmp); 2796 } 2797 tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]); 2798 if (tmp & EVERGREEN_GRPH_UPDATE_LOCK) { 2799 tmp &= ~EVERGREEN_GRPH_UPDATE_LOCK; 2800 WREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i], tmp); 2801 } 2802 tmp = RREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i]); 2803 if (tmp & 1) { 2804 tmp &= ~1; 2805 WREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i], tmp); 2806 } 2807 for (j = 0; j < rdev->usec_timeout; j++) { 2808 tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]); 2809 if ((tmp & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING) == 0) 2810 break; 2811 udelay(1); 2812 } 2813 } 2814 } 2815 2816 /* unblackout the MC */ 2817 tmp = RREG32(MC_SHARED_BLACKOUT_CNTL); 2818 tmp &= ~BLACKOUT_MODE_MASK; 2819 WREG32(MC_SHARED_BLACKOUT_CNTL, tmp); 2820 /* allow CPU access */ 2821 WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN); 2822 2823 for (i = 0; i < rdev->num_crtc; i++) { 2824 if (save->crtc_enabled[i]) { 2825 if (ASIC_IS_DCE6(rdev)) { 2826 tmp = RREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i]); 2827 tmp &= ~EVERGREEN_CRTC_BLANK_DATA_EN; 2828 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1); 2829 WREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i], tmp); 2830 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0); 2831 } else { 2832 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]); 2833 tmp &= ~EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE; 2834 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1); 2835 WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp); 2836 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0); 2837 } 2838 /* wait for the next frame */ 2839 frame_count = radeon_get_vblank_counter(rdev, i); 2840 for (j = 0; j < rdev->usec_timeout; j++) { 2841 if (radeon_get_vblank_counter(rdev, i) != frame_count) 2842 break; 2843 udelay(1); 2844 } 2845 } 2846 } 2847 if (!ASIC_IS_NODCE(rdev)) { 2848 /* Unlock vga access */ 2849 WREG32(VGA_HDP_CONTROL, save->vga_hdp_control); 2850 mdelay(1); 2851 WREG32(VGA_RENDER_CONTROL, save->vga_render_control); 2852 } 2853} 2854 2855void evergreen_mc_program(struct radeon_device *rdev) 2856{ 2857 struct evergreen_mc_save save; 2858 u32 tmp; 2859 int i, j; 2860 2861 /* Initialize HDP */ 2862 for (i = 0, j = 0; i < 32; i++, j += 0x18) { 2863 WREG32((0x2c14 + j), 0x00000000); 2864 WREG32((0x2c18 + j), 0x00000000); 2865 WREG32((0x2c1c + j), 0x00000000); 2866 WREG32((0x2c20 + j), 0x00000000); 2867 WREG32((0x2c24 + j), 0x00000000); 2868 } 2869 WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0); 2870 2871 evergreen_mc_stop(rdev, &save); 2872 if (evergreen_mc_wait_for_idle(rdev)) { 2873 dev_warn(rdev->dev, "Wait for MC idle timedout !\n"); 2874 } 2875 /* Lockout access through VGA aperture*/ 2876 WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE); 2877 /* Update configuration */ 2878 if (rdev->flags & RADEON_IS_AGP) { 2879 if (rdev->mc.vram_start < rdev->mc.gtt_start) { 2880 /* VRAM before AGP */ 2881 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR, 2882 rdev->mc.vram_start >> 12); 2883 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR, 2884 rdev->mc.gtt_end >> 12); 2885 } else { 2886 /* VRAM after AGP */ 2887 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR, 2888 rdev->mc.gtt_start >> 12); 2889 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR, 2890 rdev->mc.vram_end >> 12); 2891 } 2892 } else { 2893 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR, 2894 rdev->mc.vram_start >> 12); 2895 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR, 2896 rdev->mc.vram_end >> 12); 2897 } 2898 WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR, rdev->vram_scratch.gpu_addr >> 12); 2899 /* llano/ontario only */ 2900 if ((rdev->family == CHIP_PALM) || 2901 (rdev->family == CHIP_SUMO) || 2902 (rdev->family == CHIP_SUMO2)) { 2903 tmp = RREG32(MC_FUS_VM_FB_OFFSET) & 0x000FFFFF; 2904 tmp |= ((rdev->mc.vram_end >> 20) & 0xF) << 24; 2905 tmp |= ((rdev->mc.vram_start >> 20) & 0xF) << 20; 2906 WREG32(MC_FUS_VM_FB_OFFSET, tmp); 2907 } 2908 tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16; 2909 tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF); 2910 WREG32(MC_VM_FB_LOCATION, tmp); 2911 WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8)); 2912 WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30)); 2913 WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF); 2914 if (rdev->flags & RADEON_IS_AGP) { 2915 WREG32(MC_VM_AGP_TOP, rdev->mc.gtt_end >> 16); 2916 WREG32(MC_VM_AGP_BOT, rdev->mc.gtt_start >> 16); 2917 WREG32(MC_VM_AGP_BASE, rdev->mc.agp_base >> 22); 2918 } else { 2919 WREG32(MC_VM_AGP_BASE, 0); 2920 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF); 2921 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF); 2922 } 2923 if (evergreen_mc_wait_for_idle(rdev)) { 2924 dev_warn(rdev->dev, "Wait for MC idle timedout !\n"); 2925 } 2926 evergreen_mc_resume(rdev, &save); 2927 /* we need to own VRAM, so turn off the VGA renderer here 2928 * to stop it overwriting our objects */ 2929 rv515_vga_render_disable(rdev); 2930} 2931 2932/* 2933 * CP. 2934 */ 2935void evergreen_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib) 2936{ 2937 struct radeon_ring *ring = &rdev->ring[ib->ring]; 2938 u32 next_rptr; 2939 2940 /* set to DX10/11 mode */ 2941 radeon_ring_write(ring, PACKET3(PACKET3_MODE_CONTROL, 0)); 2942 radeon_ring_write(ring, 1); 2943 2944 if (ring->rptr_save_reg) { 2945 next_rptr = ring->wptr + 3 + 4; 2946 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1)); 2947 radeon_ring_write(ring, ((ring->rptr_save_reg - 2948 PACKET3_SET_CONFIG_REG_START) >> 2)); 2949 radeon_ring_write(ring, next_rptr); 2950 } else if (rdev->wb.enabled) { 2951 next_rptr = ring->wptr + 5 + 4; 2952 radeon_ring_write(ring, PACKET3(PACKET3_MEM_WRITE, 3)); 2953 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc); 2954 radeon_ring_write(ring, (upper_32_bits(ring->next_rptr_gpu_addr) & 0xff) | (1 << 18)); 2955 radeon_ring_write(ring, next_rptr); 2956 radeon_ring_write(ring, 0); 2957 } 2958 2959 radeon_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2)); 2960 radeon_ring_write(ring, 2961#ifdef __BIG_ENDIAN 2962 (2 << 0) | 2963#endif 2964 (ib->gpu_addr & 0xFFFFFFFC)); 2965 radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFF); 2966 radeon_ring_write(ring, ib->length_dw); 2967} 2968 2969 2970static int evergreen_cp_load_microcode(struct radeon_device *rdev) 2971{ 2972 const __be32 *fw_data; 2973 int i; 2974 2975 if (!rdev->me_fw || !rdev->pfp_fw) 2976 return -EINVAL; 2977 2978 r700_cp_stop(rdev); 2979 WREG32(CP_RB_CNTL, 2980#ifdef __BIG_ENDIAN 2981 BUF_SWAP_32BIT | 2982#endif 2983 RB_NO_UPDATE | RB_BLKSZ(15) | RB_BUFSZ(3)); 2984 2985 fw_data = (const __be32 *)rdev->pfp_fw->data; 2986 WREG32(CP_PFP_UCODE_ADDR, 0); 2987 for (i = 0; i < EVERGREEN_PFP_UCODE_SIZE; i++) 2988 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++)); 2989 WREG32(CP_PFP_UCODE_ADDR, 0); 2990 2991 fw_data = (const __be32 *)rdev->me_fw->data; 2992 WREG32(CP_ME_RAM_WADDR, 0); 2993 for (i = 0; i < EVERGREEN_PM4_UCODE_SIZE; i++) 2994 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++)); 2995 2996 WREG32(CP_PFP_UCODE_ADDR, 0); 2997 WREG32(CP_ME_RAM_WADDR, 0); 2998 WREG32(CP_ME_RAM_RADDR, 0); 2999 return 0; 3000} 3001 3002static int evergreen_cp_start(struct radeon_device *rdev) 3003{ 3004 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]; 3005 int r, i; 3006 uint32_t cp_me; 3007 3008 r = radeon_ring_lock(rdev, ring, 7); 3009 if (r) { 3010 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r); 3011 return r; 3012 } 3013 radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5)); 3014 radeon_ring_write(ring, 0x1); 3015 radeon_ring_write(ring, 0x0); 3016 radeon_ring_write(ring, rdev->config.evergreen.max_hw_contexts - 1); 3017 radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1)); 3018 radeon_ring_write(ring, 0); 3019 radeon_ring_write(ring, 0); 3020 radeon_ring_unlock_commit(rdev, ring, false); 3021 3022 cp_me = 0xff; 3023 WREG32(CP_ME_CNTL, cp_me); 3024 3025 r = radeon_ring_lock(rdev, ring, evergreen_default_size + 19); 3026 if (r) { 3027 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r); 3028 return r; 3029 } 3030 3031 /* setup clear context state */ 3032 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0)); 3033 radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE); 3034 3035 for (i = 0; i < evergreen_default_size; i++) 3036 radeon_ring_write(ring, evergreen_default_state[i]); 3037 3038 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0)); 3039 radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE); 3040 3041 /* set clear context state */ 3042 radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0)); 3043 radeon_ring_write(ring, 0); 3044 3045 /* SQ_VTX_BASE_VTX_LOC */ 3046 radeon_ring_write(ring, 0xc0026f00); 3047 radeon_ring_write(ring, 0x00000000); 3048 radeon_ring_write(ring, 0x00000000); 3049 radeon_ring_write(ring, 0x00000000); 3050 3051 /* Clear consts */ 3052 radeon_ring_write(ring, 0xc0036f00); 3053 radeon_ring_write(ring, 0x00000bc4); 3054 radeon_ring_write(ring, 0xffffffff); 3055 radeon_ring_write(ring, 0xffffffff); 3056 radeon_ring_write(ring, 0xffffffff); 3057 3058 radeon_ring_write(ring, 0xc0026900); 3059 radeon_ring_write(ring, 0x00000316); 3060 radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */ 3061 radeon_ring_write(ring, 0x00000010); /* */ 3062 3063 radeon_ring_unlock_commit(rdev, ring, false); 3064 3065 return 0; 3066} 3067 3068static int evergreen_cp_resume(struct radeon_device *rdev) 3069{ 3070 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]; 3071 u32 tmp; 3072 u32 rb_bufsz; 3073 int r; 3074 3075 /* Reset cp; if cp is reset, then PA, SH, VGT also need to be reset */ 3076 WREG32(GRBM_SOFT_RESET, (SOFT_RESET_CP | 3077 SOFT_RESET_PA | 3078 SOFT_RESET_SH | 3079 SOFT_RESET_VGT | 3080 SOFT_RESET_SPI | 3081 SOFT_RESET_SX)); 3082 RREG32(GRBM_SOFT_RESET); 3083 mdelay(15); 3084 WREG32(GRBM_SOFT_RESET, 0); 3085 RREG32(GRBM_SOFT_RESET); 3086 3087 /* Set ring buffer size */ 3088 rb_bufsz = order_base_2(ring->ring_size / 8); 3089 tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz; 3090#ifdef __BIG_ENDIAN 3091 tmp |= BUF_SWAP_32BIT; 3092#endif 3093 WREG32(CP_RB_CNTL, tmp); 3094 WREG32(CP_SEM_WAIT_TIMER, 0x0); 3095 WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0); 3096 3097 /* Set the write pointer delay */ 3098 WREG32(CP_RB_WPTR_DELAY, 0); 3099 3100 /* Initialize the ring buffer's read and write pointers */ 3101 WREG32(CP_RB_CNTL, tmp | RB_RPTR_WR_ENA); 3102 WREG32(CP_RB_RPTR_WR, 0); 3103 ring->wptr = 0; 3104 WREG32(CP_RB_WPTR, ring->wptr); 3105 3106 /* set the wb address whether it's enabled or not */ 3107 WREG32(CP_RB_RPTR_ADDR, 3108 ((rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC)); 3109 WREG32(CP_RB_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF); 3110 WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF); 3111 3112 if (rdev->wb.enabled) 3113 WREG32(SCRATCH_UMSK, 0xff); 3114 else { 3115 tmp |= RB_NO_UPDATE; 3116 WREG32(SCRATCH_UMSK, 0); 3117 } 3118 3119 mdelay(1); 3120 WREG32(CP_RB_CNTL, tmp); 3121 3122 WREG32(CP_RB_BASE, ring->gpu_addr >> 8); 3123 WREG32(CP_DEBUG, (1 << 27) | (1 << 28)); 3124 3125 evergreen_cp_start(rdev); 3126 ring->ready = true; 3127 r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, ring); 3128 if (r) { 3129 ring->ready = false; 3130 return r; 3131 } 3132 return 0; 3133} 3134 3135/* 3136 * Core functions 3137 */ 3138static void evergreen_gpu_init(struct radeon_device *rdev) 3139{ 3140 u32 gb_addr_config; 3141 u32 mc_shared_chmap __unused, mc_arb_ramcfg; 3142 u32 sx_debug_1; 3143 u32 smx_dc_ctl0; 3144 u32 sq_config; 3145 u32 sq_lds_resource_mgmt; 3146 u32 sq_gpr_resource_mgmt_1; 3147 u32 sq_gpr_resource_mgmt_2; 3148 u32 sq_gpr_resource_mgmt_3; 3149 u32 sq_thread_resource_mgmt; 3150 u32 sq_thread_resource_mgmt_2; 3151 u32 sq_stack_resource_mgmt_1; 3152 u32 sq_stack_resource_mgmt_2; 3153 u32 sq_stack_resource_mgmt_3; 3154 u32 vgt_cache_invalidation; 3155 u32 hdp_host_path_cntl, tmp; 3156 u32 disabled_rb_mask; 3157 int i, j, ps_thread_count; 3158 3159 switch (rdev->family) { 3160 case CHIP_CYPRESS: 3161 case CHIP_HEMLOCK: 3162 rdev->config.evergreen.num_ses = 2; 3163 rdev->config.evergreen.max_pipes = 4; 3164 rdev->config.evergreen.max_tile_pipes = 8; 3165 rdev->config.evergreen.max_simds = 10; 3166 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses; 3167 rdev->config.evergreen.max_gprs = 256; 3168 rdev->config.evergreen.max_threads = 248; 3169 rdev->config.evergreen.max_gs_threads = 32; 3170 rdev->config.evergreen.max_stack_entries = 512; 3171 rdev->config.evergreen.sx_num_of_sets = 4; 3172 rdev->config.evergreen.sx_max_export_size = 256; 3173 rdev->config.evergreen.sx_max_export_pos_size = 64; 3174 rdev->config.evergreen.sx_max_export_smx_size = 192; 3175 rdev->config.evergreen.max_hw_contexts = 8; 3176 rdev->config.evergreen.sq_num_cf_insts = 2; 3177 3178 rdev->config.evergreen.sc_prim_fifo_size = 0x100; 3179 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30; 3180 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130; 3181 gb_addr_config = CYPRESS_GB_ADDR_CONFIG_GOLDEN; 3182 break; 3183 case CHIP_JUNIPER: 3184 rdev->config.evergreen.num_ses = 1; 3185 rdev->config.evergreen.max_pipes = 4; 3186 rdev->config.evergreen.max_tile_pipes = 4; 3187 rdev->config.evergreen.max_simds = 10; 3188 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses; 3189 rdev->config.evergreen.max_gprs = 256; 3190 rdev->config.evergreen.max_threads = 248; 3191 rdev->config.evergreen.max_gs_threads = 32; 3192 rdev->config.evergreen.max_stack_entries = 512; 3193 rdev->config.evergreen.sx_num_of_sets = 4; 3194 rdev->config.evergreen.sx_max_export_size = 256; 3195 rdev->config.evergreen.sx_max_export_pos_size = 64; 3196 rdev->config.evergreen.sx_max_export_smx_size = 192; 3197 rdev->config.evergreen.max_hw_contexts = 8; 3198 rdev->config.evergreen.sq_num_cf_insts = 2; 3199 3200 rdev->config.evergreen.sc_prim_fifo_size = 0x100; 3201 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30; 3202 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130; 3203 gb_addr_config = JUNIPER_GB_ADDR_CONFIG_GOLDEN; 3204 break; 3205 case CHIP_REDWOOD: 3206 rdev->config.evergreen.num_ses = 1; 3207 rdev->config.evergreen.max_pipes = 4; 3208 rdev->config.evergreen.max_tile_pipes = 4; 3209 rdev->config.evergreen.max_simds = 5; 3210 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses; 3211 rdev->config.evergreen.max_gprs = 256; 3212 rdev->config.evergreen.max_threads = 248; 3213 rdev->config.evergreen.max_gs_threads = 32; 3214 rdev->config.evergreen.max_stack_entries = 256; 3215 rdev->config.evergreen.sx_num_of_sets = 4; 3216 rdev->config.evergreen.sx_max_export_size = 256; 3217 rdev->config.evergreen.sx_max_export_pos_size = 64; 3218 rdev->config.evergreen.sx_max_export_smx_size = 192; 3219 rdev->config.evergreen.max_hw_contexts = 8; 3220 rdev->config.evergreen.sq_num_cf_insts = 2; 3221 3222 rdev->config.evergreen.sc_prim_fifo_size = 0x100; 3223 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30; 3224 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130; 3225 gb_addr_config = REDWOOD_GB_ADDR_CONFIG_GOLDEN; 3226 break; 3227 case CHIP_CEDAR: 3228 default: 3229 rdev->config.evergreen.num_ses = 1; 3230 rdev->config.evergreen.max_pipes = 2; 3231 rdev->config.evergreen.max_tile_pipes = 2; 3232 rdev->config.evergreen.max_simds = 2; 3233 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses; 3234 rdev->config.evergreen.max_gprs = 256; 3235 rdev->config.evergreen.max_threads = 192; 3236 rdev->config.evergreen.max_gs_threads = 16; 3237 rdev->config.evergreen.max_stack_entries = 256; 3238 rdev->config.evergreen.sx_num_of_sets = 4; 3239 rdev->config.evergreen.sx_max_export_size = 128; 3240 rdev->config.evergreen.sx_max_export_pos_size = 32; 3241 rdev->config.evergreen.sx_max_export_smx_size = 96; 3242 rdev->config.evergreen.max_hw_contexts = 4; 3243 rdev->config.evergreen.sq_num_cf_insts = 1; 3244 3245 rdev->config.evergreen.sc_prim_fifo_size = 0x40; 3246 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30; 3247 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130; 3248 gb_addr_config = CEDAR_GB_ADDR_CONFIG_GOLDEN; 3249 break; 3250 case CHIP_PALM: 3251 rdev->config.evergreen.num_ses = 1; 3252 rdev->config.evergreen.max_pipes = 2; 3253 rdev->config.evergreen.max_tile_pipes = 2; 3254 rdev->config.evergreen.max_simds = 2; 3255 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses; 3256 rdev->config.evergreen.max_gprs = 256; 3257 rdev->config.evergreen.max_threads = 192; 3258 rdev->config.evergreen.max_gs_threads = 16; 3259 rdev->config.evergreen.max_stack_entries = 256; 3260 rdev->config.evergreen.sx_num_of_sets = 4; 3261 rdev->config.evergreen.sx_max_export_size = 128; 3262 rdev->config.evergreen.sx_max_export_pos_size = 32; 3263 rdev->config.evergreen.sx_max_export_smx_size = 96; 3264 rdev->config.evergreen.max_hw_contexts = 4; 3265 rdev->config.evergreen.sq_num_cf_insts = 1; 3266 3267 rdev->config.evergreen.sc_prim_fifo_size = 0x40; 3268 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30; 3269 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130; 3270 gb_addr_config = CEDAR_GB_ADDR_CONFIG_GOLDEN; 3271 break; 3272 case CHIP_SUMO: 3273 rdev->config.evergreen.num_ses = 1; 3274 rdev->config.evergreen.max_pipes = 4; 3275 rdev->config.evergreen.max_tile_pipes = 4; 3276 if (rdev->pdev->device == 0x9648) 3277 rdev->config.evergreen.max_simds = 3; 3278 else if ((rdev->pdev->device == 0x9647) || 3279 (rdev->pdev->device == 0x964a)) 3280 rdev->config.evergreen.max_simds = 4; 3281 else 3282 rdev->config.evergreen.max_simds = 5; 3283 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses; 3284 rdev->config.evergreen.max_gprs = 256; 3285 rdev->config.evergreen.max_threads = 248; 3286 rdev->config.evergreen.max_gs_threads = 32; 3287 rdev->config.evergreen.max_stack_entries = 256; 3288 rdev->config.evergreen.sx_num_of_sets = 4; 3289 rdev->config.evergreen.sx_max_export_size = 256; 3290 rdev->config.evergreen.sx_max_export_pos_size = 64; 3291 rdev->config.evergreen.sx_max_export_smx_size = 192; 3292 rdev->config.evergreen.max_hw_contexts = 8; 3293 rdev->config.evergreen.sq_num_cf_insts = 2; 3294 3295 rdev->config.evergreen.sc_prim_fifo_size = 0x40; 3296 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30; 3297 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130; 3298 gb_addr_config = SUMO_GB_ADDR_CONFIG_GOLDEN; 3299 break; 3300 case CHIP_SUMO2: 3301 rdev->config.evergreen.num_ses = 1; 3302 rdev->config.evergreen.max_pipes = 4; 3303 rdev->config.evergreen.max_tile_pipes = 4; 3304 rdev->config.evergreen.max_simds = 2; 3305 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses; 3306 rdev->config.evergreen.max_gprs = 256; 3307 rdev->config.evergreen.max_threads = 248; 3308 rdev->config.evergreen.max_gs_threads = 32; 3309 rdev->config.evergreen.max_stack_entries = 512; 3310 rdev->config.evergreen.sx_num_of_sets = 4; 3311 rdev->config.evergreen.sx_max_export_size = 256; 3312 rdev->config.evergreen.sx_max_export_pos_size = 64; 3313 rdev->config.evergreen.sx_max_export_smx_size = 192; 3314 rdev->config.evergreen.max_hw_contexts = 4; 3315 rdev->config.evergreen.sq_num_cf_insts = 2; 3316 3317 rdev->config.evergreen.sc_prim_fifo_size = 0x40; 3318 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30; 3319 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130; 3320 gb_addr_config = SUMO2_GB_ADDR_CONFIG_GOLDEN; 3321 break; 3322 case CHIP_BARTS: 3323 rdev->config.evergreen.num_ses = 2; 3324 rdev->config.evergreen.max_pipes = 4; 3325 rdev->config.evergreen.max_tile_pipes = 8; 3326 rdev->config.evergreen.max_simds = 7; 3327 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses; 3328 rdev->config.evergreen.max_gprs = 256; 3329 rdev->config.evergreen.max_threads = 248; 3330 rdev->config.evergreen.max_gs_threads = 32; 3331 rdev->config.evergreen.max_stack_entries = 512; 3332 rdev->config.evergreen.sx_num_of_sets = 4; 3333 rdev->config.evergreen.sx_max_export_size = 256; 3334 rdev->config.evergreen.sx_max_export_pos_size = 64; 3335 rdev->config.evergreen.sx_max_export_smx_size = 192; 3336 rdev->config.evergreen.max_hw_contexts = 8; 3337 rdev->config.evergreen.sq_num_cf_insts = 2; 3338 3339 rdev->config.evergreen.sc_prim_fifo_size = 0x100; 3340 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30; 3341 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130; 3342 gb_addr_config = BARTS_GB_ADDR_CONFIG_GOLDEN; 3343 break; 3344 case CHIP_TURKS: 3345 rdev->config.evergreen.num_ses = 1; 3346 rdev->config.evergreen.max_pipes = 4; 3347 rdev->config.evergreen.max_tile_pipes = 4; 3348 rdev->config.evergreen.max_simds = 6; 3349 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses; 3350 rdev->config.evergreen.max_gprs = 256; 3351 rdev->config.evergreen.max_threads = 248; 3352 rdev->config.evergreen.max_gs_threads = 32; 3353 rdev->config.evergreen.max_stack_entries = 256; 3354 rdev->config.evergreen.sx_num_of_sets = 4; 3355 rdev->config.evergreen.sx_max_export_size = 256; 3356 rdev->config.evergreen.sx_max_export_pos_size = 64; 3357 rdev->config.evergreen.sx_max_export_smx_size = 192; 3358 rdev->config.evergreen.max_hw_contexts = 8; 3359 rdev->config.evergreen.sq_num_cf_insts = 2; 3360 3361 rdev->config.evergreen.sc_prim_fifo_size = 0x100; 3362 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30; 3363 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130; 3364 gb_addr_config = TURKS_GB_ADDR_CONFIG_GOLDEN; 3365 break; 3366 case CHIP_CAICOS: 3367 rdev->config.evergreen.num_ses = 1; 3368 rdev->config.evergreen.max_pipes = 2; 3369 rdev->config.evergreen.max_tile_pipes = 2; 3370 rdev->config.evergreen.max_simds = 2; 3371 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses; 3372 rdev->config.evergreen.max_gprs = 256; 3373 rdev->config.evergreen.max_threads = 192; 3374 rdev->config.evergreen.max_gs_threads = 16; 3375 rdev->config.evergreen.max_stack_entries = 256; 3376 rdev->config.evergreen.sx_num_of_sets = 4; 3377 rdev->config.evergreen.sx_max_export_size = 128; 3378 rdev->config.evergreen.sx_max_export_pos_size = 32; 3379 rdev->config.evergreen.sx_max_export_smx_size = 96; 3380 rdev->config.evergreen.max_hw_contexts = 4; 3381 rdev->config.evergreen.sq_num_cf_insts = 1; 3382 3383 rdev->config.evergreen.sc_prim_fifo_size = 0x40; 3384 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30; 3385 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130; 3386 gb_addr_config = CAICOS_GB_ADDR_CONFIG_GOLDEN; 3387 break; 3388 } 3389 3390 /* Initialize HDP */ 3391 for (i = 0, j = 0; i < 32; i++, j += 0x18) { 3392 WREG32((0x2c14 + j), 0x00000000); 3393 WREG32((0x2c18 + j), 0x00000000); 3394 WREG32((0x2c1c + j), 0x00000000); 3395 WREG32((0x2c20 + j), 0x00000000); 3396 WREG32((0x2c24 + j), 0x00000000); 3397 } 3398 3399 WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff)); 3400 WREG32(SRBM_INT_CNTL, 0x1); 3401 WREG32(SRBM_INT_ACK, 0x1); 3402 3403 evergreen_fix_pci_max_read_req_size(rdev); 3404 3405 mc_shared_chmap = RREG32(MC_SHARED_CHMAP); 3406 if ((rdev->family == CHIP_PALM) || 3407 (rdev->family == CHIP_SUMO) || 3408 (rdev->family == CHIP_SUMO2)) 3409 mc_arb_ramcfg = RREG32(FUS_MC_ARB_RAMCFG); 3410 else 3411 mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG); 3412 3413 /* setup tiling info dword. gb_addr_config is not adequate since it does 3414 * not have bank info, so create a custom tiling dword. 3415 * bits 3:0 num_pipes 3416 * bits 7:4 num_banks 3417 * bits 11:8 group_size 3418 * bits 15:12 row_size 3419 */ 3420 rdev->config.evergreen.tile_config = 0; 3421 switch (rdev->config.evergreen.max_tile_pipes) { 3422 case 1: 3423 default: 3424 rdev->config.evergreen.tile_config |= (0 << 0); 3425 break; 3426 case 2: 3427 rdev->config.evergreen.tile_config |= (1 << 0); 3428 break; 3429 case 4: 3430 rdev->config.evergreen.tile_config |= (2 << 0); 3431 break; 3432 case 8: 3433 rdev->config.evergreen.tile_config |= (3 << 0); 3434 break; 3435 } 3436 /* num banks is 8 on all fusion asics. 0 = 4, 1 = 8, 2 = 16 */ 3437 if (rdev->flags & RADEON_IS_IGP) 3438 rdev->config.evergreen.tile_config |= 1 << 4; 3439 else { 3440 switch ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) { 3441 case 0: /* four banks */ 3442 rdev->config.evergreen.tile_config |= 0 << 4; 3443 break; 3444 case 1: /* eight banks */ 3445 rdev->config.evergreen.tile_config |= 1 << 4; 3446 break; 3447 case 2: /* sixteen banks */ 3448 default: 3449 rdev->config.evergreen.tile_config |= 2 << 4; 3450 break; 3451 } 3452 } 3453 rdev->config.evergreen.tile_config |= 0 << 8; 3454 rdev->config.evergreen.tile_config |= 3455 ((gb_addr_config & 0x30000000) >> 28) << 12; 3456 3457 if ((rdev->family >= CHIP_CEDAR) && (rdev->family <= CHIP_HEMLOCK)) { 3458 u32 efuse_straps_4; 3459 u32 efuse_straps_3; 3460 3461 efuse_straps_4 = RREG32_RCU(0x204); 3462 efuse_straps_3 = RREG32_RCU(0x203); 3463 tmp = (((efuse_straps_4 & 0xf) << 4) | 3464 ((efuse_straps_3 & 0xf0000000) >> 28)); 3465 } else { 3466 tmp = 0; 3467 for (i = (rdev->config.evergreen.num_ses - 1); i >= 0; i--) { 3468 u32 rb_disable_bitmap; 3469 3470 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i)); 3471 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i)); 3472 rb_disable_bitmap = (RREG32(CC_RB_BACKEND_DISABLE) & 0x00ff0000) >> 16; 3473 tmp <<= 4; 3474 tmp |= rb_disable_bitmap; 3475 } 3476 } 3477 /* enabled rb are just the one not disabled :) */ 3478 disabled_rb_mask = tmp; 3479 tmp = 0; 3480 for (i = 0; i < rdev->config.evergreen.max_backends; i++) 3481 tmp |= (1 << i); 3482 /* if all the backends are disabled, fix it up here */ 3483 if ((disabled_rb_mask & tmp) == tmp) { 3484 for (i = 0; i < rdev->config.evergreen.max_backends; i++) 3485 disabled_rb_mask &= ~(1 << i); 3486 } 3487 3488 for (i = 0; i < rdev->config.evergreen.num_ses; i++) { 3489 u32 simd_disable_bitmap; 3490 3491 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i)); 3492 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i)); 3493 simd_disable_bitmap = (RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0xffff0000) >> 16; 3494 simd_disable_bitmap |= 0xffffffff << rdev->config.evergreen.max_simds; 3495 tmp <<= 16; 3496 tmp |= simd_disable_bitmap; 3497 } 3498 rdev->config.evergreen.active_simds = hweight32(~tmp); 3499 3500 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES); 3501 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES); 3502 3503 WREG32(GB_ADDR_CONFIG, gb_addr_config); 3504 WREG32(DMIF_ADDR_CONFIG, gb_addr_config); 3505 WREG32(HDP_ADDR_CONFIG, gb_addr_config); 3506 WREG32(DMA_TILING_CONFIG, gb_addr_config); 3507 WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config); 3508 WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config); 3509 WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config); 3510 3511 if ((rdev->config.evergreen.max_backends == 1) && 3512 (rdev->flags & RADEON_IS_IGP)) { 3513 if ((disabled_rb_mask & 3) == 1) { 3514 /* RB0 disabled, RB1 enabled */ 3515 tmp = 0x11111111; 3516 } else { 3517 /* RB1 disabled, RB0 enabled */ 3518 tmp = 0x00000000; 3519 } 3520 } else { 3521 tmp = gb_addr_config & NUM_PIPES_MASK; 3522 tmp = r6xx_remap_render_backend(rdev, tmp, rdev->config.evergreen.max_backends, 3523 EVERGREEN_MAX_BACKENDS, disabled_rb_mask); 3524 } 3525 rdev->config.evergreen.backend_map = tmp; 3526 WREG32(GB_BACKEND_MAP, tmp); 3527 3528 WREG32(CGTS_SYS_TCC_DISABLE, 0); 3529 WREG32(CGTS_TCC_DISABLE, 0); 3530 WREG32(CGTS_USER_SYS_TCC_DISABLE, 0); 3531 WREG32(CGTS_USER_TCC_DISABLE, 0); 3532 3533 /* set HW defaults for 3D engine */ 3534 WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) | 3535 ROQ_IB2_START(0x2b))); 3536 3537 WREG32(CP_MEQ_THRESHOLDS, STQ_SPLIT(0x30)); 3538 3539 WREG32(TA_CNTL_AUX, (DISABLE_CUBE_ANISO | 3540 SYNC_GRADIENT | 3541 SYNC_WALKER | 3542 SYNC_ALIGNER)); 3543 3544 sx_debug_1 = RREG32(SX_DEBUG_1); 3545 sx_debug_1 |= ENABLE_NEW_SMX_ADDRESS; 3546 WREG32(SX_DEBUG_1, sx_debug_1); 3547 3548 3549 smx_dc_ctl0 = RREG32(SMX_DC_CTL0); 3550 smx_dc_ctl0 &= ~NUMBER_OF_SETS(0x1ff); 3551 smx_dc_ctl0 |= NUMBER_OF_SETS(rdev->config.evergreen.sx_num_of_sets); 3552 WREG32(SMX_DC_CTL0, smx_dc_ctl0); 3553 3554 if (rdev->family <= CHIP_SUMO2) 3555 WREG32(SMX_SAR_CTL0, 0x00010000); 3556 3557 WREG32(SX_EXPORT_BUFFER_SIZES, (COLOR_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_size / 4) - 1) | 3558 POSITION_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_pos_size / 4) - 1) | 3559 SMX_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_smx_size / 4) - 1))); 3560 3561 WREG32(PA_SC_FIFO_SIZE, (SC_PRIM_FIFO_SIZE(rdev->config.evergreen.sc_prim_fifo_size) | 3562 SC_HIZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_hiz_tile_fifo_size) | 3563 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_earlyz_tile_fifo_size))); 3564 3565 WREG32(VGT_NUM_INSTANCES, 1); 3566 WREG32(SPI_CONFIG_CNTL, 0); 3567 WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4)); 3568 WREG32(CP_PERFMON_CNTL, 0); 3569 3570 WREG32(SQ_MS_FIFO_SIZES, (CACHE_FIFO_SIZE(16 * rdev->config.evergreen.sq_num_cf_insts) | 3571 FETCH_FIFO_HIWATER(0x4) | 3572 DONE_FIFO_HIWATER(0xe0) | 3573 ALU_UPDATE_FIFO_HIWATER(0x8))); 3574 3575 sq_config = RREG32(SQ_CONFIG); 3576 sq_config &= ~(PS_PRIO(3) | 3577 VS_PRIO(3) | 3578 GS_PRIO(3) | 3579 ES_PRIO(3)); 3580 sq_config |= (VC_ENABLE | 3581 EXPORT_SRC_C | 3582 PS_PRIO(0) | 3583 VS_PRIO(1) | 3584 GS_PRIO(2) | 3585 ES_PRIO(3)); 3586 3587 switch (rdev->family) { 3588 case CHIP_CEDAR: 3589 case CHIP_PALM: 3590 case CHIP_SUMO: 3591 case CHIP_SUMO2: 3592 case CHIP_CAICOS: 3593 /* no vertex cache */ 3594 sq_config &= ~VC_ENABLE; 3595 break; 3596 default: 3597 break; 3598 } 3599 3600 sq_lds_resource_mgmt = RREG32(SQ_LDS_RESOURCE_MGMT); 3601 3602 sq_gpr_resource_mgmt_1 = NUM_PS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2))* 12 / 32); 3603 sq_gpr_resource_mgmt_1 |= NUM_VS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 6 / 32); 3604 sq_gpr_resource_mgmt_1 |= NUM_CLAUSE_TEMP_GPRS(4); 3605 sq_gpr_resource_mgmt_2 = NUM_GS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32); 3606 sq_gpr_resource_mgmt_2 |= NUM_ES_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32); 3607 sq_gpr_resource_mgmt_3 = NUM_HS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32); 3608 sq_gpr_resource_mgmt_3 |= NUM_LS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32); 3609 3610 switch (rdev->family) { 3611 case CHIP_CEDAR: 3612 case CHIP_PALM: 3613 case CHIP_SUMO: 3614 case CHIP_SUMO2: 3615 ps_thread_count = 96; 3616 break; 3617 default: 3618 ps_thread_count = 128; 3619 break; 3620 } 3621 3622 sq_thread_resource_mgmt = NUM_PS_THREADS(ps_thread_count); 3623 sq_thread_resource_mgmt |= NUM_VS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8); 3624 sq_thread_resource_mgmt |= NUM_GS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8); 3625 sq_thread_resource_mgmt |= NUM_ES_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8); 3626 sq_thread_resource_mgmt_2 = NUM_HS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8); 3627 sq_thread_resource_mgmt_2 |= NUM_LS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8); 3628 3629 sq_stack_resource_mgmt_1 = NUM_PS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6); 3630 sq_stack_resource_mgmt_1 |= NUM_VS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6); 3631 sq_stack_resource_mgmt_2 = NUM_GS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6); 3632 sq_stack_resource_mgmt_2 |= NUM_ES_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6); 3633 sq_stack_resource_mgmt_3 = NUM_HS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6); 3634 sq_stack_resource_mgmt_3 |= NUM_LS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6); 3635 3636 WREG32(SQ_CONFIG, sq_config); 3637 WREG32(SQ_GPR_RESOURCE_MGMT_1, sq_gpr_resource_mgmt_1); 3638 WREG32(SQ_GPR_RESOURCE_MGMT_2, sq_gpr_resource_mgmt_2); 3639 WREG32(SQ_GPR_RESOURCE_MGMT_3, sq_gpr_resource_mgmt_3); 3640 WREG32(SQ_THREAD_RESOURCE_MGMT, sq_thread_resource_mgmt); 3641 WREG32(SQ_THREAD_RESOURCE_MGMT_2, sq_thread_resource_mgmt_2); 3642 WREG32(SQ_STACK_RESOURCE_MGMT_1, sq_stack_resource_mgmt_1); 3643 WREG32(SQ_STACK_RESOURCE_MGMT_2, sq_stack_resource_mgmt_2); 3644 WREG32(SQ_STACK_RESOURCE_MGMT_3, sq_stack_resource_mgmt_3); 3645 WREG32(SQ_DYN_GPR_CNTL_PS_FLUSH_REQ, 0); 3646 WREG32(SQ_LDS_RESOURCE_MGMT, sq_lds_resource_mgmt); 3647 3648 WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) | 3649 FORCE_EOV_MAX_REZ_CNT(255))); 3650 3651 switch (rdev->family) { 3652 case CHIP_CEDAR: 3653 case CHIP_PALM: 3654 case CHIP_SUMO: 3655 case CHIP_SUMO2: 3656 case CHIP_CAICOS: 3657 vgt_cache_invalidation = CACHE_INVALIDATION(TC_ONLY); 3658 break; 3659 default: 3660 vgt_cache_invalidation = CACHE_INVALIDATION(VC_AND_TC); 3661 break; 3662 } 3663 vgt_cache_invalidation |= AUTO_INVLD_EN(ES_AND_GS_AUTO); 3664 WREG32(VGT_CACHE_INVALIDATION, vgt_cache_invalidation); 3665 3666 WREG32(VGT_GS_VERTEX_REUSE, 16); 3667 WREG32(PA_SU_LINE_STIPPLE_VALUE, 0); 3668 WREG32(PA_SC_LINE_STIPPLE_STATE, 0); 3669 3670 WREG32(VGT_VERTEX_REUSE_BLOCK_CNTL, 14); 3671 WREG32(VGT_OUT_DEALLOC_CNTL, 16); 3672 3673 WREG32(CB_PERF_CTR0_SEL_0, 0); 3674 WREG32(CB_PERF_CTR0_SEL_1, 0); 3675 WREG32(CB_PERF_CTR1_SEL_0, 0); 3676 WREG32(CB_PERF_CTR1_SEL_1, 0); 3677 WREG32(CB_PERF_CTR2_SEL_0, 0); 3678 WREG32(CB_PERF_CTR2_SEL_1, 0); 3679 WREG32(CB_PERF_CTR3_SEL_0, 0); 3680 WREG32(CB_PERF_CTR3_SEL_1, 0); 3681 3682 /* clear render buffer base addresses */ 3683 WREG32(CB_COLOR0_BASE, 0); 3684 WREG32(CB_COLOR1_BASE, 0); 3685 WREG32(CB_COLOR2_BASE, 0); 3686 WREG32(CB_COLOR3_BASE, 0); 3687 WREG32(CB_COLOR4_BASE, 0); 3688 WREG32(CB_COLOR5_BASE, 0); 3689 WREG32(CB_COLOR6_BASE, 0); 3690 WREG32(CB_COLOR7_BASE, 0); 3691 WREG32(CB_COLOR8_BASE, 0); 3692 WREG32(CB_COLOR9_BASE, 0); 3693 WREG32(CB_COLOR10_BASE, 0); 3694 WREG32(CB_COLOR11_BASE, 0); 3695 3696 /* set the shader const cache sizes to 0 */ 3697 for (i = SQ_ALU_CONST_BUFFER_SIZE_PS_0; i < 0x28200; i += 4) 3698 WREG32(i, 0); 3699 for (i = SQ_ALU_CONST_BUFFER_SIZE_HS_0; i < 0x29000; i += 4) 3700 WREG32(i, 0); 3701 3702 tmp = RREG32(HDP_MISC_CNTL); 3703 tmp |= HDP_FLUSH_INVALIDATE_CACHE; 3704 WREG32(HDP_MISC_CNTL, tmp); 3705 3706 hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL); 3707 WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl); 3708 3709 WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3)); 3710 3711 udelay(50); 3712 3713} 3714 3715int evergreen_mc_init(struct radeon_device *rdev) 3716{ 3717 u32 tmp; 3718 int chansize, numchan; 3719 3720 /* Get VRAM informations */ 3721 rdev->mc.vram_is_ddr = true; 3722 if ((rdev->family == CHIP_PALM) || 3723 (rdev->family == CHIP_SUMO) || 3724 (rdev->family == CHIP_SUMO2)) 3725 tmp = RREG32(FUS_MC_ARB_RAMCFG); 3726 else 3727 tmp = RREG32(MC_ARB_RAMCFG); 3728 if (tmp & CHANSIZE_OVERRIDE) { 3729 chansize = 16; 3730 } else if (tmp & CHANSIZE_MASK) { 3731 chansize = 64; 3732 } else { 3733 chansize = 32; 3734 } 3735 tmp = RREG32(MC_SHARED_CHMAP); 3736 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) { 3737 case 0: 3738 default: 3739 numchan = 1; 3740 break; 3741 case 1: 3742 numchan = 2; 3743 break; 3744 case 2: 3745 numchan = 4; 3746 break; 3747 case 3: 3748 numchan = 8; 3749 break; 3750 } 3751 rdev->mc.vram_width = numchan * chansize; 3752 /* Could aper size report 0 ? */ 3753 rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0); 3754 rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0); 3755 /* Setup GPU memory space */ 3756 if ((rdev->family == CHIP_PALM) || 3757 (rdev->family == CHIP_SUMO) || 3758 (rdev->family == CHIP_SUMO2)) { 3759 /* size in bytes on fusion */ 3760 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE); 3761 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE); 3762 } else { 3763 /* size in MB on evergreen/cayman/tn */ 3764 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL; 3765 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL; 3766 } 3767 rdev->mc.visible_vram_size = rdev->mc.aper_size; 3768 r700_vram_gtt_location(rdev, &rdev->mc); 3769 radeon_update_bandwidth_info(rdev); 3770 3771 return 0; 3772} 3773 3774void evergreen_print_gpu_status_regs(struct radeon_device *rdev) 3775{ 3776 dev_info(rdev->dev, " GRBM_STATUS = 0x%08X\n", 3777 RREG32(GRBM_STATUS)); 3778 dev_info(rdev->dev, " GRBM_STATUS_SE0 = 0x%08X\n", 3779 RREG32(GRBM_STATUS_SE0)); 3780 dev_info(rdev->dev, " GRBM_STATUS_SE1 = 0x%08X\n", 3781 RREG32(GRBM_STATUS_SE1)); 3782 dev_info(rdev->dev, " SRBM_STATUS = 0x%08X\n", 3783 RREG32(SRBM_STATUS)); 3784 dev_info(rdev->dev, " SRBM_STATUS2 = 0x%08X\n", 3785 RREG32(SRBM_STATUS2)); 3786 dev_info(rdev->dev, " R_008674_CP_STALLED_STAT1 = 0x%08X\n", 3787 RREG32(CP_STALLED_STAT1)); 3788 dev_info(rdev->dev, " R_008678_CP_STALLED_STAT2 = 0x%08X\n", 3789 RREG32(CP_STALLED_STAT2)); 3790 dev_info(rdev->dev, " R_00867C_CP_BUSY_STAT = 0x%08X\n", 3791 RREG32(CP_BUSY_STAT)); 3792 dev_info(rdev->dev, " R_008680_CP_STAT = 0x%08X\n", 3793 RREG32(CP_STAT)); 3794 dev_info(rdev->dev, " R_00D034_DMA_STATUS_REG = 0x%08X\n", 3795 RREG32(DMA_STATUS_REG)); 3796 if (rdev->family >= CHIP_CAYMAN) { 3797 dev_info(rdev->dev, " R_00D834_DMA_STATUS_REG = 0x%08X\n", 3798 RREG32(DMA_STATUS_REG + 0x800)); 3799 } 3800} 3801 3802bool evergreen_is_display_hung(struct radeon_device *rdev) 3803{ 3804 u32 crtc_hung = 0; 3805 u32 crtc_status[6]; 3806 u32 i, j, tmp; 3807 3808 for (i = 0; i < rdev->num_crtc; i++) { 3809 if (RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]) & EVERGREEN_CRTC_MASTER_EN) { 3810 crtc_status[i] = RREG32(EVERGREEN_CRTC_STATUS_HV_COUNT + crtc_offsets[i]); 3811 crtc_hung |= (1 << i); 3812 } 3813 } 3814 3815 for (j = 0; j < 10; j++) { 3816 for (i = 0; i < rdev->num_crtc; i++) { 3817 if (crtc_hung & (1 << i)) { 3818 tmp = RREG32(EVERGREEN_CRTC_STATUS_HV_COUNT + crtc_offsets[i]); 3819 if (tmp != crtc_status[i]) 3820 crtc_hung &= ~(1 << i); 3821 } 3822 } 3823 if (crtc_hung == 0) 3824 return false; 3825 udelay(100); 3826 } 3827 3828 return true; 3829} 3830 3831u32 evergreen_gpu_check_soft_reset(struct radeon_device *rdev) 3832{ 3833 u32 reset_mask = 0; 3834 u32 tmp; 3835 3836 /* GRBM_STATUS */ 3837 tmp = RREG32(GRBM_STATUS); 3838 if (tmp & (PA_BUSY | SC_BUSY | 3839 SH_BUSY | SX_BUSY | 3840 TA_BUSY | VGT_BUSY | 3841 DB_BUSY | CB_BUSY | 3842 SPI_BUSY | VGT_BUSY_NO_DMA)) 3843 reset_mask |= RADEON_RESET_GFX; 3844 3845 if (tmp & (CF_RQ_PENDING | PF_RQ_PENDING | 3846 CP_BUSY | CP_COHERENCY_BUSY)) 3847 reset_mask |= RADEON_RESET_CP; 3848 3849 if (tmp & GRBM_EE_BUSY) 3850 reset_mask |= RADEON_RESET_GRBM | RADEON_RESET_GFX | RADEON_RESET_CP; 3851 3852 /* DMA_STATUS_REG */ 3853 tmp = RREG32(DMA_STATUS_REG); 3854 if (!(tmp & DMA_IDLE)) 3855 reset_mask |= RADEON_RESET_DMA; 3856 3857 /* SRBM_STATUS2 */ 3858 tmp = RREG32(SRBM_STATUS2); 3859 if (tmp & DMA_BUSY) 3860 reset_mask |= RADEON_RESET_DMA; 3861 3862 /* SRBM_STATUS */ 3863 tmp = RREG32(SRBM_STATUS); 3864 if (tmp & (RLC_RQ_PENDING | RLC_BUSY)) 3865 reset_mask |= RADEON_RESET_RLC; 3866 3867 if (tmp & IH_BUSY) 3868 reset_mask |= RADEON_RESET_IH; 3869 3870 if (tmp & SEM_BUSY) 3871 reset_mask |= RADEON_RESET_SEM; 3872 3873 if (tmp & GRBM_RQ_PENDING) 3874 reset_mask |= RADEON_RESET_GRBM; 3875 3876 if (tmp & VMC_BUSY) 3877 reset_mask |= RADEON_RESET_VMC; 3878 3879 if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY | 3880 MCC_BUSY | MCD_BUSY)) 3881 reset_mask |= RADEON_RESET_MC; 3882 3883 if (evergreen_is_display_hung(rdev)) 3884 reset_mask |= RADEON_RESET_DISPLAY; 3885 3886 /* VM_L2_STATUS */ 3887 tmp = RREG32(VM_L2_STATUS); 3888 if (tmp & L2_BUSY) 3889 reset_mask |= RADEON_RESET_VMC; 3890 3891 /* Skip MC reset as it's mostly likely not hung, just busy */ 3892 if (reset_mask & RADEON_RESET_MC) { 3893 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask); 3894 reset_mask &= ~RADEON_RESET_MC; 3895 } 3896 3897 return reset_mask; 3898} 3899 3900static void evergreen_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask) 3901{ 3902 struct evergreen_mc_save save; 3903 u32 grbm_soft_reset = 0, srbm_soft_reset = 0; 3904 u32 tmp; 3905 3906 if (reset_mask == 0) 3907 return; 3908 3909 dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask); 3910 3911 evergreen_print_gpu_status_regs(rdev); 3912 3913 /* Disable CP parsing/prefetching */ 3914 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT); 3915 3916 if (reset_mask & RADEON_RESET_DMA) { 3917 /* Disable DMA */ 3918 tmp = RREG32(DMA_RB_CNTL); 3919 tmp &= ~DMA_RB_ENABLE; 3920 WREG32(DMA_RB_CNTL, tmp); 3921 } 3922 3923 udelay(50); 3924 3925 evergreen_mc_stop(rdev, &save); 3926 if (evergreen_mc_wait_for_idle(rdev)) { 3927 dev_warn(rdev->dev, "Wait for MC idle timedout !\n"); 3928 } 3929 3930 if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE)) { 3931 grbm_soft_reset |= SOFT_RESET_DB | 3932 SOFT_RESET_CB | 3933 SOFT_RESET_PA | 3934 SOFT_RESET_SC | 3935 SOFT_RESET_SPI | 3936 SOFT_RESET_SX | 3937 SOFT_RESET_SH | 3938 SOFT_RESET_TC | 3939 SOFT_RESET_TA | 3940 SOFT_RESET_VC | 3941 SOFT_RESET_VGT; 3942 } 3943 3944 if (reset_mask & RADEON_RESET_CP) { 3945 grbm_soft_reset |= SOFT_RESET_CP | 3946 SOFT_RESET_VGT; 3947 3948 srbm_soft_reset |= SOFT_RESET_GRBM; 3949 } 3950 3951 if (reset_mask & RADEON_RESET_DMA) 3952 srbm_soft_reset |= SOFT_RESET_DMA; 3953 3954 if (reset_mask & RADEON_RESET_DISPLAY) 3955 srbm_soft_reset |= SOFT_RESET_DC; 3956 3957 if (reset_mask & RADEON_RESET_RLC) 3958 srbm_soft_reset |= SOFT_RESET_RLC; 3959 3960 if (reset_mask & RADEON_RESET_SEM) 3961 srbm_soft_reset |= SOFT_RESET_SEM; 3962 3963 if (reset_mask & RADEON_RESET_IH) 3964 srbm_soft_reset |= SOFT_RESET_IH; 3965 3966 if (reset_mask & RADEON_RESET_GRBM) 3967 srbm_soft_reset |= SOFT_RESET_GRBM; 3968 3969 if (reset_mask & RADEON_RESET_VMC) 3970 srbm_soft_reset |= SOFT_RESET_VMC; 3971 3972 if (!(rdev->flags & RADEON_IS_IGP)) { 3973 if (reset_mask & RADEON_RESET_MC) 3974 srbm_soft_reset |= SOFT_RESET_MC; 3975 } 3976 3977 if (grbm_soft_reset) { 3978 tmp = RREG32(GRBM_SOFT_RESET); 3979 tmp |= grbm_soft_reset; 3980 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp); 3981 WREG32(GRBM_SOFT_RESET, tmp); 3982 tmp = RREG32(GRBM_SOFT_RESET); 3983 3984 udelay(50); 3985 3986 tmp &= ~grbm_soft_reset; 3987 WREG32(GRBM_SOFT_RESET, tmp); 3988 tmp = RREG32(GRBM_SOFT_RESET); 3989 } 3990 3991 if (srbm_soft_reset) { 3992 tmp = RREG32(SRBM_SOFT_RESET); 3993 tmp |= srbm_soft_reset; 3994 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp); 3995 WREG32(SRBM_SOFT_RESET, tmp); 3996 tmp = RREG32(SRBM_SOFT_RESET); 3997 3998 udelay(50); 3999 4000 tmp &= ~srbm_soft_reset; 4001 WREG32(SRBM_SOFT_RESET, tmp); 4002 tmp = RREG32(SRBM_SOFT_RESET); 4003 } 4004 4005 /* Wait a little for things to settle down */ 4006 udelay(50); 4007 4008 evergreen_mc_resume(rdev, &save); 4009 udelay(50); 4010 4011 evergreen_print_gpu_status_regs(rdev); 4012} 4013 4014void evergreen_gpu_pci_config_reset(struct radeon_device *rdev) 4015{ 4016 struct evergreen_mc_save save; 4017 u32 tmp, i; 4018 4019 dev_info(rdev->dev, "GPU pci config reset\n"); 4020 4021 /* disable dpm? */ 4022 4023 /* Disable CP parsing/prefetching */ 4024 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT); 4025 udelay(50); 4026 /* Disable DMA */ 4027 tmp = RREG32(DMA_RB_CNTL); 4028 tmp &= ~DMA_RB_ENABLE; 4029 WREG32(DMA_RB_CNTL, tmp); 4030 /* XXX other engines? */ 4031 4032 /* halt the rlc */ 4033 r600_rlc_stop(rdev); 4034 4035 udelay(50); 4036 4037 /* set mclk/sclk to bypass */ 4038 rv770_set_clk_bypass_mode(rdev); 4039 /* disable BM */ 4040 pci_clear_master(rdev->pdev); 4041 /* disable mem access */ 4042 evergreen_mc_stop(rdev, &save); 4043 if (evergreen_mc_wait_for_idle(rdev)) { 4044 dev_warn(rdev->dev, "Wait for MC idle timed out !\n"); 4045 } 4046 /* reset */ 4047 radeon_pci_config_reset(rdev); 4048 /* wait for asic to come out of reset */ 4049 for (i = 0; i < rdev->usec_timeout; i++) { 4050 if (RREG32(CONFIG_MEMSIZE) != 0xffffffff) 4051 break; 4052 udelay(1); 4053 } 4054} 4055 4056int evergreen_asic_reset(struct radeon_device *rdev, bool hard) 4057{ 4058 u32 reset_mask; 4059 4060 if (hard) { 4061 evergreen_gpu_pci_config_reset(rdev); 4062 return 0; 4063 } 4064 4065 reset_mask = evergreen_gpu_check_soft_reset(rdev); 4066 4067 if (reset_mask) 4068 r600_set_bios_scratch_engine_hung(rdev, true); 4069 4070 /* try soft reset */ 4071 evergreen_gpu_soft_reset(rdev, reset_mask); 4072 4073 reset_mask = evergreen_gpu_check_soft_reset(rdev); 4074 4075 /* try pci config reset */ 4076 if (reset_mask && radeon_hard_reset) 4077 evergreen_gpu_pci_config_reset(rdev); 4078 4079 reset_mask = evergreen_gpu_check_soft_reset(rdev); 4080 4081 if (!reset_mask) 4082 r600_set_bios_scratch_engine_hung(rdev, false); 4083 4084 return 0; 4085} 4086 4087/** 4088 * evergreen_gfx_is_lockup - Check if the GFX engine is locked up 4089 * 4090 * @rdev: radeon_device pointer 4091 * @ring: radeon_ring structure holding ring information 4092 * 4093 * Check if the GFX engine is locked up. 4094 * Returns true if the engine appears to be locked up, false if not. 4095 */ 4096bool evergreen_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring) 4097{ 4098 u32 reset_mask = evergreen_gpu_check_soft_reset(rdev); 4099 4100 if (!(reset_mask & (RADEON_RESET_GFX | 4101 RADEON_RESET_COMPUTE | 4102 RADEON_RESET_CP))) { 4103 radeon_ring_lockup_update(rdev, ring); 4104 return false; 4105 } 4106 return radeon_ring_test_lockup(rdev, ring); 4107} 4108 4109/* 4110 * RLC 4111 */ 4112#define RLC_SAVE_RESTORE_LIST_END_MARKER 0x00000000 4113#define RLC_CLEAR_STATE_END_MARKER 0x00000001 4114 4115void sumo_rlc_fini(struct radeon_device *rdev) 4116{ 4117 int r; 4118 4119 /* save restore block */ 4120 if (rdev->rlc.save_restore_obj) { 4121 r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false); 4122 if (unlikely(r != 0)) 4123 dev_warn(rdev->dev, "(%d) reserve RLC sr bo failed\n", r); 4124 radeon_bo_unpin(rdev->rlc.save_restore_obj); 4125 radeon_bo_unreserve(rdev->rlc.save_restore_obj); 4126 4127 radeon_bo_unref(&rdev->rlc.save_restore_obj); 4128 rdev->rlc.save_restore_obj = NULL; 4129 } 4130 4131 /* clear state block */ 4132 if (rdev->rlc.clear_state_obj) { 4133 r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false); 4134 if (unlikely(r != 0)) 4135 dev_warn(rdev->dev, "(%d) reserve RLC c bo failed\n", r); 4136 radeon_bo_unpin(rdev->rlc.clear_state_obj); 4137 radeon_bo_unreserve(rdev->rlc.clear_state_obj); 4138 4139 radeon_bo_unref(&rdev->rlc.clear_state_obj); 4140 rdev->rlc.clear_state_obj = NULL; 4141 } 4142 4143 /* clear state block */ 4144 if (rdev->rlc.cp_table_obj) { 4145 r = radeon_bo_reserve(rdev->rlc.cp_table_obj, false); 4146 if (unlikely(r != 0)) 4147 dev_warn(rdev->dev, "(%d) reserve RLC cp table bo failed\n", r); 4148 radeon_bo_unpin(rdev->rlc.cp_table_obj); 4149 radeon_bo_unreserve(rdev->rlc.cp_table_obj); 4150 4151 radeon_bo_unref(&rdev->rlc.cp_table_obj); 4152 rdev->rlc.cp_table_obj = NULL; 4153 } 4154} 4155 4156#define CP_ME_TABLE_SIZE 96 4157 4158int sumo_rlc_init(struct radeon_device *rdev) 4159{ 4160 const u32 *src_ptr; 4161 volatile u32 *dst_ptr; 4162 u32 dws, data, i, j, k, reg_num; 4163 u32 reg_list_num, reg_list_hdr_blk_index, reg_list_blk_index = 0; 4164 u64 reg_list_mc_addr; 4165 const struct cs_section_def *cs_data; 4166 int r; 4167 4168 src_ptr = rdev->rlc.reg_list; 4169 dws = rdev->rlc.reg_list_size; 4170 if (rdev->family >= CHIP_BONAIRE) { 4171 dws += (5 * 16) + 48 + 48 + 64; 4172 } 4173 cs_data = rdev->rlc.cs_data; 4174 4175 if (src_ptr) { 4176 /* save restore block */ 4177 if (rdev->rlc.save_restore_obj == NULL) { 4178 r = radeon_bo_create(rdev, dws * 4, PAGE_SIZE, true, 4179 RADEON_GEM_DOMAIN_VRAM, 0, NULL, 4180 NULL, &rdev->rlc.save_restore_obj); 4181 if (r) { 4182 dev_warn(rdev->dev, "(%d) create RLC sr bo failed\n", r); 4183 return r; 4184 } 4185 } 4186 4187 r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false); 4188 if (unlikely(r != 0)) { 4189 sumo_rlc_fini(rdev); 4190 return r; 4191 } 4192 r = radeon_bo_pin(rdev->rlc.save_restore_obj, RADEON_GEM_DOMAIN_VRAM, 4193 &rdev->rlc.save_restore_gpu_addr); 4194 if (r) { 4195 radeon_bo_unreserve(rdev->rlc.save_restore_obj); 4196 dev_warn(rdev->dev, "(%d) pin RLC sr bo failed\n", r); 4197 sumo_rlc_fini(rdev); 4198 return r; 4199 } 4200 4201 r = radeon_bo_kmap(rdev->rlc.save_restore_obj, (void **)__UNVOLATILE(&rdev->rlc.sr_ptr)); 4202 if (r) { 4203 dev_warn(rdev->dev, "(%d) map RLC sr bo failed\n", r); 4204 sumo_rlc_fini(rdev); 4205 return r; 4206 } 4207 /* write the sr buffer */ 4208 dst_ptr = rdev->rlc.sr_ptr; 4209 if (rdev->family >= CHIP_TAHITI) { 4210 /* SI */ 4211 for (i = 0; i < rdev->rlc.reg_list_size; i++) 4212 dst_ptr[i] = cpu_to_le32(src_ptr[i]); 4213 } else { 4214 /* ON/LN/TN */ 4215 /* format: 4216 * dw0: (reg2 << 16) | reg1 4217 * dw1: reg1 save space 4218 * dw2: reg2 save space 4219 */ 4220 for (i = 0; i < dws; i++) { 4221 data = src_ptr[i] >> 2; 4222 i++; 4223 if (i < dws) 4224 data |= (src_ptr[i] >> 2) << 16; 4225 j = (((i - 1) * 3) / 2); 4226 dst_ptr[j] = cpu_to_le32(data); 4227 } 4228 j = ((i * 3) / 2); 4229 dst_ptr[j] = cpu_to_le32(RLC_SAVE_RESTORE_LIST_END_MARKER); 4230 } 4231 radeon_bo_kunmap(rdev->rlc.save_restore_obj); 4232 radeon_bo_unreserve(rdev->rlc.save_restore_obj); 4233 } 4234 4235 if (cs_data) { 4236 /* clear state block */ 4237 if (rdev->family >= CHIP_BONAIRE) { 4238 rdev->rlc.clear_state_size = dws = cik_get_csb_size(rdev); 4239 } else if (rdev->family >= CHIP_TAHITI) { 4240 rdev->rlc.clear_state_size = si_get_csb_size(rdev); 4241 dws = rdev->rlc.clear_state_size + (256 / 4); 4242 } else { 4243 reg_list_num = 0; 4244 dws = 0; 4245 for (i = 0; cs_data[i].section != NULL; i++) { 4246 for (j = 0; cs_data[i].section[j].extent != NULL; j++) { 4247 reg_list_num++; 4248 dws += cs_data[i].section[j].reg_count; 4249 } 4250 } 4251 reg_list_blk_index = (3 * reg_list_num + 2); 4252 dws += reg_list_blk_index; 4253 rdev->rlc.clear_state_size = dws; 4254 } 4255 4256 if (rdev->rlc.clear_state_obj == NULL) { 4257 r = radeon_bo_create(rdev, dws * 4, PAGE_SIZE, true, 4258 RADEON_GEM_DOMAIN_VRAM, 0, NULL, 4259 NULL, &rdev->rlc.clear_state_obj); 4260 if (r) { 4261 dev_warn(rdev->dev, "(%d) create RLC c bo failed\n", r); 4262 sumo_rlc_fini(rdev); 4263 return r; 4264 } 4265 } 4266 r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false); 4267 if (unlikely(r != 0)) { 4268 sumo_rlc_fini(rdev); 4269 return r; 4270 } 4271 r = radeon_bo_pin(rdev->rlc.clear_state_obj, RADEON_GEM_DOMAIN_VRAM, 4272 &rdev->rlc.clear_state_gpu_addr); 4273 if (r) { 4274 radeon_bo_unreserve(rdev->rlc.clear_state_obj); 4275 dev_warn(rdev->dev, "(%d) pin RLC c bo failed\n", r); 4276 sumo_rlc_fini(rdev); 4277 return r; 4278 } 4279 4280 r = radeon_bo_kmap(rdev->rlc.clear_state_obj, (void **)__UNVOLATILE(&rdev->rlc.cs_ptr)); 4281 if (r) { 4282 dev_warn(rdev->dev, "(%d) map RLC c bo failed\n", r); 4283 sumo_rlc_fini(rdev); 4284 return r; 4285 } 4286 /* set up the cs buffer */ 4287 dst_ptr = rdev->rlc.cs_ptr; 4288 if (rdev->family >= CHIP_BONAIRE) { 4289 cik_get_csb_buffer(rdev, dst_ptr); 4290 } else if (rdev->family >= CHIP_TAHITI) { 4291 reg_list_mc_addr = rdev->rlc.clear_state_gpu_addr + 256; 4292 dst_ptr[0] = cpu_to_le32(upper_32_bits(reg_list_mc_addr)); 4293 dst_ptr[1] = cpu_to_le32(lower_32_bits(reg_list_mc_addr)); 4294 dst_ptr[2] = cpu_to_le32(rdev->rlc.clear_state_size); 4295 si_get_csb_buffer(rdev, &dst_ptr[(256/4)]); 4296 } else { 4297 reg_list_hdr_blk_index = 0; 4298 reg_list_mc_addr = rdev->rlc.clear_state_gpu_addr + (reg_list_blk_index * 4); 4299 data = upper_32_bits(reg_list_mc_addr); 4300 dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data); 4301 reg_list_hdr_blk_index++; 4302 for (i = 0; cs_data[i].section != NULL; i++) { 4303 for (j = 0; cs_data[i].section[j].extent != NULL; j++) { 4304 reg_num = cs_data[i].section[j].reg_count; 4305 data = reg_list_mc_addr & 0xffffffff; 4306 dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data); 4307 reg_list_hdr_blk_index++; 4308 4309 data = (cs_data[i].section[j].reg_index * 4) & 0xffffffff; 4310 dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data); 4311 reg_list_hdr_blk_index++; 4312 4313 data = 0x08000000 | (reg_num * 4); 4314 dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data); 4315 reg_list_hdr_blk_index++; 4316 4317 for (k = 0; k < reg_num; k++) { 4318 data = cs_data[i].section[j].extent[k]; 4319 dst_ptr[reg_list_blk_index + k] = cpu_to_le32(data); 4320 } 4321 reg_list_mc_addr += reg_num * 4; 4322 reg_list_blk_index += reg_num; 4323 } 4324 } 4325 dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(RLC_CLEAR_STATE_END_MARKER); 4326 } 4327 radeon_bo_kunmap(rdev->rlc.clear_state_obj); 4328 radeon_bo_unreserve(rdev->rlc.clear_state_obj); 4329 } 4330 4331 if (rdev->rlc.cp_table_size) { 4332 if (rdev->rlc.cp_table_obj == NULL) { 4333 r = radeon_bo_create(rdev, rdev->rlc.cp_table_size, 4334 PAGE_SIZE, true, 4335 RADEON_GEM_DOMAIN_VRAM, 0, NULL, 4336 NULL, &rdev->rlc.cp_table_obj); 4337 if (r) { 4338 dev_warn(rdev->dev, "(%d) create RLC cp table bo failed\n", r); 4339 sumo_rlc_fini(rdev); 4340 return r; 4341 } 4342 } 4343 4344 r = radeon_bo_reserve(rdev->rlc.cp_table_obj, false); 4345 if (unlikely(r != 0)) { 4346 dev_warn(rdev->dev, "(%d) reserve RLC cp table bo failed\n", r); 4347 sumo_rlc_fini(rdev); 4348 return r; 4349 } 4350 r = radeon_bo_pin(rdev->rlc.cp_table_obj, RADEON_GEM_DOMAIN_VRAM, 4351 &rdev->rlc.cp_table_gpu_addr); 4352 if (r) { 4353 radeon_bo_unreserve(rdev->rlc.cp_table_obj); 4354 dev_warn(rdev->dev, "(%d) pin RLC cp_table bo failed\n", r); 4355 sumo_rlc_fini(rdev); 4356 return r; 4357 } 4358 r = radeon_bo_kmap(rdev->rlc.cp_table_obj, (void **)__UNVOLATILE(&rdev->rlc.cp_table_ptr)); 4359 if (r) { 4360 dev_warn(rdev->dev, "(%d) map RLC cp table bo failed\n", r); 4361 sumo_rlc_fini(rdev); 4362 return r; 4363 } 4364 4365 cik_init_cp_pg_table(rdev); 4366 4367 radeon_bo_kunmap(rdev->rlc.cp_table_obj); 4368 radeon_bo_unreserve(rdev->rlc.cp_table_obj); 4369 4370 } 4371 4372 return 0; 4373} 4374 4375static void evergreen_rlc_start(struct radeon_device *rdev) 4376{ 4377 u32 mask = RLC_ENABLE; 4378 4379 if (rdev->flags & RADEON_IS_IGP) { 4380 mask |= GFX_POWER_GATING_ENABLE | GFX_POWER_GATING_SRC; 4381 } 4382 4383 WREG32(RLC_CNTL, mask); 4384} 4385 4386int evergreen_rlc_resume(struct radeon_device *rdev) 4387{ 4388 u32 i; 4389 const __be32 *fw_data; 4390 4391 if (!rdev->rlc_fw) 4392 return -EINVAL; 4393 4394 r600_rlc_stop(rdev); 4395 4396 WREG32(RLC_HB_CNTL, 0); 4397 4398 if (rdev->flags & RADEON_IS_IGP) { 4399 if (rdev->family == CHIP_ARUBA) { 4400 u32 always_on_bitmap = 4401 3 | (3 << (16 * rdev->config.cayman.max_shader_engines)); 4402 /* find out the number of active simds */ 4403 u32 tmp = (RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0xffff0000) >> 16; 4404 tmp |= 0xffffffff << rdev->config.cayman.max_simds_per_se; 4405 tmp = hweight32(~tmp); 4406 if (tmp == rdev->config.cayman.max_simds_per_se) { 4407 WREG32(TN_RLC_LB_ALWAYS_ACTIVE_SIMD_MASK, always_on_bitmap); 4408 WREG32(TN_RLC_LB_PARAMS, 0x00601004); 4409 WREG32(TN_RLC_LB_INIT_SIMD_MASK, 0xffffffff); 4410 WREG32(TN_RLC_LB_CNTR_INIT, 0x00000000); 4411 WREG32(TN_RLC_LB_CNTR_MAX, 0x00002000); 4412 } 4413 } else { 4414 WREG32(RLC_HB_WPTR_LSB_ADDR, 0); 4415 WREG32(RLC_HB_WPTR_MSB_ADDR, 0); 4416 } 4417 WREG32(TN_RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8); 4418 WREG32(TN_RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8); 4419 } else { 4420 WREG32(RLC_HB_BASE, 0); 4421 WREG32(RLC_HB_RPTR, 0); 4422 WREG32(RLC_HB_WPTR, 0); 4423 WREG32(RLC_HB_WPTR_LSB_ADDR, 0); 4424 WREG32(RLC_HB_WPTR_MSB_ADDR, 0); 4425 } 4426 WREG32(RLC_MC_CNTL, 0); 4427 WREG32(RLC_UCODE_CNTL, 0); 4428 4429 fw_data = (const __be32 *)rdev->rlc_fw->data; 4430 if (rdev->family >= CHIP_ARUBA) { 4431 for (i = 0; i < ARUBA_RLC_UCODE_SIZE; i++) { 4432 WREG32(RLC_UCODE_ADDR, i); 4433 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++)); 4434 } 4435 } else if (rdev->family >= CHIP_CAYMAN) { 4436 for (i = 0; i < CAYMAN_RLC_UCODE_SIZE; i++) { 4437 WREG32(RLC_UCODE_ADDR, i); 4438 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++)); 4439 } 4440 } else { 4441 for (i = 0; i < EVERGREEN_RLC_UCODE_SIZE; i++) { 4442 WREG32(RLC_UCODE_ADDR, i); 4443 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++)); 4444 } 4445 } 4446 WREG32(RLC_UCODE_ADDR, 0); 4447 4448 evergreen_rlc_start(rdev); 4449 4450 return 0; 4451} 4452 4453/* Interrupts */ 4454 4455u32 evergreen_get_vblank_counter(struct radeon_device *rdev, int crtc) 4456{ 4457 if (crtc >= rdev->num_crtc) 4458 return 0; 4459 else 4460 return RREG32(CRTC_STATUS_FRAME_COUNT + crtc_offsets[crtc]); 4461} 4462 4463void evergreen_disable_interrupt_state(struct radeon_device *rdev) 4464{ 4465 int i; 4466 u32 tmp; 4467 4468 if (rdev->family >= CHIP_CAYMAN) { 4469 cayman_cp_int_cntl_setup(rdev, 0, 4470 CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE); 4471 cayman_cp_int_cntl_setup(rdev, 1, 0); 4472 cayman_cp_int_cntl_setup(rdev, 2, 0); 4473 tmp = RREG32(CAYMAN_DMA1_CNTL) & ~TRAP_ENABLE; 4474 WREG32(CAYMAN_DMA1_CNTL, tmp); 4475 } else 4476 WREG32(CP_INT_CNTL, CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE); 4477 tmp = RREG32(DMA_CNTL) & ~TRAP_ENABLE; 4478 WREG32(DMA_CNTL, tmp); 4479 WREG32(GRBM_INT_CNTL, 0); 4480 WREG32(SRBM_INT_CNTL, 0); 4481 for (i = 0; i < rdev->num_crtc; i++) 4482 WREG32(INT_MASK + crtc_offsets[i], 0); 4483 for (i = 0; i < rdev->num_crtc; i++) 4484 WREG32(GRPH_INT_CONTROL + crtc_offsets[i], 0); 4485 4486 /* only one DAC on DCE5 */ 4487 if (!ASIC_IS_DCE5(rdev)) 4488 WREG32(DACA_AUTODETECT_INT_CONTROL, 0); 4489 WREG32(DACB_AUTODETECT_INT_CONTROL, 0); 4490 4491 for (i = 0; i < 6; i++) 4492 WREG32_AND(DC_HPDx_INT_CONTROL(i), DC_HPDx_INT_POLARITY); 4493} 4494 4495/* Note that the order we write back regs here is important */ 4496int evergreen_irq_set(struct radeon_device *rdev) 4497{ 4498 int i; 4499 u32 cp_int_cntl = CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE; 4500 u32 cp_int_cntl1 = 0, cp_int_cntl2 = 0; 4501 u32 grbm_int_cntl = 0; 4502 u32 dma_cntl, dma_cntl1 = 0; 4503 u32 thermal_int = 0; 4504 4505 if (!rdev->irq.installed) { 4506 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n"); 4507 return -EINVAL; 4508 } 4509 /* don't enable anything if the ih is disabled */ 4510 if (!rdev->ih.enabled) { 4511 r600_disable_interrupts(rdev); 4512 /* force the active interrupt state to all disabled */ 4513 evergreen_disable_interrupt_state(rdev); 4514 return 0; 4515 } 4516 4517 if (rdev->family == CHIP_ARUBA) 4518 thermal_int = RREG32(TN_CG_THERMAL_INT_CTRL) & 4519 ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW); 4520 else 4521 thermal_int = RREG32(CG_THERMAL_INT) & 4522 ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW); 4523 4524 dma_cntl = RREG32(DMA_CNTL) & ~TRAP_ENABLE; 4525 4526 if (rdev->family >= CHIP_CAYMAN) { 4527 /* enable CP interrupts on all rings */ 4528 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) { 4529 DRM_DEBUG("evergreen_irq_set: sw int gfx\n"); 4530 cp_int_cntl |= TIME_STAMP_INT_ENABLE; 4531 } 4532 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) { 4533 DRM_DEBUG("evergreen_irq_set: sw int cp1\n"); 4534 cp_int_cntl1 |= TIME_STAMP_INT_ENABLE; 4535 } 4536 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) { 4537 DRM_DEBUG("evergreen_irq_set: sw int cp2\n"); 4538 cp_int_cntl2 |= TIME_STAMP_INT_ENABLE; 4539 } 4540 } else { 4541 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) { 4542 DRM_DEBUG("evergreen_irq_set: sw int gfx\n"); 4543 cp_int_cntl |= RB_INT_ENABLE; 4544 cp_int_cntl |= TIME_STAMP_INT_ENABLE; 4545 } 4546 } 4547 4548 if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) { 4549 DRM_DEBUG("r600_irq_set: sw int dma\n"); 4550 dma_cntl |= TRAP_ENABLE; 4551 } 4552 4553 if (rdev->family >= CHIP_CAYMAN) { 4554 dma_cntl1 = RREG32(CAYMAN_DMA1_CNTL) & ~TRAP_ENABLE; 4555 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) { 4556 DRM_DEBUG("r600_irq_set: sw int dma1\n"); 4557 dma_cntl1 |= TRAP_ENABLE; 4558 } 4559 } 4560 4561 if (rdev->irq.dpm_thermal) { 4562 DRM_DEBUG("dpm thermal\n"); 4563 thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW; 4564 } 4565 4566 if (rdev->family >= CHIP_CAYMAN) { 4567 cayman_cp_int_cntl_setup(rdev, 0, cp_int_cntl); 4568 cayman_cp_int_cntl_setup(rdev, 1, cp_int_cntl1); 4569 cayman_cp_int_cntl_setup(rdev, 2, cp_int_cntl2); 4570 } else 4571 WREG32(CP_INT_CNTL, cp_int_cntl); 4572 4573 WREG32(DMA_CNTL, dma_cntl); 4574 4575 if (rdev->family >= CHIP_CAYMAN) 4576 WREG32(CAYMAN_DMA1_CNTL, dma_cntl1); 4577 4578 WREG32(GRBM_INT_CNTL, grbm_int_cntl); 4579 4580 for (i = 0; i < rdev->num_crtc; i++) { 4581 radeon_irq_kms_set_irq_n_enabled( 4582 rdev, INT_MASK + crtc_offsets[i], 4583 VBLANK_INT_MASK, 4584 rdev->irq.crtc_vblank_int[i] || 4585 atomic_read(&rdev->irq.pflip[i]), "vblank", i); 4586 } 4587 4588 for (i = 0; i < rdev->num_crtc; i++) 4589 WREG32(GRPH_INT_CONTROL + crtc_offsets[i], GRPH_PFLIP_INT_MASK); 4590 4591 for (i = 0; i < 6; i++) { 4592 radeon_irq_kms_set_irq_n_enabled( 4593 rdev, DC_HPDx_INT_CONTROL(i), 4594 DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN, 4595 rdev->irq.hpd[i], "HPD", i); 4596 } 4597 4598 if (rdev->family == CHIP_ARUBA) 4599 WREG32(TN_CG_THERMAL_INT_CTRL, thermal_int); 4600 else 4601 WREG32(CG_THERMAL_INT, thermal_int); 4602 4603 for (i = 0; i < 6; i++) { 4604 radeon_irq_kms_set_irq_n_enabled( 4605 rdev, AFMT_AUDIO_PACKET_CONTROL + crtc_offsets[i], 4606 AFMT_AZ_FORMAT_WTRIG_MASK, 4607 rdev->irq.afmt[i], "HDMI", i); 4608 } 4609 4610 /* posting read */ 4611 RREG32(SRBM_STATUS); 4612 4613 return 0; 4614} 4615 4616/* Note that the order we write back regs here is important */ 4617static void evergreen_irq_ack(struct radeon_device *rdev) 4618{ 4619 int i, j; 4620 u32 *grph_int = rdev->irq.stat_regs.evergreen.grph_int; 4621 u32 *disp_int = rdev->irq.stat_regs.evergreen.disp_int; 4622 u32 *afmt_status = rdev->irq.stat_regs.evergreen.afmt_status; 4623 4624 for (i = 0; i < 6; i++) { 4625 disp_int[i] = RREG32(evergreen_disp_int_status[i]); 4626 afmt_status[i] = RREG32(AFMT_STATUS + crtc_offsets[i]); 4627 if (i < rdev->num_crtc) 4628 grph_int[i] = RREG32(GRPH_INT_STATUS + crtc_offsets[i]); 4629 } 4630 4631 /* We write back each interrupt register in pairs of two */ 4632 for (i = 0; i < rdev->num_crtc; i += 2) { 4633 for (j = i; j < (i + 2); j++) { 4634 if (grph_int[j] & GRPH_PFLIP_INT_OCCURRED) 4635 WREG32(GRPH_INT_STATUS + crtc_offsets[j], 4636 GRPH_PFLIP_INT_CLEAR); 4637 } 4638 4639 for (j = i; j < (i + 2); j++) { 4640 if (disp_int[j] & LB_D1_VBLANK_INTERRUPT) 4641 WREG32(VBLANK_STATUS + crtc_offsets[j], 4642 VBLANK_ACK); 4643 if (disp_int[j] & LB_D1_VLINE_INTERRUPT) 4644 WREG32(VLINE_STATUS + crtc_offsets[j], 4645 VLINE_ACK); 4646 } 4647 } 4648 4649 for (i = 0; i < 6; i++) { 4650 if (disp_int[i] & DC_HPD1_INTERRUPT) 4651 WREG32_OR(DC_HPDx_INT_CONTROL(i), DC_HPDx_INT_ACK); 4652 } 4653 4654 for (i = 0; i < 6; i++) { 4655 if (disp_int[i] & DC_HPD1_RX_INTERRUPT) 4656 WREG32_OR(DC_HPDx_INT_CONTROL(i), DC_HPDx_RX_INT_ACK); 4657 } 4658 4659 for (i = 0; i < 6; i++) { 4660 if (afmt_status[i] & AFMT_AZ_FORMAT_WTRIG) 4661 WREG32_OR(AFMT_AUDIO_PACKET_CONTROL + crtc_offsets[i], 4662 AFMT_AZ_FORMAT_WTRIG_ACK); 4663 } 4664} 4665 4666static void evergreen_irq_disable(struct radeon_device *rdev) 4667{ 4668 r600_disable_interrupts(rdev); 4669 /* Wait and acknowledge irq */ 4670 mdelay(1); 4671 evergreen_irq_ack(rdev); 4672 evergreen_disable_interrupt_state(rdev); 4673} 4674 4675void evergreen_irq_suspend(struct radeon_device *rdev) 4676{ 4677 evergreen_irq_disable(rdev); 4678 r600_rlc_stop(rdev); 4679} 4680 4681static u32 evergreen_get_ih_wptr(struct radeon_device *rdev) 4682{ 4683 u32 wptr, tmp; 4684 4685 if (rdev->wb.enabled) 4686 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]); 4687 else 4688 wptr = RREG32(IH_RB_WPTR); 4689 4690 if (wptr & RB_OVERFLOW) { 4691 wptr &= ~RB_OVERFLOW; 4692 /* When a ring buffer overflow happen start parsing interrupt 4693 * from the last not overwritten vector (wptr + 16). Hopefully 4694 * this should allow us to catchup. 4695 */ 4696 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, 0x%08X, 0x%08X)\n", 4697 wptr, rdev->ih.rptr, (wptr + 16) & rdev->ih.ptr_mask); 4698 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask; 4699 tmp = RREG32(IH_RB_CNTL); 4700 tmp |= IH_WPTR_OVERFLOW_CLEAR; 4701 WREG32(IH_RB_CNTL, tmp); 4702 } 4703 return (wptr & rdev->ih.ptr_mask); 4704} 4705 4706int evergreen_irq_process(struct radeon_device *rdev) 4707{ 4708 u32 *disp_int = rdev->irq.stat_regs.evergreen.disp_int; 4709 u32 *afmt_status = rdev->irq.stat_regs.evergreen.afmt_status; 4710 u32 crtc_idx, hpd_idx, afmt_idx; 4711 u32 mask; 4712 u32 wptr; 4713 u32 rptr; 4714 u32 src_id, src_data; 4715 u32 ring_index; 4716 bool queue_hotplug = false; 4717 bool queue_hdmi = false; 4718 bool queue_dp = false; 4719 bool queue_thermal = false; 4720 u32 status, addr; 4721 const char *event_name; 4722 4723 if (!rdev->ih.enabled || rdev->shutdown) 4724 return IRQ_NONE; 4725 4726 wptr = evergreen_get_ih_wptr(rdev); 4727 4728restart_ih: 4729 /* is somebody else already processing irqs? */ 4730 if (atomic_xchg(&rdev->ih.lock, 1)) 4731 return IRQ_NONE; 4732 4733 rptr = rdev->ih.rptr; 4734 DRM_DEBUG("evergreen_irq_process start: rptr %d, wptr %d\n", rptr, wptr); 4735 4736 /* Order reading of wptr vs. reading of IH ring data */ 4737 rmb(); 4738 4739 /* display interrupts */ 4740 evergreen_irq_ack(rdev); 4741 4742 while (rptr != wptr) { 4743 /* wptr/rptr are in bytes! */ 4744 ring_index = rptr / 4; 4745 src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff; 4746 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff; 4747 4748 switch (src_id) { 4749 case 1: /* D1 vblank/vline */ 4750 case 2: /* D2 vblank/vline */ 4751 case 3: /* D3 vblank/vline */ 4752 case 4: /* D4 vblank/vline */ 4753 case 5: /* D5 vblank/vline */ 4754 case 6: /* D6 vblank/vline */ 4755 crtc_idx = src_id - 1; 4756 4757 if (src_data == 0) { /* vblank */ 4758 mask = LB_D1_VBLANK_INTERRUPT; 4759 event_name = "vblank"; 4760 4761 if (rdev->irq.crtc_vblank_int[crtc_idx]) { 4762 drm_handle_vblank(rdev->ddev, crtc_idx); 4763#ifdef __NetBSD__ 4764 spin_lock(&rdev->irq.vblank_lock); 4765 rdev->pm.vblank_sync = true; 4766 DRM_SPIN_WAKEUP_ONE(&rdev->irq.vblank_queue, &rdev->irq.vblank_lock); 4767 spin_unlock(&rdev->irq.vblank_lock); 4768#else 4769 rdev->pm.vblank_sync = true; 4770 wake_up(&rdev->irq.vblank_queue); 4771#endif 4772 } 4773 if (atomic_read(&rdev->irq.pflip[crtc_idx])) { 4774 radeon_crtc_handle_vblank(rdev, 4775 crtc_idx); 4776 } 4777 4778 } else if (src_data == 1) { /* vline */ 4779 mask = LB_D1_VLINE_INTERRUPT; 4780 event_name = "vline"; 4781 } else { 4782 DRM_DEBUG("Unhandled interrupt: %d %d\n", 4783 src_id, src_data); 4784 break; 4785 } 4786 4787 if (!(disp_int[crtc_idx] & mask)) { 4788 DRM_DEBUG("IH: D%d %s - IH event w/o asserted irq bit?\n", 4789 crtc_idx + 1, event_name); 4790 } 4791 4792 disp_int[crtc_idx] &= ~mask; 4793 DRM_DEBUG("IH: D%d %s\n", crtc_idx + 1, event_name); 4794 4795 break; 4796 case 8: /* D1 page flip */ 4797 case 10: /* D2 page flip */ 4798 case 12: /* D3 page flip */ 4799 case 14: /* D4 page flip */ 4800 case 16: /* D5 page flip */ 4801 case 18: /* D6 page flip */ 4802 DRM_DEBUG("IH: D%d flip\n", ((src_id - 8) >> 1) + 1); 4803 if (radeon_use_pflipirq > 0) 4804 radeon_crtc_handle_flip(rdev, (src_id - 8) >> 1); 4805 break; 4806 case 42: /* HPD hotplug */ 4807 if (src_data <= 5) { 4808 hpd_idx = src_data; 4809 mask = DC_HPD1_INTERRUPT; 4810 queue_hotplug = true; 4811 event_name = "HPD"; 4812 4813 } else if (src_data <= 11) { 4814 hpd_idx = src_data - 6; 4815 mask = DC_HPD1_RX_INTERRUPT; 4816 queue_dp = true; 4817 event_name = "HPD_RX"; 4818 4819 } else { 4820 DRM_DEBUG("Unhandled interrupt: %d %d\n", 4821 src_id, src_data); 4822 break; 4823 } 4824 4825 if (!(disp_int[hpd_idx] & mask)) 4826 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n"); 4827 4828 disp_int[hpd_idx] &= ~mask; 4829 DRM_DEBUG("IH: %s%d\n", event_name, hpd_idx + 1); 4830 4831 break; 4832 case 44: /* hdmi */ 4833 afmt_idx = src_data; 4834 if (!(afmt_status[afmt_idx] & AFMT_AZ_FORMAT_WTRIG)) 4835 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n"); 4836 4837 if (afmt_idx > 5) { 4838 DRM_ERROR("Unhandled interrupt: %d %d\n", 4839 src_id, src_data); 4840 break; 4841 } 4842 afmt_status[afmt_idx] &= ~AFMT_AZ_FORMAT_WTRIG; 4843 queue_hdmi = true; 4844 DRM_DEBUG("IH: HDMI%d\n", afmt_idx + 1); 4845 break; 4846 case 96: 4847 DRM_ERROR("SRBM_READ_ERROR: 0x%x\n", RREG32(SRBM_READ_ERROR)); 4848 WREG32(SRBM_INT_ACK, 0x1); 4849 break; 4850 case 124: /* UVD */ 4851 DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data); 4852 radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX); 4853 break; 4854 case 146: 4855 case 147: 4856 addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR); 4857 status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS); 4858 /* reset addr and status */ 4859 WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1); 4860 if (addr == 0x0 && status == 0x0) 4861 break; 4862 dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data); 4863 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n", 4864 addr); 4865 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n", 4866 status); 4867 cayman_vm_decode_fault(rdev, status, addr); 4868 break; 4869 case 176: /* CP_INT in ring buffer */ 4870 case 177: /* CP_INT in IB1 */ 4871 case 178: /* CP_INT in IB2 */ 4872 DRM_DEBUG("IH: CP int: 0x%08x\n", src_data); 4873 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX); 4874 break; 4875 case 181: /* CP EOP event */ 4876 DRM_DEBUG("IH: CP EOP\n"); 4877 if (rdev->family >= CHIP_CAYMAN) { 4878 switch (src_data) { 4879 case 0: 4880 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX); 4881 break; 4882 case 1: 4883 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX); 4884 break; 4885 case 2: 4886 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX); 4887 break; 4888 } 4889 } else 4890 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX); 4891 break; 4892 case 224: /* DMA trap event */ 4893 DRM_DEBUG("IH: DMA trap\n"); 4894 radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX); 4895 break; 4896 case 230: /* thermal low to high */ 4897 DRM_DEBUG("IH: thermal low to high\n"); 4898 rdev->pm.dpm.thermal.high_to_low = false; 4899 queue_thermal = true; 4900 break; 4901 case 231: /* thermal high to low */ 4902 DRM_DEBUG("IH: thermal high to low\n"); 4903 rdev->pm.dpm.thermal.high_to_low = true; 4904 queue_thermal = true; 4905 break; 4906 case 233: /* GUI IDLE */ 4907 DRM_DEBUG("IH: GUI idle\n"); 4908 break; 4909 case 244: /* DMA trap event */ 4910 if (rdev->family >= CHIP_CAYMAN) { 4911 DRM_DEBUG("IH: DMA1 trap\n"); 4912 radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX); 4913 } 4914 break; 4915 default: 4916 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); 4917 break; 4918 } 4919 4920 /* wptr/rptr are in bytes! */ 4921 rptr += 16; 4922 rptr &= rdev->ih.ptr_mask; 4923 WREG32(IH_RB_RPTR, rptr); 4924 } 4925 if (queue_dp) 4926 schedule_work(&rdev->dp_work); 4927 if (queue_hotplug) 4928 schedule_delayed_work(&rdev->hotplug_work, 0); 4929 if (queue_hdmi) 4930 schedule_work(&rdev->audio_work); 4931 if (queue_thermal && rdev->pm.dpm_enabled) 4932 schedule_work(&rdev->pm.dpm.thermal.work); 4933 rdev->ih.rptr = rptr; 4934 atomic_set(&rdev->ih.lock, 0); 4935 4936 /* make sure wptr hasn't changed while processing */ 4937 wptr = evergreen_get_ih_wptr(rdev); 4938 if (wptr != rptr) 4939 goto restart_ih; 4940 4941 return IRQ_HANDLED; 4942} 4943 4944static void evergreen_uvd_init(struct radeon_device *rdev) 4945{ 4946 int r; 4947 4948 if (!rdev->has_uvd) 4949 return; 4950 4951 r = radeon_uvd_init(rdev); 4952 if (r) { 4953 dev_err(rdev->dev, "failed UVD (%d) init.\n", r); 4954 /* 4955 * At this point rdev->uvd.vcpu_bo is NULL which trickles down 4956 * to early fails uvd_v2_2_resume() and thus nothing happens 4957 * there. So it is pointless to try to go through that code 4958 * hence why we disable uvd here. 4959 */ 4960 rdev->has_uvd = false; 4961 return; 4962 } 4963 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_obj = NULL; 4964 r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_UVD_INDEX], 4096); 4965} 4966 4967static void evergreen_uvd_start(struct radeon_device *rdev) 4968{ 4969 int r; 4970 4971 if (!rdev->has_uvd) 4972 return; 4973 4974 r = uvd_v2_2_resume(rdev); 4975 if (r) { 4976 dev_err(rdev->dev, "failed UVD resume (%d).\n", r); 4977 goto error; 4978 } 4979 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_UVD_INDEX); 4980 if (r) { 4981 dev_err(rdev->dev, "failed initializing UVD fences (%d).\n", r); 4982 goto error; 4983 } 4984 return; 4985 4986error: 4987 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0; 4988} 4989 4990static void evergreen_uvd_resume(struct radeon_device *rdev) 4991{ 4992 struct radeon_ring *ring; 4993 int r; 4994 4995 if (!rdev->has_uvd || !rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size) 4996 return; 4997 4998 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX]; 4999 r = radeon_ring_init(rdev, ring, ring->ring_size, 0, PACKET0(UVD_NO_OP, 0)); 5000 if (r) { 5001 dev_err(rdev->dev, "failed initializing UVD ring (%d).\n", r); 5002 return; 5003 } 5004 r = uvd_v1_0_init(rdev); 5005 if (r) { 5006 dev_err(rdev->dev, "failed initializing UVD (%d).\n", r); 5007 return; 5008 } 5009} 5010 5011static int evergreen_startup(struct radeon_device *rdev) 5012{ 5013 struct radeon_ring *ring; 5014 int r; 5015 5016 /* enable pcie gen2 link */ 5017 evergreen_pcie_gen2_enable(rdev); 5018 /* enable aspm */ 5019 evergreen_program_aspm(rdev); 5020 5021 /* scratch needs to be initialized before MC */ 5022 r = r600_vram_scratch_init(rdev); 5023 if (r) 5024 return r; 5025 5026 evergreen_mc_program(rdev); 5027 5028 if (ASIC_IS_DCE5(rdev) && !rdev->pm.dpm_enabled) { 5029 r = ni_mc_load_microcode(rdev); 5030 if (r) { 5031 DRM_ERROR("Failed to load MC firmware!\n"); 5032 return r; 5033 } 5034 } 5035 5036 if (rdev->flags & RADEON_IS_AGP) { 5037 evergreen_agp_enable(rdev); 5038 } else { 5039 r = evergreen_pcie_gart_enable(rdev); 5040 if (r) 5041 return r; 5042 } 5043 evergreen_gpu_init(rdev); 5044 5045 /* allocate rlc buffers */ 5046 if (rdev->flags & RADEON_IS_IGP) { 5047 rdev->rlc.reg_list = sumo_rlc_save_restore_register_list; 5048 rdev->rlc.reg_list_size = 5049 (u32)ARRAY_SIZE(sumo_rlc_save_restore_register_list); 5050 rdev->rlc.cs_data = evergreen_cs_data; 5051 r = sumo_rlc_init(rdev); 5052 if (r) { 5053 DRM_ERROR("Failed to init rlc BOs!\n"); 5054 return r; 5055 } 5056 } 5057 5058 /* allocate wb buffer */ 5059 r = radeon_wb_init(rdev); 5060 if (r) 5061 return r; 5062 5063 r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX); 5064 if (r) { 5065 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r); 5066 return r; 5067 } 5068 5069 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX); 5070 if (r) { 5071 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r); 5072 return r; 5073 } 5074 5075 evergreen_uvd_start(rdev); 5076 5077 /* Enable IRQ */ 5078 if (!rdev->irq.installed) { 5079 r = radeon_irq_kms_init(rdev); 5080 if (r) 5081 return r; 5082 } 5083 5084 r = r600_irq_init(rdev); 5085 if (r) { 5086 DRM_ERROR("radeon: IH init failed (%d).\n", r); 5087 radeon_irq_kms_fini(rdev); 5088 return r; 5089 } 5090 evergreen_irq_set(rdev); 5091 5092 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]; 5093 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET, 5094 RADEON_CP_PACKET2); 5095 if (r) 5096 return r; 5097 5098 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX]; 5099 r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET, 5100 DMA_PACKET(DMA_PACKET_NOP, 0, 0)); 5101 if (r) 5102 return r; 5103 5104 r = evergreen_cp_load_microcode(rdev); 5105 if (r) 5106 return r; 5107 r = evergreen_cp_resume(rdev); 5108 if (r) 5109 return r; 5110 r = r600_dma_resume(rdev); 5111 if (r) 5112 return r; 5113 5114 evergreen_uvd_resume(rdev); 5115 5116 r = radeon_ib_pool_init(rdev); 5117 if (r) { 5118 dev_err(rdev->dev, "IB initialization failed (%d).\n", r); 5119 return r; 5120 } 5121 5122 r = radeon_audio_init(rdev); 5123 if (r) { 5124 DRM_ERROR("radeon: audio init failed\n"); 5125 return r; 5126 } 5127 5128 return 0; 5129} 5130 5131int evergreen_resume(struct radeon_device *rdev) 5132{ 5133 int r; 5134 5135 /* reset the asic, the gfx blocks are often in a bad state 5136 * after the driver is unloaded or after a resume 5137 */ 5138 if (radeon_asic_reset(rdev)) 5139 dev_warn(rdev->dev, "GPU reset failed !\n"); 5140 /* Do not reset GPU before posting, on rv770 hw unlike on r500 hw, 5141 * posting will perform necessary task to bring back GPU into good 5142 * shape. 5143 */ 5144 /* post card */ 5145 atom_asic_init(rdev->mode_info.atom_context); 5146 5147 /* init golden registers */ 5148 evergreen_init_golden_registers(rdev); 5149 5150 if (rdev->pm.pm_method == PM_METHOD_DPM) 5151 radeon_pm_resume(rdev); 5152 5153 rdev->accel_working = true; 5154 r = evergreen_startup(rdev); 5155 if (r) { 5156 DRM_ERROR("evergreen startup failed on resume\n"); 5157 rdev->accel_working = false; 5158 return r; 5159 } 5160 5161 return r; 5162 5163} 5164 5165int evergreen_suspend(struct radeon_device *rdev) 5166{ 5167 radeon_pm_suspend(rdev); 5168 radeon_audio_fini(rdev); 5169 if (rdev->has_uvd) { 5170 uvd_v1_0_fini(rdev); 5171 radeon_uvd_suspend(rdev); 5172 } 5173 r700_cp_stop(rdev); 5174 r600_dma_stop(rdev); 5175 evergreen_irq_suspend(rdev); 5176 radeon_wb_disable(rdev); 5177 evergreen_pcie_gart_disable(rdev); 5178 5179 return 0; 5180} 5181 5182/* Plan is to move initialization in that function and use 5183 * helper function so that radeon_device_init pretty much 5184 * do nothing more than calling asic specific function. This 5185 * should also allow to remove a bunch of callback function 5186 * like vram_info. 5187 */ 5188int evergreen_init(struct radeon_device *rdev) 5189{ 5190 int r; 5191 5192 /* Read BIOS */ 5193 if (!radeon_get_bios(rdev)) { 5194 if (ASIC_IS_AVIVO(rdev)) 5195 return -EINVAL; 5196 } 5197 /* Must be an ATOMBIOS */ 5198 if (!rdev->is_atom_bios) { 5199 dev_err(rdev->dev, "Expecting atombios for evergreen GPU\n"); 5200 return -EINVAL; 5201 } 5202 r = radeon_atombios_init(rdev); 5203 if (r) 5204 return r; 5205 /* reset the asic, the gfx blocks are often in a bad state 5206 * after the driver is unloaded or after a resume 5207 */ 5208 if (radeon_asic_reset(rdev)) 5209 dev_warn(rdev->dev, "GPU reset failed !\n"); 5210 /* Post card if necessary */ 5211 if (!radeon_card_posted(rdev)) { 5212 if (!rdev->bios) { 5213 dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n"); 5214 return -EINVAL; 5215 } 5216 DRM_INFO("GPU not posted. posting now...\n"); 5217 atom_asic_init(rdev->mode_info.atom_context); 5218 } 5219 /* init golden registers */ 5220 evergreen_init_golden_registers(rdev); 5221 /* Initialize scratch registers */ 5222 r600_scratch_init(rdev); 5223 /* Initialize surface registers */ 5224 radeon_surface_init(rdev); 5225 /* Initialize clocks */ 5226 radeon_get_clock_info(rdev->ddev); 5227 /* Fence driver */ 5228 r = radeon_fence_driver_init(rdev); 5229 if (r) 5230 return r; 5231 /* initialize AGP */ 5232 if (rdev->flags & RADEON_IS_AGP) { 5233 r = radeon_agp_init(rdev); 5234 if (r) 5235 radeon_agp_disable(rdev); 5236 } 5237 /* initialize memory controller */ 5238 r = evergreen_mc_init(rdev); 5239 if (r) 5240 return r; 5241 /* Memory manager */ 5242 r = radeon_bo_init(rdev); 5243 if (r) 5244 return r; 5245 5246 if (ASIC_IS_DCE5(rdev)) { 5247 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw || !rdev->mc_fw) { 5248 r = ni_init_microcode(rdev); 5249 if (r) { 5250 DRM_ERROR("Failed to load firmware!\n"); 5251 return r; 5252 } 5253 } 5254 } else { 5255 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) { 5256 r = r600_init_microcode(rdev); 5257 if (r) { 5258 DRM_ERROR("Failed to load firmware!\n"); 5259 return r; 5260 } 5261 } 5262 } 5263 5264 /* Initialize power management */ 5265 radeon_pm_init(rdev); 5266 5267 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ring_obj = NULL; 5268 r600_ring_init(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX], 1024 * 1024); 5269 5270 rdev->ring[R600_RING_TYPE_DMA_INDEX].ring_obj = NULL; 5271 r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_DMA_INDEX], 64 * 1024); 5272 5273 evergreen_uvd_init(rdev); 5274 5275 rdev->ih.ring_obj = NULL; 5276 r600_ih_ring_init(rdev, 64 * 1024); 5277 5278 r = r600_pcie_gart_init(rdev); 5279 if (r) 5280 return r; 5281 5282 rdev->accel_working = true; 5283 r = evergreen_startup(rdev); 5284 if (r) { 5285 dev_err(rdev->dev, "disabling GPU acceleration\n"); 5286 r700_cp_fini(rdev); 5287 r600_dma_fini(rdev); 5288 r600_irq_fini(rdev); 5289 if (rdev->flags & RADEON_IS_IGP) 5290 sumo_rlc_fini(rdev); 5291 radeon_wb_fini(rdev); 5292 radeon_ib_pool_fini(rdev); 5293 radeon_irq_kms_fini(rdev); 5294 evergreen_pcie_gart_fini(rdev); 5295 rdev->accel_working = false; 5296 } 5297 5298 /* Don't start up if the MC ucode is missing on BTC parts. 5299 * The default clocks and voltages before the MC ucode 5300 * is loaded are not suffient for advanced operations. 5301 */ 5302 if (ASIC_IS_DCE5(rdev)) { 5303 if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) { 5304 DRM_ERROR("radeon: MC ucode required for NI+.\n"); 5305 return -EINVAL; 5306 } 5307 } 5308 5309 return 0; 5310} 5311 5312void evergreen_fini(struct radeon_device *rdev) 5313{ 5314 radeon_pm_fini(rdev); 5315 radeon_audio_fini(rdev); 5316 r700_cp_fini(rdev); 5317 r600_dma_fini(rdev); 5318 r600_irq_fini(rdev); 5319 if (rdev->flags & RADEON_IS_IGP) 5320 sumo_rlc_fini(rdev); 5321 radeon_wb_fini(rdev); 5322 radeon_ib_pool_fini(rdev); 5323 radeon_irq_kms_fini(rdev); 5324 uvd_v1_0_fini(rdev); 5325 radeon_uvd_fini(rdev); 5326 evergreen_pcie_gart_fini(rdev); 5327 r600_vram_scratch_fini(rdev); 5328 radeon_gem_fini(rdev); 5329 radeon_fence_driver_fini(rdev); 5330 radeon_agp_fini(rdev); 5331 radeon_bo_fini(rdev); 5332 radeon_atombios_fini(rdev); 5333 kfree(rdev->bios); 5334 rdev->bios = NULL; 5335} 5336 5337void evergreen_pcie_gen2_enable(struct radeon_device *rdev) 5338{ 5339 u32 link_width_cntl, speed_cntl; 5340 5341 if (radeon_pcie_gen2 == 0) 5342 return; 5343 5344 if (rdev->flags & RADEON_IS_IGP) 5345 return; 5346 5347 if (!(rdev->flags & RADEON_IS_PCIE)) 5348 return; 5349 5350 /* x2 cards have a special sequence */ 5351 if (ASIC_IS_X2(rdev)) 5352 return; 5353 5354 if ((rdev->pdev->bus->max_bus_speed != PCIE_SPEED_5_0GT) && 5355 (rdev->pdev->bus->max_bus_speed != PCIE_SPEED_8_0GT)) 5356 return; 5357 5358 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL); 5359 if (speed_cntl & LC_CURRENT_DATA_RATE) { 5360 DRM_INFO("PCIE gen 2 link speeds already enabled\n"); 5361 return; 5362 } 5363 5364 DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n"); 5365 5366 if ((speed_cntl & LC_OTHER_SIDE_EVER_SENT_GEN2) || 5367 (speed_cntl & LC_OTHER_SIDE_SUPPORTS_GEN2)) { 5368 5369 link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL); 5370 link_width_cntl &= ~LC_UPCONFIGURE_DIS; 5371 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl); 5372 5373 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL); 5374 speed_cntl &= ~LC_TARGET_LINK_SPEED_OVERRIDE_EN; 5375 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl); 5376 5377 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL); 5378 speed_cntl |= LC_CLR_FAILED_SPD_CHANGE_CNT; 5379 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl); 5380 5381 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL); 5382 speed_cntl &= ~LC_CLR_FAILED_SPD_CHANGE_CNT; 5383 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl); 5384 5385 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL); 5386 speed_cntl |= LC_GEN2_EN_STRAP; 5387 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl); 5388 5389 } else { 5390 link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL); 5391 /* XXX: only disable it if gen1 bridge vendor == 0x111d or 0x1106 */ 5392 if (1) 5393 link_width_cntl |= LC_UPCONFIGURE_DIS; 5394 else 5395 link_width_cntl &= ~LC_UPCONFIGURE_DIS; 5396 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl); 5397 } 5398} 5399 5400void evergreen_program_aspm(struct radeon_device *rdev) 5401{ 5402 u32 data, orig; 5403 u32 pcie_lc_cntl, pcie_lc_cntl_old; 5404 bool disable_l0s, disable_l1 = false, disable_plloff_in_l1 = false; 5405 /* fusion_platform = true 5406 * if the system is a fusion system 5407 * (APU or DGPU in a fusion system). 5408 * todo: check if the system is a fusion platform. 5409 */ 5410 bool fusion_platform = false; 5411 5412 if (radeon_aspm == 0) 5413 return; 5414 5415 if (!(rdev->flags & RADEON_IS_PCIE)) 5416 return; 5417 5418 switch (rdev->family) { 5419 case CHIP_CYPRESS: 5420 case CHIP_HEMLOCK: 5421 case CHIP_JUNIPER: 5422 case CHIP_REDWOOD: 5423 case CHIP_CEDAR: 5424 case CHIP_SUMO: 5425 case CHIP_SUMO2: 5426 case CHIP_PALM: 5427 case CHIP_ARUBA: 5428 disable_l0s = true; 5429 break; 5430 default: 5431 disable_l0s = false; 5432 break; 5433 } 5434 5435 if (rdev->flags & RADEON_IS_IGP) 5436 fusion_platform = true; /* XXX also dGPUs in a fusion system */ 5437 5438 data = orig = RREG32_PIF_PHY0(PB0_PIF_PAIRING); 5439 if (fusion_platform) 5440 data &= ~MULTI_PIF; 5441 else 5442 data |= MULTI_PIF; 5443 if (data != orig) 5444 WREG32_PIF_PHY0(PB0_PIF_PAIRING, data); 5445 5446 data = orig = RREG32_PIF_PHY1(PB1_PIF_PAIRING); 5447 if (fusion_platform) 5448 data &= ~MULTI_PIF; 5449 else 5450 data |= MULTI_PIF; 5451 if (data != orig) 5452 WREG32_PIF_PHY1(PB1_PIF_PAIRING, data); 5453 5454 pcie_lc_cntl = pcie_lc_cntl_old = RREG32_PCIE_PORT(PCIE_LC_CNTL); 5455 pcie_lc_cntl &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK); 5456 if (!disable_l0s) { 5457 if (rdev->family >= CHIP_BARTS) 5458 pcie_lc_cntl |= LC_L0S_INACTIVITY(7); 5459 else 5460 pcie_lc_cntl |= LC_L0S_INACTIVITY(3); 5461 } 5462 5463 if (!disable_l1) { 5464 if (rdev->family >= CHIP_BARTS) 5465 pcie_lc_cntl |= LC_L1_INACTIVITY(7); 5466 else 5467 pcie_lc_cntl |= LC_L1_INACTIVITY(8); 5468 5469 if (!disable_plloff_in_l1) { 5470 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0); 5471 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK); 5472 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7); 5473 if (data != orig) 5474 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data); 5475 5476 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1); 5477 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK); 5478 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7); 5479 if (data != orig) 5480 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data); 5481 5482 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0); 5483 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK); 5484 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7); 5485 if (data != orig) 5486 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data); 5487 5488 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1); 5489 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK); 5490 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7); 5491 if (data != orig) 5492 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data); 5493 5494 if (rdev->family >= CHIP_BARTS) { 5495 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0); 5496 data &= ~PLL_RAMP_UP_TIME_0_MASK; 5497 data |= PLL_RAMP_UP_TIME_0(4); 5498 if (data != orig) 5499 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data); 5500 5501 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1); 5502 data &= ~PLL_RAMP_UP_TIME_1_MASK; 5503 data |= PLL_RAMP_UP_TIME_1(4); 5504 if (data != orig) 5505 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data); 5506 5507 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0); 5508 data &= ~PLL_RAMP_UP_TIME_0_MASK; 5509 data |= PLL_RAMP_UP_TIME_0(4); 5510 if (data != orig) 5511 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data); 5512 5513 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1); 5514 data &= ~PLL_RAMP_UP_TIME_1_MASK; 5515 data |= PLL_RAMP_UP_TIME_1(4); 5516 if (data != orig) 5517 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data); 5518 } 5519 5520 data = orig = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL); 5521 data &= ~LC_DYN_LANES_PWR_STATE_MASK; 5522 data |= LC_DYN_LANES_PWR_STATE(3); 5523 if (data != orig) 5524 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data); 5525 5526 if (rdev->family >= CHIP_BARTS) { 5527 data = orig = RREG32_PIF_PHY0(PB0_PIF_CNTL); 5528 data &= ~LS2_EXIT_TIME_MASK; 5529 data |= LS2_EXIT_TIME(1); 5530 if (data != orig) 5531 WREG32_PIF_PHY0(PB0_PIF_CNTL, data); 5532 5533 data = orig = RREG32_PIF_PHY1(PB1_PIF_CNTL); 5534 data &= ~LS2_EXIT_TIME_MASK; 5535 data |= LS2_EXIT_TIME(1); 5536 if (data != orig) 5537 WREG32_PIF_PHY1(PB1_PIF_CNTL, data); 5538 } 5539 } 5540 } 5541 5542 /* evergreen parts only */ 5543 if (rdev->family < CHIP_BARTS) 5544 pcie_lc_cntl |= LC_PMI_TO_L1_DIS; 5545 5546 if (pcie_lc_cntl != pcie_lc_cntl_old) 5547 WREG32_PCIE_PORT(PCIE_LC_CNTL, pcie_lc_cntl); 5548} 5549