Lines Matching +full:tahiti_ce +full:. +full:bin
2 * Copyright 2011 Advanced Micro Devices, Inc.
12 * all copies or substantial portions of the Software.
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
20 * OTHER DEALINGS IN THE SOFTWARE.
25 #include <sys/cdefs.h>
26 __FBSDID("$FreeBSD: releng/11.0/sys/dev/drm2/radeon/si.c 280183 2015-03-17 18:50:33Z dumbbell $");
28 #include <dev/drm2/drmP.h>
29 #include "radeon.h"
30 #include "radeon_asic.h"
31 #include <dev/drm2/radeon/radeon_drm.h>
32 #include "sid.h"
33 #include "atom.h"
34 #include "si_blit_shaders.h"
43 MODULE_FIRMWARE("radeon/TAHITI_pfp.bin");
44 MODULE_FIRMWARE("radeon/TAHITI_me.bin");
45 MODULE_FIRMWARE("radeon/TAHITI_ce.bin");
46 MODULE_FIRMWARE("radeon/TAHITI_mc.bin");
47 MODULE_FIRMWARE("radeon/TAHITI_rlc.bin");
48 MODULE_FIRMWARE("radeon/PITCAIRN_pfp.bin");
49 MODULE_FIRMWARE("radeon/PITCAIRN_me.bin");
50 MODULE_FIRMWARE("radeon/PITCAIRN_ce.bin");
51 MODULE_FIRMWARE("radeon/PITCAIRN_mc.bin");
52 MODULE_FIRMWARE("radeon/PITCAIRN_rlc.bin");
53 MODULE_FIRMWARE("radeon/VERDE_pfp.bin");
54 MODULE_FIRMWARE("radeon/VERDE_me.bin");
55 MODULE_FIRMWARE("radeon/VERDE_ce.bin");
56 MODULE_FIRMWARE("radeon/VERDE_mc.bin");
57 MODULE_FIRMWARE("radeon/VERDE_rlc.bin");
60 #ifdef FREEBSD_WIP /* FreeBSD: to please GCC 4.2. */
428 * Drop the pfp, me, rlc, mc and ce firmware image references.
429 * Called at driver shutdown.
469 * There are 3 line buffers, each one shared by 2 display controllers.
471 * the display controllers. The paritioning is done via one of four
477 * of crtcs. Ideally for multiple large displays we'd assign them to
478 * non-linked crtcs for maximum line buffer allocation.
480 if (radeon_crtc->base.enabled && mode) {
491 if (radeon_crtc->base.enabled && mode) {
555 a.full = dfixed_const(1000);
556 yclk.full = dfixed_const(wm->yclk);
557 yclk.full = dfixed_div(yclk, a);
558 dram_channels.full = dfixed_const(wm->dram_channels * 4);
559 a.full = dfixed_const(10);
560 dram_efficiency.full = dfixed_const(7);
561 dram_efficiency.full = dfixed_div(dram_efficiency, a);
562 bandwidth.full = dfixed_mul(dram_channels, yclk);
563 bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
570 /* Calculate DRAM Bandwidth and the part allocated to display. */
575 a.full = dfixed_const(1000);
576 yclk.full = dfixed_const(wm->yclk);
577 yclk.full = dfixed_div(yclk, a);
578 dram_channels.full = dfixed_const(wm->dram_channels * 4);
579 a.full = dfixed_const(10);
580 disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
581 disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
582 bandwidth.full = dfixed_mul(dram_channels, yclk);
583 bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
595 a.full = dfixed_const(1000);
596 sclk.full = dfixed_const(wm->sclk);
597 sclk.full = dfixed_div(sclk, a);
598 a.full = dfixed_const(10);
599 return_efficiency.full = dfixed_const(8);
600 return_efficiency.full = dfixed_div(return_efficiency, a);
601 a.full = dfixed_const(32);
602 bandwidth.full = dfixed_mul(a, sclk);
603 bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
621 a.full = dfixed_const(1000);
622 disp_clk.full = dfixed_const(wm->disp_clk);
623 disp_clk.full = dfixed_div(disp_clk, a);
624 a.full = dfixed_const(dce6_get_dmif_bytes_per_request(wm) / 2);
625 b1.full = dfixed_mul(a, disp_clk);
627 a.full = dfixed_const(1000);
628 sclk.full = dfixed_const(wm->sclk);
629 sclk.full = dfixed_div(sclk, a);
630 a.full = dfixed_const(dce6_get_dmif_bytes_per_request(wm));
631 b2.full = dfixed_mul(a, sclk);
633 a.full = dfixed_const(10);
634 disp_clk_request_efficiency.full = dfixed_const(8);
635 disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
639 a.full = dfixed_const(min_bandwidth);
640 bandwidth.full = dfixed_mul(a, disp_clk_request_efficiency);
647 /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
659 * timing, etc.
667 a.full = dfixed_const(1000);
668 line_time.full = dfixed_const(wm->active_time + wm->blank_time);
669 line_time.full = dfixed_div(line_time, a);
670 bpp.full = dfixed_const(wm->bytes_per_pixel);
671 src_width.full = dfixed_const(wm->src_width);
672 bandwidth.full = dfixed_mul(src_width, bpp);
673 bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
674 bandwidth.full = dfixed_div(bandwidth, line_time);
682 u32 mc_latency = 2000; /* 2000 ns. */
697 a.full = dfixed_const(2);
698 b.full = dfixed_const(1);
699 if ((wm->vsc.full > a.full) ||
700 ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
702 ((wm->vsc.full >= a.full) && wm->interlaced))
707 a.full = dfixed_const(available_bandwidth);
708 b.full = dfixed_const(wm->num_heads);
709 a.full = dfixed_div(a, b);
711 b.full = dfixed_const(mc_latency + 512);
712 c.full = dfixed_const(wm->disp_clk);
713 b.full = dfixed_div(b, c);
715 c.full = dfixed_const(dmif_size);
716 b.full = dfixed_div(c, b);
720 b.full = dfixed_const(1000);
721 c.full = dfixed_const(wm->disp_clk);
722 b.full = dfixed_div(c, b);
723 c.full = dfixed_const(wm->bytes_per_pixel);
724 b.full = dfixed_mul(b, c);
728 a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
729 b.full = dfixed_const(1000);
730 c.full = dfixed_const(lb_fill_bw);
731 b.full = dfixed_div(c, b);
732 a.full = dfixed_div(a, b);
768 a.full = dfixed_const(1);
769 if (wm->vsc.full > a.full)
790 struct drm_display_mode *mode = &radeon_crtc->base.mode;
801 if (radeon_crtc->base.enabled && num_heads && mode) {
807 wm.yclk = rdev->pm.current_mclk * 10;
808 wm.sclk = rdev->pm.current_sclk * 10;
809 wm.disp_clk = mode->clock;
810 wm.src_width = mode->crtc_hdisplay;
811 wm.active_time = mode->crtc_hdisplay * pixel_period;
812 wm.blank_time = line_time - wm.active_time;
813 wm.interlaced = false;
815 wm.interlaced = true;
816 wm.vsc = radeon_crtc->vsc;
817 wm.vtaps = 1;
819 wm.vtaps = 2;
820 wm.bytes_per_pixel = 4; /* XXX: get this from fb config */
821 wm.lb_size = lb_size;
823 wm.dram_channels = evergreen_get_number_of_dram_channels(rdev);
825 wm.dram_channels = si_get_number_of_dram_channels(rdev);
826 wm.num_heads = num_heads;
831 /* wm.yclk = low clk; wm.sclk = low clk */
835 /* should really do this at mode validation time... */
845 a.full = dfixed_const(1000);
846 b.full = dfixed_const(mode->clock);
847 b.full = dfixed_div(b, a);
848 c.full = dfixed_const(latency_watermark_a);
849 c.full = dfixed_mul(c, b);
850 c.full = dfixed_mul(c, radeon_crtc->hsc);
851 c.full = dfixed_div(c, a);
852 a.full = dfixed_const(16);
853 c.full = dfixed_div(c, a);
857 a.full = dfixed_const(1000);
858 b.full = dfixed_const(mode->clock);
859 b.full = dfixed_div(b, a);
860 c.full = dfixed_const(latency_watermark_b);
861 c.full = dfixed_mul(c, b);
862 c.full = dfixed_mul(c, radeon_crtc->hsc);
863 c.full = dfixed_div(c, a);
864 a.full = dfixed_const(16);
865 c.full = dfixed_div(c, a);
906 if (rdev->mode_info.crtcs[i]->base.enabled)
910 mode0 = &rdev->mode_info.crtcs[i]->base.mode;
911 mode1 = &rdev->mode_info.crtcs[i+1]->base.mode;
912 lb_size = dce6_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode0, mode1);
913 dce6_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
914 lb_size = dce6_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i+1], mode1, mode0);
915 dce6_program_watermarks(rdev, rdev->mode_info.crtcs[i+1], lb_size, num_heads);
927 switch (rdev->config.si.mem_row_size_in_kb) {
1034 case 9: /* Displayable maps. */
1044 case 10: /* Display 8bpp. */
1054 case 11: /* Display 16bpp. */
1064 case 12: /* Display 32bpp. */
1074 case 13: /* Thin. */
1084 case 14: /* Thin 8 bpp. */
1094 case 15: /* Thin 16 bpp. */
1104 case 16: /* Thin 32 bpp. */
1114 case 17: /* Thin 64 bpp. */
1124 case 21: /* 8 bpp PRT. */
1273 case 9: /* Displayable maps. */
1283 case 10: /* Display 8bpp. */
1293 case 11: /* Display 16bpp. */
1303 case 12: /* Display 32bpp. */
1313 case 13: /* Thin. */
1323 case 14: /* Thin 8 bpp. */
1333 case 15: /* Thin 16 bpp. */
1343 case 16: /* Thin 32 bpp. */
1353 case 17: /* Thin 64 bpp. */
1363 case 21: /* 8 bpp PRT. */
1574 rdev->config.si.max_shader_engines = 2;
1575 rdev->config.si.max_tile_pipes = 12;
1576 rdev->config.si.max_cu_per_sh = 8;
1577 rdev->config.si.max_sh_per_se = 2;
1578 rdev->config.si.max_backends_per_se = 4;
1579 rdev->config.si.max_texture_channel_caches = 12;
1580 rdev->config.si.max_gprs = 256;
1581 rdev->config.si.max_gs_threads = 32;
1582 rdev->config.si.max_hw_contexts = 8;
1584 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
1585 rdev->config.si.sc_prim_fifo_size_backend = 0x100;
1586 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
1587 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
1591 rdev->config.si.max_shader_engines = 2;
1592 rdev->config.si.max_tile_pipes = 8;
1593 rdev->config.si.max_cu_per_sh = 5;
1594 rdev->config.si.max_sh_per_se = 2;
1595 rdev->config.si.max_backends_per_se = 4;
1596 rdev->config.si.max_texture_channel_caches = 8;
1597 rdev->config.si.max_gprs = 256;
1598 rdev->config.si.max_gs_threads = 32;
1599 rdev->config.si.max_hw_contexts = 8;
1601 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
1602 rdev->config.si.sc_prim_fifo_size_backend = 0x100;
1603 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
1604 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
1609 rdev->config.si.max_shader_engines = 1;
1610 rdev->config.si.max_tile_pipes = 4;
1611 rdev->config.si.max_cu_per_sh = 2;
1612 rdev->config.si.max_sh_per_se = 2;
1613 rdev->config.si.max_backends_per_se = 4;
1614 rdev->config.si.max_texture_channel_caches = 4;
1615 rdev->config.si.max_gprs = 256;
1616 rdev->config.si.max_gs_threads = 32;
1617 rdev->config.si.max_hw_contexts = 8;
1619 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
1620 rdev->config.si.sc_prim_fifo_size_backend = 0x40;
1621 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
1622 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
1645 rdev->config.si.num_tile_pipes = rdev->config.si.max_tile_pipes;
1646 rdev->config.si.mem_max_burst_length_bytes = 256;
1648 rdev->config.si.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024;
1649 if (rdev->config.si.mem_row_size_in_kb > 4)
1650 rdev->config.si.mem_row_size_in_kb = 4;
1652 rdev->config.si.shader_engine_tile_size = 32;
1653 rdev->config.si.num_gpus = 1;
1654 rdev->config.si.multi_gpu_tile_size = 64;
1658 switch (rdev->config.si.mem_row_size_in_kb) {
1671 /* setup tiling info dword. gb_addr_config is not adequate since it does
1672 * not have bank info, so create a custom tiling dword.
1678 rdev->config.si.tile_config = 0;
1679 switch (rdev->config.si.num_tile_pipes) {
1681 rdev->config.si.tile_config |= (0 << 0);
1684 rdev->config.si.tile_config |= (1 << 0);
1687 rdev->config.si.tile_config |= (2 << 0);
1692 rdev->config.si.tile_config |= (3 << 0);
1697 rdev->config.si.tile_config |= 0 << 4;
1700 rdev->config.si.tile_config |= 1 << 4;
1704 rdev->config.si.tile_config |= 2 << 4;
1707 rdev->config.si.tile_config |=
1709 rdev->config.si.tile_config |=
1721 si_setup_rb(rdev, rdev->config.si.max_shader_engines,
1722 rdev->config.si.max_sh_per_se,
1723 rdev->config.si.max_backends_per_se);
1725 si_setup_spi(rdev, rdev->config.si.max_shader_engines,
1726 rdev->config.si.max_sh_per_se,
1727 rdev->config.si.max_cu_per_sh);
1740 WREG32(PA_SC_FIFO_SIZE, (SC_FRONTEND_PRIM_FIFO_SIZE(rdev->config.si.sc_prim_fifo_size_frontend) |
1741 SC_BACKEND_PRIM_FIFO_SIZE(rdev->config.si.sc_prim_fifo_size_backend) |
1742 SC_HIZ_TILE_FIFO_SIZE(rdev->config.si.sc_hiz_tile_fifo_size) |
1743 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.si.sc_earlyz_tile_fifo_size)));
1782 * GPU scratch registers helpers function.
1788 rdev->scratch.num_reg = 7;
1789 rdev->scratch.reg_base = SCRATCH_REG0;
1790 for (i = 0; i < rdev->scratch.num_reg; i++) {
1791 rdev->scratch.free[i] = true;
1792 rdev->scratch.reg[i] = rdev->scratch.reg_base + (i * 4);
1800 u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
1845 } else if (rdev->wb.enabled) {
1884 * CP.
1891 radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
1894 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
1895 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
1896 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
1946 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
1953 radeon_ring_write(ring, rdev->config.si.max_hw_contexts - 1);
1969 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
2051 WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
2069 WREG32(CP_RB0_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC);
2070 WREG32(CP_RB0_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
2072 if (rdev->wb.enabled)
2102 WREG32(CP_RB1_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET) & 0xFFFFFFFC);
2103 WREG32(CP_RB1_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET) & 0xFF);
2128 WREG32(CP_RB2_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET) & 0xFFFFFFFC);
2129 WREG32(CP_RB2_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET) & 0xFF);
2140 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true;
2141 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = true;
2142 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = true;
2145 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
2146 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
2147 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
2152 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
2156 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
2337 rdev->mc.vram_start >> 12);
2339 rdev->mc.vram_end >> 12);
2341 rdev->vram_scratch.gpu_addr >> 12);
2342 tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
2343 tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
2346 WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
2411 rdev->mc.gtt_base_align = 0;
2421 rdev->mc.vram_is_ddr = true;
2461 rdev->mc.vram_width = numchan * chansize;
2463 rdev->mc.aper_base = drm_get_resource_start(rdev->ddev, 0);
2464 rdev->mc.aper_size = drm_get_resource_len(rdev->ddev, 0);
2466 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024 * 1024;
2467 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024 * 1024;
2468 rdev->mc.visible_vram_size = rdev->mc.aper_size;
2491 if (rdev->gart.robj == NULL) {
2492 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
2516 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
2517 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
2518 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
2520 (u32)(rdev->dummy_page.addr >> 12));
2532 WREG32(VM_CONTEXT1_PAGE_TABLE_END_ADDR, rdev->vm_manager.max_pfn);
2535 * on the fly in the vm part of radeon_gart.c
2540 rdev->gart.table_addr >> 12);
2543 rdev->gart.table_addr >> 12);
2548 (u32)(rdev->dummy_page.addr >> 12));
2565 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
2566 (unsigned)(rdev->mc.gtt_size >> 20),
2567 (unsigned long long)rdev->gart.table_addr);
2568 rdev->gart.ready = true;
2910 pkt.idx = idx;
2911 pkt.type = CP_PACKET_GET_TYPE(ib->ptr[idx]);
2912 pkt.count = CP_PACKET_GET_COUNT(ib->ptr[idx]);
2913 pkt.one_reg_wr = 0;
2914 switch (pkt.type) {
2923 pkt.opcode = CP_PACKET3_GET_OPCODE(ib->ptr[idx]);
2941 idx += pkt.count + 2;
2944 dev_err(rdev->dev, "Unknown packet type %d !\n", pkt.type);
2961 rdev->vm_manager.nvm = 16;
2963 rdev->vm_manager.vram_base_offset = 0;
2982 * Update the page tables using the CP (cayman-si).
2988 struct radeon_ring *ring = &rdev->ring[rdev->asic->vm.pt_ring_index];
2993 if (rdev->asic->vm.pt_ring_index == RADEON_RING_TYPE_GFX_INDEX) {
3151 if (rdev->rlc.save_restore_obj) {
3152 r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false);
3155 radeon_bo_unpin(rdev->rlc.save_restore_obj);
3156 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
3158 radeon_bo_unref(&rdev->rlc.save_restore_obj);
3159 rdev->rlc.save_restore_obj = NULL;
3163 if (rdev->rlc.clear_state_obj) {
3164 r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false);
3167 radeon_bo_unpin(rdev->rlc.clear_state_obj);
3168 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
3170 radeon_bo_unref(&rdev->rlc.clear_state_obj);
3171 rdev->rlc.clear_state_obj = NULL;
3180 if (rdev->rlc.save_restore_obj == NULL) {
3183 &rdev->rlc.save_restore_obj);
3190 r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false);
3195 r = radeon_bo_pin(rdev->rlc.save_restore_obj, RADEON_GEM_DOMAIN_VRAM,
3196 &rdev->rlc.save_restore_gpu_addr);
3197 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
3205 if (rdev->rlc.clear_state_obj == NULL) {
3208 &rdev->rlc.clear_state_obj);
3215 r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false);
3220 r = radeon_bo_pin(rdev->rlc.clear_state_obj, RADEON_GEM_DOMAIN_VRAM,
3221 &rdev->rlc.clear_state_gpu_addr);
3222 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
3258 WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
3259 WREG32(RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
3285 rdev->ih.enabled = true;
3300 rdev->ih.enabled = false;
3301 rdev->ih.rptr = 0;
3378 WREG32(INTERRUPT_CNTL2, rdev->ih.gpu_addr >> 8);
3384 /* IH_REQ_NONSNOOP_EN=1 if ring is in non-cacheable memory, e.g., vram */
3388 WREG32(IH_RB_BASE, rdev->ih.gpu_addr >> 8);
3389 rb_bufsz = drm_order(rdev->ih.ring_size / 4);
3395 if (rdev->wb.enabled)
3399 WREG32(IH_RB_WPTR_ADDR_LO, (rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFFFFFFFC);
3400 WREG32(IH_RB_WPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFF);
3436 if (!rdev->irq.installed) {
3441 if (!rdev->ih.enabled) {
3459 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
3463 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
3467 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
3471 if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
3476 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
3480 if (rdev->irq.crtc_vblank_int[0] ||
3481 atomic_read(&rdev->irq.pflip[0])) {
3485 if (rdev->irq.crtc_vblank_int[1] ||
3486 atomic_read(&rdev->irq.pflip[1])) {
3490 if (rdev->irq.crtc_vblank_int[2] ||
3491 atomic_read(&rdev->irq.pflip[2])) {
3495 if (rdev->irq.crtc_vblank_int[3] ||
3496 atomic_read(&rdev->irq.pflip[3])) {
3500 if (rdev->irq.crtc_vblank_int[4] ||
3501 atomic_read(&rdev->irq.pflip[4])) {
3505 if (rdev->irq.crtc_vblank_int[5] ||
3506 atomic_read(&rdev->irq.pflip[5])) {
3510 if (rdev->irq.hpd[0]) {
3514 if (rdev->irq.hpd[1]) {
3518 if (rdev->irq.hpd[2]) {
3522 if (rdev->irq.hpd[3]) {
3526 if (rdev->irq.hpd[4]) {
3530 if (rdev->irq.hpd[5]) {
3580 rdev->irq.stat_regs.evergreen.disp_int = RREG32(DISP_INTERRUPT_STATUS);
3581 rdev->irq.stat_regs.evergreen.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
3582 rdev->irq.stat_regs.evergreen.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2);
3583 rdev->irq.stat_regs.evergreen.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3);
3584 rdev->irq.stat_regs.evergreen.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4);
3585 rdev->irq.stat_regs.evergreen.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5);
3586 rdev->irq.stat_regs.evergreen.d1grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET);
3587 rdev->irq.stat_regs.evergreen.d2grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET);
3589 rdev->irq.stat_regs.evergreen.d3grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET);
3590 rdev->irq.stat_regs.evergreen.d4grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET);
3593 rdev->irq.stat_regs.evergreen.d5grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET);
3594 rdev->irq.stat_regs.evergreen.d6grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET);
3597 if (rdev->irq.stat_regs.evergreen.d1grph_int & GRPH_PFLIP_INT_OCCURRED)
3599 if (rdev->irq.stat_regs.evergreen.d2grph_int & GRPH_PFLIP_INT_OCCURRED)
3601 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT)
3603 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT)
3605 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT)
3607 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT)
3611 if (rdev->irq.stat_regs.evergreen.d3grph_int & GRPH_PFLIP_INT_OCCURRED)
3613 if (rdev->irq.stat_regs.evergreen.d4grph_int & GRPH_PFLIP_INT_OCCURRED)
3615 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)
3617 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)
3619 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)
3621 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)
3626 if (rdev->irq.stat_regs.evergreen.d5grph_int & GRPH_PFLIP_INT_OCCURRED)
3628 if (rdev->irq.stat_regs.evergreen.d6grph_int & GRPH_PFLIP_INT_OCCURRED)
3630 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)
3632 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)
3634 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)
3636 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)
3640 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
3645 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
3650 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
3655 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
3660 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
3665 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
3697 if (rdev->wb.enabled)
3698 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
3704 * from the last not overwritten vector (wptr + 16). Hopefully
3705 * this should allow us to catchup.
3708 wptr, rdev->ih.rptr, (wptr + 16) + rdev->ih.ptr_mask);
3709 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
3714 return (wptr & rdev->ih.ptr_mask);
3735 if (!rdev->ih.enabled || rdev->shutdown)
3742 if (atomic_xchg(&rdev->ih.lock, 1))
3745 rptr = rdev->ih.rptr;
3748 /* Order reading of wptr vs. reading of IH ring data */
3757 src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
3758 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
3759 ring_id = le32_to_cpu(rdev->ih.ring[ring_index + 2]) & 0xff;
3765 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT) {
3766 if (rdev->irq.crtc_vblank_int[0]) {
3768 rdev->pm.vblank_sync = true;
3769 DRM_WAKEUP(&rdev->irq.vblank_queue);
3771 if (atomic_read(&rdev->irq.pflip[0]))
3773 rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
3778 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT) {
3779 rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VLINE_INTERRUPT;
3791 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT) {
3792 if (rdev->irq.crtc_vblank_int[1]) {
3794 rdev->pm.vblank_sync = true;
3795 DRM_WAKEUP(&rdev->irq.vblank_queue);
3797 if (atomic_read(&rdev->irq.pflip[1]))
3799 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT;
3804 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT) {
3805 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT;
3817 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) {
3818 if (rdev->irq.crtc_vblank_int[2]) {
3820 rdev->pm.vblank_sync = true;
3821 DRM_WAKEUP(&rdev->irq.vblank_queue);
3823 if (atomic_read(&rdev->irq.pflip[2]))
3825 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT;
3830 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) {
3831 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT;
3843 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) {
3844 if (rdev->irq.crtc_vblank_int[3]) {
3846 rdev->pm.vblank_sync = true;
3847 DRM_WAKEUP(&rdev->irq.vblank_queue);
3849 if (atomic_read(&rdev->irq.pflip[3]))
3851 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT;
3856 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) {
3857 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT;
3869 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) {
3870 if (rdev->irq.crtc_vblank_int[4]) {
3872 rdev->pm.vblank_sync = true;
3873 DRM_WAKEUP(&rdev->irq.vblank_queue);
3875 if (atomic_read(&rdev->irq.pflip[4]))
3877 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT;
3882 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) {
3883 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT;
3895 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) {
3896 if (rdev->irq.crtc_vblank_int[5]) {
3898 rdev->pm.vblank_sync = true;
3899 DRM_WAKEUP(&rdev->irq.vblank_queue);
3901 if (atomic_read(&rdev->irq.pflip[5]))
3903 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT;
3908 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) {
3909 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT;
3921 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
3922 rdev->irq.stat_regs.evergreen.disp_int &= ~DC_HPD1_INTERRUPT;
3928 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
3929 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~DC_HPD2_INTERRUPT;
3935 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
3936 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~DC_HPD3_INTERRUPT;
3942 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
3943 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~DC_HPD4_INTERRUPT;
3949 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
3950 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~DC_HPD5_INTERRUPT;
3956 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
3957 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~DC_HPD6_INTERRUPT;
4018 rptr &= rdev->ih.ptr_mask;
4022 rdev->ih.rptr = rptr;
4023 WREG32(IH_RB_RPTR, rdev->ih.rptr);
4024 atomic_set(&rdev->ih.lock, 0);
4043 * Copy GPU paging using the DMA engine (SI).
4045 * registered as the asic copy callback.
4053 int ring_index = rdev->asic->copy.dma_ring_index;
4061 DRM_ERROR("radeon: moving bo (%d).\n", r);
4069 DRM_ERROR("radeon: moving bo (%d).\n", r);
4163 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
4169 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
4175 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
4181 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
4187 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
4194 DRM_ERROR("radeon: IH init failed (%d).\n", r);
4250 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
4256 dev_err(rdev->dev, "vm manager initialization failed (%d).\n", r);
4269 * shape.
4272 atom_asic_init(rdev->mode_info.atom_context);
4299 * do nothing more than calling asic specific function. This
4301 * like vram_info.
4328 DRM_INFO("GPU not posted. posting now...\n");
4329 atom_asic_init(rdev->mode_info.atom_context);
4376 rdev->ih.ring_obj = NULL;
4399 /* Don't start up if the MC ucode is missing.
4401 * is loaded are not suffient for advanced operations.
4404 DRM_ERROR("radeon: MC ucode required for NI+.\n");
4440 * Fetches a GPU clock counter snapshot (SI).
4441 * Returns the 64 bit clock counter snapshot.