1/*
2 * Copyright 2012 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 * Authors: Alex Deucher
23 */
24#include <linux/firmware.h>
25#include <linux/slab.h>
26#include <linux/module.h>
27#include <linux/pci.h>
28
29#include <drm/amdgpu_drm.h>
30
31#include "amdgpu.h"
32#include "amdgpu_atombios.h"
33#include "amdgpu_ih.h"
34#include "amdgpu_uvd.h"
35#include "amdgpu_vce.h"
36#include "cikd.h"
37#include "atom.h"
38#include "amd_pcie.h"
39
40#include "cik.h"
41#include "gmc_v7_0.h"
42#include "cik_ih.h"
43#include "dce_v8_0.h"
44#include "gfx_v7_0.h"
45#include "cik_sdma.h"
46#include "uvd_v4_2.h"
47#include "vce_v2_0.h"
48#include "cik_dpm.h"
49
50#include "uvd/uvd_4_2_d.h"
51
52#include "smu/smu_7_0_1_d.h"
53#include "smu/smu_7_0_1_sh_mask.h"
54
55#include "dce/dce_8_0_d.h"
56#include "dce/dce_8_0_sh_mask.h"
57
58#include "bif/bif_4_1_d.h"
59#include "bif/bif_4_1_sh_mask.h"
60
61#include "gca/gfx_7_2_d.h"
62#include "gca/gfx_7_2_enum.h"
63#include "gca/gfx_7_2_sh_mask.h"
64
65#include "gmc/gmc_7_1_d.h"
66#include "gmc/gmc_7_1_sh_mask.h"
67
68#include "oss/oss_2_0_d.h"
69#include "oss/oss_2_0_sh_mask.h"
70
71#include "amdgpu_dm.h"
72#include "amdgpu_amdkfd.h"
73#include "amdgpu_vkms.h"
74
75static const struct amdgpu_video_codec_info cik_video_codecs_encode_array[] =
76{
77	{
78		.codec_type = AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_MPEG4_AVC,
79		.max_width = 2048,
80		.max_height = 1152,
81		.max_pixels_per_frame = 2048 * 1152,
82		.max_level = 0,
83	},
84};
85
86static const struct amdgpu_video_codecs cik_video_codecs_encode =
87{
88	.codec_count = ARRAY_SIZE(cik_video_codecs_encode_array),
89	.codec_array = cik_video_codecs_encode_array,
90};
91
92static const struct amdgpu_video_codec_info cik_video_codecs_decode_array[] =
93{
94	{
95		.codec_type = AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_MPEG2,
96		.max_width = 2048,
97		.max_height = 1152,
98		.max_pixels_per_frame = 2048 * 1152,
99		.max_level = 3,
100	},
101	{
102		.codec_type = AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_MPEG4,
103		.max_width = 2048,
104		.max_height = 1152,
105		.max_pixels_per_frame = 2048 * 1152,
106		.max_level = 5,
107	},
108	{
109		.codec_type = AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_MPEG4_AVC,
110		.max_width = 2048,
111		.max_height = 1152,
112		.max_pixels_per_frame = 2048 * 1152,
113		.max_level = 41,
114	},
115	{
116		.codec_type = AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_VC1,
117		.max_width = 2048,
118		.max_height = 1152,
119		.max_pixels_per_frame = 2048 * 1152,
120		.max_level = 4,
121	},
122};
123
124static const struct amdgpu_video_codecs cik_video_codecs_decode =
125{
126	.codec_count = ARRAY_SIZE(cik_video_codecs_decode_array),
127	.codec_array = cik_video_codecs_decode_array,
128};
129
130static int cik_query_video_codecs(struct amdgpu_device *adev, bool encode,
131				  const struct amdgpu_video_codecs **codecs)
132{
133	switch (adev->asic_type) {
134	case CHIP_BONAIRE:
135	case CHIP_HAWAII:
136	case CHIP_KAVERI:
137	case CHIP_KABINI:
138	case CHIP_MULLINS:
139		if (encode)
140			*codecs = &cik_video_codecs_encode;
141		else
142			*codecs = &cik_video_codecs_decode;
143		return 0;
144	default:
145		return -EINVAL;
146	}
147}
148
149/*
150 * Indirect registers accessor
151 */
152static u32 cik_pcie_rreg(struct amdgpu_device *adev, u32 reg)
153{
154	unsigned long flags;
155	u32 r;
156
157	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
158	WREG32(mmPCIE_INDEX, reg);
159	(void)RREG32(mmPCIE_INDEX);
160	r = RREG32(mmPCIE_DATA);
161	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
162	return r;
163}
164
165static void cik_pcie_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
166{
167	unsigned long flags;
168
169	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
170	WREG32(mmPCIE_INDEX, reg);
171	(void)RREG32(mmPCIE_INDEX);
172	WREG32(mmPCIE_DATA, v);
173	(void)RREG32(mmPCIE_DATA);
174	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
175}
176
177static u32 cik_smc_rreg(struct amdgpu_device *adev, u32 reg)
178{
179	unsigned long flags;
180	u32 r;
181
182	spin_lock_irqsave(&adev->smc_idx_lock, flags);
183	WREG32(mmSMC_IND_INDEX_0, (reg));
184	r = RREG32(mmSMC_IND_DATA_0);
185	spin_unlock_irqrestore(&adev->smc_idx_lock, flags);
186	return r;
187}
188
189static void cik_smc_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
190{
191	unsigned long flags;
192
193	spin_lock_irqsave(&adev->smc_idx_lock, flags);
194	WREG32(mmSMC_IND_INDEX_0, (reg));
195	WREG32(mmSMC_IND_DATA_0, (v));
196	spin_unlock_irqrestore(&adev->smc_idx_lock, flags);
197}
198
199static u32 cik_uvd_ctx_rreg(struct amdgpu_device *adev, u32 reg)
200{
201	unsigned long flags;
202	u32 r;
203
204	spin_lock_irqsave(&adev->uvd_ctx_idx_lock, flags);
205	WREG32(mmUVD_CTX_INDEX, ((reg) & 0x1ff));
206	r = RREG32(mmUVD_CTX_DATA);
207	spin_unlock_irqrestore(&adev->uvd_ctx_idx_lock, flags);
208	return r;
209}
210
211static void cik_uvd_ctx_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
212{
213	unsigned long flags;
214
215	spin_lock_irqsave(&adev->uvd_ctx_idx_lock, flags);
216	WREG32(mmUVD_CTX_INDEX, ((reg) & 0x1ff));
217	WREG32(mmUVD_CTX_DATA, (v));
218	spin_unlock_irqrestore(&adev->uvd_ctx_idx_lock, flags);
219}
220
221static u32 cik_didt_rreg(struct amdgpu_device *adev, u32 reg)
222{
223	unsigned long flags;
224	u32 r;
225
226	spin_lock_irqsave(&adev->didt_idx_lock, flags);
227	WREG32(mmDIDT_IND_INDEX, (reg));
228	r = RREG32(mmDIDT_IND_DATA);
229	spin_unlock_irqrestore(&adev->didt_idx_lock, flags);
230	return r;
231}
232
233static void cik_didt_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
234{
235	unsigned long flags;
236
237	spin_lock_irqsave(&adev->didt_idx_lock, flags);
238	WREG32(mmDIDT_IND_INDEX, (reg));
239	WREG32(mmDIDT_IND_DATA, (v));
240	spin_unlock_irqrestore(&adev->didt_idx_lock, flags);
241}
242
243static const u32 bonaire_golden_spm_registers[] =
244{
245	0xc200, 0xe0ffffff, 0xe0000000
246};
247
248static const u32 bonaire_golden_common_registers[] =
249{
250	0x31dc, 0xffffffff, 0x00000800,
251	0x31dd, 0xffffffff, 0x00000800,
252	0x31e6, 0xffffffff, 0x00007fbf,
253	0x31e7, 0xffffffff, 0x00007faf
254};
255
256static const u32 bonaire_golden_registers[] =
257{
258	0xcd5, 0x00000333, 0x00000333,
259	0xcd4, 0x000c0fc0, 0x00040200,
260	0x2684, 0x00010000, 0x00058208,
261	0xf000, 0xffff1fff, 0x00140000,
262	0xf080, 0xfdfc0fff, 0x00000100,
263	0xf08d, 0x40000000, 0x40000200,
264	0x260c, 0xffffffff, 0x00000000,
265	0x260d, 0xf00fffff, 0x00000400,
266	0x260e, 0x0002021c, 0x00020200,
267	0x31e, 0x00000080, 0x00000000,
268	0x16ec, 0x000000f0, 0x00000070,
269	0x16f0, 0xf0311fff, 0x80300000,
270	0x263e, 0x73773777, 0x12010001,
271	0xd43, 0x00810000, 0x408af000,
272	0x1c0c, 0x31000111, 0x00000011,
273	0xbd2, 0x73773777, 0x12010001,
274	0x883, 0x00007fb6, 0x0021a1b1,
275	0x884, 0x00007fb6, 0x002021b1,
276	0x860, 0x00007fb6, 0x00002191,
277	0x886, 0x00007fb6, 0x002121b1,
278	0x887, 0x00007fb6, 0x002021b1,
279	0x877, 0x00007fb6, 0x00002191,
280	0x878, 0x00007fb6, 0x00002191,
281	0xd8a, 0x0000003f, 0x0000000a,
282	0xd8b, 0x0000003f, 0x0000000a,
283	0xab9, 0x00073ffe, 0x000022a2,
284	0x903, 0x000007ff, 0x00000000,
285	0x2285, 0xf000003f, 0x00000007,
286	0x22fc, 0x00002001, 0x00000001,
287	0x22c9, 0xffffffff, 0x00ffffff,
288	0xc281, 0x0000ff0f, 0x00000000,
289	0xa293, 0x07ffffff, 0x06000000,
290	0x136, 0x00000fff, 0x00000100,
291	0xf9e, 0x00000001, 0x00000002,
292	0x2440, 0x03000000, 0x0362c688,
293	0x2300, 0x000000ff, 0x00000001,
294	0x390, 0x00001fff, 0x00001fff,
295	0x2418, 0x0000007f, 0x00000020,
296	0x2542, 0x00010000, 0x00010000,
297	0x2b05, 0x000003ff, 0x000000f3,
298	0x2b03, 0xffffffff, 0x00001032
299};
300
301static const u32 bonaire_mgcg_cgcg_init[] =
302{
303	0x3108, 0xffffffff, 0xfffffffc,
304	0xc200, 0xffffffff, 0xe0000000,
305	0xf0a8, 0xffffffff, 0x00000100,
306	0xf082, 0xffffffff, 0x00000100,
307	0xf0b0, 0xffffffff, 0xc0000100,
308	0xf0b2, 0xffffffff, 0xc0000100,
309	0xf0b1, 0xffffffff, 0xc0000100,
310	0x1579, 0xffffffff, 0x00600100,
311	0xf0a0, 0xffffffff, 0x00000100,
312	0xf085, 0xffffffff, 0x06000100,
313	0xf088, 0xffffffff, 0x00000100,
314	0xf086, 0xffffffff, 0x06000100,
315	0xf081, 0xffffffff, 0x00000100,
316	0xf0b8, 0xffffffff, 0x00000100,
317	0xf089, 0xffffffff, 0x00000100,
318	0xf080, 0xffffffff, 0x00000100,
319	0xf08c, 0xffffffff, 0x00000100,
320	0xf08d, 0xffffffff, 0x00000100,
321	0xf094, 0xffffffff, 0x00000100,
322	0xf095, 0xffffffff, 0x00000100,
323	0xf096, 0xffffffff, 0x00000100,
324	0xf097, 0xffffffff, 0x00000100,
325	0xf098, 0xffffffff, 0x00000100,
326	0xf09f, 0xffffffff, 0x00000100,
327	0xf09e, 0xffffffff, 0x00000100,
328	0xf084, 0xffffffff, 0x06000100,
329	0xf0a4, 0xffffffff, 0x00000100,
330	0xf09d, 0xffffffff, 0x00000100,
331	0xf0ad, 0xffffffff, 0x00000100,
332	0xf0ac, 0xffffffff, 0x00000100,
333	0xf09c, 0xffffffff, 0x00000100,
334	0xc200, 0xffffffff, 0xe0000000,
335	0xf008, 0xffffffff, 0x00010000,
336	0xf009, 0xffffffff, 0x00030002,
337	0xf00a, 0xffffffff, 0x00040007,
338	0xf00b, 0xffffffff, 0x00060005,
339	0xf00c, 0xffffffff, 0x00090008,
340	0xf00d, 0xffffffff, 0x00010000,
341	0xf00e, 0xffffffff, 0x00030002,
342	0xf00f, 0xffffffff, 0x00040007,
343	0xf010, 0xffffffff, 0x00060005,
344	0xf011, 0xffffffff, 0x00090008,
345	0xf012, 0xffffffff, 0x00010000,
346	0xf013, 0xffffffff, 0x00030002,
347	0xf014, 0xffffffff, 0x00040007,
348	0xf015, 0xffffffff, 0x00060005,
349	0xf016, 0xffffffff, 0x00090008,
350	0xf017, 0xffffffff, 0x00010000,
351	0xf018, 0xffffffff, 0x00030002,
352	0xf019, 0xffffffff, 0x00040007,
353	0xf01a, 0xffffffff, 0x00060005,
354	0xf01b, 0xffffffff, 0x00090008,
355	0xf01c, 0xffffffff, 0x00010000,
356	0xf01d, 0xffffffff, 0x00030002,
357	0xf01e, 0xffffffff, 0x00040007,
358	0xf01f, 0xffffffff, 0x00060005,
359	0xf020, 0xffffffff, 0x00090008,
360	0xf021, 0xffffffff, 0x00010000,
361	0xf022, 0xffffffff, 0x00030002,
362	0xf023, 0xffffffff, 0x00040007,
363	0xf024, 0xffffffff, 0x00060005,
364	0xf025, 0xffffffff, 0x00090008,
365	0xf026, 0xffffffff, 0x00010000,
366	0xf027, 0xffffffff, 0x00030002,
367	0xf028, 0xffffffff, 0x00040007,
368	0xf029, 0xffffffff, 0x00060005,
369	0xf02a, 0xffffffff, 0x00090008,
370	0xf000, 0xffffffff, 0x96e00200,
371	0x21c2, 0xffffffff, 0x00900100,
372	0x3109, 0xffffffff, 0x0020003f,
373	0xe, 0xffffffff, 0x0140001c,
374	0xf, 0x000f0000, 0x000f0000,
375	0x88, 0xffffffff, 0xc060000c,
376	0x89, 0xc0000fff, 0x00000100,
377	0x3e4, 0xffffffff, 0x00000100,
378	0x3e6, 0x00000101, 0x00000000,
379	0x82a, 0xffffffff, 0x00000104,
380	0x1579, 0xff000fff, 0x00000100,
381	0xc33, 0xc0000fff, 0x00000104,
382	0x3079, 0x00000001, 0x00000001,
383	0x3403, 0xff000ff0, 0x00000100,
384	0x3603, 0xff000ff0, 0x00000100
385};
386
387static const u32 spectre_golden_spm_registers[] =
388{
389	0xc200, 0xe0ffffff, 0xe0000000
390};
391
392static const u32 spectre_golden_common_registers[] =
393{
394	0x31dc, 0xffffffff, 0x00000800,
395	0x31dd, 0xffffffff, 0x00000800,
396	0x31e6, 0xffffffff, 0x00007fbf,
397	0x31e7, 0xffffffff, 0x00007faf
398};
399
400static const u32 spectre_golden_registers[] =
401{
402	0xf000, 0xffff1fff, 0x96940200,
403	0xf003, 0xffff0001, 0xff000000,
404	0xf080, 0xfffc0fff, 0x00000100,
405	0x1bb6, 0x00010101, 0x00010000,
406	0x260d, 0xf00fffff, 0x00000400,
407	0x260e, 0xfffffffc, 0x00020200,
408	0x16ec, 0x000000f0, 0x00000070,
409	0x16f0, 0xf0311fff, 0x80300000,
410	0x263e, 0x73773777, 0x12010001,
411	0x26df, 0x00ff0000, 0x00fc0000,
412	0xbd2, 0x73773777, 0x12010001,
413	0x2285, 0xf000003f, 0x00000007,
414	0x22c9, 0xffffffff, 0x00ffffff,
415	0xa0d4, 0x3f3f3fff, 0x00000082,
416	0xa0d5, 0x0000003f, 0x00000000,
417	0xf9e, 0x00000001, 0x00000002,
418	0x244f, 0xffff03df, 0x00000004,
419	0x31da, 0x00000008, 0x00000008,
420	0x2300, 0x000008ff, 0x00000800,
421	0x2542, 0x00010000, 0x00010000,
422	0x2b03, 0xffffffff, 0x54763210,
423	0x853e, 0x01ff01ff, 0x00000002,
424	0x8526, 0x007ff800, 0x00200000,
425	0x8057, 0xffffffff, 0x00000f40,
426	0xc24d, 0xffffffff, 0x00000001
427};
428
429static const u32 spectre_mgcg_cgcg_init[] =
430{
431	0x3108, 0xffffffff, 0xfffffffc,
432	0xc200, 0xffffffff, 0xe0000000,
433	0xf0a8, 0xffffffff, 0x00000100,
434	0xf082, 0xffffffff, 0x00000100,
435	0xf0b0, 0xffffffff, 0x00000100,
436	0xf0b2, 0xffffffff, 0x00000100,
437	0xf0b1, 0xffffffff, 0x00000100,
438	0x1579, 0xffffffff, 0x00600100,
439	0xf0a0, 0xffffffff, 0x00000100,
440	0xf085, 0xffffffff, 0x06000100,
441	0xf088, 0xffffffff, 0x00000100,
442	0xf086, 0xffffffff, 0x06000100,
443	0xf081, 0xffffffff, 0x00000100,
444	0xf0b8, 0xffffffff, 0x00000100,
445	0xf089, 0xffffffff, 0x00000100,
446	0xf080, 0xffffffff, 0x00000100,
447	0xf08c, 0xffffffff, 0x00000100,
448	0xf08d, 0xffffffff, 0x00000100,
449	0xf094, 0xffffffff, 0x00000100,
450	0xf095, 0xffffffff, 0x00000100,
451	0xf096, 0xffffffff, 0x00000100,
452	0xf097, 0xffffffff, 0x00000100,
453	0xf098, 0xffffffff, 0x00000100,
454	0xf09f, 0xffffffff, 0x00000100,
455	0xf09e, 0xffffffff, 0x00000100,
456	0xf084, 0xffffffff, 0x06000100,
457	0xf0a4, 0xffffffff, 0x00000100,
458	0xf09d, 0xffffffff, 0x00000100,
459	0xf0ad, 0xffffffff, 0x00000100,
460	0xf0ac, 0xffffffff, 0x00000100,
461	0xf09c, 0xffffffff, 0x00000100,
462	0xc200, 0xffffffff, 0xe0000000,
463	0xf008, 0xffffffff, 0x00010000,
464	0xf009, 0xffffffff, 0x00030002,
465	0xf00a, 0xffffffff, 0x00040007,
466	0xf00b, 0xffffffff, 0x00060005,
467	0xf00c, 0xffffffff, 0x00090008,
468	0xf00d, 0xffffffff, 0x00010000,
469	0xf00e, 0xffffffff, 0x00030002,
470	0xf00f, 0xffffffff, 0x00040007,
471	0xf010, 0xffffffff, 0x00060005,
472	0xf011, 0xffffffff, 0x00090008,
473	0xf012, 0xffffffff, 0x00010000,
474	0xf013, 0xffffffff, 0x00030002,
475	0xf014, 0xffffffff, 0x00040007,
476	0xf015, 0xffffffff, 0x00060005,
477	0xf016, 0xffffffff, 0x00090008,
478	0xf017, 0xffffffff, 0x00010000,
479	0xf018, 0xffffffff, 0x00030002,
480	0xf019, 0xffffffff, 0x00040007,
481	0xf01a, 0xffffffff, 0x00060005,
482	0xf01b, 0xffffffff, 0x00090008,
483	0xf01c, 0xffffffff, 0x00010000,
484	0xf01d, 0xffffffff, 0x00030002,
485	0xf01e, 0xffffffff, 0x00040007,
486	0xf01f, 0xffffffff, 0x00060005,
487	0xf020, 0xffffffff, 0x00090008,
488	0xf021, 0xffffffff, 0x00010000,
489	0xf022, 0xffffffff, 0x00030002,
490	0xf023, 0xffffffff, 0x00040007,
491	0xf024, 0xffffffff, 0x00060005,
492	0xf025, 0xffffffff, 0x00090008,
493	0xf026, 0xffffffff, 0x00010000,
494	0xf027, 0xffffffff, 0x00030002,
495	0xf028, 0xffffffff, 0x00040007,
496	0xf029, 0xffffffff, 0x00060005,
497	0xf02a, 0xffffffff, 0x00090008,
498	0xf02b, 0xffffffff, 0x00010000,
499	0xf02c, 0xffffffff, 0x00030002,
500	0xf02d, 0xffffffff, 0x00040007,
501	0xf02e, 0xffffffff, 0x00060005,
502	0xf02f, 0xffffffff, 0x00090008,
503	0xf000, 0xffffffff, 0x96e00200,
504	0x21c2, 0xffffffff, 0x00900100,
505	0x3109, 0xffffffff, 0x0020003f,
506	0xe, 0xffffffff, 0x0140001c,
507	0xf, 0x000f0000, 0x000f0000,
508	0x88, 0xffffffff, 0xc060000c,
509	0x89, 0xc0000fff, 0x00000100,
510	0x3e4, 0xffffffff, 0x00000100,
511	0x3e6, 0x00000101, 0x00000000,
512	0x82a, 0xffffffff, 0x00000104,
513	0x1579, 0xff000fff, 0x00000100,
514	0xc33, 0xc0000fff, 0x00000104,
515	0x3079, 0x00000001, 0x00000001,
516	0x3403, 0xff000ff0, 0x00000100,
517	0x3603, 0xff000ff0, 0x00000100
518};
519
520static const u32 kalindi_golden_spm_registers[] =
521{
522	0xc200, 0xe0ffffff, 0xe0000000
523};
524
525static const u32 kalindi_golden_common_registers[] =
526{
527	0x31dc, 0xffffffff, 0x00000800,
528	0x31dd, 0xffffffff, 0x00000800,
529	0x31e6, 0xffffffff, 0x00007fbf,
530	0x31e7, 0xffffffff, 0x00007faf
531};
532
533static const u32 kalindi_golden_registers[] =
534{
535	0xf000, 0xffffdfff, 0x6e944040,
536	0x1579, 0xff607fff, 0xfc000100,
537	0xf088, 0xff000fff, 0x00000100,
538	0xf089, 0xff000fff, 0x00000100,
539	0xf080, 0xfffc0fff, 0x00000100,
540	0x1bb6, 0x00010101, 0x00010000,
541	0x260c, 0xffffffff, 0x00000000,
542	0x260d, 0xf00fffff, 0x00000400,
543	0x16ec, 0x000000f0, 0x00000070,
544	0x16f0, 0xf0311fff, 0x80300000,
545	0x263e, 0x73773777, 0x12010001,
546	0x263f, 0xffffffff, 0x00000010,
547	0x26df, 0x00ff0000, 0x00fc0000,
548	0x200c, 0x00001f0f, 0x0000100a,
549	0xbd2, 0x73773777, 0x12010001,
550	0x902, 0x000fffff, 0x000c007f,
551	0x2285, 0xf000003f, 0x00000007,
552	0x22c9, 0x3fff3fff, 0x00ffcfff,
553	0xc281, 0x0000ff0f, 0x00000000,
554	0xa293, 0x07ffffff, 0x06000000,
555	0x136, 0x00000fff, 0x00000100,
556	0xf9e, 0x00000001, 0x00000002,
557	0x31da, 0x00000008, 0x00000008,
558	0x2300, 0x000000ff, 0x00000003,
559	0x853e, 0x01ff01ff, 0x00000002,
560	0x8526, 0x007ff800, 0x00200000,
561	0x8057, 0xffffffff, 0x00000f40,
562	0x2231, 0x001f3ae3, 0x00000082,
563	0x2235, 0x0000001f, 0x00000010,
564	0xc24d, 0xffffffff, 0x00000000
565};
566
567static const u32 kalindi_mgcg_cgcg_init[] =
568{
569	0x3108, 0xffffffff, 0xfffffffc,
570	0xc200, 0xffffffff, 0xe0000000,
571	0xf0a8, 0xffffffff, 0x00000100,
572	0xf082, 0xffffffff, 0x00000100,
573	0xf0b0, 0xffffffff, 0x00000100,
574	0xf0b2, 0xffffffff, 0x00000100,
575	0xf0b1, 0xffffffff, 0x00000100,
576	0x1579, 0xffffffff, 0x00600100,
577	0xf0a0, 0xffffffff, 0x00000100,
578	0xf085, 0xffffffff, 0x06000100,
579	0xf088, 0xffffffff, 0x00000100,
580	0xf086, 0xffffffff, 0x06000100,
581	0xf081, 0xffffffff, 0x00000100,
582	0xf0b8, 0xffffffff, 0x00000100,
583	0xf089, 0xffffffff, 0x00000100,
584	0xf080, 0xffffffff, 0x00000100,
585	0xf08c, 0xffffffff, 0x00000100,
586	0xf08d, 0xffffffff, 0x00000100,
587	0xf094, 0xffffffff, 0x00000100,
588	0xf095, 0xffffffff, 0x00000100,
589	0xf096, 0xffffffff, 0x00000100,
590	0xf097, 0xffffffff, 0x00000100,
591	0xf098, 0xffffffff, 0x00000100,
592	0xf09f, 0xffffffff, 0x00000100,
593	0xf09e, 0xffffffff, 0x00000100,
594	0xf084, 0xffffffff, 0x06000100,
595	0xf0a4, 0xffffffff, 0x00000100,
596	0xf09d, 0xffffffff, 0x00000100,
597	0xf0ad, 0xffffffff, 0x00000100,
598	0xf0ac, 0xffffffff, 0x00000100,
599	0xf09c, 0xffffffff, 0x00000100,
600	0xc200, 0xffffffff, 0xe0000000,
601	0xf008, 0xffffffff, 0x00010000,
602	0xf009, 0xffffffff, 0x00030002,
603	0xf00a, 0xffffffff, 0x00040007,
604	0xf00b, 0xffffffff, 0x00060005,
605	0xf00c, 0xffffffff, 0x00090008,
606	0xf00d, 0xffffffff, 0x00010000,
607	0xf00e, 0xffffffff, 0x00030002,
608	0xf00f, 0xffffffff, 0x00040007,
609	0xf010, 0xffffffff, 0x00060005,
610	0xf011, 0xffffffff, 0x00090008,
611	0xf000, 0xffffffff, 0x96e00200,
612	0x21c2, 0xffffffff, 0x00900100,
613	0x3109, 0xffffffff, 0x0020003f,
614	0xe, 0xffffffff, 0x0140001c,
615	0xf, 0x000f0000, 0x000f0000,
616	0x88, 0xffffffff, 0xc060000c,
617	0x89, 0xc0000fff, 0x00000100,
618	0x82a, 0xffffffff, 0x00000104,
619	0x1579, 0xff000fff, 0x00000100,
620	0xc33, 0xc0000fff, 0x00000104,
621	0x3079, 0x00000001, 0x00000001,
622	0x3403, 0xff000ff0, 0x00000100,
623	0x3603, 0xff000ff0, 0x00000100
624};
625
626static const u32 hawaii_golden_spm_registers[] =
627{
628	0xc200, 0xe0ffffff, 0xe0000000
629};
630
631static const u32 hawaii_golden_common_registers[] =
632{
633	0xc200, 0xffffffff, 0xe0000000,
634	0xa0d4, 0xffffffff, 0x3a00161a,
635	0xa0d5, 0xffffffff, 0x0000002e,
636	0x2684, 0xffffffff, 0x00018208,
637	0x263e, 0xffffffff, 0x12011003
638};
639
640static const u32 hawaii_golden_registers[] =
641{
642	0xcd5, 0x00000333, 0x00000333,
643	0x2684, 0x00010000, 0x00058208,
644	0x260c, 0xffffffff, 0x00000000,
645	0x260d, 0xf00fffff, 0x00000400,
646	0x260e, 0x0002021c, 0x00020200,
647	0x31e, 0x00000080, 0x00000000,
648	0x16ec, 0x000000f0, 0x00000070,
649	0x16f0, 0xf0311fff, 0x80300000,
650	0xd43, 0x00810000, 0x408af000,
651	0x1c0c, 0x31000111, 0x00000011,
652	0xbd2, 0x73773777, 0x12010001,
653	0x848, 0x0000007f, 0x0000001b,
654	0x877, 0x00007fb6, 0x00002191,
655	0xd8a, 0x0000003f, 0x0000000a,
656	0xd8b, 0x0000003f, 0x0000000a,
657	0xab9, 0x00073ffe, 0x000022a2,
658	0x903, 0x000007ff, 0x00000000,
659	0x22fc, 0x00002001, 0x00000001,
660	0x22c9, 0xffffffff, 0x00ffffff,
661	0xc281, 0x0000ff0f, 0x00000000,
662	0xa293, 0x07ffffff, 0x06000000,
663	0xf9e, 0x00000001, 0x00000002,
664	0x31da, 0x00000008, 0x00000008,
665	0x31dc, 0x00000f00, 0x00000800,
666	0x31dd, 0x00000f00, 0x00000800,
667	0x31e6, 0x00ffffff, 0x00ff7fbf,
668	0x31e7, 0x00ffffff, 0x00ff7faf,
669	0x2300, 0x000000ff, 0x00000800,
670	0x390, 0x00001fff, 0x00001fff,
671	0x2418, 0x0000007f, 0x00000020,
672	0x2542, 0x00010000, 0x00010000,
673	0x2b80, 0x00100000, 0x000ff07c,
674	0x2b05, 0x000003ff, 0x0000000f,
675	0x2b04, 0xffffffff, 0x7564fdec,
676	0x2b03, 0xffffffff, 0x3120b9a8,
677	0x2b02, 0x20000000, 0x0f9c0000
678};
679
680static const u32 hawaii_mgcg_cgcg_init[] =
681{
682	0x3108, 0xffffffff, 0xfffffffd,
683	0xc200, 0xffffffff, 0xe0000000,
684	0xf0a8, 0xffffffff, 0x00000100,
685	0xf082, 0xffffffff, 0x00000100,
686	0xf0b0, 0xffffffff, 0x00000100,
687	0xf0b2, 0xffffffff, 0x00000100,
688	0xf0b1, 0xffffffff, 0x00000100,
689	0x1579, 0xffffffff, 0x00200100,
690	0xf0a0, 0xffffffff, 0x00000100,
691	0xf085, 0xffffffff, 0x06000100,
692	0xf088, 0xffffffff, 0x00000100,
693	0xf086, 0xffffffff, 0x06000100,
694	0xf081, 0xffffffff, 0x00000100,
695	0xf0b8, 0xffffffff, 0x00000100,
696	0xf089, 0xffffffff, 0x00000100,
697	0xf080, 0xffffffff, 0x00000100,
698	0xf08c, 0xffffffff, 0x00000100,
699	0xf08d, 0xffffffff, 0x00000100,
700	0xf094, 0xffffffff, 0x00000100,
701	0xf095, 0xffffffff, 0x00000100,
702	0xf096, 0xffffffff, 0x00000100,
703	0xf097, 0xffffffff, 0x00000100,
704	0xf098, 0xffffffff, 0x00000100,
705	0xf09f, 0xffffffff, 0x00000100,
706	0xf09e, 0xffffffff, 0x00000100,
707	0xf084, 0xffffffff, 0x06000100,
708	0xf0a4, 0xffffffff, 0x00000100,
709	0xf09d, 0xffffffff, 0x00000100,
710	0xf0ad, 0xffffffff, 0x00000100,
711	0xf0ac, 0xffffffff, 0x00000100,
712	0xf09c, 0xffffffff, 0x00000100,
713	0xc200, 0xffffffff, 0xe0000000,
714	0xf008, 0xffffffff, 0x00010000,
715	0xf009, 0xffffffff, 0x00030002,
716	0xf00a, 0xffffffff, 0x00040007,
717	0xf00b, 0xffffffff, 0x00060005,
718	0xf00c, 0xffffffff, 0x00090008,
719	0xf00d, 0xffffffff, 0x00010000,
720	0xf00e, 0xffffffff, 0x00030002,
721	0xf00f, 0xffffffff, 0x00040007,
722	0xf010, 0xffffffff, 0x00060005,
723	0xf011, 0xffffffff, 0x00090008,
724	0xf012, 0xffffffff, 0x00010000,
725	0xf013, 0xffffffff, 0x00030002,
726	0xf014, 0xffffffff, 0x00040007,
727	0xf015, 0xffffffff, 0x00060005,
728	0xf016, 0xffffffff, 0x00090008,
729	0xf017, 0xffffffff, 0x00010000,
730	0xf018, 0xffffffff, 0x00030002,
731	0xf019, 0xffffffff, 0x00040007,
732	0xf01a, 0xffffffff, 0x00060005,
733	0xf01b, 0xffffffff, 0x00090008,
734	0xf01c, 0xffffffff, 0x00010000,
735	0xf01d, 0xffffffff, 0x00030002,
736	0xf01e, 0xffffffff, 0x00040007,
737	0xf01f, 0xffffffff, 0x00060005,
738	0xf020, 0xffffffff, 0x00090008,
739	0xf021, 0xffffffff, 0x00010000,
740	0xf022, 0xffffffff, 0x00030002,
741	0xf023, 0xffffffff, 0x00040007,
742	0xf024, 0xffffffff, 0x00060005,
743	0xf025, 0xffffffff, 0x00090008,
744	0xf026, 0xffffffff, 0x00010000,
745	0xf027, 0xffffffff, 0x00030002,
746	0xf028, 0xffffffff, 0x00040007,
747	0xf029, 0xffffffff, 0x00060005,
748	0xf02a, 0xffffffff, 0x00090008,
749	0xf02b, 0xffffffff, 0x00010000,
750	0xf02c, 0xffffffff, 0x00030002,
751	0xf02d, 0xffffffff, 0x00040007,
752	0xf02e, 0xffffffff, 0x00060005,
753	0xf02f, 0xffffffff, 0x00090008,
754	0xf030, 0xffffffff, 0x00010000,
755	0xf031, 0xffffffff, 0x00030002,
756	0xf032, 0xffffffff, 0x00040007,
757	0xf033, 0xffffffff, 0x00060005,
758	0xf034, 0xffffffff, 0x00090008,
759	0xf035, 0xffffffff, 0x00010000,
760	0xf036, 0xffffffff, 0x00030002,
761	0xf037, 0xffffffff, 0x00040007,
762	0xf038, 0xffffffff, 0x00060005,
763	0xf039, 0xffffffff, 0x00090008,
764	0xf03a, 0xffffffff, 0x00010000,
765	0xf03b, 0xffffffff, 0x00030002,
766	0xf03c, 0xffffffff, 0x00040007,
767	0xf03d, 0xffffffff, 0x00060005,
768	0xf03e, 0xffffffff, 0x00090008,
769	0x30c6, 0xffffffff, 0x00020200,
770	0xcd4, 0xffffffff, 0x00000200,
771	0x570, 0xffffffff, 0x00000400,
772	0x157a, 0xffffffff, 0x00000000,
773	0xbd4, 0xffffffff, 0x00000902,
774	0xf000, 0xffffffff, 0x96940200,
775	0x21c2, 0xffffffff, 0x00900100,
776	0x3109, 0xffffffff, 0x0020003f,
777	0xe, 0xffffffff, 0x0140001c,
778	0xf, 0x000f0000, 0x000f0000,
779	0x88, 0xffffffff, 0xc060000c,
780	0x89, 0xc0000fff, 0x00000100,
781	0x3e4, 0xffffffff, 0x00000100,
782	0x3e6, 0x00000101, 0x00000000,
783	0x82a, 0xffffffff, 0x00000104,
784	0x1579, 0xff000fff, 0x00000100,
785	0xc33, 0xc0000fff, 0x00000104,
786	0x3079, 0x00000001, 0x00000001,
787	0x3403, 0xff000ff0, 0x00000100,
788	0x3603, 0xff000ff0, 0x00000100
789};
790
791static const u32 godavari_golden_registers[] =
792{
793	0x1579, 0xff607fff, 0xfc000100,
794	0x1bb6, 0x00010101, 0x00010000,
795	0x260c, 0xffffffff, 0x00000000,
796	0x260c0, 0xf00fffff, 0x00000400,
797	0x184c, 0xffffffff, 0x00010000,
798	0x16ec, 0x000000f0, 0x00000070,
799	0x16f0, 0xf0311fff, 0x80300000,
800	0x263e, 0x73773777, 0x12010001,
801	0x263f, 0xffffffff, 0x00000010,
802	0x200c, 0x00001f0f, 0x0000100a,
803	0xbd2, 0x73773777, 0x12010001,
804	0x902, 0x000fffff, 0x000c007f,
805	0x2285, 0xf000003f, 0x00000007,
806	0x22c9, 0xffffffff, 0x00ff0fff,
807	0xc281, 0x0000ff0f, 0x00000000,
808	0xa293, 0x07ffffff, 0x06000000,
809	0x136, 0x00000fff, 0x00000100,
810	0x3405, 0x00010000, 0x00810001,
811	0x3605, 0x00010000, 0x00810001,
812	0xf9e, 0x00000001, 0x00000002,
813	0x31da, 0x00000008, 0x00000008,
814	0x31dc, 0x00000f00, 0x00000800,
815	0x31dd, 0x00000f00, 0x00000800,
816	0x31e6, 0x00ffffff, 0x00ff7fbf,
817	0x31e7, 0x00ffffff, 0x00ff7faf,
818	0x2300, 0x000000ff, 0x00000001,
819	0x853e, 0x01ff01ff, 0x00000002,
820	0x8526, 0x007ff800, 0x00200000,
821	0x8057, 0xffffffff, 0x00000f40,
822	0x2231, 0x001f3ae3, 0x00000082,
823	0x2235, 0x0000001f, 0x00000010,
824	0xc24d, 0xffffffff, 0x00000000
825};
826
827static void cik_init_golden_registers(struct amdgpu_device *adev)
828{
829	/* Some of the registers might be dependent on GRBM_GFX_INDEX */
830	mutex_lock(&adev->grbm_idx_mutex);
831
832	switch (adev->asic_type) {
833	case CHIP_BONAIRE:
834		amdgpu_device_program_register_sequence(adev,
835							bonaire_mgcg_cgcg_init,
836							ARRAY_SIZE(bonaire_mgcg_cgcg_init));
837		amdgpu_device_program_register_sequence(adev,
838							bonaire_golden_registers,
839							ARRAY_SIZE(bonaire_golden_registers));
840		amdgpu_device_program_register_sequence(adev,
841							bonaire_golden_common_registers,
842							ARRAY_SIZE(bonaire_golden_common_registers));
843		amdgpu_device_program_register_sequence(adev,
844							bonaire_golden_spm_registers,
845							ARRAY_SIZE(bonaire_golden_spm_registers));
846		break;
847	case CHIP_KABINI:
848		amdgpu_device_program_register_sequence(adev,
849							kalindi_mgcg_cgcg_init,
850							ARRAY_SIZE(kalindi_mgcg_cgcg_init));
851		amdgpu_device_program_register_sequence(adev,
852							kalindi_golden_registers,
853							ARRAY_SIZE(kalindi_golden_registers));
854		amdgpu_device_program_register_sequence(adev,
855							kalindi_golden_common_registers,
856							ARRAY_SIZE(kalindi_golden_common_registers));
857		amdgpu_device_program_register_sequence(adev,
858							kalindi_golden_spm_registers,
859							ARRAY_SIZE(kalindi_golden_spm_registers));
860		break;
861	case CHIP_MULLINS:
862		amdgpu_device_program_register_sequence(adev,
863							kalindi_mgcg_cgcg_init,
864							ARRAY_SIZE(kalindi_mgcg_cgcg_init));
865		amdgpu_device_program_register_sequence(adev,
866							godavari_golden_registers,
867							ARRAY_SIZE(godavari_golden_registers));
868		amdgpu_device_program_register_sequence(adev,
869							kalindi_golden_common_registers,
870							ARRAY_SIZE(kalindi_golden_common_registers));
871		amdgpu_device_program_register_sequence(adev,
872							kalindi_golden_spm_registers,
873							ARRAY_SIZE(kalindi_golden_spm_registers));
874		break;
875	case CHIP_KAVERI:
876		amdgpu_device_program_register_sequence(adev,
877							spectre_mgcg_cgcg_init,
878							ARRAY_SIZE(spectre_mgcg_cgcg_init));
879		amdgpu_device_program_register_sequence(adev,
880							spectre_golden_registers,
881							ARRAY_SIZE(spectre_golden_registers));
882		amdgpu_device_program_register_sequence(adev,
883							spectre_golden_common_registers,
884							ARRAY_SIZE(spectre_golden_common_registers));
885		amdgpu_device_program_register_sequence(adev,
886							spectre_golden_spm_registers,
887							ARRAY_SIZE(spectre_golden_spm_registers));
888		break;
889	case CHIP_HAWAII:
890		amdgpu_device_program_register_sequence(adev,
891							hawaii_mgcg_cgcg_init,
892							ARRAY_SIZE(hawaii_mgcg_cgcg_init));
893		amdgpu_device_program_register_sequence(adev,
894							hawaii_golden_registers,
895							ARRAY_SIZE(hawaii_golden_registers));
896		amdgpu_device_program_register_sequence(adev,
897							hawaii_golden_common_registers,
898							ARRAY_SIZE(hawaii_golden_common_registers));
899		amdgpu_device_program_register_sequence(adev,
900							hawaii_golden_spm_registers,
901							ARRAY_SIZE(hawaii_golden_spm_registers));
902		break;
903	default:
904		break;
905	}
906	mutex_unlock(&adev->grbm_idx_mutex);
907}
908
909/**
910 * cik_get_xclk - get the xclk
911 *
912 * @adev: amdgpu_device pointer
913 *
914 * Returns the reference clock used by the gfx engine
915 * (CIK).
916 */
917static u32 cik_get_xclk(struct amdgpu_device *adev)
918{
919	u32 reference_clock = adev->clock.spll.reference_freq;
920
921	if (adev->flags & AMD_IS_APU) {
922		if (RREG32_SMC(ixGENERAL_PWRMGT) & GENERAL_PWRMGT__GPU_COUNTER_CLK_MASK)
923			return reference_clock / 2;
924	} else {
925		if (RREG32_SMC(ixCG_CLKPIN_CNTL) & CG_CLKPIN_CNTL__XTALIN_DIVIDE_MASK)
926			return reference_clock / 4;
927	}
928	return reference_clock;
929}
930
931/**
932 * cik_srbm_select - select specific register instances
933 *
934 * @adev: amdgpu_device pointer
935 * @me: selected ME (micro engine)
936 * @pipe: pipe
937 * @queue: queue
938 * @vmid: VMID
939 *
940 * Switches the currently active registers instances.  Some
941 * registers are instanced per VMID, others are instanced per
942 * me/pipe/queue combination.
943 */
944void cik_srbm_select(struct amdgpu_device *adev,
945		     u32 me, u32 pipe, u32 queue, u32 vmid)
946{
947	u32 srbm_gfx_cntl =
948		(((pipe << SRBM_GFX_CNTL__PIPEID__SHIFT) & SRBM_GFX_CNTL__PIPEID_MASK)|
949		((me << SRBM_GFX_CNTL__MEID__SHIFT) & SRBM_GFX_CNTL__MEID_MASK)|
950		((vmid << SRBM_GFX_CNTL__VMID__SHIFT) & SRBM_GFX_CNTL__VMID_MASK)|
951		((queue << SRBM_GFX_CNTL__QUEUEID__SHIFT) & SRBM_GFX_CNTL__QUEUEID_MASK));
952	WREG32(mmSRBM_GFX_CNTL, srbm_gfx_cntl);
953}
954
955static void cik_vga_set_state(struct amdgpu_device *adev, bool state)
956{
957	uint32_t tmp;
958
959	tmp = RREG32(mmCONFIG_CNTL);
960	if (!state)
961		tmp |= CONFIG_CNTL__VGA_DIS_MASK;
962	else
963		tmp &= ~CONFIG_CNTL__VGA_DIS_MASK;
964	WREG32(mmCONFIG_CNTL, tmp);
965}
966
967static bool cik_read_disabled_bios(struct amdgpu_device *adev)
968{
969	u32 bus_cntl;
970	u32 d1vga_control = 0;
971	u32 d2vga_control = 0;
972	u32 vga_render_control = 0;
973	u32 rom_cntl;
974	bool r;
975
976	bus_cntl = RREG32(mmBUS_CNTL);
977	if (adev->mode_info.num_crtc) {
978		d1vga_control = RREG32(mmD1VGA_CONTROL);
979		d2vga_control = RREG32(mmD2VGA_CONTROL);
980		vga_render_control = RREG32(mmVGA_RENDER_CONTROL);
981	}
982	rom_cntl = RREG32_SMC(ixROM_CNTL);
983
984	/* enable the rom */
985	WREG32(mmBUS_CNTL, (bus_cntl & ~BUS_CNTL__BIOS_ROM_DIS_MASK));
986	if (adev->mode_info.num_crtc) {
987		/* Disable VGA mode */
988		WREG32(mmD1VGA_CONTROL,
989		       (d1vga_control & ~(D1VGA_CONTROL__D1VGA_MODE_ENABLE_MASK |
990					  D1VGA_CONTROL__D1VGA_TIMING_SELECT_MASK)));
991		WREG32(mmD2VGA_CONTROL,
992		       (d2vga_control & ~(D1VGA_CONTROL__D1VGA_MODE_ENABLE_MASK |
993					  D1VGA_CONTROL__D1VGA_TIMING_SELECT_MASK)));
994		WREG32(mmVGA_RENDER_CONTROL,
995		       (vga_render_control & ~VGA_RENDER_CONTROL__VGA_VSTATUS_CNTL_MASK));
996	}
997	WREG32_SMC(ixROM_CNTL, rom_cntl | ROM_CNTL__SCK_OVERWRITE_MASK);
998
999	r = amdgpu_read_bios(adev);
1000
1001	/* restore regs */
1002	WREG32(mmBUS_CNTL, bus_cntl);
1003	if (adev->mode_info.num_crtc) {
1004		WREG32(mmD1VGA_CONTROL, d1vga_control);
1005		WREG32(mmD2VGA_CONTROL, d2vga_control);
1006		WREG32(mmVGA_RENDER_CONTROL, vga_render_control);
1007	}
1008	WREG32_SMC(ixROM_CNTL, rom_cntl);
1009	return r;
1010}
1011
1012static bool cik_read_bios_from_rom(struct amdgpu_device *adev,
1013				   u8 *bios, u32 length_bytes)
1014{
1015	u32 *dw_ptr;
1016	unsigned long flags;
1017	u32 i, length_dw;
1018
1019	if (bios == NULL)
1020		return false;
1021	if (length_bytes == 0)
1022		return false;
1023	/* APU vbios image is part of sbios image */
1024	if (adev->flags & AMD_IS_APU)
1025		return false;
1026
1027	dw_ptr = (u32 *)bios;
1028	length_dw = ALIGN(length_bytes, 4) / 4;
1029	/* take the smc lock since we are using the smc index */
1030	spin_lock_irqsave(&adev->smc_idx_lock, flags);
1031	/* set rom index to 0 */
1032	WREG32(mmSMC_IND_INDEX_0, ixROM_INDEX);
1033	WREG32(mmSMC_IND_DATA_0, 0);
1034	/* set index to data for continous read */
1035	WREG32(mmSMC_IND_INDEX_0, ixROM_DATA);
1036	for (i = 0; i < length_dw; i++)
1037		dw_ptr[i] = RREG32(mmSMC_IND_DATA_0);
1038	spin_unlock_irqrestore(&adev->smc_idx_lock, flags);
1039
1040	return true;
1041}
1042
1043static const struct amdgpu_allowed_register_entry cik_allowed_read_registers[] = {
1044	{mmGRBM_STATUS},
1045	{mmGRBM_STATUS2},
1046	{mmGRBM_STATUS_SE0},
1047	{mmGRBM_STATUS_SE1},
1048	{mmGRBM_STATUS_SE2},
1049	{mmGRBM_STATUS_SE3},
1050	{mmSRBM_STATUS},
1051	{mmSRBM_STATUS2},
1052	{mmSDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET},
1053	{mmSDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET},
1054	{mmCP_STAT},
1055	{mmCP_STALLED_STAT1},
1056	{mmCP_STALLED_STAT2},
1057	{mmCP_STALLED_STAT3},
1058	{mmCP_CPF_BUSY_STAT},
1059	{mmCP_CPF_STALLED_STAT1},
1060	{mmCP_CPF_STATUS},
1061	{mmCP_CPC_BUSY_STAT},
1062	{mmCP_CPC_STALLED_STAT1},
1063	{mmCP_CPC_STATUS},
1064	{mmGB_ADDR_CONFIG},
1065	{mmMC_ARB_RAMCFG},
1066	{mmGB_TILE_MODE0},
1067	{mmGB_TILE_MODE1},
1068	{mmGB_TILE_MODE2},
1069	{mmGB_TILE_MODE3},
1070	{mmGB_TILE_MODE4},
1071	{mmGB_TILE_MODE5},
1072	{mmGB_TILE_MODE6},
1073	{mmGB_TILE_MODE7},
1074	{mmGB_TILE_MODE8},
1075	{mmGB_TILE_MODE9},
1076	{mmGB_TILE_MODE10},
1077	{mmGB_TILE_MODE11},
1078	{mmGB_TILE_MODE12},
1079	{mmGB_TILE_MODE13},
1080	{mmGB_TILE_MODE14},
1081	{mmGB_TILE_MODE15},
1082	{mmGB_TILE_MODE16},
1083	{mmGB_TILE_MODE17},
1084	{mmGB_TILE_MODE18},
1085	{mmGB_TILE_MODE19},
1086	{mmGB_TILE_MODE20},
1087	{mmGB_TILE_MODE21},
1088	{mmGB_TILE_MODE22},
1089	{mmGB_TILE_MODE23},
1090	{mmGB_TILE_MODE24},
1091	{mmGB_TILE_MODE25},
1092	{mmGB_TILE_MODE26},
1093	{mmGB_TILE_MODE27},
1094	{mmGB_TILE_MODE28},
1095	{mmGB_TILE_MODE29},
1096	{mmGB_TILE_MODE30},
1097	{mmGB_TILE_MODE31},
1098	{mmGB_MACROTILE_MODE0},
1099	{mmGB_MACROTILE_MODE1},
1100	{mmGB_MACROTILE_MODE2},
1101	{mmGB_MACROTILE_MODE3},
1102	{mmGB_MACROTILE_MODE4},
1103	{mmGB_MACROTILE_MODE5},
1104	{mmGB_MACROTILE_MODE6},
1105	{mmGB_MACROTILE_MODE7},
1106	{mmGB_MACROTILE_MODE8},
1107	{mmGB_MACROTILE_MODE9},
1108	{mmGB_MACROTILE_MODE10},
1109	{mmGB_MACROTILE_MODE11},
1110	{mmGB_MACROTILE_MODE12},
1111	{mmGB_MACROTILE_MODE13},
1112	{mmGB_MACROTILE_MODE14},
1113	{mmGB_MACROTILE_MODE15},
1114	{mmCC_RB_BACKEND_DISABLE, true},
1115	{mmGC_USER_RB_BACKEND_DISABLE, true},
1116	{mmGB_BACKEND_MAP, false},
1117	{mmPA_SC_RASTER_CONFIG, true},
1118	{mmPA_SC_RASTER_CONFIG_1, true},
1119};
1120
1121
1122static uint32_t cik_get_register_value(struct amdgpu_device *adev,
1123				       bool indexed, u32 se_num,
1124				       u32 sh_num, u32 reg_offset)
1125{
1126	if (indexed) {
1127		uint32_t val;
1128		unsigned se_idx = (se_num == 0xffffffff) ? 0 : se_num;
1129		unsigned sh_idx = (sh_num == 0xffffffff) ? 0 : sh_num;
1130
1131		switch (reg_offset) {
1132		case mmCC_RB_BACKEND_DISABLE:
1133			return adev->gfx.config.rb_config[se_idx][sh_idx].rb_backend_disable;
1134		case mmGC_USER_RB_BACKEND_DISABLE:
1135			return adev->gfx.config.rb_config[se_idx][sh_idx].user_rb_backend_disable;
1136		case mmPA_SC_RASTER_CONFIG:
1137			return adev->gfx.config.rb_config[se_idx][sh_idx].raster_config;
1138		case mmPA_SC_RASTER_CONFIG_1:
1139			return adev->gfx.config.rb_config[se_idx][sh_idx].raster_config_1;
1140		}
1141
1142		mutex_lock(&adev->grbm_idx_mutex);
1143		if (se_num != 0xffffffff || sh_num != 0xffffffff)
1144			amdgpu_gfx_select_se_sh(adev, se_num, sh_num, 0xffffffff, 0);
1145
1146		val = RREG32(reg_offset);
1147
1148		if (se_num != 0xffffffff || sh_num != 0xffffffff)
1149			amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0);
1150		mutex_unlock(&adev->grbm_idx_mutex);
1151		return val;
1152	} else {
1153		unsigned idx;
1154
1155		switch (reg_offset) {
1156		case mmGB_ADDR_CONFIG:
1157			return adev->gfx.config.gb_addr_config;
1158		case mmMC_ARB_RAMCFG:
1159			return adev->gfx.config.mc_arb_ramcfg;
1160		case mmGB_TILE_MODE0:
1161		case mmGB_TILE_MODE1:
1162		case mmGB_TILE_MODE2:
1163		case mmGB_TILE_MODE3:
1164		case mmGB_TILE_MODE4:
1165		case mmGB_TILE_MODE5:
1166		case mmGB_TILE_MODE6:
1167		case mmGB_TILE_MODE7:
1168		case mmGB_TILE_MODE8:
1169		case mmGB_TILE_MODE9:
1170		case mmGB_TILE_MODE10:
1171		case mmGB_TILE_MODE11:
1172		case mmGB_TILE_MODE12:
1173		case mmGB_TILE_MODE13:
1174		case mmGB_TILE_MODE14:
1175		case mmGB_TILE_MODE15:
1176		case mmGB_TILE_MODE16:
1177		case mmGB_TILE_MODE17:
1178		case mmGB_TILE_MODE18:
1179		case mmGB_TILE_MODE19:
1180		case mmGB_TILE_MODE20:
1181		case mmGB_TILE_MODE21:
1182		case mmGB_TILE_MODE22:
1183		case mmGB_TILE_MODE23:
1184		case mmGB_TILE_MODE24:
1185		case mmGB_TILE_MODE25:
1186		case mmGB_TILE_MODE26:
1187		case mmGB_TILE_MODE27:
1188		case mmGB_TILE_MODE28:
1189		case mmGB_TILE_MODE29:
1190		case mmGB_TILE_MODE30:
1191		case mmGB_TILE_MODE31:
1192			idx = (reg_offset - mmGB_TILE_MODE0);
1193			return adev->gfx.config.tile_mode_array[idx];
1194		case mmGB_MACROTILE_MODE0:
1195		case mmGB_MACROTILE_MODE1:
1196		case mmGB_MACROTILE_MODE2:
1197		case mmGB_MACROTILE_MODE3:
1198		case mmGB_MACROTILE_MODE4:
1199		case mmGB_MACROTILE_MODE5:
1200		case mmGB_MACROTILE_MODE6:
1201		case mmGB_MACROTILE_MODE7:
1202		case mmGB_MACROTILE_MODE8:
1203		case mmGB_MACROTILE_MODE9:
1204		case mmGB_MACROTILE_MODE10:
1205		case mmGB_MACROTILE_MODE11:
1206		case mmGB_MACROTILE_MODE12:
1207		case mmGB_MACROTILE_MODE13:
1208		case mmGB_MACROTILE_MODE14:
1209		case mmGB_MACROTILE_MODE15:
1210			idx = (reg_offset - mmGB_MACROTILE_MODE0);
1211			return adev->gfx.config.macrotile_mode_array[idx];
1212		default:
1213			return RREG32(reg_offset);
1214		}
1215	}
1216}
1217
1218static int cik_read_register(struct amdgpu_device *adev, u32 se_num,
1219			     u32 sh_num, u32 reg_offset, u32 *value)
1220{
1221	uint32_t i;
1222
1223	*value = 0;
1224	for (i = 0; i < ARRAY_SIZE(cik_allowed_read_registers); i++) {
1225		bool indexed = cik_allowed_read_registers[i].grbm_indexed;
1226
1227		if (reg_offset != cik_allowed_read_registers[i].reg_offset)
1228			continue;
1229
1230		*value = cik_get_register_value(adev, indexed, se_num, sh_num,
1231						reg_offset);
1232		return 0;
1233	}
1234	return -EINVAL;
1235}
1236
1237struct kv_reset_save_regs {
1238	u32 gmcon_reng_execute;
1239	u32 gmcon_misc;
1240	u32 gmcon_misc3;
1241};
1242
1243static void kv_save_regs_for_reset(struct amdgpu_device *adev,
1244				   struct kv_reset_save_regs *save)
1245{
1246	save->gmcon_reng_execute = RREG32(mmGMCON_RENG_EXECUTE);
1247	save->gmcon_misc = RREG32(mmGMCON_MISC);
1248	save->gmcon_misc3 = RREG32(mmGMCON_MISC3);
1249
1250	WREG32(mmGMCON_RENG_EXECUTE, save->gmcon_reng_execute &
1251		~GMCON_RENG_EXECUTE__RENG_EXECUTE_ON_PWR_UP_MASK);
1252	WREG32(mmGMCON_MISC, save->gmcon_misc &
1253		~(GMCON_MISC__RENG_EXECUTE_ON_REG_UPDATE_MASK |
1254			GMCON_MISC__STCTRL_STUTTER_EN_MASK));
1255}
1256
1257static void kv_restore_regs_for_reset(struct amdgpu_device *adev,
1258				      struct kv_reset_save_regs *save)
1259{
1260	int i;
1261
1262	WREG32(mmGMCON_PGFSM_WRITE, 0);
1263	WREG32(mmGMCON_PGFSM_CONFIG, 0x200010ff);
1264
1265	for (i = 0; i < 5; i++)
1266		WREG32(mmGMCON_PGFSM_WRITE, 0);
1267
1268	WREG32(mmGMCON_PGFSM_WRITE, 0);
1269	WREG32(mmGMCON_PGFSM_CONFIG, 0x300010ff);
1270
1271	for (i = 0; i < 5; i++)
1272		WREG32(mmGMCON_PGFSM_WRITE, 0);
1273
1274	WREG32(mmGMCON_PGFSM_WRITE, 0x210000);
1275	WREG32(mmGMCON_PGFSM_CONFIG, 0xa00010ff);
1276
1277	for (i = 0; i < 5; i++)
1278		WREG32(mmGMCON_PGFSM_WRITE, 0);
1279
1280	WREG32(mmGMCON_PGFSM_WRITE, 0x21003);
1281	WREG32(mmGMCON_PGFSM_CONFIG, 0xb00010ff);
1282
1283	for (i = 0; i < 5; i++)
1284		WREG32(mmGMCON_PGFSM_WRITE, 0);
1285
1286	WREG32(mmGMCON_PGFSM_WRITE, 0x2b00);
1287	WREG32(mmGMCON_PGFSM_CONFIG, 0xc00010ff);
1288
1289	for (i = 0; i < 5; i++)
1290		WREG32(mmGMCON_PGFSM_WRITE, 0);
1291
1292	WREG32(mmGMCON_PGFSM_WRITE, 0);
1293	WREG32(mmGMCON_PGFSM_CONFIG, 0xd00010ff);
1294
1295	for (i = 0; i < 5; i++)
1296		WREG32(mmGMCON_PGFSM_WRITE, 0);
1297
1298	WREG32(mmGMCON_PGFSM_WRITE, 0x420000);
1299	WREG32(mmGMCON_PGFSM_CONFIG, 0x100010ff);
1300
1301	for (i = 0; i < 5; i++)
1302		WREG32(mmGMCON_PGFSM_WRITE, 0);
1303
1304	WREG32(mmGMCON_PGFSM_WRITE, 0x120202);
1305	WREG32(mmGMCON_PGFSM_CONFIG, 0x500010ff);
1306
1307	for (i = 0; i < 5; i++)
1308		WREG32(mmGMCON_PGFSM_WRITE, 0);
1309
1310	WREG32(mmGMCON_PGFSM_WRITE, 0x3e3e36);
1311	WREG32(mmGMCON_PGFSM_CONFIG, 0x600010ff);
1312
1313	for (i = 0; i < 5; i++)
1314		WREG32(mmGMCON_PGFSM_WRITE, 0);
1315
1316	WREG32(mmGMCON_PGFSM_WRITE, 0x373f3e);
1317	WREG32(mmGMCON_PGFSM_CONFIG, 0x700010ff);
1318
1319	for (i = 0; i < 5; i++)
1320		WREG32(mmGMCON_PGFSM_WRITE, 0);
1321
1322	WREG32(mmGMCON_PGFSM_WRITE, 0x3e1332);
1323	WREG32(mmGMCON_PGFSM_CONFIG, 0xe00010ff);
1324
1325	WREG32(mmGMCON_MISC3, save->gmcon_misc3);
1326	WREG32(mmGMCON_MISC, save->gmcon_misc);
1327	WREG32(mmGMCON_RENG_EXECUTE, save->gmcon_reng_execute);
1328}
1329
1330/**
1331 * cik_asic_pci_config_reset - soft reset GPU
1332 *
1333 * @adev: amdgpu_device pointer
1334 *
1335 * Use PCI Config method to reset the GPU.
1336 *
1337 * Returns 0 for success.
1338 */
1339static int cik_asic_pci_config_reset(struct amdgpu_device *adev)
1340{
1341	struct kv_reset_save_regs kv_save = { 0 };
1342	u32 i;
1343	int r = -EINVAL;
1344
1345	amdgpu_atombios_scratch_regs_engine_hung(adev, true);
1346
1347	if (adev->flags & AMD_IS_APU)
1348		kv_save_regs_for_reset(adev, &kv_save);
1349
1350	/* disable BM */
1351	pci_clear_master(adev->pdev);
1352	/* reset */
1353	amdgpu_device_pci_config_reset(adev);
1354
1355	udelay(100);
1356
1357	/* wait for asic to come out of reset */
1358	for (i = 0; i < adev->usec_timeout; i++) {
1359		if (RREG32(mmCONFIG_MEMSIZE) != 0xffffffff) {
1360			/* enable BM */
1361			pci_set_master(adev->pdev);
1362			adev->has_hw_reset = true;
1363			r = 0;
1364			break;
1365		}
1366		udelay(1);
1367	}
1368
1369	/* does asic init need to be run first??? */
1370	if (adev->flags & AMD_IS_APU)
1371		kv_restore_regs_for_reset(adev, &kv_save);
1372
1373	amdgpu_atombios_scratch_regs_engine_hung(adev, false);
1374
1375	return r;
1376}
1377
1378static bool cik_asic_supports_baco(struct amdgpu_device *adev)
1379{
1380	switch (adev->asic_type) {
1381	case CHIP_BONAIRE:
1382	case CHIP_HAWAII:
1383		return amdgpu_dpm_is_baco_supported(adev);
1384	default:
1385		return false;
1386	}
1387}
1388
1389static enum amd_reset_method
1390cik_asic_reset_method(struct amdgpu_device *adev)
1391{
1392	bool baco_reset;
1393
1394	if (amdgpu_reset_method == AMD_RESET_METHOD_LEGACY ||
1395	    amdgpu_reset_method == AMD_RESET_METHOD_BACO)
1396		return amdgpu_reset_method;
1397
1398	if (amdgpu_reset_method != -1)
1399		dev_warn(adev->dev, "Specified reset:%d isn't supported, using AUTO instead.\n",
1400				  amdgpu_reset_method);
1401
1402	switch (adev->asic_type) {
1403	case CHIP_BONAIRE:
1404	case CHIP_HAWAII:
1405		baco_reset = cik_asic_supports_baco(adev);
1406		break;
1407	default:
1408		baco_reset = false;
1409		break;
1410	}
1411
1412	if (baco_reset)
1413		return AMD_RESET_METHOD_BACO;
1414	else
1415		return AMD_RESET_METHOD_LEGACY;
1416}
1417
1418/**
1419 * cik_asic_reset - soft reset GPU
1420 *
1421 * @adev: amdgpu_device pointer
1422 *
1423 * Look up which blocks are hung and attempt
1424 * to reset them.
1425 * Returns 0 for success.
1426 */
1427static int cik_asic_reset(struct amdgpu_device *adev)
1428{
1429	int r;
1430
1431	/* APUs don't have full asic reset */
1432	if (adev->flags & AMD_IS_APU)
1433		return 0;
1434
1435	if (cik_asic_reset_method(adev) == AMD_RESET_METHOD_BACO) {
1436		dev_info(adev->dev, "BACO reset\n");
1437		r = amdgpu_dpm_baco_reset(adev);
1438	} else {
1439		dev_info(adev->dev, "PCI CONFIG reset\n");
1440		r = cik_asic_pci_config_reset(adev);
1441	}
1442
1443	return r;
1444}
1445
1446static u32 cik_get_config_memsize(struct amdgpu_device *adev)
1447{
1448	return RREG32(mmCONFIG_MEMSIZE);
1449}
1450
1451static int cik_set_uvd_clock(struct amdgpu_device *adev, u32 clock,
1452			      u32 cntl_reg, u32 status_reg)
1453{
1454	int r, i;
1455	struct atom_clock_dividers dividers;
1456	uint32_t tmp;
1457
1458	r = amdgpu_atombios_get_clock_dividers(adev,
1459					       COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
1460					       clock, false, &dividers);
1461	if (r)
1462		return r;
1463
1464	tmp = RREG32_SMC(cntl_reg);
1465	tmp &= ~(CG_DCLK_CNTL__DCLK_DIR_CNTL_EN_MASK |
1466		CG_DCLK_CNTL__DCLK_DIVIDER_MASK);
1467	tmp |= dividers.post_divider;
1468	WREG32_SMC(cntl_reg, tmp);
1469
1470	for (i = 0; i < 100; i++) {
1471		if (RREG32_SMC(status_reg) & CG_DCLK_STATUS__DCLK_STATUS_MASK)
1472			break;
1473		mdelay(10);
1474	}
1475	if (i == 100)
1476		return -ETIMEDOUT;
1477
1478	return 0;
1479}
1480
1481static int cik_set_uvd_clocks(struct amdgpu_device *adev, u32 vclk, u32 dclk)
1482{
1483	int r = 0;
1484
1485	r = cik_set_uvd_clock(adev, vclk, ixCG_VCLK_CNTL, ixCG_VCLK_STATUS);
1486	if (r)
1487		return r;
1488
1489	r = cik_set_uvd_clock(adev, dclk, ixCG_DCLK_CNTL, ixCG_DCLK_STATUS);
1490	return r;
1491}
1492
1493static int cik_set_vce_clocks(struct amdgpu_device *adev, u32 evclk, u32 ecclk)
1494{
1495	int r, i;
1496	struct atom_clock_dividers dividers;
1497	u32 tmp;
1498
1499	r = amdgpu_atombios_get_clock_dividers(adev,
1500					       COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
1501					       ecclk, false, &dividers);
1502	if (r)
1503		return r;
1504
1505	for (i = 0; i < 100; i++) {
1506		if (RREG32_SMC(ixCG_ECLK_STATUS) & CG_ECLK_STATUS__ECLK_STATUS_MASK)
1507			break;
1508		mdelay(10);
1509	}
1510	if (i == 100)
1511		return -ETIMEDOUT;
1512
1513	tmp = RREG32_SMC(ixCG_ECLK_CNTL);
1514	tmp &= ~(CG_ECLK_CNTL__ECLK_DIR_CNTL_EN_MASK |
1515		CG_ECLK_CNTL__ECLK_DIVIDER_MASK);
1516	tmp |= dividers.post_divider;
1517	WREG32_SMC(ixCG_ECLK_CNTL, tmp);
1518
1519	for (i = 0; i < 100; i++) {
1520		if (RREG32_SMC(ixCG_ECLK_STATUS) & CG_ECLK_STATUS__ECLK_STATUS_MASK)
1521			break;
1522		mdelay(10);
1523	}
1524	if (i == 100)
1525		return -ETIMEDOUT;
1526
1527	return 0;
1528}
1529
1530static void cik_pcie_gen3_enable(struct amdgpu_device *adev)
1531{
1532	struct pci_dev *root = adev->pdev->bus->self;
1533	u32 speed_cntl, current_data_rate;
1534	int i;
1535	u16 tmp16;
1536
1537	if (pci_is_root_bus(adev->pdev->bus))
1538		return;
1539
1540	if (amdgpu_pcie_gen2 == 0)
1541		return;
1542
1543	if (adev->flags & AMD_IS_APU)
1544		return;
1545
1546	if (!(adev->pm.pcie_gen_mask & (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 |
1547					CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3)))
1548		return;
1549
1550	speed_cntl = RREG32_PCIE(ixPCIE_LC_SPEED_CNTL);
1551	current_data_rate = (speed_cntl & PCIE_LC_SPEED_CNTL__LC_CURRENT_DATA_RATE_MASK) >>
1552		PCIE_LC_SPEED_CNTL__LC_CURRENT_DATA_RATE__SHIFT;
1553	if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3) {
1554		if (current_data_rate == 2) {
1555			DRM_INFO("PCIE gen 3 link speeds already enabled\n");
1556			return;
1557		}
1558		DRM_INFO("enabling PCIE gen 3 link speeds, disable with amdgpu.pcie_gen2=0\n");
1559	} else if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2) {
1560		if (current_data_rate == 1) {
1561			DRM_INFO("PCIE gen 2 link speeds already enabled\n");
1562			return;
1563		}
1564		DRM_INFO("enabling PCIE gen 2 link speeds, disable with amdgpu.pcie_gen2=0\n");
1565	}
1566
1567	if (!pci_is_pcie(root) || !pci_is_pcie(adev->pdev))
1568		return;
1569
1570	if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3) {
1571		/* re-try equalization if gen3 is not already enabled */
1572		if (current_data_rate != 2) {
1573			u16 bridge_cfg, gpu_cfg;
1574			u16 bridge_cfg2, gpu_cfg2;
1575			u32 max_lw, current_lw, tmp;
1576
1577			pcie_capability_set_word(root, PCI_EXP_LNKCTL, PCI_EXP_LNKCTL_HAWD);
1578			pcie_capability_set_word(adev->pdev, PCI_EXP_LNKCTL, PCI_EXP_LNKCTL_HAWD);
1579
1580			tmp = RREG32_PCIE(ixPCIE_LC_STATUS1);
1581			max_lw = (tmp & PCIE_LC_STATUS1__LC_DETECTED_LINK_WIDTH_MASK) >>
1582				PCIE_LC_STATUS1__LC_DETECTED_LINK_WIDTH__SHIFT;
1583			current_lw = (tmp & PCIE_LC_STATUS1__LC_OPERATING_LINK_WIDTH_MASK)
1584				>> PCIE_LC_STATUS1__LC_OPERATING_LINK_WIDTH__SHIFT;
1585
1586			if (current_lw < max_lw) {
1587				tmp = RREG32_PCIE(ixPCIE_LC_LINK_WIDTH_CNTL);
1588				if (tmp & PCIE_LC_LINK_WIDTH_CNTL__LC_RENEGOTIATION_SUPPORT_MASK) {
1589					tmp &= ~(PCIE_LC_LINK_WIDTH_CNTL__LC_LINK_WIDTH_MASK |
1590						PCIE_LC_LINK_WIDTH_CNTL__LC_UPCONFIGURE_DIS_MASK);
1591					tmp |= (max_lw <<
1592						PCIE_LC_LINK_WIDTH_CNTL__LC_LINK_WIDTH__SHIFT);
1593					tmp |= PCIE_LC_LINK_WIDTH_CNTL__LC_UPCONFIGURE_SUPPORT_MASK |
1594					PCIE_LC_LINK_WIDTH_CNTL__LC_RENEGOTIATE_EN_MASK |
1595					PCIE_LC_LINK_WIDTH_CNTL__LC_RECONFIG_NOW_MASK;
1596					WREG32_PCIE(ixPCIE_LC_LINK_WIDTH_CNTL, tmp);
1597				}
1598			}
1599
1600			for (i = 0; i < 10; i++) {
1601				/* check status */
1602				pcie_capability_read_word(adev->pdev,
1603							  PCI_EXP_DEVSTA,
1604							  &tmp16);
1605				if (tmp16 & PCI_EXP_DEVSTA_TRPND)
1606					break;
1607
1608				pcie_capability_read_word(root, PCI_EXP_LNKCTL,
1609							  &bridge_cfg);
1610				pcie_capability_read_word(adev->pdev,
1611							  PCI_EXP_LNKCTL,
1612							  &gpu_cfg);
1613
1614				pcie_capability_read_word(root, PCI_EXP_LNKCTL2,
1615							  &bridge_cfg2);
1616				pcie_capability_read_word(adev->pdev,
1617							  PCI_EXP_LNKCTL2,
1618							  &gpu_cfg2);
1619
1620				tmp = RREG32_PCIE(ixPCIE_LC_CNTL4);
1621				tmp |= PCIE_LC_CNTL4__LC_SET_QUIESCE_MASK;
1622				WREG32_PCIE(ixPCIE_LC_CNTL4, tmp);
1623
1624				tmp = RREG32_PCIE(ixPCIE_LC_CNTL4);
1625				tmp |= PCIE_LC_CNTL4__LC_REDO_EQ_MASK;
1626				WREG32_PCIE(ixPCIE_LC_CNTL4, tmp);
1627
1628				drm_msleep(100);
1629
1630				/* linkctl */
1631				pcie_capability_clear_and_set_word(root, PCI_EXP_LNKCTL,
1632								   PCI_EXP_LNKCTL_HAWD,
1633								   bridge_cfg &
1634								   PCI_EXP_LNKCTL_HAWD);
1635				pcie_capability_clear_and_set_word(adev->pdev, PCI_EXP_LNKCTL,
1636								   PCI_EXP_LNKCTL_HAWD,
1637								   gpu_cfg &
1638								   PCI_EXP_LNKCTL_HAWD);
1639
1640				/* linkctl2 */
1641				pcie_capability_read_word(root, PCI_EXP_LNKCTL2,
1642							  &tmp16);
1643				tmp16 &= ~(PCI_EXP_LNKCTL2_ENTER_COMP |
1644					   PCI_EXP_LNKCTL2_TX_MARGIN);
1645				tmp16 |= (bridge_cfg2 &
1646					  (PCI_EXP_LNKCTL2_ENTER_COMP |
1647					   PCI_EXP_LNKCTL2_TX_MARGIN));
1648				pcie_capability_write_word(root,
1649							   PCI_EXP_LNKCTL2,
1650							   tmp16);
1651
1652				pcie_capability_read_word(adev->pdev,
1653							  PCI_EXP_LNKCTL2,
1654							  &tmp16);
1655				tmp16 &= ~(PCI_EXP_LNKCTL2_ENTER_COMP |
1656					   PCI_EXP_LNKCTL2_TX_MARGIN);
1657				tmp16 |= (gpu_cfg2 &
1658					  (PCI_EXP_LNKCTL2_ENTER_COMP |
1659					   PCI_EXP_LNKCTL2_TX_MARGIN));
1660				pcie_capability_write_word(adev->pdev,
1661							   PCI_EXP_LNKCTL2,
1662							   tmp16);
1663
1664				tmp = RREG32_PCIE(ixPCIE_LC_CNTL4);
1665				tmp &= ~PCIE_LC_CNTL4__LC_SET_QUIESCE_MASK;
1666				WREG32_PCIE(ixPCIE_LC_CNTL4, tmp);
1667			}
1668		}
1669	}
1670
1671	/* set the link speed */
1672	speed_cntl |= PCIE_LC_SPEED_CNTL__LC_FORCE_EN_SW_SPEED_CHANGE_MASK |
1673		PCIE_LC_SPEED_CNTL__LC_FORCE_DIS_HW_SPEED_CHANGE_MASK;
1674	speed_cntl &= ~PCIE_LC_SPEED_CNTL__LC_FORCE_DIS_SW_SPEED_CHANGE_MASK;
1675	WREG32_PCIE(ixPCIE_LC_SPEED_CNTL, speed_cntl);
1676
1677	pcie_capability_read_word(adev->pdev, PCI_EXP_LNKCTL2, &tmp16);
1678	tmp16 &= ~PCI_EXP_LNKCTL2_TLS;
1679
1680	if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3)
1681		tmp16 |= PCI_EXP_LNKCTL2_TLS_8_0GT; /* gen3 */
1682	else if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2)
1683		tmp16 |= PCI_EXP_LNKCTL2_TLS_5_0GT; /* gen2 */
1684	else
1685		tmp16 |= PCI_EXP_LNKCTL2_TLS_2_5GT; /* gen1 */
1686	pcie_capability_write_word(adev->pdev, PCI_EXP_LNKCTL2, tmp16);
1687
1688	speed_cntl = RREG32_PCIE(ixPCIE_LC_SPEED_CNTL);
1689	speed_cntl |= PCIE_LC_SPEED_CNTL__LC_INITIATE_LINK_SPEED_CHANGE_MASK;
1690	WREG32_PCIE(ixPCIE_LC_SPEED_CNTL, speed_cntl);
1691
1692	for (i = 0; i < adev->usec_timeout; i++) {
1693		speed_cntl = RREG32_PCIE(ixPCIE_LC_SPEED_CNTL);
1694		if ((speed_cntl & PCIE_LC_SPEED_CNTL__LC_INITIATE_LINK_SPEED_CHANGE_MASK) == 0)
1695			break;
1696		udelay(1);
1697	}
1698}
1699
1700static void cik_program_aspm(struct amdgpu_device *adev)
1701{
1702	u32 data, orig;
1703	bool disable_l0s = false, disable_l1 = false, disable_plloff_in_l1 = false;
1704	bool disable_clkreq = false;
1705
1706	if (!amdgpu_device_should_use_aspm(adev))
1707		return;
1708
1709	if (pci_is_root_bus(adev->pdev->bus))
1710		return;
1711
1712	/* XXX double check APUs */
1713	if (adev->flags & AMD_IS_APU)
1714		return;
1715
1716	orig = data = RREG32_PCIE(ixPCIE_LC_N_FTS_CNTL);
1717	data &= ~PCIE_LC_N_FTS_CNTL__LC_XMIT_N_FTS_MASK;
1718	data |= (0x24 << PCIE_LC_N_FTS_CNTL__LC_XMIT_N_FTS__SHIFT) |
1719		PCIE_LC_N_FTS_CNTL__LC_XMIT_N_FTS_OVERRIDE_EN_MASK;
1720	if (orig != data)
1721		WREG32_PCIE(ixPCIE_LC_N_FTS_CNTL, data);
1722
1723	orig = data = RREG32_PCIE(ixPCIE_LC_CNTL3);
1724	data |= PCIE_LC_CNTL3__LC_GO_TO_RECOVERY_MASK;
1725	if (orig != data)
1726		WREG32_PCIE(ixPCIE_LC_CNTL3, data);
1727
1728	orig = data = RREG32_PCIE(ixPCIE_P_CNTL);
1729	data |= PCIE_P_CNTL__P_IGNORE_EDB_ERR_MASK;
1730	if (orig != data)
1731		WREG32_PCIE(ixPCIE_P_CNTL, data);
1732
1733	orig = data = RREG32_PCIE(ixPCIE_LC_CNTL);
1734	data &= ~(PCIE_LC_CNTL__LC_L0S_INACTIVITY_MASK |
1735		PCIE_LC_CNTL__LC_L1_INACTIVITY_MASK);
1736	data |= PCIE_LC_CNTL__LC_PMI_TO_L1_DIS_MASK;
1737	if (!disable_l0s)
1738		data |= (7 << PCIE_LC_CNTL__LC_L0S_INACTIVITY__SHIFT);
1739
1740	if (!disable_l1) {
1741		data |= (7 << PCIE_LC_CNTL__LC_L1_INACTIVITY__SHIFT);
1742		data &= ~PCIE_LC_CNTL__LC_PMI_TO_L1_DIS_MASK;
1743		if (orig != data)
1744			WREG32_PCIE(ixPCIE_LC_CNTL, data);
1745
1746		if (!disable_plloff_in_l1) {
1747			bool clk_req_support;
1748
1749			orig = data = RREG32_PCIE(ixPB0_PIF_PWRDOWN_0);
1750			data &= ~(PB0_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_OFF_0_MASK |
1751				PB0_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_TXS2_0_MASK);
1752			data |= (7 << PB0_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_OFF_0__SHIFT) |
1753				(7 << PB0_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_TXS2_0__SHIFT);
1754			if (orig != data)
1755				WREG32_PCIE(ixPB0_PIF_PWRDOWN_0, data);
1756
1757			orig = data = RREG32_PCIE(ixPB0_PIF_PWRDOWN_1);
1758			data &= ~(PB0_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_OFF_1_MASK |
1759				PB0_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_TXS2_1_MASK);
1760			data |= (7 << PB0_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_OFF_1__SHIFT) |
1761				(7 << PB0_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_TXS2_1__SHIFT);
1762			if (orig != data)
1763				WREG32_PCIE(ixPB0_PIF_PWRDOWN_1, data);
1764
1765			orig = data = RREG32_PCIE(ixPB1_PIF_PWRDOWN_0);
1766			data &= ~(PB1_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_OFF_0_MASK |
1767				PB1_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_TXS2_0_MASK);
1768			data |= (7 << PB1_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_OFF_0__SHIFT) |
1769				(7 << PB1_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_TXS2_0__SHIFT);
1770			if (orig != data)
1771				WREG32_PCIE(ixPB1_PIF_PWRDOWN_0, data);
1772
1773			orig = data = RREG32_PCIE(ixPB1_PIF_PWRDOWN_1);
1774			data &= ~(PB1_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_OFF_1_MASK |
1775				PB1_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_TXS2_1_MASK);
1776			data |= (7 << PB1_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_OFF_1__SHIFT) |
1777				(7 << PB1_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_TXS2_1__SHIFT);
1778			if (orig != data)
1779				WREG32_PCIE(ixPB1_PIF_PWRDOWN_1, data);
1780
1781			orig = data = RREG32_PCIE(ixPCIE_LC_LINK_WIDTH_CNTL);
1782			data &= ~PCIE_LC_LINK_WIDTH_CNTL__LC_DYN_LANES_PWR_STATE_MASK;
1783			data |= ~(3 << PCIE_LC_LINK_WIDTH_CNTL__LC_DYN_LANES_PWR_STATE__SHIFT);
1784			if (orig != data)
1785				WREG32_PCIE(ixPCIE_LC_LINK_WIDTH_CNTL, data);
1786
1787			if (!disable_clkreq) {
1788				struct pci_dev *root = adev->pdev->bus->self;
1789				u32 lnkcap;
1790
1791				clk_req_support = false;
1792				pcie_capability_read_dword(root, PCI_EXP_LNKCAP, &lnkcap);
1793				if (lnkcap & PCI_EXP_LNKCAP_CLKPM)
1794					clk_req_support = true;
1795			} else {
1796				clk_req_support = false;
1797			}
1798
1799			if (clk_req_support) {
1800				orig = data = RREG32_PCIE(ixPCIE_LC_CNTL2);
1801				data |= PCIE_LC_CNTL2__LC_ALLOW_PDWN_IN_L1_MASK |
1802					PCIE_LC_CNTL2__LC_ALLOW_PDWN_IN_L23_MASK;
1803				if (orig != data)
1804					WREG32_PCIE(ixPCIE_LC_CNTL2, data);
1805
1806				orig = data = RREG32_SMC(ixTHM_CLK_CNTL);
1807				data &= ~(THM_CLK_CNTL__CMON_CLK_SEL_MASK |
1808					THM_CLK_CNTL__TMON_CLK_SEL_MASK);
1809				data |= (1 << THM_CLK_CNTL__CMON_CLK_SEL__SHIFT) |
1810					(1 << THM_CLK_CNTL__TMON_CLK_SEL__SHIFT);
1811				if (orig != data)
1812					WREG32_SMC(ixTHM_CLK_CNTL, data);
1813
1814				orig = data = RREG32_SMC(ixMISC_CLK_CTRL);
1815				data &= ~(MISC_CLK_CTRL__DEEP_SLEEP_CLK_SEL_MASK |
1816					MISC_CLK_CTRL__ZCLK_SEL_MASK);
1817				data |= (1 << MISC_CLK_CTRL__DEEP_SLEEP_CLK_SEL__SHIFT) |
1818					(1 << MISC_CLK_CTRL__ZCLK_SEL__SHIFT);
1819				if (orig != data)
1820					WREG32_SMC(ixMISC_CLK_CTRL, data);
1821
1822				orig = data = RREG32_SMC(ixCG_CLKPIN_CNTL);
1823				data &= ~CG_CLKPIN_CNTL__BCLK_AS_XCLK_MASK;
1824				if (orig != data)
1825					WREG32_SMC(ixCG_CLKPIN_CNTL, data);
1826
1827				orig = data = RREG32_SMC(ixCG_CLKPIN_CNTL_2);
1828				data &= ~CG_CLKPIN_CNTL_2__FORCE_BIF_REFCLK_EN_MASK;
1829				if (orig != data)
1830					WREG32_SMC(ixCG_CLKPIN_CNTL_2, data);
1831
1832				orig = data = RREG32_SMC(ixMPLL_BYPASSCLK_SEL);
1833				data &= ~MPLL_BYPASSCLK_SEL__MPLL_CLKOUT_SEL_MASK;
1834				data |= (4 << MPLL_BYPASSCLK_SEL__MPLL_CLKOUT_SEL__SHIFT);
1835				if (orig != data)
1836					WREG32_SMC(ixMPLL_BYPASSCLK_SEL, data);
1837			}
1838		}
1839	} else {
1840		if (orig != data)
1841			WREG32_PCIE(ixPCIE_LC_CNTL, data);
1842	}
1843
1844	orig = data = RREG32_PCIE(ixPCIE_CNTL2);
1845	data |= PCIE_CNTL2__SLV_MEM_LS_EN_MASK |
1846		PCIE_CNTL2__MST_MEM_LS_EN_MASK |
1847		PCIE_CNTL2__REPLAY_MEM_LS_EN_MASK;
1848	if (orig != data)
1849		WREG32_PCIE(ixPCIE_CNTL2, data);
1850
1851	if (!disable_l0s) {
1852		data = RREG32_PCIE(ixPCIE_LC_N_FTS_CNTL);
1853		if ((data & PCIE_LC_N_FTS_CNTL__LC_N_FTS_MASK) ==
1854				PCIE_LC_N_FTS_CNTL__LC_N_FTS_MASK) {
1855			data = RREG32_PCIE(ixPCIE_LC_STATUS1);
1856			if ((data & PCIE_LC_STATUS1__LC_REVERSE_XMIT_MASK) &&
1857			(data & PCIE_LC_STATUS1__LC_REVERSE_RCVR_MASK)) {
1858				orig = data = RREG32_PCIE(ixPCIE_LC_CNTL);
1859				data &= ~PCIE_LC_CNTL__LC_L0S_INACTIVITY_MASK;
1860				if (orig != data)
1861					WREG32_PCIE(ixPCIE_LC_CNTL, data);
1862			}
1863		}
1864	}
1865}
1866
1867static uint32_t cik_get_rev_id(struct amdgpu_device *adev)
1868{
1869	return (RREG32(mmCC_DRM_ID_STRAPS) & CC_DRM_ID_STRAPS__ATI_REV_ID_MASK)
1870		>> CC_DRM_ID_STRAPS__ATI_REV_ID__SHIFT;
1871}
1872
1873static void cik_flush_hdp(struct amdgpu_device *adev, struct amdgpu_ring *ring)
1874{
1875	if (!ring || !ring->funcs->emit_wreg) {
1876		WREG32(mmHDP_MEM_COHERENCY_FLUSH_CNTL, 1);
1877		RREG32(mmHDP_MEM_COHERENCY_FLUSH_CNTL);
1878	} else {
1879		amdgpu_ring_emit_wreg(ring, mmHDP_MEM_COHERENCY_FLUSH_CNTL, 1);
1880	}
1881}
1882
1883static void cik_invalidate_hdp(struct amdgpu_device *adev,
1884			       struct amdgpu_ring *ring)
1885{
1886	if (!ring || !ring->funcs->emit_wreg) {
1887		WREG32(mmHDP_DEBUG0, 1);
1888		RREG32(mmHDP_DEBUG0);
1889	} else {
1890		amdgpu_ring_emit_wreg(ring, mmHDP_DEBUG0, 1);
1891	}
1892}
1893
1894static bool cik_need_full_reset(struct amdgpu_device *adev)
1895{
1896	/* change this when we support soft reset */
1897	return true;
1898}
1899
1900static void cik_get_pcie_usage(struct amdgpu_device *adev, uint64_t *count0,
1901			       uint64_t *count1)
1902{
1903	uint32_t perfctr = 0;
1904	uint64_t cnt0_of, cnt1_of;
1905	int tmp;
1906
1907	/* This reports 0 on APUs, so return to avoid writing/reading registers
1908	 * that may or may not be different from their GPU counterparts
1909	 */
1910	if (adev->flags & AMD_IS_APU)
1911		return;
1912
1913	/* Set the 2 events that we wish to watch, defined above */
1914	/* Reg 40 is # received msgs, Reg 104 is # of posted requests sent */
1915	perfctr = REG_SET_FIELD(perfctr, PCIE_PERF_CNTL_TXCLK, EVENT0_SEL, 40);
1916	perfctr = REG_SET_FIELD(perfctr, PCIE_PERF_CNTL_TXCLK, EVENT1_SEL, 104);
1917
1918	/* Write to enable desired perf counters */
1919	WREG32_PCIE(ixPCIE_PERF_CNTL_TXCLK, perfctr);
1920	/* Zero out and enable the perf counters
1921	 * Write 0x5:
1922	 * Bit 0 = Start all counters(1)
1923	 * Bit 2 = Global counter reset enable(1)
1924	 */
1925	WREG32_PCIE(ixPCIE_PERF_COUNT_CNTL, 0x00000005);
1926
1927	drm_msleep(1000);
1928
1929	/* Load the shadow and disable the perf counters
1930	 * Write 0x2:
1931	 * Bit 0 = Stop counters(0)
1932	 * Bit 1 = Load the shadow counters(1)
1933	 */
1934	WREG32_PCIE(ixPCIE_PERF_COUNT_CNTL, 0x00000002);
1935
1936	/* Read register values to get any >32bit overflow */
1937	tmp = RREG32_PCIE(ixPCIE_PERF_CNTL_TXCLK);
1938	cnt0_of = REG_GET_FIELD(tmp, PCIE_PERF_CNTL_TXCLK, COUNTER0_UPPER);
1939	cnt1_of = REG_GET_FIELD(tmp, PCIE_PERF_CNTL_TXCLK, COUNTER1_UPPER);
1940
1941	/* Get the values and add the overflow */
1942	*count0 = RREG32_PCIE(ixPCIE_PERF_COUNT0_TXCLK) | (cnt0_of << 32);
1943	*count1 = RREG32_PCIE(ixPCIE_PERF_COUNT1_TXCLK) | (cnt1_of << 32);
1944}
1945
1946static bool cik_need_reset_on_init(struct amdgpu_device *adev)
1947{
1948	u32 clock_cntl, pc;
1949
1950	if (adev->flags & AMD_IS_APU)
1951		return false;
1952
1953	/* check if the SMC is already running */
1954	clock_cntl = RREG32_SMC(ixSMC_SYSCON_CLOCK_CNTL_0);
1955	pc = RREG32_SMC(ixSMC_PC_C);
1956	if ((0 == REG_GET_FIELD(clock_cntl, SMC_SYSCON_CLOCK_CNTL_0, ck_disable)) &&
1957	    (0x20100 <= pc))
1958		return true;
1959
1960	return false;
1961}
1962
1963static uint64_t cik_get_pcie_replay_count(struct amdgpu_device *adev)
1964{
1965	uint64_t nak_r, nak_g;
1966
1967	/* Get the number of NAKs received and generated */
1968	nak_r = RREG32_PCIE(ixPCIE_RX_NUM_NAK);
1969	nak_g = RREG32_PCIE(ixPCIE_RX_NUM_NAK_GENERATED);
1970
1971	/* Add the total number of NAKs, i.e the number of replays */
1972	return (nak_r + nak_g);
1973}
1974
1975static void cik_pre_asic_init(struct amdgpu_device *adev)
1976{
1977}
1978
1979static const struct amdgpu_asic_funcs cik_asic_funcs =
1980{
1981	.read_disabled_bios = &cik_read_disabled_bios,
1982	.read_bios_from_rom = &cik_read_bios_from_rom,
1983	.read_register = &cik_read_register,
1984	.reset = &cik_asic_reset,
1985	.reset_method = &cik_asic_reset_method,
1986	.set_vga_state = &cik_vga_set_state,
1987	.get_xclk = &cik_get_xclk,
1988	.set_uvd_clocks = &cik_set_uvd_clocks,
1989	.set_vce_clocks = &cik_set_vce_clocks,
1990	.get_config_memsize = &cik_get_config_memsize,
1991	.flush_hdp = &cik_flush_hdp,
1992	.invalidate_hdp = &cik_invalidate_hdp,
1993	.need_full_reset = &cik_need_full_reset,
1994	.init_doorbell_index = &legacy_doorbell_index_init,
1995	.get_pcie_usage = &cik_get_pcie_usage,
1996	.need_reset_on_init = &cik_need_reset_on_init,
1997	.get_pcie_replay_count = &cik_get_pcie_replay_count,
1998	.supports_baco = &cik_asic_supports_baco,
1999	.pre_asic_init = &cik_pre_asic_init,
2000	.query_video_codecs = &cik_query_video_codecs,
2001};
2002
2003static int cik_common_early_init(void *handle)
2004{
2005	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2006
2007	adev->smc_rreg = &cik_smc_rreg;
2008	adev->smc_wreg = &cik_smc_wreg;
2009	adev->pcie_rreg = &cik_pcie_rreg;
2010	adev->pcie_wreg = &cik_pcie_wreg;
2011	adev->uvd_ctx_rreg = &cik_uvd_ctx_rreg;
2012	adev->uvd_ctx_wreg = &cik_uvd_ctx_wreg;
2013	adev->didt_rreg = &cik_didt_rreg;
2014	adev->didt_wreg = &cik_didt_wreg;
2015
2016	adev->asic_funcs = &cik_asic_funcs;
2017
2018	adev->rev_id = cik_get_rev_id(adev);
2019	adev->external_rev_id = 0xFF;
2020	switch (adev->asic_type) {
2021	case CHIP_BONAIRE:
2022		adev->cg_flags =
2023			AMD_CG_SUPPORT_GFX_MGCG |
2024			AMD_CG_SUPPORT_GFX_MGLS |
2025			/*AMD_CG_SUPPORT_GFX_CGCG |*/
2026			AMD_CG_SUPPORT_GFX_CGLS |
2027			AMD_CG_SUPPORT_GFX_CGTS |
2028			AMD_CG_SUPPORT_GFX_CGTS_LS |
2029			AMD_CG_SUPPORT_GFX_CP_LS |
2030			AMD_CG_SUPPORT_MC_LS |
2031			AMD_CG_SUPPORT_MC_MGCG |
2032			AMD_CG_SUPPORT_SDMA_MGCG |
2033			AMD_CG_SUPPORT_SDMA_LS |
2034			AMD_CG_SUPPORT_BIF_LS |
2035			AMD_CG_SUPPORT_VCE_MGCG |
2036			AMD_CG_SUPPORT_UVD_MGCG |
2037			AMD_CG_SUPPORT_HDP_LS |
2038			AMD_CG_SUPPORT_HDP_MGCG;
2039		adev->pg_flags = 0;
2040		adev->external_rev_id = adev->rev_id + 0x14;
2041		break;
2042	case CHIP_HAWAII:
2043		adev->cg_flags =
2044			AMD_CG_SUPPORT_GFX_MGCG |
2045			AMD_CG_SUPPORT_GFX_MGLS |
2046			/*AMD_CG_SUPPORT_GFX_CGCG |*/
2047			AMD_CG_SUPPORT_GFX_CGLS |
2048			AMD_CG_SUPPORT_GFX_CGTS |
2049			AMD_CG_SUPPORT_GFX_CP_LS |
2050			AMD_CG_SUPPORT_MC_LS |
2051			AMD_CG_SUPPORT_MC_MGCG |
2052			AMD_CG_SUPPORT_SDMA_MGCG |
2053			AMD_CG_SUPPORT_SDMA_LS |
2054			AMD_CG_SUPPORT_BIF_LS |
2055			AMD_CG_SUPPORT_VCE_MGCG |
2056			AMD_CG_SUPPORT_UVD_MGCG |
2057			AMD_CG_SUPPORT_HDP_LS |
2058			AMD_CG_SUPPORT_HDP_MGCG;
2059		adev->pg_flags = 0;
2060		adev->external_rev_id = 0x28;
2061		break;
2062	case CHIP_KAVERI:
2063		adev->cg_flags =
2064			AMD_CG_SUPPORT_GFX_MGCG |
2065			AMD_CG_SUPPORT_GFX_MGLS |
2066			/*AMD_CG_SUPPORT_GFX_CGCG |*/
2067			AMD_CG_SUPPORT_GFX_CGLS |
2068			AMD_CG_SUPPORT_GFX_CGTS |
2069			AMD_CG_SUPPORT_GFX_CGTS_LS |
2070			AMD_CG_SUPPORT_GFX_CP_LS |
2071			AMD_CG_SUPPORT_SDMA_MGCG |
2072			AMD_CG_SUPPORT_SDMA_LS |
2073			AMD_CG_SUPPORT_BIF_LS |
2074			AMD_CG_SUPPORT_VCE_MGCG |
2075			AMD_CG_SUPPORT_UVD_MGCG |
2076			AMD_CG_SUPPORT_HDP_LS |
2077			AMD_CG_SUPPORT_HDP_MGCG;
2078		adev->pg_flags =
2079			/*AMD_PG_SUPPORT_GFX_PG |
2080			  AMD_PG_SUPPORT_GFX_SMG |
2081			  AMD_PG_SUPPORT_GFX_DMG |*/
2082			AMD_PG_SUPPORT_UVD |
2083			AMD_PG_SUPPORT_VCE |
2084			/*  AMD_PG_SUPPORT_CP |
2085			  AMD_PG_SUPPORT_GDS |
2086			  AMD_PG_SUPPORT_RLC_SMU_HS |
2087			  AMD_PG_SUPPORT_ACP |
2088			  AMD_PG_SUPPORT_SAMU |*/
2089			0;
2090		if (adev->pdev->device == 0x1312 ||
2091			adev->pdev->device == 0x1316 ||
2092			adev->pdev->device == 0x1317)
2093			adev->external_rev_id = 0x41;
2094		else
2095			adev->external_rev_id = 0x1;
2096		break;
2097	case CHIP_KABINI:
2098	case CHIP_MULLINS:
2099		adev->cg_flags =
2100			AMD_CG_SUPPORT_GFX_MGCG |
2101			AMD_CG_SUPPORT_GFX_MGLS |
2102			/*AMD_CG_SUPPORT_GFX_CGCG |*/
2103			AMD_CG_SUPPORT_GFX_CGLS |
2104			AMD_CG_SUPPORT_GFX_CGTS |
2105			AMD_CG_SUPPORT_GFX_CGTS_LS |
2106			AMD_CG_SUPPORT_GFX_CP_LS |
2107			AMD_CG_SUPPORT_SDMA_MGCG |
2108			AMD_CG_SUPPORT_SDMA_LS |
2109			AMD_CG_SUPPORT_BIF_LS |
2110			AMD_CG_SUPPORT_VCE_MGCG |
2111			AMD_CG_SUPPORT_UVD_MGCG |
2112			AMD_CG_SUPPORT_HDP_LS |
2113			AMD_CG_SUPPORT_HDP_MGCG;
2114		adev->pg_flags =
2115			/*AMD_PG_SUPPORT_GFX_PG |
2116			  AMD_PG_SUPPORT_GFX_SMG | */
2117			AMD_PG_SUPPORT_UVD |
2118			/*AMD_PG_SUPPORT_VCE |
2119			  AMD_PG_SUPPORT_CP |
2120			  AMD_PG_SUPPORT_GDS |
2121			  AMD_PG_SUPPORT_RLC_SMU_HS |
2122			  AMD_PG_SUPPORT_SAMU |*/
2123			0;
2124		if (adev->asic_type == CHIP_KABINI) {
2125			if (adev->rev_id == 0)
2126				adev->external_rev_id = 0x81;
2127			else if (adev->rev_id == 1)
2128				adev->external_rev_id = 0x82;
2129			else if (adev->rev_id == 2)
2130				adev->external_rev_id = 0x85;
2131		} else
2132			adev->external_rev_id = adev->rev_id + 0xa1;
2133		break;
2134	default:
2135		/* FIXME: not supported yet */
2136		return -EINVAL;
2137	}
2138
2139	return 0;
2140}
2141
2142static int cik_common_sw_init(void *handle)
2143{
2144	return 0;
2145}
2146
2147static int cik_common_sw_fini(void *handle)
2148{
2149	return 0;
2150}
2151
2152static int cik_common_hw_init(void *handle)
2153{
2154	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2155
2156	/* move the golden regs per IP block */
2157	cik_init_golden_registers(adev);
2158	/* enable pcie gen2/3 link */
2159	cik_pcie_gen3_enable(adev);
2160	/* enable aspm */
2161	cik_program_aspm(adev);
2162
2163	return 0;
2164}
2165
2166static int cik_common_hw_fini(void *handle)
2167{
2168	return 0;
2169}
2170
2171static int cik_common_suspend(void *handle)
2172{
2173	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2174
2175	return cik_common_hw_fini(adev);
2176}
2177
2178static int cik_common_resume(void *handle)
2179{
2180	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2181
2182	return cik_common_hw_init(adev);
2183}
2184
2185static bool cik_common_is_idle(void *handle)
2186{
2187	return true;
2188}
2189
2190static int cik_common_wait_for_idle(void *handle)
2191{
2192	return 0;
2193}
2194
2195static int cik_common_soft_reset(void *handle)
2196{
2197	/* XXX hard reset?? */
2198	return 0;
2199}
2200
2201static int cik_common_set_clockgating_state(void *handle,
2202					    enum amd_clockgating_state state)
2203{
2204	return 0;
2205}
2206
2207static int cik_common_set_powergating_state(void *handle,
2208					    enum amd_powergating_state state)
2209{
2210	return 0;
2211}
2212
2213static const struct amd_ip_funcs cik_common_ip_funcs = {
2214	.name = "cik_common",
2215	.early_init = cik_common_early_init,
2216	.late_init = NULL,
2217	.sw_init = cik_common_sw_init,
2218	.sw_fini = cik_common_sw_fini,
2219	.hw_init = cik_common_hw_init,
2220	.hw_fini = cik_common_hw_fini,
2221	.suspend = cik_common_suspend,
2222	.resume = cik_common_resume,
2223	.is_idle = cik_common_is_idle,
2224	.wait_for_idle = cik_common_wait_for_idle,
2225	.soft_reset = cik_common_soft_reset,
2226	.set_clockgating_state = cik_common_set_clockgating_state,
2227	.set_powergating_state = cik_common_set_powergating_state,
2228};
2229
2230static const struct amdgpu_ip_block_version cik_common_ip_block =
2231{
2232	.type = AMD_IP_BLOCK_TYPE_COMMON,
2233	.major = 1,
2234	.minor = 0,
2235	.rev = 0,
2236	.funcs = &cik_common_ip_funcs,
2237};
2238
2239int cik_set_ip_blocks(struct amdgpu_device *adev)
2240{
2241	switch (adev->asic_type) {
2242	case CHIP_BONAIRE:
2243		amdgpu_device_ip_block_add(adev, &cik_common_ip_block);
2244		amdgpu_device_ip_block_add(adev, &gmc_v7_0_ip_block);
2245		amdgpu_device_ip_block_add(adev, &cik_ih_ip_block);
2246		amdgpu_device_ip_block_add(adev, &gfx_v7_2_ip_block);
2247		amdgpu_device_ip_block_add(adev, &cik_sdma_ip_block);
2248		amdgpu_device_ip_block_add(adev, &pp_smu_ip_block);
2249		if (adev->enable_virtual_display)
2250			amdgpu_device_ip_block_add(adev, &amdgpu_vkms_ip_block);
2251#if defined(CONFIG_DRM_AMD_DC)
2252		else if (amdgpu_device_has_dc_support(adev))
2253			amdgpu_device_ip_block_add(adev, &dm_ip_block);
2254#endif
2255		else
2256			amdgpu_device_ip_block_add(adev, &dce_v8_2_ip_block);
2257		amdgpu_device_ip_block_add(adev, &uvd_v4_2_ip_block);
2258		amdgpu_device_ip_block_add(adev, &vce_v2_0_ip_block);
2259		break;
2260	case CHIP_HAWAII:
2261		amdgpu_device_ip_block_add(adev, &cik_common_ip_block);
2262		amdgpu_device_ip_block_add(adev, &gmc_v7_0_ip_block);
2263		amdgpu_device_ip_block_add(adev, &cik_ih_ip_block);
2264		amdgpu_device_ip_block_add(adev, &gfx_v7_3_ip_block);
2265		amdgpu_device_ip_block_add(adev, &cik_sdma_ip_block);
2266		amdgpu_device_ip_block_add(adev, &pp_smu_ip_block);
2267		if (adev->enable_virtual_display)
2268			amdgpu_device_ip_block_add(adev, &amdgpu_vkms_ip_block);
2269#if defined(CONFIG_DRM_AMD_DC)
2270		else if (amdgpu_device_has_dc_support(adev))
2271			amdgpu_device_ip_block_add(adev, &dm_ip_block);
2272#endif
2273		else
2274			amdgpu_device_ip_block_add(adev, &dce_v8_5_ip_block);
2275		amdgpu_device_ip_block_add(adev, &uvd_v4_2_ip_block);
2276		amdgpu_device_ip_block_add(adev, &vce_v2_0_ip_block);
2277		break;
2278	case CHIP_KAVERI:
2279		amdgpu_device_ip_block_add(adev, &cik_common_ip_block);
2280		amdgpu_device_ip_block_add(adev, &gmc_v7_0_ip_block);
2281		amdgpu_device_ip_block_add(adev, &cik_ih_ip_block);
2282		amdgpu_device_ip_block_add(adev, &gfx_v7_1_ip_block);
2283		amdgpu_device_ip_block_add(adev, &cik_sdma_ip_block);
2284		amdgpu_device_ip_block_add(adev, &kv_smu_ip_block);
2285		if (adev->enable_virtual_display)
2286			amdgpu_device_ip_block_add(adev, &amdgpu_vkms_ip_block);
2287#if defined(CONFIG_DRM_AMD_DC)
2288		else if (amdgpu_device_has_dc_support(adev))
2289			amdgpu_device_ip_block_add(adev, &dm_ip_block);
2290#endif
2291		else
2292			amdgpu_device_ip_block_add(adev, &dce_v8_1_ip_block);
2293
2294		amdgpu_device_ip_block_add(adev, &uvd_v4_2_ip_block);
2295		amdgpu_device_ip_block_add(adev, &vce_v2_0_ip_block);
2296		break;
2297	case CHIP_KABINI:
2298	case CHIP_MULLINS:
2299		amdgpu_device_ip_block_add(adev, &cik_common_ip_block);
2300		amdgpu_device_ip_block_add(adev, &gmc_v7_0_ip_block);
2301		amdgpu_device_ip_block_add(adev, &cik_ih_ip_block);
2302		amdgpu_device_ip_block_add(adev, &gfx_v7_2_ip_block);
2303		amdgpu_device_ip_block_add(adev, &cik_sdma_ip_block);
2304		amdgpu_device_ip_block_add(adev, &kv_smu_ip_block);
2305		if (adev->enable_virtual_display)
2306			amdgpu_device_ip_block_add(adev, &amdgpu_vkms_ip_block);
2307#if defined(CONFIG_DRM_AMD_DC)
2308		else if (amdgpu_device_has_dc_support(adev))
2309			amdgpu_device_ip_block_add(adev, &dm_ip_block);
2310#endif
2311		else
2312			amdgpu_device_ip_block_add(adev, &dce_v8_3_ip_block);
2313		amdgpu_device_ip_block_add(adev, &uvd_v4_2_ip_block);
2314		amdgpu_device_ip_block_add(adev, &vce_v2_0_ip_block);
2315		break;
2316	default:
2317		/* FIXME: not supported yet */
2318		return -EINVAL;
2319	}
2320	return 0;
2321}
2322