1/*	$NetBSD: amdgpu_cik.c,v 1.7 2023/09/30 10:46:45 mrg Exp $	*/
2
3/*
4 * Copyright 2012 Advanced Micro Devices, Inc.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 * and/or sell copies of the Software, and to permit persons to whom the
11 * Software is furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
19 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
22 * OTHER DEALINGS IN THE SOFTWARE.
23 *
24 * Authors: Alex Deucher
25 */
26#include <sys/cdefs.h>
27__KERNEL_RCSID(0, "$NetBSD: amdgpu_cik.c,v 1.7 2023/09/30 10:46:45 mrg Exp $");
28
29#include <linux/firmware.h>
30#include <linux/slab.h>
31#include <linux/module.h>
32#include <linux/pci.h>
33
34#include "amdgpu.h"
35#include "amdgpu_atombios.h"
36#include "amdgpu_ih.h"
37#include "amdgpu_uvd.h"
38#include "amdgpu_vce.h"
39#include "cikd.h"
40#include "atom.h"
41#include "amd_pcie.h"
42
43#include "cik.h"
44#include "gmc_v7_0.h"
45#include "cik_ih.h"
46#include "dce_v8_0.h"
47#include "gfx_v7_0.h"
48#include "cik_sdma.h"
49#include "uvd_v4_2.h"
50#include "vce_v2_0.h"
51#include "cik_dpm.h"
52
53#include "uvd/uvd_4_2_d.h"
54
55#include "smu/smu_7_0_1_d.h"
56#include "smu/smu_7_0_1_sh_mask.h"
57
58#include "dce/dce_8_0_d.h"
59#include "dce/dce_8_0_sh_mask.h"
60
61#include "bif/bif_4_1_d.h"
62#include "bif/bif_4_1_sh_mask.h"
63
64#include "gca/gfx_7_2_d.h"
65#include "gca/gfx_7_2_enum.h"
66#include "gca/gfx_7_2_sh_mask.h"
67
68#include "gmc/gmc_7_1_d.h"
69#include "gmc/gmc_7_1_sh_mask.h"
70
71#include "oss/oss_2_0_d.h"
72#include "oss/oss_2_0_sh_mask.h"
73
74#include "amdgpu_dm.h"
75#include "amdgpu_amdkfd.h"
76#include "dce_virtual.h"
77
78#include <linux/nbsd-namespace.h>
79
80/*
81 * Indirect registers accessor
82 */
83static u32 cik_pcie_rreg(struct amdgpu_device *adev, u32 reg)
84{
85	unsigned long flags;
86	u32 r;
87
88	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
89	WREG32(mmPCIE_INDEX, reg);
90	(void)RREG32(mmPCIE_INDEX);
91	r = RREG32(mmPCIE_DATA);
92	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
93	return r;
94}
95
96static void cik_pcie_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
97{
98	unsigned long flags;
99
100	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
101	WREG32(mmPCIE_INDEX, reg);
102	(void)RREG32(mmPCIE_INDEX);
103	WREG32(mmPCIE_DATA, v);
104	(void)RREG32(mmPCIE_DATA);
105	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
106}
107
108static u32 cik_smc_rreg(struct amdgpu_device *adev, u32 reg)
109{
110	unsigned long flags;
111	u32 r;
112
113	spin_lock_irqsave(&adev->smc_idx_lock, flags);
114	WREG32(mmSMC_IND_INDEX_0, (reg));
115	r = RREG32(mmSMC_IND_DATA_0);
116	spin_unlock_irqrestore(&adev->smc_idx_lock, flags);
117	return r;
118}
119
120static void cik_smc_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
121{
122	unsigned long flags;
123
124	spin_lock_irqsave(&adev->smc_idx_lock, flags);
125	WREG32(mmSMC_IND_INDEX_0, (reg));
126	WREG32(mmSMC_IND_DATA_0, (v));
127	spin_unlock_irqrestore(&adev->smc_idx_lock, flags);
128}
129
130static u32 cik_uvd_ctx_rreg(struct amdgpu_device *adev, u32 reg)
131{
132	unsigned long flags;
133	u32 r;
134
135	spin_lock_irqsave(&adev->uvd_ctx_idx_lock, flags);
136	WREG32(mmUVD_CTX_INDEX, ((reg) & 0x1ff));
137	r = RREG32(mmUVD_CTX_DATA);
138	spin_unlock_irqrestore(&adev->uvd_ctx_idx_lock, flags);
139	return r;
140}
141
142static void cik_uvd_ctx_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
143{
144	unsigned long flags;
145
146	spin_lock_irqsave(&adev->uvd_ctx_idx_lock, flags);
147	WREG32(mmUVD_CTX_INDEX, ((reg) & 0x1ff));
148	WREG32(mmUVD_CTX_DATA, (v));
149	spin_unlock_irqrestore(&adev->uvd_ctx_idx_lock, flags);
150}
151
152static u32 cik_didt_rreg(struct amdgpu_device *adev, u32 reg)
153{
154	unsigned long flags;
155	u32 r;
156
157	spin_lock_irqsave(&adev->didt_idx_lock, flags);
158	WREG32(mmDIDT_IND_INDEX, (reg));
159	r = RREG32(mmDIDT_IND_DATA);
160	spin_unlock_irqrestore(&adev->didt_idx_lock, flags);
161	return r;
162}
163
164static void cik_didt_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
165{
166	unsigned long flags;
167
168	spin_lock_irqsave(&adev->didt_idx_lock, flags);
169	WREG32(mmDIDT_IND_INDEX, (reg));
170	WREG32(mmDIDT_IND_DATA, (v));
171	spin_unlock_irqrestore(&adev->didt_idx_lock, flags);
172}
173
174static const u32 bonaire_golden_spm_registers[] =
175{
176	0xc200, 0xe0ffffff, 0xe0000000
177};
178
179static const u32 bonaire_golden_common_registers[] =
180{
181	0x31dc, 0xffffffff, 0x00000800,
182	0x31dd, 0xffffffff, 0x00000800,
183	0x31e6, 0xffffffff, 0x00007fbf,
184	0x31e7, 0xffffffff, 0x00007faf
185};
186
187static const u32 bonaire_golden_registers[] =
188{
189	0xcd5, 0x00000333, 0x00000333,
190	0xcd4, 0x000c0fc0, 0x00040200,
191	0x2684, 0x00010000, 0x00058208,
192	0xf000, 0xffff1fff, 0x00140000,
193	0xf080, 0xfdfc0fff, 0x00000100,
194	0xf08d, 0x40000000, 0x40000200,
195	0x260c, 0xffffffff, 0x00000000,
196	0x260d, 0xf00fffff, 0x00000400,
197	0x260e, 0x0002021c, 0x00020200,
198	0x31e, 0x00000080, 0x00000000,
199	0x16ec, 0x000000f0, 0x00000070,
200	0x16f0, 0xf0311fff, 0x80300000,
201	0x263e, 0x73773777, 0x12010001,
202	0xd43, 0x00810000, 0x408af000,
203	0x1c0c, 0x31000111, 0x00000011,
204	0xbd2, 0x73773777, 0x12010001,
205	0x883, 0x00007fb6, 0x0021a1b1,
206	0x884, 0x00007fb6, 0x002021b1,
207	0x860, 0x00007fb6, 0x00002191,
208	0x886, 0x00007fb6, 0x002121b1,
209	0x887, 0x00007fb6, 0x002021b1,
210	0x877, 0x00007fb6, 0x00002191,
211	0x878, 0x00007fb6, 0x00002191,
212	0xd8a, 0x0000003f, 0x0000000a,
213	0xd8b, 0x0000003f, 0x0000000a,
214	0xab9, 0x00073ffe, 0x000022a2,
215	0x903, 0x000007ff, 0x00000000,
216	0x2285, 0xf000003f, 0x00000007,
217	0x22fc, 0x00002001, 0x00000001,
218	0x22c9, 0xffffffff, 0x00ffffff,
219	0xc281, 0x0000ff0f, 0x00000000,
220	0xa293, 0x07ffffff, 0x06000000,
221	0x136, 0x00000fff, 0x00000100,
222	0xf9e, 0x00000001, 0x00000002,
223	0x2440, 0x03000000, 0x0362c688,
224	0x2300, 0x000000ff, 0x00000001,
225	0x390, 0x00001fff, 0x00001fff,
226	0x2418, 0x0000007f, 0x00000020,
227	0x2542, 0x00010000, 0x00010000,
228	0x2b05, 0x000003ff, 0x000000f3,
229	0x2b03, 0xffffffff, 0x00001032
230};
231
232static const u32 bonaire_mgcg_cgcg_init[] =
233{
234	0x3108, 0xffffffff, 0xfffffffc,
235	0xc200, 0xffffffff, 0xe0000000,
236	0xf0a8, 0xffffffff, 0x00000100,
237	0xf082, 0xffffffff, 0x00000100,
238	0xf0b0, 0xffffffff, 0xc0000100,
239	0xf0b2, 0xffffffff, 0xc0000100,
240	0xf0b1, 0xffffffff, 0xc0000100,
241	0x1579, 0xffffffff, 0x00600100,
242	0xf0a0, 0xffffffff, 0x00000100,
243	0xf085, 0xffffffff, 0x06000100,
244	0xf088, 0xffffffff, 0x00000100,
245	0xf086, 0xffffffff, 0x06000100,
246	0xf081, 0xffffffff, 0x00000100,
247	0xf0b8, 0xffffffff, 0x00000100,
248	0xf089, 0xffffffff, 0x00000100,
249	0xf080, 0xffffffff, 0x00000100,
250	0xf08c, 0xffffffff, 0x00000100,
251	0xf08d, 0xffffffff, 0x00000100,
252	0xf094, 0xffffffff, 0x00000100,
253	0xf095, 0xffffffff, 0x00000100,
254	0xf096, 0xffffffff, 0x00000100,
255	0xf097, 0xffffffff, 0x00000100,
256	0xf098, 0xffffffff, 0x00000100,
257	0xf09f, 0xffffffff, 0x00000100,
258	0xf09e, 0xffffffff, 0x00000100,
259	0xf084, 0xffffffff, 0x06000100,
260	0xf0a4, 0xffffffff, 0x00000100,
261	0xf09d, 0xffffffff, 0x00000100,
262	0xf0ad, 0xffffffff, 0x00000100,
263	0xf0ac, 0xffffffff, 0x00000100,
264	0xf09c, 0xffffffff, 0x00000100,
265	0xc200, 0xffffffff, 0xe0000000,
266	0xf008, 0xffffffff, 0x00010000,
267	0xf009, 0xffffffff, 0x00030002,
268	0xf00a, 0xffffffff, 0x00040007,
269	0xf00b, 0xffffffff, 0x00060005,
270	0xf00c, 0xffffffff, 0x00090008,
271	0xf00d, 0xffffffff, 0x00010000,
272	0xf00e, 0xffffffff, 0x00030002,
273	0xf00f, 0xffffffff, 0x00040007,
274	0xf010, 0xffffffff, 0x00060005,
275	0xf011, 0xffffffff, 0x00090008,
276	0xf012, 0xffffffff, 0x00010000,
277	0xf013, 0xffffffff, 0x00030002,
278	0xf014, 0xffffffff, 0x00040007,
279	0xf015, 0xffffffff, 0x00060005,
280	0xf016, 0xffffffff, 0x00090008,
281	0xf017, 0xffffffff, 0x00010000,
282	0xf018, 0xffffffff, 0x00030002,
283	0xf019, 0xffffffff, 0x00040007,
284	0xf01a, 0xffffffff, 0x00060005,
285	0xf01b, 0xffffffff, 0x00090008,
286	0xf01c, 0xffffffff, 0x00010000,
287	0xf01d, 0xffffffff, 0x00030002,
288	0xf01e, 0xffffffff, 0x00040007,
289	0xf01f, 0xffffffff, 0x00060005,
290	0xf020, 0xffffffff, 0x00090008,
291	0xf021, 0xffffffff, 0x00010000,
292	0xf022, 0xffffffff, 0x00030002,
293	0xf023, 0xffffffff, 0x00040007,
294	0xf024, 0xffffffff, 0x00060005,
295	0xf025, 0xffffffff, 0x00090008,
296	0xf026, 0xffffffff, 0x00010000,
297	0xf027, 0xffffffff, 0x00030002,
298	0xf028, 0xffffffff, 0x00040007,
299	0xf029, 0xffffffff, 0x00060005,
300	0xf02a, 0xffffffff, 0x00090008,
301	0xf000, 0xffffffff, 0x96e00200,
302	0x21c2, 0xffffffff, 0x00900100,
303	0x3109, 0xffffffff, 0x0020003f,
304	0xe, 0xffffffff, 0x0140001c,
305	0xf, 0x000f0000, 0x000f0000,
306	0x88, 0xffffffff, 0xc060000c,
307	0x89, 0xc0000fff, 0x00000100,
308	0x3e4, 0xffffffff, 0x00000100,
309	0x3e6, 0x00000101, 0x00000000,
310	0x82a, 0xffffffff, 0x00000104,
311	0x1579, 0xff000fff, 0x00000100,
312	0xc33, 0xc0000fff, 0x00000104,
313	0x3079, 0x00000001, 0x00000001,
314	0x3403, 0xff000ff0, 0x00000100,
315	0x3603, 0xff000ff0, 0x00000100
316};
317
318static const u32 spectre_golden_spm_registers[] =
319{
320	0xc200, 0xe0ffffff, 0xe0000000
321};
322
323static const u32 spectre_golden_common_registers[] =
324{
325	0x31dc, 0xffffffff, 0x00000800,
326	0x31dd, 0xffffffff, 0x00000800,
327	0x31e6, 0xffffffff, 0x00007fbf,
328	0x31e7, 0xffffffff, 0x00007faf
329};
330
331static const u32 spectre_golden_registers[] =
332{
333	0xf000, 0xffff1fff, 0x96940200,
334	0xf003, 0xffff0001, 0xff000000,
335	0xf080, 0xfffc0fff, 0x00000100,
336	0x1bb6, 0x00010101, 0x00010000,
337	0x260d, 0xf00fffff, 0x00000400,
338	0x260e, 0xfffffffc, 0x00020200,
339	0x16ec, 0x000000f0, 0x00000070,
340	0x16f0, 0xf0311fff, 0x80300000,
341	0x263e, 0x73773777, 0x12010001,
342	0x26df, 0x00ff0000, 0x00fc0000,
343	0xbd2, 0x73773777, 0x12010001,
344	0x2285, 0xf000003f, 0x00000007,
345	0x22c9, 0xffffffff, 0x00ffffff,
346	0xa0d4, 0x3f3f3fff, 0x00000082,
347	0xa0d5, 0x0000003f, 0x00000000,
348	0xf9e, 0x00000001, 0x00000002,
349	0x244f, 0xffff03df, 0x00000004,
350	0x31da, 0x00000008, 0x00000008,
351	0x2300, 0x000008ff, 0x00000800,
352	0x2542, 0x00010000, 0x00010000,
353	0x2b03, 0xffffffff, 0x54763210,
354	0x853e, 0x01ff01ff, 0x00000002,
355	0x8526, 0x007ff800, 0x00200000,
356	0x8057, 0xffffffff, 0x00000f40,
357	0xc24d, 0xffffffff, 0x00000001
358};
359
360static const u32 spectre_mgcg_cgcg_init[] =
361{
362	0x3108, 0xffffffff, 0xfffffffc,
363	0xc200, 0xffffffff, 0xe0000000,
364	0xf0a8, 0xffffffff, 0x00000100,
365	0xf082, 0xffffffff, 0x00000100,
366	0xf0b0, 0xffffffff, 0x00000100,
367	0xf0b2, 0xffffffff, 0x00000100,
368	0xf0b1, 0xffffffff, 0x00000100,
369	0x1579, 0xffffffff, 0x00600100,
370	0xf0a0, 0xffffffff, 0x00000100,
371	0xf085, 0xffffffff, 0x06000100,
372	0xf088, 0xffffffff, 0x00000100,
373	0xf086, 0xffffffff, 0x06000100,
374	0xf081, 0xffffffff, 0x00000100,
375	0xf0b8, 0xffffffff, 0x00000100,
376	0xf089, 0xffffffff, 0x00000100,
377	0xf080, 0xffffffff, 0x00000100,
378	0xf08c, 0xffffffff, 0x00000100,
379	0xf08d, 0xffffffff, 0x00000100,
380	0xf094, 0xffffffff, 0x00000100,
381	0xf095, 0xffffffff, 0x00000100,
382	0xf096, 0xffffffff, 0x00000100,
383	0xf097, 0xffffffff, 0x00000100,
384	0xf098, 0xffffffff, 0x00000100,
385	0xf09f, 0xffffffff, 0x00000100,
386	0xf09e, 0xffffffff, 0x00000100,
387	0xf084, 0xffffffff, 0x06000100,
388	0xf0a4, 0xffffffff, 0x00000100,
389	0xf09d, 0xffffffff, 0x00000100,
390	0xf0ad, 0xffffffff, 0x00000100,
391	0xf0ac, 0xffffffff, 0x00000100,
392	0xf09c, 0xffffffff, 0x00000100,
393	0xc200, 0xffffffff, 0xe0000000,
394	0xf008, 0xffffffff, 0x00010000,
395	0xf009, 0xffffffff, 0x00030002,
396	0xf00a, 0xffffffff, 0x00040007,
397	0xf00b, 0xffffffff, 0x00060005,
398	0xf00c, 0xffffffff, 0x00090008,
399	0xf00d, 0xffffffff, 0x00010000,
400	0xf00e, 0xffffffff, 0x00030002,
401	0xf00f, 0xffffffff, 0x00040007,
402	0xf010, 0xffffffff, 0x00060005,
403	0xf011, 0xffffffff, 0x00090008,
404	0xf012, 0xffffffff, 0x00010000,
405	0xf013, 0xffffffff, 0x00030002,
406	0xf014, 0xffffffff, 0x00040007,
407	0xf015, 0xffffffff, 0x00060005,
408	0xf016, 0xffffffff, 0x00090008,
409	0xf017, 0xffffffff, 0x00010000,
410	0xf018, 0xffffffff, 0x00030002,
411	0xf019, 0xffffffff, 0x00040007,
412	0xf01a, 0xffffffff, 0x00060005,
413	0xf01b, 0xffffffff, 0x00090008,
414	0xf01c, 0xffffffff, 0x00010000,
415	0xf01d, 0xffffffff, 0x00030002,
416	0xf01e, 0xffffffff, 0x00040007,
417	0xf01f, 0xffffffff, 0x00060005,
418	0xf020, 0xffffffff, 0x00090008,
419	0xf021, 0xffffffff, 0x00010000,
420	0xf022, 0xffffffff, 0x00030002,
421	0xf023, 0xffffffff, 0x00040007,
422	0xf024, 0xffffffff, 0x00060005,
423	0xf025, 0xffffffff, 0x00090008,
424	0xf026, 0xffffffff, 0x00010000,
425	0xf027, 0xffffffff, 0x00030002,
426	0xf028, 0xffffffff, 0x00040007,
427	0xf029, 0xffffffff, 0x00060005,
428	0xf02a, 0xffffffff, 0x00090008,
429	0xf02b, 0xffffffff, 0x00010000,
430	0xf02c, 0xffffffff, 0x00030002,
431	0xf02d, 0xffffffff, 0x00040007,
432	0xf02e, 0xffffffff, 0x00060005,
433	0xf02f, 0xffffffff, 0x00090008,
434	0xf000, 0xffffffff, 0x96e00200,
435	0x21c2, 0xffffffff, 0x00900100,
436	0x3109, 0xffffffff, 0x0020003f,
437	0xe, 0xffffffff, 0x0140001c,
438	0xf, 0x000f0000, 0x000f0000,
439	0x88, 0xffffffff, 0xc060000c,
440	0x89, 0xc0000fff, 0x00000100,
441	0x3e4, 0xffffffff, 0x00000100,
442	0x3e6, 0x00000101, 0x00000000,
443	0x82a, 0xffffffff, 0x00000104,
444	0x1579, 0xff000fff, 0x00000100,
445	0xc33, 0xc0000fff, 0x00000104,
446	0x3079, 0x00000001, 0x00000001,
447	0x3403, 0xff000ff0, 0x00000100,
448	0x3603, 0xff000ff0, 0x00000100
449};
450
451static const u32 kalindi_golden_spm_registers[] =
452{
453	0xc200, 0xe0ffffff, 0xe0000000
454};
455
456static const u32 kalindi_golden_common_registers[] =
457{
458	0x31dc, 0xffffffff, 0x00000800,
459	0x31dd, 0xffffffff, 0x00000800,
460	0x31e6, 0xffffffff, 0x00007fbf,
461	0x31e7, 0xffffffff, 0x00007faf
462};
463
464static const u32 kalindi_golden_registers[] =
465{
466	0xf000, 0xffffdfff, 0x6e944040,
467	0x1579, 0xff607fff, 0xfc000100,
468	0xf088, 0xff000fff, 0x00000100,
469	0xf089, 0xff000fff, 0x00000100,
470	0xf080, 0xfffc0fff, 0x00000100,
471	0x1bb6, 0x00010101, 0x00010000,
472	0x260c, 0xffffffff, 0x00000000,
473	0x260d, 0xf00fffff, 0x00000400,
474	0x16ec, 0x000000f0, 0x00000070,
475	0x16f0, 0xf0311fff, 0x80300000,
476	0x263e, 0x73773777, 0x12010001,
477	0x263f, 0xffffffff, 0x00000010,
478	0x26df, 0x00ff0000, 0x00fc0000,
479	0x200c, 0x00001f0f, 0x0000100a,
480	0xbd2, 0x73773777, 0x12010001,
481	0x902, 0x000fffff, 0x000c007f,
482	0x2285, 0xf000003f, 0x00000007,
483	0x22c9, 0x3fff3fff, 0x00ffcfff,
484	0xc281, 0x0000ff0f, 0x00000000,
485	0xa293, 0x07ffffff, 0x06000000,
486	0x136, 0x00000fff, 0x00000100,
487	0xf9e, 0x00000001, 0x00000002,
488	0x31da, 0x00000008, 0x00000008,
489	0x2300, 0x000000ff, 0x00000003,
490	0x853e, 0x01ff01ff, 0x00000002,
491	0x8526, 0x007ff800, 0x00200000,
492	0x8057, 0xffffffff, 0x00000f40,
493	0x2231, 0x001f3ae3, 0x00000082,
494	0x2235, 0x0000001f, 0x00000010,
495	0xc24d, 0xffffffff, 0x00000000
496};
497
498static const u32 kalindi_mgcg_cgcg_init[] =
499{
500	0x3108, 0xffffffff, 0xfffffffc,
501	0xc200, 0xffffffff, 0xe0000000,
502	0xf0a8, 0xffffffff, 0x00000100,
503	0xf082, 0xffffffff, 0x00000100,
504	0xf0b0, 0xffffffff, 0x00000100,
505	0xf0b2, 0xffffffff, 0x00000100,
506	0xf0b1, 0xffffffff, 0x00000100,
507	0x1579, 0xffffffff, 0x00600100,
508	0xf0a0, 0xffffffff, 0x00000100,
509	0xf085, 0xffffffff, 0x06000100,
510	0xf088, 0xffffffff, 0x00000100,
511	0xf086, 0xffffffff, 0x06000100,
512	0xf081, 0xffffffff, 0x00000100,
513	0xf0b8, 0xffffffff, 0x00000100,
514	0xf089, 0xffffffff, 0x00000100,
515	0xf080, 0xffffffff, 0x00000100,
516	0xf08c, 0xffffffff, 0x00000100,
517	0xf08d, 0xffffffff, 0x00000100,
518	0xf094, 0xffffffff, 0x00000100,
519	0xf095, 0xffffffff, 0x00000100,
520	0xf096, 0xffffffff, 0x00000100,
521	0xf097, 0xffffffff, 0x00000100,
522	0xf098, 0xffffffff, 0x00000100,
523	0xf09f, 0xffffffff, 0x00000100,
524	0xf09e, 0xffffffff, 0x00000100,
525	0xf084, 0xffffffff, 0x06000100,
526	0xf0a4, 0xffffffff, 0x00000100,
527	0xf09d, 0xffffffff, 0x00000100,
528	0xf0ad, 0xffffffff, 0x00000100,
529	0xf0ac, 0xffffffff, 0x00000100,
530	0xf09c, 0xffffffff, 0x00000100,
531	0xc200, 0xffffffff, 0xe0000000,
532	0xf008, 0xffffffff, 0x00010000,
533	0xf009, 0xffffffff, 0x00030002,
534	0xf00a, 0xffffffff, 0x00040007,
535	0xf00b, 0xffffffff, 0x00060005,
536	0xf00c, 0xffffffff, 0x00090008,
537	0xf00d, 0xffffffff, 0x00010000,
538	0xf00e, 0xffffffff, 0x00030002,
539	0xf00f, 0xffffffff, 0x00040007,
540	0xf010, 0xffffffff, 0x00060005,
541	0xf011, 0xffffffff, 0x00090008,
542	0xf000, 0xffffffff, 0x96e00200,
543	0x21c2, 0xffffffff, 0x00900100,
544	0x3109, 0xffffffff, 0x0020003f,
545	0xe, 0xffffffff, 0x0140001c,
546	0xf, 0x000f0000, 0x000f0000,
547	0x88, 0xffffffff, 0xc060000c,
548	0x89, 0xc0000fff, 0x00000100,
549	0x82a, 0xffffffff, 0x00000104,
550	0x1579, 0xff000fff, 0x00000100,
551	0xc33, 0xc0000fff, 0x00000104,
552	0x3079, 0x00000001, 0x00000001,
553	0x3403, 0xff000ff0, 0x00000100,
554	0x3603, 0xff000ff0, 0x00000100
555};
556
557static const u32 hawaii_golden_spm_registers[] =
558{
559	0xc200, 0xe0ffffff, 0xe0000000
560};
561
562static const u32 hawaii_golden_common_registers[] =
563{
564	0xc200, 0xffffffff, 0xe0000000,
565	0xa0d4, 0xffffffff, 0x3a00161a,
566	0xa0d5, 0xffffffff, 0x0000002e,
567	0x2684, 0xffffffff, 0x00018208,
568	0x263e, 0xffffffff, 0x12011003
569};
570
571static const u32 hawaii_golden_registers[] =
572{
573	0xcd5, 0x00000333, 0x00000333,
574	0x2684, 0x00010000, 0x00058208,
575	0x260c, 0xffffffff, 0x00000000,
576	0x260d, 0xf00fffff, 0x00000400,
577	0x260e, 0x0002021c, 0x00020200,
578	0x31e, 0x00000080, 0x00000000,
579	0x16ec, 0x000000f0, 0x00000070,
580	0x16f0, 0xf0311fff, 0x80300000,
581	0xd43, 0x00810000, 0x408af000,
582	0x1c0c, 0x31000111, 0x00000011,
583	0xbd2, 0x73773777, 0x12010001,
584	0x848, 0x0000007f, 0x0000001b,
585	0x877, 0x00007fb6, 0x00002191,
586	0xd8a, 0x0000003f, 0x0000000a,
587	0xd8b, 0x0000003f, 0x0000000a,
588	0xab9, 0x00073ffe, 0x000022a2,
589	0x903, 0x000007ff, 0x00000000,
590	0x22fc, 0x00002001, 0x00000001,
591	0x22c9, 0xffffffff, 0x00ffffff,
592	0xc281, 0x0000ff0f, 0x00000000,
593	0xa293, 0x07ffffff, 0x06000000,
594	0xf9e, 0x00000001, 0x00000002,
595	0x31da, 0x00000008, 0x00000008,
596	0x31dc, 0x00000f00, 0x00000800,
597	0x31dd, 0x00000f00, 0x00000800,
598	0x31e6, 0x00ffffff, 0x00ff7fbf,
599	0x31e7, 0x00ffffff, 0x00ff7faf,
600	0x2300, 0x000000ff, 0x00000800,
601	0x390, 0x00001fff, 0x00001fff,
602	0x2418, 0x0000007f, 0x00000020,
603	0x2542, 0x00010000, 0x00010000,
604	0x2b80, 0x00100000, 0x000ff07c,
605	0x2b05, 0x000003ff, 0x0000000f,
606	0x2b04, 0xffffffff, 0x7564fdec,
607	0x2b03, 0xffffffff, 0x3120b9a8,
608	0x2b02, 0x20000000, 0x0f9c0000
609};
610
611static const u32 hawaii_mgcg_cgcg_init[] =
612{
613	0x3108, 0xffffffff, 0xfffffffd,
614	0xc200, 0xffffffff, 0xe0000000,
615	0xf0a8, 0xffffffff, 0x00000100,
616	0xf082, 0xffffffff, 0x00000100,
617	0xf0b0, 0xffffffff, 0x00000100,
618	0xf0b2, 0xffffffff, 0x00000100,
619	0xf0b1, 0xffffffff, 0x00000100,
620	0x1579, 0xffffffff, 0x00200100,
621	0xf0a0, 0xffffffff, 0x00000100,
622	0xf085, 0xffffffff, 0x06000100,
623	0xf088, 0xffffffff, 0x00000100,
624	0xf086, 0xffffffff, 0x06000100,
625	0xf081, 0xffffffff, 0x00000100,
626	0xf0b8, 0xffffffff, 0x00000100,
627	0xf089, 0xffffffff, 0x00000100,
628	0xf080, 0xffffffff, 0x00000100,
629	0xf08c, 0xffffffff, 0x00000100,
630	0xf08d, 0xffffffff, 0x00000100,
631	0xf094, 0xffffffff, 0x00000100,
632	0xf095, 0xffffffff, 0x00000100,
633	0xf096, 0xffffffff, 0x00000100,
634	0xf097, 0xffffffff, 0x00000100,
635	0xf098, 0xffffffff, 0x00000100,
636	0xf09f, 0xffffffff, 0x00000100,
637	0xf09e, 0xffffffff, 0x00000100,
638	0xf084, 0xffffffff, 0x06000100,
639	0xf0a4, 0xffffffff, 0x00000100,
640	0xf09d, 0xffffffff, 0x00000100,
641	0xf0ad, 0xffffffff, 0x00000100,
642	0xf0ac, 0xffffffff, 0x00000100,
643	0xf09c, 0xffffffff, 0x00000100,
644	0xc200, 0xffffffff, 0xe0000000,
645	0xf008, 0xffffffff, 0x00010000,
646	0xf009, 0xffffffff, 0x00030002,
647	0xf00a, 0xffffffff, 0x00040007,
648	0xf00b, 0xffffffff, 0x00060005,
649	0xf00c, 0xffffffff, 0x00090008,
650	0xf00d, 0xffffffff, 0x00010000,
651	0xf00e, 0xffffffff, 0x00030002,
652	0xf00f, 0xffffffff, 0x00040007,
653	0xf010, 0xffffffff, 0x00060005,
654	0xf011, 0xffffffff, 0x00090008,
655	0xf012, 0xffffffff, 0x00010000,
656	0xf013, 0xffffffff, 0x00030002,
657	0xf014, 0xffffffff, 0x00040007,
658	0xf015, 0xffffffff, 0x00060005,
659	0xf016, 0xffffffff, 0x00090008,
660	0xf017, 0xffffffff, 0x00010000,
661	0xf018, 0xffffffff, 0x00030002,
662	0xf019, 0xffffffff, 0x00040007,
663	0xf01a, 0xffffffff, 0x00060005,
664	0xf01b, 0xffffffff, 0x00090008,
665	0xf01c, 0xffffffff, 0x00010000,
666	0xf01d, 0xffffffff, 0x00030002,
667	0xf01e, 0xffffffff, 0x00040007,
668	0xf01f, 0xffffffff, 0x00060005,
669	0xf020, 0xffffffff, 0x00090008,
670	0xf021, 0xffffffff, 0x00010000,
671	0xf022, 0xffffffff, 0x00030002,
672	0xf023, 0xffffffff, 0x00040007,
673	0xf024, 0xffffffff, 0x00060005,
674	0xf025, 0xffffffff, 0x00090008,
675	0xf026, 0xffffffff, 0x00010000,
676	0xf027, 0xffffffff, 0x00030002,
677	0xf028, 0xffffffff, 0x00040007,
678	0xf029, 0xffffffff, 0x00060005,
679	0xf02a, 0xffffffff, 0x00090008,
680	0xf02b, 0xffffffff, 0x00010000,
681	0xf02c, 0xffffffff, 0x00030002,
682	0xf02d, 0xffffffff, 0x00040007,
683	0xf02e, 0xffffffff, 0x00060005,
684	0xf02f, 0xffffffff, 0x00090008,
685	0xf030, 0xffffffff, 0x00010000,
686	0xf031, 0xffffffff, 0x00030002,
687	0xf032, 0xffffffff, 0x00040007,
688	0xf033, 0xffffffff, 0x00060005,
689	0xf034, 0xffffffff, 0x00090008,
690	0xf035, 0xffffffff, 0x00010000,
691	0xf036, 0xffffffff, 0x00030002,
692	0xf037, 0xffffffff, 0x00040007,
693	0xf038, 0xffffffff, 0x00060005,
694	0xf039, 0xffffffff, 0x00090008,
695	0xf03a, 0xffffffff, 0x00010000,
696	0xf03b, 0xffffffff, 0x00030002,
697	0xf03c, 0xffffffff, 0x00040007,
698	0xf03d, 0xffffffff, 0x00060005,
699	0xf03e, 0xffffffff, 0x00090008,
700	0x30c6, 0xffffffff, 0x00020200,
701	0xcd4, 0xffffffff, 0x00000200,
702	0x570, 0xffffffff, 0x00000400,
703	0x157a, 0xffffffff, 0x00000000,
704	0xbd4, 0xffffffff, 0x00000902,
705	0xf000, 0xffffffff, 0x96940200,
706	0x21c2, 0xffffffff, 0x00900100,
707	0x3109, 0xffffffff, 0x0020003f,
708	0xe, 0xffffffff, 0x0140001c,
709	0xf, 0x000f0000, 0x000f0000,
710	0x88, 0xffffffff, 0xc060000c,
711	0x89, 0xc0000fff, 0x00000100,
712	0x3e4, 0xffffffff, 0x00000100,
713	0x3e6, 0x00000101, 0x00000000,
714	0x82a, 0xffffffff, 0x00000104,
715	0x1579, 0xff000fff, 0x00000100,
716	0xc33, 0xc0000fff, 0x00000104,
717	0x3079, 0x00000001, 0x00000001,
718	0x3403, 0xff000ff0, 0x00000100,
719	0x3603, 0xff000ff0, 0x00000100
720};
721
722static const u32 godavari_golden_registers[] =
723{
724	0x1579, 0xff607fff, 0xfc000100,
725	0x1bb6, 0x00010101, 0x00010000,
726	0x260c, 0xffffffff, 0x00000000,
727	0x260c0, 0xf00fffff, 0x00000400,
728	0x184c, 0xffffffff, 0x00010000,
729	0x16ec, 0x000000f0, 0x00000070,
730	0x16f0, 0xf0311fff, 0x80300000,
731	0x263e, 0x73773777, 0x12010001,
732	0x263f, 0xffffffff, 0x00000010,
733	0x200c, 0x00001f0f, 0x0000100a,
734	0xbd2, 0x73773777, 0x12010001,
735	0x902, 0x000fffff, 0x000c007f,
736	0x2285, 0xf000003f, 0x00000007,
737	0x22c9, 0xffffffff, 0x00ff0fff,
738	0xc281, 0x0000ff0f, 0x00000000,
739	0xa293, 0x07ffffff, 0x06000000,
740	0x136, 0x00000fff, 0x00000100,
741	0x3405, 0x00010000, 0x00810001,
742	0x3605, 0x00010000, 0x00810001,
743	0xf9e, 0x00000001, 0x00000002,
744	0x31da, 0x00000008, 0x00000008,
745	0x31dc, 0x00000f00, 0x00000800,
746	0x31dd, 0x00000f00, 0x00000800,
747	0x31e6, 0x00ffffff, 0x00ff7fbf,
748	0x31e7, 0x00ffffff, 0x00ff7faf,
749	0x2300, 0x000000ff, 0x00000001,
750	0x853e, 0x01ff01ff, 0x00000002,
751	0x8526, 0x007ff800, 0x00200000,
752	0x8057, 0xffffffff, 0x00000f40,
753	0x2231, 0x001f3ae3, 0x00000082,
754	0x2235, 0x0000001f, 0x00000010,
755	0xc24d, 0xffffffff, 0x00000000
756};
757
758static void cik_init_golden_registers(struct amdgpu_device *adev)
759{
760	/* Some of the registers might be dependent on GRBM_GFX_INDEX */
761	mutex_lock(&adev->grbm_idx_mutex);
762
763	switch (adev->asic_type) {
764	case CHIP_BONAIRE:
765		amdgpu_device_program_register_sequence(adev,
766							bonaire_mgcg_cgcg_init,
767							ARRAY_SIZE(bonaire_mgcg_cgcg_init));
768		amdgpu_device_program_register_sequence(adev,
769							bonaire_golden_registers,
770							ARRAY_SIZE(bonaire_golden_registers));
771		amdgpu_device_program_register_sequence(adev,
772							bonaire_golden_common_registers,
773							ARRAY_SIZE(bonaire_golden_common_registers));
774		amdgpu_device_program_register_sequence(adev,
775							bonaire_golden_spm_registers,
776							ARRAY_SIZE(bonaire_golden_spm_registers));
777		break;
778	case CHIP_KABINI:
779		amdgpu_device_program_register_sequence(adev,
780							kalindi_mgcg_cgcg_init,
781							ARRAY_SIZE(kalindi_mgcg_cgcg_init));
782		amdgpu_device_program_register_sequence(adev,
783							kalindi_golden_registers,
784							ARRAY_SIZE(kalindi_golden_registers));
785		amdgpu_device_program_register_sequence(adev,
786							kalindi_golden_common_registers,
787							ARRAY_SIZE(kalindi_golden_common_registers));
788		amdgpu_device_program_register_sequence(adev,
789							kalindi_golden_spm_registers,
790							ARRAY_SIZE(kalindi_golden_spm_registers));
791		break;
792	case CHIP_MULLINS:
793		amdgpu_device_program_register_sequence(adev,
794							kalindi_mgcg_cgcg_init,
795							ARRAY_SIZE(kalindi_mgcg_cgcg_init));
796		amdgpu_device_program_register_sequence(adev,
797							godavari_golden_registers,
798							ARRAY_SIZE(godavari_golden_registers));
799		amdgpu_device_program_register_sequence(adev,
800							kalindi_golden_common_registers,
801							ARRAY_SIZE(kalindi_golden_common_registers));
802		amdgpu_device_program_register_sequence(adev,
803							kalindi_golden_spm_registers,
804							ARRAY_SIZE(kalindi_golden_spm_registers));
805		break;
806	case CHIP_KAVERI:
807		amdgpu_device_program_register_sequence(adev,
808							spectre_mgcg_cgcg_init,
809							ARRAY_SIZE(spectre_mgcg_cgcg_init));
810		amdgpu_device_program_register_sequence(adev,
811							spectre_golden_registers,
812							ARRAY_SIZE(spectre_golden_registers));
813		amdgpu_device_program_register_sequence(adev,
814							spectre_golden_common_registers,
815							ARRAY_SIZE(spectre_golden_common_registers));
816		amdgpu_device_program_register_sequence(adev,
817							spectre_golden_spm_registers,
818							ARRAY_SIZE(spectre_golden_spm_registers));
819		break;
820	case CHIP_HAWAII:
821		amdgpu_device_program_register_sequence(adev,
822							hawaii_mgcg_cgcg_init,
823							ARRAY_SIZE(hawaii_mgcg_cgcg_init));
824		amdgpu_device_program_register_sequence(adev,
825							hawaii_golden_registers,
826							ARRAY_SIZE(hawaii_golden_registers));
827		amdgpu_device_program_register_sequence(adev,
828							hawaii_golden_common_registers,
829							ARRAY_SIZE(hawaii_golden_common_registers));
830		amdgpu_device_program_register_sequence(adev,
831							hawaii_golden_spm_registers,
832							ARRAY_SIZE(hawaii_golden_spm_registers));
833		break;
834	default:
835		break;
836	}
837	mutex_unlock(&adev->grbm_idx_mutex);
838}
839
840/**
841 * cik_get_xclk - get the xclk
842 *
843 * @adev: amdgpu_device pointer
844 *
845 * Returns the reference clock used by the gfx engine
846 * (CIK).
847 */
848static u32 cik_get_xclk(struct amdgpu_device *adev)
849{
850	u32 reference_clock = adev->clock.spll.reference_freq;
851
852	if (adev->flags & AMD_IS_APU) {
853		if (RREG32_SMC(ixGENERAL_PWRMGT) & GENERAL_PWRMGT__GPU_COUNTER_CLK_MASK)
854			return reference_clock / 2;
855	} else {
856		if (RREG32_SMC(ixCG_CLKPIN_CNTL) & CG_CLKPIN_CNTL__XTALIN_DIVIDE_MASK)
857			return reference_clock / 4;
858	}
859	return reference_clock;
860}
861
862/**
863 * cik_srbm_select - select specific register instances
864 *
865 * @adev: amdgpu_device pointer
866 * @me: selected ME (micro engine)
867 * @pipe: pipe
868 * @queue: queue
869 * @vmid: VMID
870 *
871 * Switches the currently active registers instances.  Some
872 * registers are instanced per VMID, others are instanced per
873 * me/pipe/queue combination.
874 */
875void cik_srbm_select(struct amdgpu_device *adev,
876		     u32 me, u32 pipe, u32 queue, u32 vmid)
877{
878	u32 srbm_gfx_cntl =
879		(((pipe << SRBM_GFX_CNTL__PIPEID__SHIFT) & SRBM_GFX_CNTL__PIPEID_MASK)|
880		((me << SRBM_GFX_CNTL__MEID__SHIFT) & SRBM_GFX_CNTL__MEID_MASK)|
881		((vmid << SRBM_GFX_CNTL__VMID__SHIFT) & SRBM_GFX_CNTL__VMID_MASK)|
882		((queue << SRBM_GFX_CNTL__QUEUEID__SHIFT) & SRBM_GFX_CNTL__QUEUEID_MASK));
883	WREG32(mmSRBM_GFX_CNTL, srbm_gfx_cntl);
884}
885
886static void cik_vga_set_state(struct amdgpu_device *adev, bool state)
887{
888	uint32_t tmp;
889
890	tmp = RREG32(mmCONFIG_CNTL);
891	if (!state)
892		tmp |= CONFIG_CNTL__VGA_DIS_MASK;
893	else
894		tmp &= ~CONFIG_CNTL__VGA_DIS_MASK;
895	WREG32(mmCONFIG_CNTL, tmp);
896}
897
898static bool cik_read_disabled_bios(struct amdgpu_device *adev)
899{
900	u32 bus_cntl;
901	u32 d1vga_control = 0;
902	u32 d2vga_control = 0;
903	u32 vga_render_control = 0;
904	u32 rom_cntl;
905	bool r;
906
907	bus_cntl = RREG32(mmBUS_CNTL);
908	if (adev->mode_info.num_crtc) {
909		d1vga_control = RREG32(mmD1VGA_CONTROL);
910		d2vga_control = RREG32(mmD2VGA_CONTROL);
911		vga_render_control = RREG32(mmVGA_RENDER_CONTROL);
912	}
913	rom_cntl = RREG32_SMC(ixROM_CNTL);
914
915	/* enable the rom */
916	WREG32(mmBUS_CNTL, (bus_cntl & ~BUS_CNTL__BIOS_ROM_DIS_MASK));
917	if (adev->mode_info.num_crtc) {
918		/* Disable VGA mode */
919		WREG32(mmD1VGA_CONTROL,
920		       (d1vga_control & ~(D1VGA_CONTROL__D1VGA_MODE_ENABLE_MASK |
921					  D1VGA_CONTROL__D1VGA_TIMING_SELECT_MASK)));
922		WREG32(mmD2VGA_CONTROL,
923		       (d2vga_control & ~(D1VGA_CONTROL__D1VGA_MODE_ENABLE_MASK |
924					  D1VGA_CONTROL__D1VGA_TIMING_SELECT_MASK)));
925		WREG32(mmVGA_RENDER_CONTROL,
926		       (vga_render_control & ~VGA_RENDER_CONTROL__VGA_VSTATUS_CNTL_MASK));
927	}
928	WREG32_SMC(ixROM_CNTL, rom_cntl | ROM_CNTL__SCK_OVERWRITE_MASK);
929
930	r = amdgpu_read_bios(adev);
931
932	/* restore regs */
933	WREG32(mmBUS_CNTL, bus_cntl);
934	if (adev->mode_info.num_crtc) {
935		WREG32(mmD1VGA_CONTROL, d1vga_control);
936		WREG32(mmD2VGA_CONTROL, d2vga_control);
937		WREG32(mmVGA_RENDER_CONTROL, vga_render_control);
938	}
939	WREG32_SMC(ixROM_CNTL, rom_cntl);
940	return r;
941}
942
943static bool cik_read_bios_from_rom(struct amdgpu_device *adev,
944				   u8 *bios, u32 length_bytes)
945{
946	u32 *dw_ptr;
947	unsigned long flags;
948	u32 i, length_dw;
949
950	if (bios == NULL)
951		return false;
952	if (length_bytes == 0)
953		return false;
954	/* APU vbios image is part of sbios image */
955	if (adev->flags & AMD_IS_APU)
956		return false;
957
958	dw_ptr = (u32 *)bios;
959	length_dw = ALIGN(length_bytes, 4) / 4;
960	/* take the smc lock since we are using the smc index */
961	spin_lock_irqsave(&adev->smc_idx_lock, flags);
962	/* set rom index to 0 */
963	WREG32(mmSMC_IND_INDEX_0, ixROM_INDEX);
964	WREG32(mmSMC_IND_DATA_0, 0);
965	/* set index to data for continous read */
966	WREG32(mmSMC_IND_INDEX_0, ixROM_DATA);
967	for (i = 0; i < length_dw; i++)
968		dw_ptr[i] = RREG32(mmSMC_IND_DATA_0);
969	spin_unlock_irqrestore(&adev->smc_idx_lock, flags);
970
971	return true;
972}
973
974static const struct amdgpu_allowed_register_entry cik_allowed_read_registers[] = {
975	{mmGRBM_STATUS},
976	{mmGRBM_STATUS2},
977	{mmGRBM_STATUS_SE0},
978	{mmGRBM_STATUS_SE1},
979	{mmGRBM_STATUS_SE2},
980	{mmGRBM_STATUS_SE3},
981	{mmSRBM_STATUS},
982	{mmSRBM_STATUS2},
983	{mmSDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET},
984	{mmSDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET},
985	{mmCP_STAT},
986	{mmCP_STALLED_STAT1},
987	{mmCP_STALLED_STAT2},
988	{mmCP_STALLED_STAT3},
989	{mmCP_CPF_BUSY_STAT},
990	{mmCP_CPF_STALLED_STAT1},
991	{mmCP_CPF_STATUS},
992	{mmCP_CPC_BUSY_STAT},
993	{mmCP_CPC_STALLED_STAT1},
994	{mmCP_CPC_STATUS},
995	{mmGB_ADDR_CONFIG},
996	{mmMC_ARB_RAMCFG},
997	{mmGB_TILE_MODE0},
998	{mmGB_TILE_MODE1},
999	{mmGB_TILE_MODE2},
1000	{mmGB_TILE_MODE3},
1001	{mmGB_TILE_MODE4},
1002	{mmGB_TILE_MODE5},
1003	{mmGB_TILE_MODE6},
1004	{mmGB_TILE_MODE7},
1005	{mmGB_TILE_MODE8},
1006	{mmGB_TILE_MODE9},
1007	{mmGB_TILE_MODE10},
1008	{mmGB_TILE_MODE11},
1009	{mmGB_TILE_MODE12},
1010	{mmGB_TILE_MODE13},
1011	{mmGB_TILE_MODE14},
1012	{mmGB_TILE_MODE15},
1013	{mmGB_TILE_MODE16},
1014	{mmGB_TILE_MODE17},
1015	{mmGB_TILE_MODE18},
1016	{mmGB_TILE_MODE19},
1017	{mmGB_TILE_MODE20},
1018	{mmGB_TILE_MODE21},
1019	{mmGB_TILE_MODE22},
1020	{mmGB_TILE_MODE23},
1021	{mmGB_TILE_MODE24},
1022	{mmGB_TILE_MODE25},
1023	{mmGB_TILE_MODE26},
1024	{mmGB_TILE_MODE27},
1025	{mmGB_TILE_MODE28},
1026	{mmGB_TILE_MODE29},
1027	{mmGB_TILE_MODE30},
1028	{mmGB_TILE_MODE31},
1029	{mmGB_MACROTILE_MODE0},
1030	{mmGB_MACROTILE_MODE1},
1031	{mmGB_MACROTILE_MODE2},
1032	{mmGB_MACROTILE_MODE3},
1033	{mmGB_MACROTILE_MODE4},
1034	{mmGB_MACROTILE_MODE5},
1035	{mmGB_MACROTILE_MODE6},
1036	{mmGB_MACROTILE_MODE7},
1037	{mmGB_MACROTILE_MODE8},
1038	{mmGB_MACROTILE_MODE9},
1039	{mmGB_MACROTILE_MODE10},
1040	{mmGB_MACROTILE_MODE11},
1041	{mmGB_MACROTILE_MODE12},
1042	{mmGB_MACROTILE_MODE13},
1043	{mmGB_MACROTILE_MODE14},
1044	{mmGB_MACROTILE_MODE15},
1045	{mmCC_RB_BACKEND_DISABLE, true},
1046	{mmGC_USER_RB_BACKEND_DISABLE, true},
1047	{mmGB_BACKEND_MAP, false},
1048	{mmPA_SC_RASTER_CONFIG, true},
1049	{mmPA_SC_RASTER_CONFIG_1, true},
1050};
1051
1052
1053static uint32_t cik_get_register_value(struct amdgpu_device *adev,
1054				       bool indexed, u32 se_num,
1055				       u32 sh_num, u32 reg_offset)
1056{
1057	if (indexed) {
1058		uint32_t val;
1059		unsigned se_idx = (se_num == 0xffffffff) ? 0 : se_num;
1060		unsigned sh_idx = (sh_num == 0xffffffff) ? 0 : sh_num;
1061
1062		switch (reg_offset) {
1063		case mmCC_RB_BACKEND_DISABLE:
1064			return adev->gfx.config.rb_config[se_idx][sh_idx].rb_backend_disable;
1065		case mmGC_USER_RB_BACKEND_DISABLE:
1066			return adev->gfx.config.rb_config[se_idx][sh_idx].user_rb_backend_disable;
1067		case mmPA_SC_RASTER_CONFIG:
1068			return adev->gfx.config.rb_config[se_idx][sh_idx].raster_config;
1069		case mmPA_SC_RASTER_CONFIG_1:
1070			return adev->gfx.config.rb_config[se_idx][sh_idx].raster_config_1;
1071		}
1072
1073		mutex_lock(&adev->grbm_idx_mutex);
1074		if (se_num != 0xffffffff || sh_num != 0xffffffff)
1075			amdgpu_gfx_select_se_sh(adev, se_num, sh_num, 0xffffffff);
1076
1077		val = RREG32(reg_offset);
1078
1079		if (se_num != 0xffffffff || sh_num != 0xffffffff)
1080			amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
1081		mutex_unlock(&adev->grbm_idx_mutex);
1082		return val;
1083	} else {
1084		unsigned idx;
1085
1086		switch (reg_offset) {
1087		case mmGB_ADDR_CONFIG:
1088			return adev->gfx.config.gb_addr_config;
1089		case mmMC_ARB_RAMCFG:
1090			return adev->gfx.config.mc_arb_ramcfg;
1091		case mmGB_TILE_MODE0:
1092		case mmGB_TILE_MODE1:
1093		case mmGB_TILE_MODE2:
1094		case mmGB_TILE_MODE3:
1095		case mmGB_TILE_MODE4:
1096		case mmGB_TILE_MODE5:
1097		case mmGB_TILE_MODE6:
1098		case mmGB_TILE_MODE7:
1099		case mmGB_TILE_MODE8:
1100		case mmGB_TILE_MODE9:
1101		case mmGB_TILE_MODE10:
1102		case mmGB_TILE_MODE11:
1103		case mmGB_TILE_MODE12:
1104		case mmGB_TILE_MODE13:
1105		case mmGB_TILE_MODE14:
1106		case mmGB_TILE_MODE15:
1107		case mmGB_TILE_MODE16:
1108		case mmGB_TILE_MODE17:
1109		case mmGB_TILE_MODE18:
1110		case mmGB_TILE_MODE19:
1111		case mmGB_TILE_MODE20:
1112		case mmGB_TILE_MODE21:
1113		case mmGB_TILE_MODE22:
1114		case mmGB_TILE_MODE23:
1115		case mmGB_TILE_MODE24:
1116		case mmGB_TILE_MODE25:
1117		case mmGB_TILE_MODE26:
1118		case mmGB_TILE_MODE27:
1119		case mmGB_TILE_MODE28:
1120		case mmGB_TILE_MODE29:
1121		case mmGB_TILE_MODE30:
1122		case mmGB_TILE_MODE31:
1123			idx = (reg_offset - mmGB_TILE_MODE0);
1124			return adev->gfx.config.tile_mode_array[idx];
1125		case mmGB_MACROTILE_MODE0:
1126		case mmGB_MACROTILE_MODE1:
1127		case mmGB_MACROTILE_MODE2:
1128		case mmGB_MACROTILE_MODE3:
1129		case mmGB_MACROTILE_MODE4:
1130		case mmGB_MACROTILE_MODE5:
1131		case mmGB_MACROTILE_MODE6:
1132		case mmGB_MACROTILE_MODE7:
1133		case mmGB_MACROTILE_MODE8:
1134		case mmGB_MACROTILE_MODE9:
1135		case mmGB_MACROTILE_MODE10:
1136		case mmGB_MACROTILE_MODE11:
1137		case mmGB_MACROTILE_MODE12:
1138		case mmGB_MACROTILE_MODE13:
1139		case mmGB_MACROTILE_MODE14:
1140		case mmGB_MACROTILE_MODE15:
1141			idx = (reg_offset - mmGB_MACROTILE_MODE0);
1142			return adev->gfx.config.macrotile_mode_array[idx];
1143		default:
1144			return RREG32(reg_offset);
1145		}
1146	}
1147}
1148
1149static int cik_read_register(struct amdgpu_device *adev, u32 se_num,
1150			     u32 sh_num, u32 reg_offset, u32 *value)
1151{
1152	uint32_t i;
1153
1154	*value = 0;
1155	for (i = 0; i < ARRAY_SIZE(cik_allowed_read_registers); i++) {
1156		bool indexed = cik_allowed_read_registers[i].grbm_indexed;
1157
1158		if (reg_offset != cik_allowed_read_registers[i].reg_offset)
1159			continue;
1160
1161		*value = cik_get_register_value(adev, indexed, se_num, sh_num,
1162						reg_offset);
1163		return 0;
1164	}
1165	return -EINVAL;
1166}
1167
1168struct kv_reset_save_regs {
1169	u32 gmcon_reng_execute;
1170	u32 gmcon_misc;
1171	u32 gmcon_misc3;
1172};
1173
1174static void kv_save_regs_for_reset(struct amdgpu_device *adev,
1175				   struct kv_reset_save_regs *save)
1176{
1177	save->gmcon_reng_execute = RREG32(mmGMCON_RENG_EXECUTE);
1178	save->gmcon_misc = RREG32(mmGMCON_MISC);
1179	save->gmcon_misc3 = RREG32(mmGMCON_MISC3);
1180
1181	WREG32(mmGMCON_RENG_EXECUTE, save->gmcon_reng_execute &
1182		~GMCON_RENG_EXECUTE__RENG_EXECUTE_ON_PWR_UP_MASK);
1183	WREG32(mmGMCON_MISC, save->gmcon_misc &
1184		~(GMCON_MISC__RENG_EXECUTE_ON_REG_UPDATE_MASK |
1185			GMCON_MISC__STCTRL_STUTTER_EN_MASK));
1186}
1187
1188static void kv_restore_regs_for_reset(struct amdgpu_device *adev,
1189				      struct kv_reset_save_regs *save)
1190{
1191	int i;
1192
1193	WREG32(mmGMCON_PGFSM_WRITE, 0);
1194	WREG32(mmGMCON_PGFSM_CONFIG, 0x200010ff);
1195
1196	for (i = 0; i < 5; i++)
1197		WREG32(mmGMCON_PGFSM_WRITE, 0);
1198
1199	WREG32(mmGMCON_PGFSM_WRITE, 0);
1200	WREG32(mmGMCON_PGFSM_CONFIG, 0x300010ff);
1201
1202	for (i = 0; i < 5; i++)
1203		WREG32(mmGMCON_PGFSM_WRITE, 0);
1204
1205	WREG32(mmGMCON_PGFSM_WRITE, 0x210000);
1206	WREG32(mmGMCON_PGFSM_CONFIG, 0xa00010ff);
1207
1208	for (i = 0; i < 5; i++)
1209		WREG32(mmGMCON_PGFSM_WRITE, 0);
1210
1211	WREG32(mmGMCON_PGFSM_WRITE, 0x21003);
1212	WREG32(mmGMCON_PGFSM_CONFIG, 0xb00010ff);
1213
1214	for (i = 0; i < 5; i++)
1215		WREG32(mmGMCON_PGFSM_WRITE, 0);
1216
1217	WREG32(mmGMCON_PGFSM_WRITE, 0x2b00);
1218	WREG32(mmGMCON_PGFSM_CONFIG, 0xc00010ff);
1219
1220	for (i = 0; i < 5; i++)
1221		WREG32(mmGMCON_PGFSM_WRITE, 0);
1222
1223	WREG32(mmGMCON_PGFSM_WRITE, 0);
1224	WREG32(mmGMCON_PGFSM_CONFIG, 0xd00010ff);
1225
1226	for (i = 0; i < 5; i++)
1227		WREG32(mmGMCON_PGFSM_WRITE, 0);
1228
1229	WREG32(mmGMCON_PGFSM_WRITE, 0x420000);
1230	WREG32(mmGMCON_PGFSM_CONFIG, 0x100010ff);
1231
1232	for (i = 0; i < 5; i++)
1233		WREG32(mmGMCON_PGFSM_WRITE, 0);
1234
1235	WREG32(mmGMCON_PGFSM_WRITE, 0x120202);
1236	WREG32(mmGMCON_PGFSM_CONFIG, 0x500010ff);
1237
1238	for (i = 0; i < 5; i++)
1239		WREG32(mmGMCON_PGFSM_WRITE, 0);
1240
1241	WREG32(mmGMCON_PGFSM_WRITE, 0x3e3e36);
1242	WREG32(mmGMCON_PGFSM_CONFIG, 0x600010ff);
1243
1244	for (i = 0; i < 5; i++)
1245		WREG32(mmGMCON_PGFSM_WRITE, 0);
1246
1247	WREG32(mmGMCON_PGFSM_WRITE, 0x373f3e);
1248	WREG32(mmGMCON_PGFSM_CONFIG, 0x700010ff);
1249
1250	for (i = 0; i < 5; i++)
1251		WREG32(mmGMCON_PGFSM_WRITE, 0);
1252
1253	WREG32(mmGMCON_PGFSM_WRITE, 0x3e1332);
1254	WREG32(mmGMCON_PGFSM_CONFIG, 0xe00010ff);
1255
1256	WREG32(mmGMCON_MISC3, save->gmcon_misc3);
1257	WREG32(mmGMCON_MISC, save->gmcon_misc);
1258	WREG32(mmGMCON_RENG_EXECUTE, save->gmcon_reng_execute);
1259}
1260
1261static int cik_gpu_pci_config_reset(struct amdgpu_device *adev)
1262{
1263	struct kv_reset_save_regs kv_save = { 0 };
1264	u32 i;
1265	int r = -EINVAL;
1266
1267	dev_info(adev->dev, "GPU pci config reset\n");
1268
1269	if (adev->flags & AMD_IS_APU)
1270		kv_save_regs_for_reset(adev, &kv_save);
1271
1272	/* disable BM */
1273	pci_clear_master(adev->pdev);
1274	/* reset */
1275	amdgpu_device_pci_config_reset(adev);
1276
1277	udelay(100);
1278
1279	/* wait for asic to come out of reset */
1280	for (i = 0; i < adev->usec_timeout; i++) {
1281		if (RREG32(mmCONFIG_MEMSIZE) != 0xffffffff) {
1282			/* enable BM */
1283			pci_set_master(adev->pdev);
1284			adev->has_hw_reset = true;
1285			r = 0;
1286			break;
1287		}
1288		udelay(1);
1289	}
1290
1291	/* does asic init need to be run first??? */
1292	if (adev->flags & AMD_IS_APU)
1293		kv_restore_regs_for_reset(adev, &kv_save);
1294
1295	return r;
1296}
1297
1298/**
1299 * cik_asic_pci_config_reset - soft reset GPU
1300 *
1301 * @adev: amdgpu_device pointer
1302 *
1303 * Use PCI Config method to reset the GPU.
1304 *
1305 * Returns 0 for success.
1306 */
1307static int cik_asic_pci_config_reset(struct amdgpu_device *adev)
1308{
1309	int r;
1310
1311	amdgpu_atombios_scratch_regs_engine_hung(adev, true);
1312
1313	r = cik_gpu_pci_config_reset(adev);
1314
1315	amdgpu_atombios_scratch_regs_engine_hung(adev, false);
1316
1317	return r;
1318}
1319
1320static bool cik_asic_supports_baco(struct amdgpu_device *adev)
1321{
1322	switch (adev->asic_type) {
1323	case CHIP_BONAIRE:
1324	case CHIP_HAWAII:
1325		return amdgpu_dpm_is_baco_supported(adev);
1326	default:
1327		return false;
1328	}
1329}
1330
1331static enum amd_reset_method
1332cik_asic_reset_method(struct amdgpu_device *adev)
1333{
1334	bool baco_reset;
1335
1336	switch (adev->asic_type) {
1337	case CHIP_BONAIRE:
1338	case CHIP_HAWAII:
1339		/* disable baco reset until it works */
1340		/* smu7_asic_get_baco_capability(adev, &baco_reset); */
1341		baco_reset = false;
1342		break;
1343	default:
1344		baco_reset = false;
1345		break;
1346	}
1347
1348	if (baco_reset)
1349		return AMD_RESET_METHOD_BACO;
1350	else
1351		return AMD_RESET_METHOD_LEGACY;
1352}
1353
1354/**
1355 * cik_asic_reset - soft reset GPU
1356 *
1357 * @adev: amdgpu_device pointer
1358 *
1359 * Look up which blocks are hung and attempt
1360 * to reset them.
1361 * Returns 0 for success.
1362 */
1363static int cik_asic_reset(struct amdgpu_device *adev)
1364{
1365	int r;
1366
1367	if (cik_asic_reset_method(adev) == AMD_RESET_METHOD_BACO) {
1368		if (!adev->in_suspend)
1369			amdgpu_inc_vram_lost(adev);
1370		r = amdgpu_dpm_baco_reset(adev);
1371	} else {
1372		r = cik_asic_pci_config_reset(adev);
1373	}
1374
1375	return r;
1376}
1377
1378static u32 cik_get_config_memsize(struct amdgpu_device *adev)
1379{
1380	return RREG32(mmCONFIG_MEMSIZE);
1381}
1382
1383static int cik_set_uvd_clock(struct amdgpu_device *adev, u32 clock,
1384			      u32 cntl_reg, u32 status_reg)
1385{
1386	int r, i;
1387	struct atom_clock_dividers dividers;
1388	uint32_t tmp;
1389
1390	r = amdgpu_atombios_get_clock_dividers(adev,
1391					       COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
1392					       clock, false, &dividers);
1393	if (r)
1394		return r;
1395
1396	tmp = RREG32_SMC(cntl_reg);
1397	tmp &= ~(CG_DCLK_CNTL__DCLK_DIR_CNTL_EN_MASK |
1398		CG_DCLK_CNTL__DCLK_DIVIDER_MASK);
1399	tmp |= dividers.post_divider;
1400	WREG32_SMC(cntl_reg, tmp);
1401
1402	for (i = 0; i < 100; i++) {
1403		if (RREG32_SMC(status_reg) & CG_DCLK_STATUS__DCLK_STATUS_MASK)
1404			break;
1405		mdelay(10);
1406	}
1407	if (i == 100)
1408		return -ETIMEDOUT;
1409
1410	return 0;
1411}
1412
1413static int cik_set_uvd_clocks(struct amdgpu_device *adev, u32 vclk, u32 dclk)
1414{
1415	int r = 0;
1416
1417	r = cik_set_uvd_clock(adev, vclk, ixCG_VCLK_CNTL, ixCG_VCLK_STATUS);
1418	if (r)
1419		return r;
1420
1421	r = cik_set_uvd_clock(adev, dclk, ixCG_DCLK_CNTL, ixCG_DCLK_STATUS);
1422	return r;
1423}
1424
1425static int cik_set_vce_clocks(struct amdgpu_device *adev, u32 evclk, u32 ecclk)
1426{
1427	int r, i;
1428	struct atom_clock_dividers dividers;
1429	u32 tmp;
1430
1431	r = amdgpu_atombios_get_clock_dividers(adev,
1432					       COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
1433					       ecclk, false, &dividers);
1434	if (r)
1435		return r;
1436
1437	for (i = 0; i < 100; i++) {
1438		if (RREG32_SMC(ixCG_ECLK_STATUS) & CG_ECLK_STATUS__ECLK_STATUS_MASK)
1439			break;
1440		mdelay(10);
1441	}
1442	if (i == 100)
1443		return -ETIMEDOUT;
1444
1445	tmp = RREG32_SMC(ixCG_ECLK_CNTL);
1446	tmp &= ~(CG_ECLK_CNTL__ECLK_DIR_CNTL_EN_MASK |
1447		CG_ECLK_CNTL__ECLK_DIVIDER_MASK);
1448	tmp |= dividers.post_divider;
1449	WREG32_SMC(ixCG_ECLK_CNTL, tmp);
1450
1451	for (i = 0; i < 100; i++) {
1452		if (RREG32_SMC(ixCG_ECLK_STATUS) & CG_ECLK_STATUS__ECLK_STATUS_MASK)
1453			break;
1454		mdelay(10);
1455	}
1456	if (i == 100)
1457		return -ETIMEDOUT;
1458
1459	return 0;
1460}
1461
1462static void cik_pcie_gen3_enable(struct amdgpu_device *adev)
1463{
1464	struct pci_dev *root = adev->pdev->bus->self;
1465	u32 speed_cntl, current_data_rate;
1466	int i;
1467	u16 tmp16;
1468
1469	if (pci_is_root_bus(adev->pdev->bus))
1470		return;
1471
1472	if (amdgpu_pcie_gen2 == 0)
1473		return;
1474
1475	if (adev->flags & AMD_IS_APU)
1476		return;
1477
1478	if (!(adev->pm.pcie_gen_mask & (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 |
1479					CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3)))
1480		return;
1481
1482	speed_cntl = RREG32_PCIE(ixPCIE_LC_SPEED_CNTL);
1483	current_data_rate = (speed_cntl & PCIE_LC_SPEED_CNTL__LC_CURRENT_DATA_RATE_MASK) >>
1484		PCIE_LC_SPEED_CNTL__LC_CURRENT_DATA_RATE__SHIFT;
1485	if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3) {
1486		if (current_data_rate == 2) {
1487			DRM_INFO("PCIE gen 3 link speeds already enabled\n");
1488			return;
1489		}
1490		DRM_INFO("enabling PCIE gen 3 link speeds, disable with amdgpu.pcie_gen2=0\n");
1491	} else if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2) {
1492		if (current_data_rate == 1) {
1493			DRM_INFO("PCIE gen 2 link speeds already enabled\n");
1494			return;
1495		}
1496		DRM_INFO("enabling PCIE gen 2 link speeds, disable with amdgpu.pcie_gen2=0\n");
1497	}
1498
1499	if (!pci_is_pcie(root) || !pci_is_pcie(adev->pdev))
1500		return;
1501
1502	if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3) {
1503		/* re-try equalization if gen3 is not already enabled */
1504		if (current_data_rate != 2) {
1505			u16 bridge_cfg, gpu_cfg;
1506			u16 bridge_cfg2, gpu_cfg2;
1507			u32 max_lw, current_lw, tmp;
1508
1509			pcie_capability_read_word(root, PCI_EXP_LNKCTL,
1510						  &bridge_cfg);
1511			pcie_capability_read_word(adev->pdev, PCI_EXP_LNKCTL,
1512						  &gpu_cfg);
1513
1514			tmp16 = bridge_cfg | PCI_EXP_LNKCTL_HAWD;
1515			pcie_capability_write_word(root, PCI_EXP_LNKCTL, tmp16);
1516
1517			tmp16 = gpu_cfg | PCI_EXP_LNKCTL_HAWD;
1518			pcie_capability_write_word(adev->pdev, PCI_EXP_LNKCTL,
1519						   tmp16);
1520
1521			tmp = RREG32_PCIE(ixPCIE_LC_STATUS1);
1522			max_lw = (tmp & PCIE_LC_STATUS1__LC_DETECTED_LINK_WIDTH_MASK) >>
1523				PCIE_LC_STATUS1__LC_DETECTED_LINK_WIDTH__SHIFT;
1524			current_lw = (tmp & PCIE_LC_STATUS1__LC_OPERATING_LINK_WIDTH_MASK)
1525				>> PCIE_LC_STATUS1__LC_OPERATING_LINK_WIDTH__SHIFT;
1526
1527			if (current_lw < max_lw) {
1528				tmp = RREG32_PCIE(ixPCIE_LC_LINK_WIDTH_CNTL);
1529				if (tmp & PCIE_LC_LINK_WIDTH_CNTL__LC_RENEGOTIATION_SUPPORT_MASK) {
1530					tmp &= ~(PCIE_LC_LINK_WIDTH_CNTL__LC_LINK_WIDTH_MASK |
1531						PCIE_LC_LINK_WIDTH_CNTL__LC_UPCONFIGURE_DIS_MASK);
1532					tmp |= (max_lw <<
1533						PCIE_LC_LINK_WIDTH_CNTL__LC_LINK_WIDTH__SHIFT);
1534					tmp |= PCIE_LC_LINK_WIDTH_CNTL__LC_UPCONFIGURE_SUPPORT_MASK |
1535					PCIE_LC_LINK_WIDTH_CNTL__LC_RENEGOTIATE_EN_MASK |
1536					PCIE_LC_LINK_WIDTH_CNTL__LC_RECONFIG_NOW_MASK;
1537					WREG32_PCIE(ixPCIE_LC_LINK_WIDTH_CNTL, tmp);
1538				}
1539			}
1540
1541			for (i = 0; i < 10; i++) {
1542				/* check status */
1543				pcie_capability_read_word(adev->pdev,
1544							  PCI_EXP_DEVSTA,
1545							  &tmp16);
1546				if (tmp16 & PCI_EXP_DEVSTA_TRPND)
1547					break;
1548
1549				pcie_capability_read_word(root, PCI_EXP_LNKCTL,
1550							  &bridge_cfg);
1551				pcie_capability_read_word(adev->pdev,
1552							  PCI_EXP_LNKCTL,
1553							  &gpu_cfg);
1554
1555				pcie_capability_read_word(root, PCI_EXP_LNKCTL2,
1556							  &bridge_cfg2);
1557				pcie_capability_read_word(adev->pdev,
1558							  PCI_EXP_LNKCTL2,
1559							  &gpu_cfg2);
1560
1561				tmp = RREG32_PCIE(ixPCIE_LC_CNTL4);
1562				tmp |= PCIE_LC_CNTL4__LC_SET_QUIESCE_MASK;
1563				WREG32_PCIE(ixPCIE_LC_CNTL4, tmp);
1564
1565				tmp = RREG32_PCIE(ixPCIE_LC_CNTL4);
1566				tmp |= PCIE_LC_CNTL4__LC_REDO_EQ_MASK;
1567				WREG32_PCIE(ixPCIE_LC_CNTL4, tmp);
1568
1569				msleep(100);
1570
1571				/* linkctl */
1572				pcie_capability_read_word(root, PCI_EXP_LNKCTL,
1573							  &tmp16);
1574				tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
1575				tmp16 |= (bridge_cfg & PCI_EXP_LNKCTL_HAWD);
1576				pcie_capability_write_word(root, PCI_EXP_LNKCTL,
1577							   tmp16);
1578
1579				pcie_capability_read_word(adev->pdev,
1580							  PCI_EXP_LNKCTL,
1581							  &tmp16);
1582				tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
1583				tmp16 |= (gpu_cfg & PCI_EXP_LNKCTL_HAWD);
1584				pcie_capability_write_word(adev->pdev,
1585							   PCI_EXP_LNKCTL,
1586							   tmp16);
1587
1588				/* linkctl2 */
1589				pcie_capability_read_word(root, PCI_EXP_LNKCTL2,
1590							  &tmp16);
1591				tmp16 &= ~(PCI_EXP_LNKCTL2_ENTER_COMP |
1592					   PCI_EXP_LNKCTL2_TX_MARGIN);
1593				tmp16 |= (bridge_cfg2 &
1594					  (PCI_EXP_LNKCTL2_ENTER_COMP |
1595					   PCI_EXP_LNKCTL2_TX_MARGIN));
1596				pcie_capability_write_word(root,
1597							   PCI_EXP_LNKCTL2,
1598							   tmp16);
1599
1600				pcie_capability_read_word(adev->pdev,
1601							  PCI_EXP_LNKCTL2,
1602							  &tmp16);
1603				tmp16 &= ~(PCI_EXP_LNKCTL2_ENTER_COMP |
1604					   PCI_EXP_LNKCTL2_TX_MARGIN);
1605				tmp16 |= (gpu_cfg2 &
1606					  (PCI_EXP_LNKCTL2_ENTER_COMP |
1607					   PCI_EXP_LNKCTL2_TX_MARGIN));
1608				pcie_capability_write_word(adev->pdev,
1609							   PCI_EXP_LNKCTL2,
1610							   tmp16);
1611
1612				tmp = RREG32_PCIE(ixPCIE_LC_CNTL4);
1613				tmp &= ~PCIE_LC_CNTL4__LC_SET_QUIESCE_MASK;
1614				WREG32_PCIE(ixPCIE_LC_CNTL4, tmp);
1615			}
1616		}
1617	}
1618
1619	/* set the link speed */
1620	speed_cntl |= PCIE_LC_SPEED_CNTL__LC_FORCE_EN_SW_SPEED_CHANGE_MASK |
1621		PCIE_LC_SPEED_CNTL__LC_FORCE_DIS_HW_SPEED_CHANGE_MASK;
1622	speed_cntl &= ~PCIE_LC_SPEED_CNTL__LC_FORCE_DIS_SW_SPEED_CHANGE_MASK;
1623	WREG32_PCIE(ixPCIE_LC_SPEED_CNTL, speed_cntl);
1624
1625	pcie_capability_read_word(adev->pdev, PCI_EXP_LNKCTL2, &tmp16);
1626	tmp16 &= ~PCI_EXP_LNKCTL2_TLS;
1627
1628	if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3)
1629		tmp16 |= PCI_EXP_LNKCTL2_TLS_8_0GT; /* gen3 */
1630	else if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2)
1631		tmp16 |= PCI_EXP_LNKCTL2_TLS_5_0GT; /* gen2 */
1632	else
1633		tmp16 |= PCI_EXP_LNKCTL2_TLS_2_5GT; /* gen1 */
1634	pcie_capability_write_word(adev->pdev, PCI_EXP_LNKCTL2, tmp16);
1635
1636	speed_cntl = RREG32_PCIE(ixPCIE_LC_SPEED_CNTL);
1637	speed_cntl |= PCIE_LC_SPEED_CNTL__LC_INITIATE_LINK_SPEED_CHANGE_MASK;
1638	WREG32_PCIE(ixPCIE_LC_SPEED_CNTL, speed_cntl);
1639
1640	for (i = 0; i < adev->usec_timeout; i++) {
1641		speed_cntl = RREG32_PCIE(ixPCIE_LC_SPEED_CNTL);
1642		if ((speed_cntl & PCIE_LC_SPEED_CNTL__LC_INITIATE_LINK_SPEED_CHANGE_MASK) == 0)
1643			break;
1644		udelay(1);
1645	}
1646}
1647
1648static void cik_program_aspm(struct amdgpu_device *adev)
1649{
1650	u32 data, orig;
1651	bool disable_l0s = false, disable_l1 = false, disable_plloff_in_l1 = false;
1652	bool disable_clkreq = false;
1653
1654	if (amdgpu_aspm == 0)
1655		return;
1656
1657	if (pci_is_root_bus(adev->pdev->bus))
1658		return;
1659
1660	/* XXX double check APUs */
1661	if (adev->flags & AMD_IS_APU)
1662		return;
1663
1664	orig = data = RREG32_PCIE(ixPCIE_LC_N_FTS_CNTL);
1665	data &= ~PCIE_LC_N_FTS_CNTL__LC_XMIT_N_FTS_MASK;
1666	data |= (0x24 << PCIE_LC_N_FTS_CNTL__LC_XMIT_N_FTS__SHIFT) |
1667		PCIE_LC_N_FTS_CNTL__LC_XMIT_N_FTS_OVERRIDE_EN_MASK;
1668	if (orig != data)
1669		WREG32_PCIE(ixPCIE_LC_N_FTS_CNTL, data);
1670
1671	orig = data = RREG32_PCIE(ixPCIE_LC_CNTL3);
1672	data |= PCIE_LC_CNTL3__LC_GO_TO_RECOVERY_MASK;
1673	if (orig != data)
1674		WREG32_PCIE(ixPCIE_LC_CNTL3, data);
1675
1676	orig = data = RREG32_PCIE(ixPCIE_P_CNTL);
1677	data |= PCIE_P_CNTL__P_IGNORE_EDB_ERR_MASK;
1678	if (orig != data)
1679		WREG32_PCIE(ixPCIE_P_CNTL, data);
1680
1681	orig = data = RREG32_PCIE(ixPCIE_LC_CNTL);
1682	data &= ~(PCIE_LC_CNTL__LC_L0S_INACTIVITY_MASK |
1683		PCIE_LC_CNTL__LC_L1_INACTIVITY_MASK);
1684	data |= PCIE_LC_CNTL__LC_PMI_TO_L1_DIS_MASK;
1685	if (!disable_l0s)
1686		data |= (7 << PCIE_LC_CNTL__LC_L0S_INACTIVITY__SHIFT);
1687
1688	if (!disable_l1) {
1689		data |= (7 << PCIE_LC_CNTL__LC_L1_INACTIVITY__SHIFT);
1690		data &= ~PCIE_LC_CNTL__LC_PMI_TO_L1_DIS_MASK;
1691		if (orig != data)
1692			WREG32_PCIE(ixPCIE_LC_CNTL, data);
1693
1694		if (!disable_plloff_in_l1) {
1695			bool clk_req_support;
1696
1697			orig = data = RREG32_PCIE(ixPB0_PIF_PWRDOWN_0);
1698			data &= ~(PB0_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_OFF_0_MASK |
1699				PB0_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_TXS2_0_MASK);
1700			data |= (7 << PB0_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_OFF_0__SHIFT) |
1701				(7 << PB0_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_TXS2_0__SHIFT);
1702			if (orig != data)
1703				WREG32_PCIE(ixPB0_PIF_PWRDOWN_0, data);
1704
1705			orig = data = RREG32_PCIE(ixPB0_PIF_PWRDOWN_1);
1706			data &= ~(PB0_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_OFF_1_MASK |
1707				PB0_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_TXS2_1_MASK);
1708			data |= (7 << PB0_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_OFF_1__SHIFT) |
1709				(7 << PB0_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_TXS2_1__SHIFT);
1710			if (orig != data)
1711				WREG32_PCIE(ixPB0_PIF_PWRDOWN_1, data);
1712
1713			orig = data = RREG32_PCIE(ixPB1_PIF_PWRDOWN_0);
1714			data &= ~(PB1_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_OFF_0_MASK |
1715				PB1_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_TXS2_0_MASK);
1716			data |= (7 << PB1_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_OFF_0__SHIFT) |
1717				(7 << PB1_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_TXS2_0__SHIFT);
1718			if (orig != data)
1719				WREG32_PCIE(ixPB1_PIF_PWRDOWN_0, data);
1720
1721			orig = data = RREG32_PCIE(ixPB1_PIF_PWRDOWN_1);
1722			data &= ~(PB1_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_OFF_1_MASK |
1723				PB1_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_TXS2_1_MASK);
1724			data |= (7 << PB1_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_OFF_1__SHIFT) |
1725				(7 << PB1_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_TXS2_1__SHIFT);
1726			if (orig != data)
1727				WREG32_PCIE(ixPB1_PIF_PWRDOWN_1, data);
1728
1729			orig = data = RREG32_PCIE(ixPCIE_LC_LINK_WIDTH_CNTL);
1730			data &= ~PCIE_LC_LINK_WIDTH_CNTL__LC_DYN_LANES_PWR_STATE_MASK;
1731			data |= ~(3 << PCIE_LC_LINK_WIDTH_CNTL__LC_DYN_LANES_PWR_STATE__SHIFT);
1732			if (orig != data)
1733				WREG32_PCIE(ixPCIE_LC_LINK_WIDTH_CNTL, data);
1734
1735			if (!disable_clkreq) {
1736				struct pci_dev *root = adev->pdev->bus->self;
1737				u32 lnkcap;
1738
1739				clk_req_support = false;
1740				pcie_capability_read_dword(root, PCI_EXP_LNKCAP, &lnkcap);
1741				if (lnkcap & PCI_EXP_LNKCAP_CLKPM)
1742					clk_req_support = true;
1743			} else {
1744				clk_req_support = false;
1745			}
1746
1747			if (clk_req_support) {
1748				orig = data = RREG32_PCIE(ixPCIE_LC_CNTL2);
1749				data |= PCIE_LC_CNTL2__LC_ALLOW_PDWN_IN_L1_MASK |
1750					PCIE_LC_CNTL2__LC_ALLOW_PDWN_IN_L23_MASK;
1751				if (orig != data)
1752					WREG32_PCIE(ixPCIE_LC_CNTL2, data);
1753
1754				orig = data = RREG32_SMC(ixTHM_CLK_CNTL);
1755				data &= ~(THM_CLK_CNTL__CMON_CLK_SEL_MASK |
1756					THM_CLK_CNTL__TMON_CLK_SEL_MASK);
1757				data |= (1 << THM_CLK_CNTL__CMON_CLK_SEL__SHIFT) |
1758					(1 << THM_CLK_CNTL__TMON_CLK_SEL__SHIFT);
1759				if (orig != data)
1760					WREG32_SMC(ixTHM_CLK_CNTL, data);
1761
1762				orig = data = RREG32_SMC(ixMISC_CLK_CTRL);
1763				data &= ~(MISC_CLK_CTRL__DEEP_SLEEP_CLK_SEL_MASK |
1764					MISC_CLK_CTRL__ZCLK_SEL_MASK);
1765				data |= (1 << MISC_CLK_CTRL__DEEP_SLEEP_CLK_SEL__SHIFT) |
1766					(1 << MISC_CLK_CTRL__ZCLK_SEL__SHIFT);
1767				if (orig != data)
1768					WREG32_SMC(ixMISC_CLK_CTRL, data);
1769
1770				orig = data = RREG32_SMC(ixCG_CLKPIN_CNTL);
1771				data &= ~CG_CLKPIN_CNTL__BCLK_AS_XCLK_MASK;
1772				if (orig != data)
1773					WREG32_SMC(ixCG_CLKPIN_CNTL, data);
1774
1775				orig = data = RREG32_SMC(ixCG_CLKPIN_CNTL_2);
1776				data &= ~CG_CLKPIN_CNTL_2__FORCE_BIF_REFCLK_EN_MASK;
1777				if (orig != data)
1778					WREG32_SMC(ixCG_CLKPIN_CNTL_2, data);
1779
1780				orig = data = RREG32_SMC(ixMPLL_BYPASSCLK_SEL);
1781				data &= ~MPLL_BYPASSCLK_SEL__MPLL_CLKOUT_SEL_MASK;
1782				data |= (4 << MPLL_BYPASSCLK_SEL__MPLL_CLKOUT_SEL__SHIFT);
1783				if (orig != data)
1784					WREG32_SMC(ixMPLL_BYPASSCLK_SEL, data);
1785			}
1786		}
1787	} else {
1788		if (orig != data)
1789			WREG32_PCIE(ixPCIE_LC_CNTL, data);
1790	}
1791
1792	orig = data = RREG32_PCIE(ixPCIE_CNTL2);
1793	data |= PCIE_CNTL2__SLV_MEM_LS_EN_MASK |
1794		PCIE_CNTL2__MST_MEM_LS_EN_MASK |
1795		PCIE_CNTL2__REPLAY_MEM_LS_EN_MASK;
1796	if (orig != data)
1797		WREG32_PCIE(ixPCIE_CNTL2, data);
1798
1799	if (!disable_l0s) {
1800		data = RREG32_PCIE(ixPCIE_LC_N_FTS_CNTL);
1801		if ((data & PCIE_LC_N_FTS_CNTL__LC_N_FTS_MASK) ==
1802				PCIE_LC_N_FTS_CNTL__LC_N_FTS_MASK) {
1803			data = RREG32_PCIE(ixPCIE_LC_STATUS1);
1804			if ((data & PCIE_LC_STATUS1__LC_REVERSE_XMIT_MASK) &&
1805			(data & PCIE_LC_STATUS1__LC_REVERSE_RCVR_MASK)) {
1806				orig = data = RREG32_PCIE(ixPCIE_LC_CNTL);
1807				data &= ~PCIE_LC_CNTL__LC_L0S_INACTIVITY_MASK;
1808				if (orig != data)
1809					WREG32_PCIE(ixPCIE_LC_CNTL, data);
1810			}
1811		}
1812	}
1813}
1814
1815static uint32_t cik_get_rev_id(struct amdgpu_device *adev)
1816{
1817	return (RREG32(mmCC_DRM_ID_STRAPS) & CC_DRM_ID_STRAPS__ATI_REV_ID_MASK)
1818		>> CC_DRM_ID_STRAPS__ATI_REV_ID__SHIFT;
1819}
1820
1821static void cik_detect_hw_virtualization(struct amdgpu_device *adev)
1822{
1823	if (is_virtual_machine()) /* passthrough mode */
1824		adev->virt.caps |= AMDGPU_PASSTHROUGH_MODE;
1825}
1826
1827static void cik_flush_hdp(struct amdgpu_device *adev, struct amdgpu_ring *ring)
1828{
1829	if (!ring || !ring->funcs->emit_wreg) {
1830		WREG32(mmHDP_MEM_COHERENCY_FLUSH_CNTL, 1);
1831		RREG32(mmHDP_MEM_COHERENCY_FLUSH_CNTL);
1832	} else {
1833		amdgpu_ring_emit_wreg(ring, mmHDP_MEM_COHERENCY_FLUSH_CNTL, 1);
1834	}
1835}
1836
1837static void cik_invalidate_hdp(struct amdgpu_device *adev,
1838			       struct amdgpu_ring *ring)
1839{
1840	if (!ring || !ring->funcs->emit_wreg) {
1841		WREG32(mmHDP_DEBUG0, 1);
1842		RREG32(mmHDP_DEBUG0);
1843	} else {
1844		amdgpu_ring_emit_wreg(ring, mmHDP_DEBUG0, 1);
1845	}
1846}
1847
1848static bool cik_need_full_reset(struct amdgpu_device *adev)
1849{
1850	/* change this when we support soft reset */
1851	return true;
1852}
1853
1854static void cik_get_pcie_usage(struct amdgpu_device *adev, uint64_t *count0,
1855			       uint64_t *count1)
1856{
1857	uint32_t perfctr = 0;
1858	uint64_t cnt0_of, cnt1_of;
1859	int tmp;
1860
1861	/* This reports 0 on APUs, so return to avoid writing/reading registers
1862	 * that may or may not be different from their GPU counterparts
1863	 */
1864	if (adev->flags & AMD_IS_APU)
1865		return;
1866
1867	/* Set the 2 events that we wish to watch, defined above */
1868	/* Reg 40 is # received msgs, Reg 104 is # of posted requests sent */
1869	perfctr = REG_SET_FIELD(perfctr, PCIE_PERF_CNTL_TXCLK, EVENT0_SEL, 40);
1870	perfctr = REG_SET_FIELD(perfctr, PCIE_PERF_CNTL_TXCLK, EVENT1_SEL, 104);
1871
1872	/* Write to enable desired perf counters */
1873	WREG32_PCIE(ixPCIE_PERF_CNTL_TXCLK, perfctr);
1874	/* Zero out and enable the perf counters
1875	 * Write 0x5:
1876	 * Bit 0 = Start all counters(1)
1877	 * Bit 2 = Global counter reset enable(1)
1878	 */
1879	WREG32_PCIE(ixPCIE_PERF_COUNT_CNTL, 0x00000005);
1880
1881	msleep(1000);
1882
1883	/* Load the shadow and disable the perf counters
1884	 * Write 0x2:
1885	 * Bit 0 = Stop counters(0)
1886	 * Bit 1 = Load the shadow counters(1)
1887	 */
1888	WREG32_PCIE(ixPCIE_PERF_COUNT_CNTL, 0x00000002);
1889
1890	/* Read register values to get any >32bit overflow */
1891	tmp = RREG32_PCIE(ixPCIE_PERF_CNTL_TXCLK);
1892	cnt0_of = REG_GET_FIELD(tmp, PCIE_PERF_CNTL_TXCLK, COUNTER0_UPPER);
1893	cnt1_of = REG_GET_FIELD(tmp, PCIE_PERF_CNTL_TXCLK, COUNTER1_UPPER);
1894
1895	/* Get the values and add the overflow */
1896	*count0 = RREG32_PCIE(ixPCIE_PERF_COUNT0_TXCLK) | (cnt0_of << 32);
1897	*count1 = RREG32_PCIE(ixPCIE_PERF_COUNT1_TXCLK) | (cnt1_of << 32);
1898}
1899
1900static bool cik_need_reset_on_init(struct amdgpu_device *adev)
1901{
1902	u32 clock_cntl, pc;
1903
1904	if (adev->flags & AMD_IS_APU)
1905		return false;
1906
1907	/* check if the SMC is already running */
1908	clock_cntl = RREG32_SMC(ixSMC_SYSCON_CLOCK_CNTL_0);
1909	pc = RREG32_SMC(ixSMC_PC_C);
1910	if ((0 == REG_GET_FIELD(clock_cntl, SMC_SYSCON_CLOCK_CNTL_0, ck_disable)) &&
1911	    (0x20100 <= pc))
1912		return true;
1913
1914	return false;
1915}
1916
1917static uint64_t cik_get_pcie_replay_count(struct amdgpu_device *adev)
1918{
1919	uint64_t nak_r, nak_g;
1920
1921	/* Get the number of NAKs received and generated */
1922	nak_r = RREG32_PCIE(ixPCIE_RX_NUM_NAK);
1923	nak_g = RREG32_PCIE(ixPCIE_RX_NUM_NAK_GENERATED);
1924
1925	/* Add the total number of NAKs, i.e the number of replays */
1926	return (nak_r + nak_g);
1927}
1928
1929static const struct amdgpu_asic_funcs cik_asic_funcs =
1930{
1931	.read_disabled_bios = &cik_read_disabled_bios,
1932	.read_bios_from_rom = &cik_read_bios_from_rom,
1933	.read_register = &cik_read_register,
1934	.reset = &cik_asic_reset,
1935	.reset_method = &cik_asic_reset_method,
1936	.set_vga_state = &cik_vga_set_state,
1937	.get_xclk = &cik_get_xclk,
1938	.set_uvd_clocks = &cik_set_uvd_clocks,
1939	.set_vce_clocks = &cik_set_vce_clocks,
1940	.get_config_memsize = &cik_get_config_memsize,
1941	.flush_hdp = &cik_flush_hdp,
1942	.invalidate_hdp = &cik_invalidate_hdp,
1943	.need_full_reset = &cik_need_full_reset,
1944	.init_doorbell_index = &legacy_doorbell_index_init,
1945	.get_pcie_usage = &cik_get_pcie_usage,
1946	.need_reset_on_init = &cik_need_reset_on_init,
1947	.get_pcie_replay_count = &cik_get_pcie_replay_count,
1948	.supports_baco = &cik_asic_supports_baco,
1949};
1950
1951static int cik_common_early_init(void *handle)
1952{
1953	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1954
1955	adev->smc_rreg = &cik_smc_rreg;
1956	adev->smc_wreg = &cik_smc_wreg;
1957	adev->pcie_rreg = &cik_pcie_rreg;
1958	adev->pcie_wreg = &cik_pcie_wreg;
1959	adev->uvd_ctx_rreg = &cik_uvd_ctx_rreg;
1960	adev->uvd_ctx_wreg = &cik_uvd_ctx_wreg;
1961	adev->didt_rreg = &cik_didt_rreg;
1962	adev->didt_wreg = &cik_didt_wreg;
1963
1964	adev->asic_funcs = &cik_asic_funcs;
1965
1966	adev->rev_id = cik_get_rev_id(adev);
1967	adev->external_rev_id = 0xFF;
1968	switch (adev->asic_type) {
1969	case CHIP_BONAIRE:
1970		adev->cg_flags =
1971			AMD_CG_SUPPORT_GFX_MGCG |
1972			AMD_CG_SUPPORT_GFX_MGLS |
1973			/*AMD_CG_SUPPORT_GFX_CGCG |*/
1974			AMD_CG_SUPPORT_GFX_CGLS |
1975			AMD_CG_SUPPORT_GFX_CGTS |
1976			AMD_CG_SUPPORT_GFX_CGTS_LS |
1977			AMD_CG_SUPPORT_GFX_CP_LS |
1978			AMD_CG_SUPPORT_MC_LS |
1979			AMD_CG_SUPPORT_MC_MGCG |
1980			AMD_CG_SUPPORT_SDMA_MGCG |
1981			AMD_CG_SUPPORT_SDMA_LS |
1982			AMD_CG_SUPPORT_BIF_LS |
1983			AMD_CG_SUPPORT_VCE_MGCG |
1984			AMD_CG_SUPPORT_UVD_MGCG |
1985			AMD_CG_SUPPORT_HDP_LS |
1986			AMD_CG_SUPPORT_HDP_MGCG;
1987		adev->pg_flags = 0;
1988		adev->external_rev_id = adev->rev_id + 0x14;
1989		break;
1990	case CHIP_HAWAII:
1991		adev->cg_flags =
1992			AMD_CG_SUPPORT_GFX_MGCG |
1993			AMD_CG_SUPPORT_GFX_MGLS |
1994			/*AMD_CG_SUPPORT_GFX_CGCG |*/
1995			AMD_CG_SUPPORT_GFX_CGLS |
1996			AMD_CG_SUPPORT_GFX_CGTS |
1997			AMD_CG_SUPPORT_GFX_CP_LS |
1998			AMD_CG_SUPPORT_MC_LS |
1999			AMD_CG_SUPPORT_MC_MGCG |
2000			AMD_CG_SUPPORT_SDMA_MGCG |
2001			AMD_CG_SUPPORT_SDMA_LS |
2002			AMD_CG_SUPPORT_BIF_LS |
2003			AMD_CG_SUPPORT_VCE_MGCG |
2004			AMD_CG_SUPPORT_UVD_MGCG |
2005			AMD_CG_SUPPORT_HDP_LS |
2006			AMD_CG_SUPPORT_HDP_MGCG;
2007		adev->pg_flags = 0;
2008		adev->external_rev_id = 0x28;
2009		break;
2010	case CHIP_KAVERI:
2011		adev->cg_flags =
2012			AMD_CG_SUPPORT_GFX_MGCG |
2013			AMD_CG_SUPPORT_GFX_MGLS |
2014			/*AMD_CG_SUPPORT_GFX_CGCG |*/
2015			AMD_CG_SUPPORT_GFX_CGLS |
2016			AMD_CG_SUPPORT_GFX_CGTS |
2017			AMD_CG_SUPPORT_GFX_CGTS_LS |
2018			AMD_CG_SUPPORT_GFX_CP_LS |
2019			AMD_CG_SUPPORT_SDMA_MGCG |
2020			AMD_CG_SUPPORT_SDMA_LS |
2021			AMD_CG_SUPPORT_BIF_LS |
2022			AMD_CG_SUPPORT_VCE_MGCG |
2023			AMD_CG_SUPPORT_UVD_MGCG |
2024			AMD_CG_SUPPORT_HDP_LS |
2025			AMD_CG_SUPPORT_HDP_MGCG;
2026		adev->pg_flags =
2027			/*AMD_PG_SUPPORT_GFX_PG |
2028			  AMD_PG_SUPPORT_GFX_SMG |
2029			  AMD_PG_SUPPORT_GFX_DMG |*/
2030			AMD_PG_SUPPORT_UVD |
2031			AMD_PG_SUPPORT_VCE |
2032			/*  AMD_PG_SUPPORT_CP |
2033			  AMD_PG_SUPPORT_GDS |
2034			  AMD_PG_SUPPORT_RLC_SMU_HS |
2035			  AMD_PG_SUPPORT_ACP |
2036			  AMD_PG_SUPPORT_SAMU |*/
2037			0;
2038		if (adev->pdev->device == 0x1312 ||
2039			adev->pdev->device == 0x1316 ||
2040			adev->pdev->device == 0x1317)
2041			adev->external_rev_id = 0x41;
2042		else
2043			adev->external_rev_id = 0x1;
2044		break;
2045	case CHIP_KABINI:
2046	case CHIP_MULLINS:
2047		adev->cg_flags =
2048			AMD_CG_SUPPORT_GFX_MGCG |
2049			AMD_CG_SUPPORT_GFX_MGLS |
2050			/*AMD_CG_SUPPORT_GFX_CGCG |*/
2051			AMD_CG_SUPPORT_GFX_CGLS |
2052			AMD_CG_SUPPORT_GFX_CGTS |
2053			AMD_CG_SUPPORT_GFX_CGTS_LS |
2054			AMD_CG_SUPPORT_GFX_CP_LS |
2055			AMD_CG_SUPPORT_SDMA_MGCG |
2056			AMD_CG_SUPPORT_SDMA_LS |
2057			AMD_CG_SUPPORT_BIF_LS |
2058			AMD_CG_SUPPORT_VCE_MGCG |
2059			AMD_CG_SUPPORT_UVD_MGCG |
2060			AMD_CG_SUPPORT_HDP_LS |
2061			AMD_CG_SUPPORT_HDP_MGCG;
2062		adev->pg_flags =
2063			/*AMD_PG_SUPPORT_GFX_PG |
2064			  AMD_PG_SUPPORT_GFX_SMG | */
2065			AMD_PG_SUPPORT_UVD |
2066			/*AMD_PG_SUPPORT_VCE |
2067			  AMD_PG_SUPPORT_CP |
2068			  AMD_PG_SUPPORT_GDS |
2069			  AMD_PG_SUPPORT_RLC_SMU_HS |
2070			  AMD_PG_SUPPORT_SAMU |*/
2071			0;
2072		if (adev->asic_type == CHIP_KABINI) {
2073			if (adev->rev_id == 0)
2074				adev->external_rev_id = 0x81;
2075			else if (adev->rev_id == 1)
2076				adev->external_rev_id = 0x82;
2077			else if (adev->rev_id == 2)
2078				adev->external_rev_id = 0x85;
2079		} else
2080			adev->external_rev_id = adev->rev_id + 0xa1;
2081		break;
2082	default:
2083		/* FIXME: not supported yet */
2084		return -EINVAL;
2085	}
2086
2087	return 0;
2088}
2089
2090static int cik_common_sw_init(void *handle)
2091{
2092	return 0;
2093}
2094
2095static int cik_common_sw_fini(void *handle)
2096{
2097	return 0;
2098}
2099
2100static int cik_common_hw_init(void *handle)
2101{
2102	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2103
2104	/* move the golden regs per IP block */
2105	cik_init_golden_registers(adev);
2106	/* enable pcie gen2/3 link */
2107	cik_pcie_gen3_enable(adev);
2108	/* enable aspm */
2109	cik_program_aspm(adev);
2110
2111	return 0;
2112}
2113
2114static int cik_common_hw_fini(void *handle)
2115{
2116	return 0;
2117}
2118
2119static int cik_common_suspend(void *handle)
2120{
2121	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2122
2123	return cik_common_hw_fini(adev);
2124}
2125
2126static int cik_common_resume(void *handle)
2127{
2128	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2129
2130	return cik_common_hw_init(adev);
2131}
2132
2133static bool cik_common_is_idle(void *handle)
2134{
2135	return true;
2136}
2137
2138static int cik_common_wait_for_idle(void *handle)
2139{
2140	return 0;
2141}
2142
2143static int cik_common_soft_reset(void *handle)
2144{
2145	/* XXX hard reset?? */
2146	return 0;
2147}
2148
2149static int cik_common_set_clockgating_state(void *handle,
2150					    enum amd_clockgating_state state)
2151{
2152	return 0;
2153}
2154
2155static int cik_common_set_powergating_state(void *handle,
2156					    enum amd_powergating_state state)
2157{
2158	return 0;
2159}
2160
2161static const struct amd_ip_funcs cik_common_ip_funcs = {
2162	.name = "cik_common",
2163	.early_init = cik_common_early_init,
2164	.late_init = NULL,
2165	.sw_init = cik_common_sw_init,
2166	.sw_fini = cik_common_sw_fini,
2167	.hw_init = cik_common_hw_init,
2168	.hw_fini = cik_common_hw_fini,
2169	.suspend = cik_common_suspend,
2170	.resume = cik_common_resume,
2171	.is_idle = cik_common_is_idle,
2172	.wait_for_idle = cik_common_wait_for_idle,
2173	.soft_reset = cik_common_soft_reset,
2174	.set_clockgating_state = cik_common_set_clockgating_state,
2175	.set_powergating_state = cik_common_set_powergating_state,
2176};
2177
2178static const struct amdgpu_ip_block_version cik_common_ip_block =
2179{
2180	.type = AMD_IP_BLOCK_TYPE_COMMON,
2181	.major = 1,
2182	.minor = 0,
2183	.rev = 0,
2184	.funcs = &cik_common_ip_funcs,
2185};
2186
2187int cik_set_ip_blocks(struct amdgpu_device *adev)
2188{
2189	cik_detect_hw_virtualization(adev);
2190
2191	switch (adev->asic_type) {
2192	case CHIP_BONAIRE:
2193		amdgpu_device_ip_block_add(adev, &cik_common_ip_block);
2194		amdgpu_device_ip_block_add(adev, &gmc_v7_0_ip_block);
2195		amdgpu_device_ip_block_add(adev, &cik_ih_ip_block);
2196		amdgpu_device_ip_block_add(adev, &gfx_v7_2_ip_block);
2197		amdgpu_device_ip_block_add(adev, &cik_sdma_ip_block);
2198		amdgpu_device_ip_block_add(adev, &pp_smu_ip_block);
2199		if (adev->enable_virtual_display)
2200			amdgpu_device_ip_block_add(adev, &dce_virtual_ip_block);
2201#if defined(CONFIG_DRM_AMD_DC)
2202		else if (amdgpu_device_has_dc_support(adev))
2203			amdgpu_device_ip_block_add(adev, &dm_ip_block);
2204#endif
2205		else
2206			amdgpu_device_ip_block_add(adev, &dce_v8_2_ip_block);
2207		amdgpu_device_ip_block_add(adev, &uvd_v4_2_ip_block);
2208		amdgpu_device_ip_block_add(adev, &vce_v2_0_ip_block);
2209		break;
2210	case CHIP_HAWAII:
2211		amdgpu_device_ip_block_add(adev, &cik_common_ip_block);
2212		amdgpu_device_ip_block_add(adev, &gmc_v7_0_ip_block);
2213		amdgpu_device_ip_block_add(adev, &cik_ih_ip_block);
2214		amdgpu_device_ip_block_add(adev, &gfx_v7_3_ip_block);
2215		amdgpu_device_ip_block_add(adev, &cik_sdma_ip_block);
2216		amdgpu_device_ip_block_add(adev, &pp_smu_ip_block);
2217		if (adev->enable_virtual_display)
2218			amdgpu_device_ip_block_add(adev, &dce_virtual_ip_block);
2219#if defined(CONFIG_DRM_AMD_DC)
2220		else if (amdgpu_device_has_dc_support(adev))
2221			amdgpu_device_ip_block_add(adev, &dm_ip_block);
2222#endif
2223		else
2224			amdgpu_device_ip_block_add(adev, &dce_v8_5_ip_block);
2225		amdgpu_device_ip_block_add(adev, &uvd_v4_2_ip_block);
2226		amdgpu_device_ip_block_add(adev, &vce_v2_0_ip_block);
2227		break;
2228	case CHIP_KAVERI:
2229		amdgpu_device_ip_block_add(adev, &cik_common_ip_block);
2230		amdgpu_device_ip_block_add(adev, &gmc_v7_0_ip_block);
2231		amdgpu_device_ip_block_add(adev, &cik_ih_ip_block);
2232		amdgpu_device_ip_block_add(adev, &gfx_v7_1_ip_block);
2233		amdgpu_device_ip_block_add(adev, &cik_sdma_ip_block);
2234		amdgpu_device_ip_block_add(adev, &kv_smu_ip_block);
2235		if (adev->enable_virtual_display)
2236			amdgpu_device_ip_block_add(adev, &dce_virtual_ip_block);
2237#if defined(CONFIG_DRM_AMD_DC)
2238		else if (amdgpu_device_has_dc_support(adev))
2239			amdgpu_device_ip_block_add(adev, &dm_ip_block);
2240#endif
2241		else
2242			amdgpu_device_ip_block_add(adev, &dce_v8_1_ip_block);
2243
2244		amdgpu_device_ip_block_add(adev, &uvd_v4_2_ip_block);
2245		amdgpu_device_ip_block_add(adev, &vce_v2_0_ip_block);
2246		break;
2247	case CHIP_KABINI:
2248	case CHIP_MULLINS:
2249		amdgpu_device_ip_block_add(adev, &cik_common_ip_block);
2250		amdgpu_device_ip_block_add(adev, &gmc_v7_0_ip_block);
2251		amdgpu_device_ip_block_add(adev, &cik_ih_ip_block);
2252		amdgpu_device_ip_block_add(adev, &gfx_v7_2_ip_block);
2253		amdgpu_device_ip_block_add(adev, &cik_sdma_ip_block);
2254		amdgpu_device_ip_block_add(adev, &kv_smu_ip_block);
2255		if (adev->enable_virtual_display)
2256			amdgpu_device_ip_block_add(adev, &dce_virtual_ip_block);
2257#if defined(CONFIG_DRM_AMD_DC)
2258		else if (amdgpu_device_has_dc_support(adev))
2259			amdgpu_device_ip_block_add(adev, &dm_ip_block);
2260#endif
2261		else
2262			amdgpu_device_ip_block_add(adev, &dce_v8_3_ip_block);
2263		amdgpu_device_ip_block_add(adev, &uvd_v4_2_ip_block);
2264		amdgpu_device_ip_block_add(adev, &vce_v2_0_ip_block);
2265		break;
2266	default:
2267		/* FIXME: not supported yet */
2268		return -EINVAL;
2269	}
2270	return 0;
2271}
2272