radeon_cp.c revision 162132 
193618Sjake/* radeon_cp.c -- CP support for Radeon -*- linux-c -*- */
293618Sjake/*-
393618Sjake * Copyright 2000 Precision Insight, Inc., Cedar Park, Texas.
493618Sjake * Copyright 2000 VA Linux Systems, Inc., Fremont, California.
593618Sjake * All Rights Reserved.
693618Sjake *
793618Sjake * Permission is hereby granted, free of charge, to any person obtaining a
893618Sjake * copy of this software and associated documentation files (the "Software"),
993618Sjake * to deal in the Software without restriction, including without limitation
1093618Sjake * the rights to use, copy, modify, merge, publish, distribute, sublicense,
1193618Sjake * and/or sell copies of the Software, and to permit persons to whom the
1293618Sjake * Software is furnished to do so, subject to the following conditions:
1393618Sjake *
1493618Sjake * The above copyright notice and this permission notice (including the next
1593618Sjake * paragraph) shall be included in all copies or substantial portions of the
1693618Sjake * Software.
1793618Sjake *
1893618Sjake * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
1993618Sjake * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
2093618Sjake * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
2193618Sjake * PRECISION INSIGHT AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
2293618Sjake * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
2393618Sjake * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
2493618Sjake * DEALINGS IN THE SOFTWARE.
2593618Sjake *
2693618Sjake * Authors:
2793618Sjake *    Kevin E. Martin <martin@valinux.com>
28147370Sru *    Gareth Hughes <gareth@valinux.com>
2993618Sjake */
3093618Sjake
3193618Sjake#include <sys/cdefs.h>
3293618Sjake__FBSDID("$FreeBSD: head/sys/dev/drm/radeon_cp.c 162132 2006-09-07 23:04:47Z anholt $");
3393618Sjake
3493618Sjake#include "dev/drm/drmP.h"
3593618Sjake#include "dev/drm/drm.h"
36217873Sdchagin#include "dev/drm/radeon_drm.h"
3793618Sjake#include "dev/drm/radeon_drv.h"
38143275Shmp#include "dev/drm/r300_reg.h"
3993618Sjake
4093618Sjake#define RADEON_FIFO_DEBUG	0
4193618Sjake
4293618Sjakestatic int radeon_do_cleanup_cp(drm_device_t * dev);
4393618Sjake
4499970Scharnier/* CP microcode (from ATI) */
4593618Sjakestatic const u32 R200_cp_microcode[][2] = {
4693618Sjake	{0x21007000, 0000000000},
4797600Sru	{0x20007000, 0000000000},
4893618Sjake	{0x000000ab, 0x00000004},
4997600Sru	{0x000000af, 0x00000004},
5093618Sjake	{0x66544a49, 0000000000},
5193618Sjake	{0x49494174, 0000000000},
52129559Srwatson	{0x54517d83, 0000000000},
53131754Sru	{0x498d8b64, 0000000000},
54129574Srwatson	{0x49494949, 0000000000},
55129574Srwatson	{0x49da493c, 0000000000},
5693618Sjake	{0x49989898, 0000000000},
5793618Sjake	{0xd34949d5, 0000000000},
58217873Sdchagin	{0x9dc90e11, 0000000000},
59217873Sdchagin	{0xce9b9b9b, 0000000000},
60143275Shmp	{0x000f0000, 0x00000016},
61143275Shmp	{0x352e232c, 0000000000},
62143275Shmp	{0x00000013, 0x00000004},
63143275Shmp	{0x000f0000, 0x00000016},
6497600Sru	{0x352e272c, 0000000000},
6593618Sjake	{0x000f0001, 0x00000016},
6693618Sjake	{0x3239362f, 0000000000},
6793618Sjake	{0x000077ef, 0x00000002},
6893618Sjake	{0x00061000, 0x00000002},
6993618Sjake	{0x00000020, 0x0000001a},
7093618Sjake	{0x00004000, 0x0000001e},
7193618Sjake	{0x00061000, 0x00000002},
7293618Sjake	{0x00000020, 0x0000001a},
7393618Sjake	{0x00004000, 0x0000001e},
7497600Sru	{0x00061000, 0x00000002},
7593618Sjake	{0x00000020, 0x0000001a},
7693618Sjake	{0x00004000, 0x0000001e},
7793618Sjake	{0x00000016, 0x00000004},
7893618Sjake	{0x0003802a, 0x00000002},
7993618Sjake	{0x040067e0, 0x00000002},
8093618Sjake	{0x00000016, 0x00000004},
8193618Sjake	{0x000077e0, 0x00000002},
8299970Scharnier	{0x00065000, 0x00000002},
8393618Sjake	{0x000037e1, 0x00000002},
8493618Sjake	{0x040067e1, 0x00000006},
8593618Sjake	{0x000077e0, 0x00000002},
8693618Sjake	{0x000077e1, 0x00000002},
8793618Sjake	{0x000077e1, 0x00000006},
8899970Scharnier	{0xffffffff, 0000000000},
8993618Sjake	{0x10000000, 0000000000},
9093618Sjake	{0x0003802a, 0x00000002},
9193618Sjake	{0x040067e0, 0x00000006},
9293618Sjake	{0x00007675, 0x00000002},
93	{0x00007676, 0x00000002},
94	{0x00007677, 0x00000002},
95	{0x00007678, 0x00000006},
96	{0x0003802b, 0x00000002},
97	{0x04002676, 0x00000002},
98	{0x00007677, 0x00000002},
99	{0x00007678, 0x00000006},
100	{0x0000002e, 0x00000018},
101	{0x0000002e, 0x00000018},
102	{0000000000, 0x00000006},
103	{0x0000002f, 0x00000018},
104	{0x0000002f, 0x00000018},
105	{0000000000, 0x00000006},
106	{0x01605000, 0x00000002},
107	{0x00065000, 0x00000002},
108	{0x00098000, 0x00000002},
109	{0x00061000, 0x00000002},
110	{0x64c0603d, 0x00000004},
111	{0x00080000, 0x00000016},
112	{0000000000, 0000000000},
113	{0x0400251d, 0x00000002},
114	{0x00007580, 0x00000002},
115	{0x00067581, 0x00000002},
116	{0x04002580, 0x00000002},
117	{0x00067581, 0x00000002},
118	{0x00000046, 0x00000004},
119	{0x00005000, 0000000000},
120	{0x00061000, 0x00000002},
121	{0x0000750e, 0x00000002},
122	{0x00019000, 0x00000002},
123	{0x00011055, 0x00000014},
124	{0x00000055, 0x00000012},
125	{0x0400250f, 0x00000002},
126	{0x0000504a, 0x00000004},
127	{0x00007565, 0x00000002},
128	{0x00007566, 0x00000002},
129	{0x00000051, 0x00000004},
130	{0x01e655b4, 0x00000002},
131	{0x4401b0dc, 0x00000002},
132	{0x01c110dc, 0x00000002},
133	{0x2666705d, 0x00000018},
134	{0x040c2565, 0x00000002},
135	{0x0000005d, 0x00000018},
136	{0x04002564, 0x00000002},
137	{0x00007566, 0x00000002},
138	{0x00000054, 0x00000004},
139	{0x00401060, 0x00000008},
140	{0x00101000, 0x00000002},
141	{0x000d80ff, 0x00000002},
142	{0x00800063, 0x00000008},
143	{0x000f9000, 0x00000002},
144	{0x000e00ff, 0x00000002},
145	{0000000000, 0x00000006},
146	{0x00000080, 0x00000018},
147	{0x00000054, 0x00000004},
148	{0x00007576, 0x00000002},
149	{0x00065000, 0x00000002},
150	{0x00009000, 0x00000002},
151	{0x00041000, 0x00000002},
152	{0x0c00350e, 0x00000002},
153	{0x00049000, 0x00000002},
154	{0x00051000, 0x00000002},
155	{0x01e785f8, 0x00000002},
156	{0x00200000, 0x00000002},
157	{0x00600073, 0x0000000c},
158	{0x00007563, 0x00000002},
159	{0x006075f0, 0x00000021},
160	{0x20007068, 0x00000004},
161	{0x00005068, 0x00000004},
162	{0x00007576, 0x00000002},
163	{0x00007577, 0x00000002},
164	{0x0000750e, 0x00000002},
165	{0x0000750f, 0x00000002},
166	{0x00a05000, 0x00000002},
167	{0x00600076, 0x0000000c},
168	{0x006075f0, 0x00000021},
169	{0x000075f8, 0x00000002},
170	{0x00000076, 0x00000004},
171	{0x000a750e, 0x00000002},
172	{0x0020750f, 0x00000002},
173	{0x00600079, 0x00000004},
174	{0x00007570, 0x00000002},
175	{0x00007571, 0x00000002},
176	{0x00007572, 0x00000006},
177	{0x00005000, 0x00000002},
178	{0x00a05000, 0x00000002},
179	{0x00007568, 0x00000002},
180	{0x00061000, 0x00000002},
181	{0x00000084, 0x0000000c},
182	{0x00058000, 0x00000002},
183	{0x0c607562, 0x00000002},
184	{0x00000086, 0x00000004},
185	{0x00600085, 0x00000004},
186	{0x400070dd, 0000000000},
187	{0x000380dd, 0x00000002},
188	{0x00000093, 0x0000001c},
189	{0x00065095, 0x00000018},
190	{0x040025bb, 0x00000002},
191	{0x00061096, 0x00000018},
192	{0x040075bc, 0000000000},
193	{0x000075bb, 0x00000002},
194	{0x000075bc, 0000000000},
195	{0x00090000, 0x00000006},
196	{0x00090000, 0x00000002},
197	{0x000d8002, 0x00000006},
198	{0x00005000, 0x00000002},
199	{0x00007821, 0x00000002},
200	{0x00007800, 0000000000},
201	{0x00007821, 0x00000002},
202	{0x00007800, 0000000000},
203	{0x01665000, 0x00000002},
204	{0x000a0000, 0x00000002},
205	{0x000671cc, 0x00000002},
206	{0x0286f1cd, 0x00000002},
207	{0x000000a3, 0x00000010},
208	{0x21007000, 0000000000},
209	{0x000000aa, 0x0000001c},
210	{0x00065000, 0x00000002},
211	{0x000a0000, 0x00000002},
212	{0x00061000, 0x00000002},
213	{0x000b0000, 0x00000002},
214	{0x38067000, 0x00000002},
215	{0x000a00a6, 0x00000004},
216	{0x20007000, 0000000000},
217	{0x01200000, 0x00000002},
218	{0x20077000, 0x00000002},
219	{0x01200000, 0x00000002},
220	{0x20007000, 0000000000},
221	{0x00061000, 0x00000002},
222	{0x0120751b, 0x00000002},
223	{0x8040750a, 0x00000002},
224	{0x8040750b, 0x00000002},
225	{0x00110000, 0x00000002},
226	{0x000380dd, 0x00000002},
227	{0x000000bd, 0x0000001c},
228	{0x00061096, 0x00000018},
229	{0x844075bd, 0x00000002},
230	{0x00061095, 0x00000018},
231	{0x840075bb, 0x00000002},
232	{0x00061096, 0x00000018},
233	{0x844075bc, 0x00000002},
234	{0x000000c0, 0x00000004},
235	{0x804075bd, 0x00000002},
236	{0x800075bb, 0x00000002},
237	{0x804075bc, 0x00000002},
238	{0x00108000, 0x00000002},
239	{0x01400000, 0x00000002},
240	{0x006000c4, 0x0000000c},
241	{0x20c07000, 0x00000020},
242	{0x000000c6, 0x00000012},
243	{0x00800000, 0x00000006},
244	{0x0080751d, 0x00000006},
245	{0x000025bb, 0x00000002},
246	{0x000040c0, 0x00000004},
247	{0x0000775c, 0x00000002},
248	{0x00a05000, 0x00000002},
249	{0x00661000, 0x00000002},
250	{0x0460275d, 0x00000020},
251	{0x00004000, 0000000000},
252	{0x00007999, 0x00000002},
253	{0x00a05000, 0x00000002},
254	{0x00661000, 0x00000002},
255	{0x0460299b, 0x00000020},
256	{0x00004000, 0000000000},
257	{0x01e00830, 0x00000002},
258	{0x21007000, 0000000000},
259	{0x00005000, 0x00000002},
260	{0x00038042, 0x00000002},
261	{0x040025e0, 0x00000002},
262	{0x000075e1, 0000000000},
263	{0x00000001, 0000000000},
264	{0x000380d9, 0x00000002},
265	{0x04007394, 0000000000},
266	{0000000000, 0000000000},
267	{0000000000, 0000000000},
268	{0000000000, 0000000000},
269	{0000000000, 0000000000},
270	{0000000000, 0000000000},
271	{0000000000, 0000000000},
272	{0000000000, 0000000000},
273	{0000000000, 0000000000},
274	{0000000000, 0000000000},
275	{0000000000, 0000000000},
276	{0000000000, 0000000000},
277	{0000000000, 0000000000},
278	{0000000000, 0000000000},
279	{0000000000, 0000000000},
280	{0000000000, 0000000000},
281	{0000000000, 0000000000},
282	{0000000000, 0000000000},
283	{0000000000, 0000000000},
284	{0000000000, 0000000000},
285	{0000000000, 0000000000},
286	{0000000000, 0000000000},
287	{0000000000, 0000000000},
288	{0000000000, 0000000000},
289	{0000000000, 0000000000},
290	{0000000000, 0000000000},
291	{0000000000, 0000000000},
292	{0000000000, 0000000000},
293	{0000000000, 0000000000},
294	{0000000000, 0000000000},
295	{0000000000, 0000000000},
296	{0000000000, 0000000000},
297	{0000000000, 0000000000},
298	{0000000000, 0000000000},
299	{0000000000, 0000000000},
300	{0000000000, 0000000000},
301	{0000000000, 0000000000},
302};
303
304static const u32 radeon_cp_microcode[][2] = {
305	{0x21007000, 0000000000},
306	{0x20007000, 0000000000},
307	{0x000000b4, 0x00000004},
308	{0x000000b8, 0x00000004},
309	{0x6f5b4d4c, 0000000000},
310	{0x4c4c427f, 0000000000},
311	{0x5b568a92, 0000000000},
312	{0x4ca09c6d, 0000000000},
313	{0xad4c4c4c, 0000000000},
314	{0x4ce1af3d, 0000000000},
315	{0xd8afafaf, 0000000000},
316	{0xd64c4cdc, 0000000000},
317	{0x4cd10d10, 0000000000},
318	{0x000f0000, 0x00000016},
319	{0x362f242d, 0000000000},
320	{0x00000012, 0x00000004},
321	{0x000f0000, 0x00000016},
322	{0x362f282d, 0000000000},
323	{0x000380e7, 0x00000002},
324	{0x04002c97, 0x00000002},
325	{0x000f0001, 0x00000016},
326	{0x333a3730, 0000000000},
327	{0x000077ef, 0x00000002},
328	{0x00061000, 0x00000002},
329	{0x00000021, 0x0000001a},
330	{0x00004000, 0x0000001e},
331	{0x00061000, 0x00000002},
332	{0x00000021, 0x0000001a},
333	{0x00004000, 0x0000001e},
334	{0x00061000, 0x00000002},
335	{0x00000021, 0x0000001a},
336	{0x00004000, 0x0000001e},
337	{0x00000017, 0x00000004},
338	{0x0003802b, 0x00000002},
339	{0x040067e0, 0x00000002},
340	{0x00000017, 0x00000004},
341	{0x000077e0, 0x00000002},
342	{0x00065000, 0x00000002},
343	{0x000037e1, 0x00000002},
344	{0x040067e1, 0x00000006},
345	{0x000077e0, 0x00000002},
346	{0x000077e1, 0x00000002},
347	{0x000077e1, 0x00000006},
348	{0xffffffff, 0000000000},
349	{0x10000000, 0000000000},
350	{0x0003802b, 0x00000002},
351	{0x040067e0, 0x00000006},
352	{0x00007675, 0x00000002},
353	{0x00007676, 0x00000002},
354	{0x00007677, 0x00000002},
355	{0x00007678, 0x00000006},
356	{0x0003802c, 0x00000002},
357	{0x04002676, 0x00000002},
358	{0x00007677, 0x00000002},
359	{0x00007678, 0x00000006},
360	{0x0000002f, 0x00000018},
361	{0x0000002f, 0x00000018},
362	{0000000000, 0x00000006},
363	{0x00000030, 0x00000018},
364	{0x00000030, 0x00000018},
365	{0000000000, 0x00000006},
366	{0x01605000, 0x00000002},
367	{0x00065000, 0x00000002},
368	{0x00098000, 0x00000002},
369	{0x00061000, 0x00000002},
370	{0x64c0603e, 0x00000004},
371	{0x000380e6, 0x00000002},
372	{0x040025c5, 0x00000002},
373	{0x00080000, 0x00000016},
374	{0000000000, 0000000000},
375	{0x0400251d, 0x00000002},
376	{0x00007580, 0x00000002},
377	{0x00067581, 0x00000002},
378	{0x04002580, 0x00000002},
379	{0x00067581, 0x00000002},
380	{0x00000049, 0x00000004},
381	{0x00005000, 0000000000},
382	{0x000380e6, 0x00000002},
383	{0x040025c5, 0x00000002},
384	{0x00061000, 0x00000002},
385	{0x0000750e, 0x00000002},
386	{0x00019000, 0x00000002},
387	{0x00011055, 0x00000014},
388	{0x00000055, 0x00000012},
389	{0x0400250f, 0x00000002},
390	{0x0000504f, 0x00000004},
391	{0x000380e6, 0x00000002},
392	{0x040025c5, 0x00000002},
393	{0x00007565, 0x00000002},
394	{0x00007566, 0x00000002},
395	{0x00000058, 0x00000004},
396	{0x000380e6, 0x00000002},
397	{0x040025c5, 0x00000002},
398	{0x01e655b4, 0x00000002},
399	{0x4401b0e4, 0x00000002},
400	{0x01c110e4, 0x00000002},
401	{0x26667066, 0x00000018},
402	{0x040c2565, 0x00000002},
403	{0x00000066, 0x00000018},
404	{0x04002564, 0x00000002},
405	{0x00007566, 0x00000002},
406	{0x0000005d, 0x00000004},
407	{0x00401069, 0x00000008},
408	{0x00101000, 0x00000002},
409	{0x000d80ff, 0x00000002},
410	{0x0080006c, 0x00000008},
411	{0x000f9000, 0x00000002},
412	{0x000e00ff, 0x00000002},
413	{0000000000, 0x00000006},
414	{0x0000008f, 0x00000018},
415	{0x0000005b, 0x00000004},
416	{0x000380e6, 0x00000002},
417	{0x040025c5, 0x00000002},
418	{0x00007576, 0x00000002},
419	{0x00065000, 0x00000002},
420	{0x00009000, 0x00000002},
421	{0x00041000, 0x00000002},
422	{0x0c00350e, 0x00000002},
423	{0x00049000, 0x00000002},
424	{0x00051000, 0x00000002},
425	{0x01e785f8, 0x00000002},
426	{0x00200000, 0x00000002},
427	{0x0060007e, 0x0000000c},
428	{0x00007563, 0x00000002},
429	{0x006075f0, 0x00000021},
430	{0x20007073, 0x00000004},
431	{0x00005073, 0x00000004},
432	{0x000380e6, 0x00000002},
433	{0x040025c5, 0x00000002},
434	{0x00007576, 0x00000002},
435	{0x00007577, 0x00000002},
436	{0x0000750e, 0x00000002},
437	{0x0000750f, 0x00000002},
438	{0x00a05000, 0x00000002},
439	{0x00600083, 0x0000000c},
440	{0x006075f0, 0x00000021},
441	{0x000075f8, 0x00000002},
442	{0x00000083, 0x00000004},
443	{0x000a750e, 0x00000002},
444	{0x000380e6, 0x00000002},
445	{0x040025c5, 0x00000002},
446	{0x0020750f, 0x00000002},
447	{0x00600086, 0x00000004},
448	{0x00007570, 0x00000002},
449	{0x00007571, 0x00000002},
450	{0x00007572, 0x00000006},
451	{0x000380e6, 0x00000002},
452	{0x040025c5, 0x00000002},
453	{0x00005000, 0x00000002},
454	{0x00a05000, 0x00000002},
455	{0x00007568, 0x00000002},
456	{0x00061000, 0x00000002},
457	{0x00000095, 0x0000000c},
458	{0x00058000, 0x00000002},
459	{0x0c607562, 0x00000002},
460	{0x00000097, 0x00000004},
461	{0x000380e6, 0x00000002},
462	{0x040025c5, 0x00000002},
463	{0x00600096, 0x00000004},
464	{0x400070e5, 0000000000},
465	{0x000380e6, 0x00000002},
466	{0x040025c5, 0x00000002},
467	{0x000380e5, 0x00000002},
468	{0x000000a8, 0x0000001c},
469	{0x000650aa, 0x00000018},
470	{0x040025bb, 0x00000002},
471	{0x000610ab, 0x00000018},
472	{0x040075bc, 0000000000},
473	{0x000075bb, 0x00000002},
474	{0x000075bc, 0000000000},
475	{0x00090000, 0x00000006},
476	{0x00090000, 0x00000002},
477	{0x000d8002, 0x00000006},
478	{0x00007832, 0x00000002},
479	{0x00005000, 0x00000002},
480	{0x000380e7, 0x00000002},
481	{0x04002c97, 0x00000002},
482	{0x00007820, 0x00000002},
483	{0x00007821, 0x00000002},
484	{0x00007800, 0000000000},
485	{0x01200000, 0x00000002},
486	{0x20077000, 0x00000002},
487	{0x01200000, 0x00000002},
488	{0x20007000, 0x00000002},
489	{0x00061000, 0x00000002},
490	{0x0120751b, 0x00000002},
491	{0x8040750a, 0x00000002},
492	{0x8040750b, 0x00000002},
493	{0x00110000, 0x00000002},
494	{0x000380e5, 0x00000002},
495	{0x000000c6, 0x0000001c},
496	{0x000610ab, 0x00000018},
497	{0x844075bd, 0x00000002},
498	{0x000610aa, 0x00000018},
499	{0x840075bb, 0x00000002},
500	{0x000610ab, 0x00000018},
501	{0x844075bc, 0x00000002},
502	{0x000000c9, 0x00000004},
503	{0x804075bd, 0x00000002},
504	{0x800075bb, 0x00000002},
505	{0x804075bc, 0x00000002},
506	{0x00108000, 0x00000002},
507	{0x01400000, 0x00000002},
508	{0x006000cd, 0x0000000c},
509	{0x20c07000, 0x00000020},
510	{0x000000cf, 0x00000012},
511	{0x00800000, 0x00000006},
512	{0x0080751d, 0x00000006},
513	{0000000000, 0000000000},
514	{0x0000775c, 0x00000002},
515	{0x00a05000, 0x00000002},
516	{0x00661000, 0x00000002},
517	{0x0460275d, 0x00000020},
518	{0x00004000, 0000000000},
519	{0x01e00830, 0x00000002},
520	{0x21007000, 0000000000},
521	{0x6464614d, 0000000000},
522	{0x69687420, 0000000000},
523	{0x00000073, 0000000000},
524	{0000000000, 0000000000},
525	{0x00005000, 0x00000002},
526	{0x000380d0, 0x00000002},
527	{0x040025e0, 0x00000002},
528	{0x000075e1, 0000000000},
529	{0x00000001, 0000000000},
530	{0x000380e0, 0x00000002},
531	{0x04002394, 0x00000002},
532	{0x00005000, 0000000000},
533	{0000000000, 0000000000},
534	{0000000000, 0000000000},
535	{0x00000008, 0000000000},
536	{0x00000004, 0000000000},
537	{0000000000, 0000000000},
538	{0000000000, 0000000000},
539	{0000000000, 0000000000},
540	{0000000000, 0000000000},
541	{0000000000, 0000000000},
542	{0000000000, 0000000000},
543	{0000000000, 0000000000},
544	{0000000000, 0000000000},
545	{0000000000, 0000000000},
546	{0000000000, 0000000000},
547	{0000000000, 0000000000},
548	{0000000000, 0000000000},
549	{0000000000, 0000000000},
550	{0000000000, 0000000000},
551	{0000000000, 0000000000},
552	{0000000000, 0000000000},
553	{0000000000, 0000000000},
554	{0000000000, 0000000000},
555	{0000000000, 0000000000},
556	{0000000000, 0000000000},
557	{0000000000, 0000000000},
558	{0000000000, 0000000000},
559	{0000000000, 0000000000},
560	{0000000000, 0000000000},
561};
562
563static const u32 R300_cp_microcode[][2] = {
564	{ 0x4200e000, 0000000000 },
565	{ 0x4000e000, 0000000000 },
566	{ 0x000000af, 0x00000008 },
567	{ 0x000000b3, 0x00000008 },
568	{ 0x6c5a504f, 0000000000 },
569	{ 0x4f4f497a, 0000000000 },
570	{ 0x5a578288, 0000000000 },
571	{ 0x4f91906a, 0000000000 },
572	{ 0x4f4f4f4f, 0000000000 },
573	{ 0x4fe24f44, 0000000000 },
574	{ 0x4f9c9c9c, 0000000000 },
575	{ 0xdc4f4fde, 0000000000 },
576	{ 0xa1cd4f4f, 0000000000 },
577	{ 0xd29d9d9d, 0000000000 },
578	{ 0x4f0f9fd7, 0000000000 },
579	{ 0x000ca000, 0x00000004 },
580	{ 0x000d0012, 0x00000038 },
581	{ 0x0000e8b4, 0x00000004 },
582	{ 0x000d0014, 0x00000038 },
583	{ 0x0000e8b6, 0x00000004 },
584	{ 0x000d0016, 0x00000038 },
585	{ 0x0000e854, 0x00000004 },
586	{ 0x000d0018, 0x00000038 },
587	{ 0x0000e855, 0x00000004 },
588	{ 0x000d001a, 0x00000038 },
589	{ 0x0000e856, 0x00000004 },
590	{ 0x000d001c, 0x00000038 },
591	{ 0x0000e857, 0x00000004 },
592	{ 0x000d001e, 0x00000038 },
593	{ 0x0000e824, 0x00000004 },
594	{ 0x000d0020, 0x00000038 },
595	{ 0x0000e825, 0x00000004 },
596	{ 0x000d0022, 0x00000038 },
597	{ 0x0000e830, 0x00000004 },
598	{ 0x000d0024, 0x00000038 },
599	{ 0x0000f0c0, 0x00000004 },
600	{ 0x000d0026, 0x00000038 },
601	{ 0x0000f0c1, 0x00000004 },
602	{ 0x000d0028, 0x00000038 },
603	{ 0x0000f041, 0x00000004 },
604	{ 0x000d002a, 0x00000038 },
605	{ 0x0000f184, 0x00000004 },
606	{ 0x000d002c, 0x00000038 },
607	{ 0x0000f185, 0x00000004 },
608	{ 0x000d002e, 0x00000038 },
609	{ 0x0000f186, 0x00000004 },
610	{ 0x000d0030, 0x00000038 },
611	{ 0x0000f187, 0x00000004 },
612	{ 0x000d0032, 0x00000038 },
613	{ 0x0000f180, 0x00000004 },
614	{ 0x000d0034, 0x00000038 },
615	{ 0x0000f393, 0x00000004 },
616	{ 0x000d0036, 0x00000038 },
617	{ 0x0000f38a, 0x00000004 },
618	{ 0x000d0038, 0x00000038 },
619	{ 0x0000f38e, 0x00000004 },
620	{ 0x0000e821, 0x00000004 },
621	{ 0x0140a000, 0x00000004 },
622	{ 0x00000043, 0x00000018 },
623	{ 0x00cce800, 0x00000004 },
624	{ 0x001b0001, 0x00000004 },
625	{ 0x08004800, 0x00000004 },
626	{ 0x001b0001, 0x00000004 },
627	{ 0x08004800, 0x00000004 },
628	{ 0x001b0001, 0x00000004 },
629	{ 0x08004800, 0x00000004 },
630	{ 0x0000003a, 0x00000008 },
631	{ 0x0000a000, 0000000000 },
632	{ 0x02c0a000, 0x00000004 },
633	{ 0x000ca000, 0x00000004 },
634	{ 0x00130000, 0x00000004 },
635	{ 0x000c2000, 0x00000004 },
636	{ 0xc980c045, 0x00000008 },
637	{ 0x2000451d, 0x00000004 },
638	{ 0x0000e580, 0x00000004 },
639	{ 0x000ce581, 0x00000004 },
640	{ 0x08004580, 0x00000004 },
641	{ 0x000ce581, 0x00000004 },
642	{ 0x0000004c, 0x00000008 },
643	{ 0x0000a000, 0000000000 },
644	{ 0x000c2000, 0x00000004 },
645	{ 0x0000e50e, 0x00000004 },
646	{ 0x00032000, 0x00000004 },
647	{ 0x00022056, 0x00000028 },
648	{ 0x00000056, 0x00000024 },
649	{ 0x0800450f, 0x00000004 },
650	{ 0x0000a050, 0x00000008 },
651	{ 0x0000e565, 0x00000004 },
652	{ 0x0000e566, 0x00000004 },
653	{ 0x00000057, 0x00000008 },
654	{ 0x03cca5b4, 0x00000004 },
655	{ 0x05432000, 0x00000004 },
656	{ 0x00022000, 0x00000004 },
657	{ 0x4ccce063, 0x00000030 },
658	{ 0x08274565, 0x00000004 },
659	{ 0x00000063, 0x00000030 },
660	{ 0x08004564, 0x00000004 },
661	{ 0x0000e566, 0x00000004 },
662	{ 0x0000005a, 0x00000008 },
663	{ 0x00802066, 0x00000010 },
664	{ 0x00202000, 0x00000004 },
665	{ 0x001b00ff, 0x00000004 },
666	{ 0x01000069, 0x00000010 },
667	{ 0x001f2000, 0x00000004 },
668	{ 0x001c00ff, 0x00000004 },
669	{ 0000000000, 0x0000000c },
670	{ 0x00000085, 0x00000030 },
671	{ 0x0000005a, 0x00000008 },
672	{ 0x0000e576, 0x00000004 },
673	{ 0x000ca000, 0x00000004 },
674	{ 0x00012000, 0x00000004 },
675	{ 0x00082000, 0x00000004 },
676	{ 0x1800650e, 0x00000004 },
677	{ 0x00092000, 0x00000004 },
678	{ 0x000a2000, 0x00000004 },
679	{ 0x000f0000, 0x00000004 },
680	{ 0x00400000, 0x00000004 },
681	{ 0x00000079, 0x00000018 },
682	{ 0x0000e563, 0x00000004 },
683	{ 0x00c0e5f9, 0x000000c2 },
684	{ 0x0000006e, 0x00000008 },
685	{ 0x0000a06e, 0x00000008 },
686	{ 0x0000e576, 0x00000004 },
687	{ 0x0000e577, 0x00000004 },
688	{ 0x0000e50e, 0x00000004 },
689	{ 0x0000e50f, 0x00000004 },
690	{ 0x0140a000, 0x00000004 },
691	{ 0x0000007c, 0x00000018 },
692	{ 0x00c0e5f9, 0x000000c2 },
693	{ 0x0000007c, 0x00000008 },
694	{ 0x0014e50e, 0x00000004 },
695	{ 0x0040e50f, 0x00000004 },
696	{ 0x00c0007f, 0x00000008 },
697	{ 0x0000e570, 0x00000004 },
698	{ 0x0000e571, 0x00000004 },
699	{ 0x0000e572, 0x0000000c },
700	{ 0x0000a000, 0x00000004 },
701	{ 0x0140a000, 0x00000004 },
702	{ 0x0000e568, 0x00000004 },
703	{ 0x000c2000, 0x00000004 },
704	{ 0x00000089, 0x00000018 },
705	{ 0x000b0000, 0x00000004 },
706	{ 0x18c0e562, 0x00000004 },
707	{ 0x0000008b, 0x00000008 },
708	{ 0x00c0008a, 0x00000008 },
709	{ 0x000700e4, 0x00000004 },
710	{ 0x00000097, 0x00000038 },
711	{ 0x000ca099, 0x00000030 },
712	{ 0x080045bb, 0x00000004 },
713	{ 0x000c209a, 0x00000030 },
714	{ 0x0800e5bc, 0000000000 },
715	{ 0x0000e5bb, 0x00000004 },
716	{ 0x0000e5bc, 0000000000 },
717	{ 0x00120000, 0x0000000c },
718	{ 0x00120000, 0x00000004 },
719	{ 0x001b0002, 0x0000000c },
720	{ 0x0000a000, 0x00000004 },
721	{ 0x0000e821, 0x00000004 },
722	{ 0x0000e800, 0000000000 },
723	{ 0x0000e821, 0x00000004 },
724	{ 0x0000e82e, 0000000000 },
725	{ 0x02cca000, 0x00000004 },
726	{ 0x00140000, 0x00000004 },
727	{ 0x000ce1cc, 0x00000004 },
728	{ 0x050de1cd, 0x00000004 },
729	{ 0x000000a7, 0x00000020 },
730	{ 0x4200e000, 0000000000 },
731	{ 0x000000ae, 0x00000038 },
732	{ 0x000ca000, 0x00000004 },
733	{ 0x00140000, 0x00000004 },
734	{ 0x000c2000, 0x00000004 },
735	{ 0x00160000, 0x00000004 },
736	{ 0x700ce000, 0x00000004 },
737	{ 0x001400aa, 0x00000008 },
738	{ 0x4000e000, 0000000000 },
739	{ 0x02400000, 0x00000004 },
740	{ 0x400ee000, 0x00000004 },
741	{ 0x02400000, 0x00000004 },
742	{ 0x4000e000, 0000000000 },
743	{ 0x000c2000, 0x00000004 },
744	{ 0x0240e51b, 0x00000004 },
745	{ 0x0080e50a, 0x00000005 },
746	{ 0x0080e50b, 0x00000005 },
747	{ 0x00220000, 0x00000004 },
748	{ 0x000700e4, 0x00000004 },
749	{ 0x000000c1, 0x00000038 },
750	{ 0x000c209a, 0x00000030 },
751	{ 0x0880e5bd, 0x00000005 },
752	{ 0x000c2099, 0x00000030 },
753	{ 0x0800e5bb, 0x00000005 },
754	{ 0x000c209a, 0x00000030 },
755	{ 0x0880e5bc, 0x00000005 },
756	{ 0x000000c4, 0x00000008 },
757	{ 0x0080e5bd, 0x00000005 },
758	{ 0x0000e5bb, 0x00000005 },
759	{ 0x0080e5bc, 0x00000005 },
760	{ 0x00210000, 0x00000004 },
761	{ 0x02800000, 0x00000004 },
762	{ 0x00c000c8, 0x00000018 },
763	{ 0x4180e000, 0x00000040 },
764	{ 0x000000ca, 0x00000024 },
765	{ 0x01000000, 0x0000000c },
766	{ 0x0100e51d, 0x0000000c },
767	{ 0x000045bb, 0x00000004 },
768	{ 0x000080c4, 0x00000008 },
769	{ 0x0000f3ce, 0x00000004 },
770	{ 0x0140a000, 0x00000004 },
771	{ 0x00cc2000, 0x00000004 },
772	{ 0x08c053cf, 0x00000040 },
773	{ 0x00008000, 0000000000 },
774	{ 0x0000f3d2, 0x00000004 },
775	{ 0x0140a000, 0x00000004 },
776	{ 0x00cc2000, 0x00000004 },
777	{ 0x08c053d3, 0x00000040 },
778	{ 0x00008000, 0000000000 },
779	{ 0x0000f39d, 0x00000004 },
780	{ 0x0140a000, 0x00000004 },
781	{ 0x00cc2000, 0x00000004 },
782	{ 0x08c0539e, 0x00000040 },
783	{ 0x00008000, 0000000000 },
784	{ 0x03c00830, 0x00000004 },
785	{ 0x4200e000, 0000000000 },
786	{ 0x0000a000, 0x00000004 },
787	{ 0x200045e0, 0x00000004 },
788	{ 0x0000e5e1, 0000000000 },
789	{ 0x00000001, 0000000000 },
790	{ 0x000700e1, 0x00000004 },
791	{ 0x0800e394, 0000000000 },
792	{ 0000000000, 0000000000 },
793	{ 0000000000, 0000000000 },
794	{ 0000000000, 0000000000 },
795	{ 0000000000, 0000000000 },
796	{ 0000000000, 0000000000 },
797	{ 0000000000, 0000000000 },
798	{ 0000000000, 0000000000 },
799	{ 0000000000, 0000000000 },
800	{ 0000000000, 0000000000 },
801	{ 0000000000, 0000000000 },
802	{ 0000000000, 0000000000 },
803	{ 0000000000, 0000000000 },
804	{ 0000000000, 0000000000 },
805	{ 0000000000, 0000000000 },
806	{ 0000000000, 0000000000 },
807	{ 0000000000, 0000000000 },
808	{ 0000000000, 0000000000 },
809	{ 0000000000, 0000000000 },
810	{ 0000000000, 0000000000 },
811	{ 0000000000, 0000000000 },
812	{ 0000000000, 0000000000 },
813	{ 0000000000, 0000000000 },
814	{ 0000000000, 0000000000 },
815	{ 0000000000, 0000000000 },
816	{ 0000000000, 0000000000 },
817	{ 0000000000, 0000000000 },
818	{ 0000000000, 0000000000 },
819	{ 0000000000, 0000000000 },
820};
821
822static int RADEON_READ_PLL(drm_device_t * dev, int addr)
823{
824	drm_radeon_private_t *dev_priv = dev->dev_private;
825
826	RADEON_WRITE8(RADEON_CLOCK_CNTL_INDEX, addr & 0x1f);
827	return RADEON_READ(RADEON_CLOCK_CNTL_DATA);
828}
829
830static int RADEON_READ_PCIE(drm_radeon_private_t *dev_priv, int addr)
831{
832	RADEON_WRITE8(RADEON_PCIE_INDEX, addr & 0xff);
833	return RADEON_READ(RADEON_PCIE_DATA);
834}
835
836#if RADEON_FIFO_DEBUG
837static void radeon_status(drm_radeon_private_t * dev_priv)
838{
839	printk("%s:\n", __FUNCTION__);
840	printk("RBBM_STATUS = 0x%08x\n",
841	       (unsigned int)RADEON_READ(RADEON_RBBM_STATUS));
842	printk("CP_RB_RTPR = 0x%08x\n",
843	       (unsigned int)RADEON_READ(RADEON_CP_RB_RPTR));
844	printk("CP_RB_WTPR = 0x%08x\n",
845	       (unsigned int)RADEON_READ(RADEON_CP_RB_WPTR));
846	printk("AIC_CNTL = 0x%08x\n",
847	       (unsigned int)RADEON_READ(RADEON_AIC_CNTL));
848	printk("AIC_STAT = 0x%08x\n",
849	       (unsigned int)RADEON_READ(RADEON_AIC_STAT));
850	printk("AIC_PT_BASE = 0x%08x\n",
851	       (unsigned int)RADEON_READ(RADEON_AIC_PT_BASE));
852	printk("TLB_ADDR = 0x%08x\n",
853	       (unsigned int)RADEON_READ(RADEON_AIC_TLB_ADDR));
854	printk("TLB_DATA = 0x%08x\n",
855	       (unsigned int)RADEON_READ(RADEON_AIC_TLB_DATA));
856}
857#endif
858
859/* ================================================================
860 * Engine, FIFO control
861 */
862
863static int radeon_do_pixcache_flush(drm_radeon_private_t * dev_priv)
864{
865	u32 tmp;
866	int i;
867
868	dev_priv->stats.boxes |= RADEON_BOX_WAIT_IDLE;
869
870	tmp = RADEON_READ(RADEON_RB3D_DSTCACHE_CTLSTAT);
871	tmp |= RADEON_RB3D_DC_FLUSH_ALL;
872	RADEON_WRITE(RADEON_RB3D_DSTCACHE_CTLSTAT, tmp);
873
874	for (i = 0; i < dev_priv->usec_timeout; i++) {
875		if (!(RADEON_READ(RADEON_RB3D_DSTCACHE_CTLSTAT)
876		      & RADEON_RB3D_DC_BUSY)) {
877			return 0;
878		}
879		DRM_UDELAY(1);
880	}
881
882#if RADEON_FIFO_DEBUG
883	DRM_ERROR("failed!\n");
884	radeon_status(dev_priv);
885#endif
886	return DRM_ERR(EBUSY);
887}
888
889static int radeon_do_wait_for_fifo(drm_radeon_private_t * dev_priv, int entries)
890{
891	int i;
892
893	dev_priv->stats.boxes |= RADEON_BOX_WAIT_IDLE;
894
895	for (i = 0; i < dev_priv->usec_timeout; i++) {
896		int slots = (RADEON_READ(RADEON_RBBM_STATUS)
897			     & RADEON_RBBM_FIFOCNT_MASK);
898		if (slots >= entries)
899			return 0;
900		DRM_UDELAY(1);
901	}
902
903#if RADEON_FIFO_DEBUG
904	DRM_ERROR("failed!\n");
905	radeon_status(dev_priv);
906#endif
907	return DRM_ERR(EBUSY);
908}
909
910static int radeon_do_wait_for_idle(drm_radeon_private_t * dev_priv)
911{
912	int i, ret;
913
914	dev_priv->stats.boxes |= RADEON_BOX_WAIT_IDLE;
915
916	ret = radeon_do_wait_for_fifo(dev_priv, 64);
917	if (ret)
918		return ret;
919
920	for (i = 0; i < dev_priv->usec_timeout; i++) {
921		if (!(RADEON_READ(RADEON_RBBM_STATUS)
922		      & RADEON_RBBM_ACTIVE)) {
923			radeon_do_pixcache_flush(dev_priv);
924			return 0;
925		}
926		DRM_UDELAY(1);
927	}
928
929#if RADEON_FIFO_DEBUG
930	DRM_ERROR("failed!\n");
931	radeon_status(dev_priv);
932#endif
933	return DRM_ERR(EBUSY);
934}
935
936/* ================================================================
937 * CP control, initialization
938 */
939
940/* Load the microcode for the CP */
941static void radeon_cp_load_microcode(drm_radeon_private_t * dev_priv)
942{
943	int i;
944	DRM_DEBUG("\n");
945
946	radeon_do_wait_for_idle(dev_priv);
947
948	RADEON_WRITE(RADEON_CP_ME_RAM_ADDR, 0);
949
950	if (dev_priv->microcode_version == UCODE_R200) {
951		DRM_INFO("Loading R200 Microcode\n");
952		for (i = 0; i < 256; i++) {
953			RADEON_WRITE(RADEON_CP_ME_RAM_DATAH,
954				     R200_cp_microcode[i][1]);
955			RADEON_WRITE(RADEON_CP_ME_RAM_DATAL,
956				     R200_cp_microcode[i][0]);
957		}
958	} else if (dev_priv->microcode_version == UCODE_R300) {
959		DRM_INFO("Loading R300 Microcode\n");
960		for (i = 0; i < 256; i++) {
961			RADEON_WRITE(RADEON_CP_ME_RAM_DATAH,
962				     R300_cp_microcode[i][1]);
963			RADEON_WRITE(RADEON_CP_ME_RAM_DATAL,
964				     R300_cp_microcode[i][0]);
965		}
966	} else {
967		for (i = 0; i < 256; i++) {
968			RADEON_WRITE(RADEON_CP_ME_RAM_DATAH,
969				     radeon_cp_microcode[i][1]);
970			RADEON_WRITE(RADEON_CP_ME_RAM_DATAL,
971				     radeon_cp_microcode[i][0]);
972		}
973	}
974}
975
976/* Flush any pending commands to the CP.  This should only be used just
977 * prior to a wait for idle, as it informs the engine that the command
978 * stream is ending.
979 */
980static void radeon_do_cp_flush(drm_radeon_private_t * dev_priv)
981{
982	DRM_DEBUG("\n");
983#if 0
984	u32 tmp;
985
986	tmp = RADEON_READ(RADEON_CP_RB_WPTR) | (1 << 31);
987	RADEON_WRITE(RADEON_CP_RB_WPTR, tmp);
988#endif
989}
990
991/* Wait for the CP to go idle.
992 */
993int radeon_do_cp_idle(drm_radeon_private_t * dev_priv)
994{
995	RING_LOCALS;
996	DRM_DEBUG("\n");
997
998	BEGIN_RING(6);
999
1000	RADEON_PURGE_CACHE();
1001	RADEON_PURGE_ZCACHE();
1002	RADEON_WAIT_UNTIL_IDLE();
1003
1004	ADVANCE_RING();
1005	COMMIT_RING();
1006
1007	return radeon_do_wait_for_idle(dev_priv);
1008}
1009
1010/* Start the Command Processor.
1011 */
1012static void radeon_do_cp_start(drm_radeon_private_t * dev_priv)
1013{
1014	RING_LOCALS;
1015	DRM_DEBUG("\n");
1016
1017	radeon_do_wait_for_idle(dev_priv);
1018
1019	RADEON_WRITE(RADEON_CP_CSQ_CNTL, dev_priv->cp_mode);
1020
1021	dev_priv->cp_running = 1;
1022
1023	BEGIN_RING(6);
1024
1025	RADEON_PURGE_CACHE();
1026	RADEON_PURGE_ZCACHE();
1027	RADEON_WAIT_UNTIL_IDLE();
1028
1029	ADVANCE_RING();
1030	COMMIT_RING();
1031}
1032
1033/* Reset the Command Processor.  This will not flush any pending
1034 * commands, so you must wait for the CP command stream to complete
1035 * before calling this routine.
1036 */
1037static void radeon_do_cp_reset(drm_radeon_private_t * dev_priv)
1038{
1039	u32 cur_read_ptr;
1040	DRM_DEBUG("\n");
1041
1042	cur_read_ptr = RADEON_READ(RADEON_CP_RB_RPTR);
1043	RADEON_WRITE(RADEON_CP_RB_WPTR, cur_read_ptr);
1044	SET_RING_HEAD(dev_priv, cur_read_ptr);
1045	dev_priv->ring.tail = cur_read_ptr;
1046}
1047
1048/* Stop the Command Processor.  This will not flush any pending
1049 * commands, so you must flush the command stream and wait for the CP
1050 * to go idle before calling this routine.
1051 */
1052static void radeon_do_cp_stop(drm_radeon_private_t * dev_priv)
1053{
1054	DRM_DEBUG("\n");
1055
1056	RADEON_WRITE(RADEON_CP_CSQ_CNTL, RADEON_CSQ_PRIDIS_INDDIS);
1057
1058	dev_priv->cp_running = 0;
1059}
1060
1061/* Reset the engine.  This will stop the CP if it is running.
1062 */
1063static int radeon_do_engine_reset(drm_device_t * dev)
1064{
1065	drm_radeon_private_t *dev_priv = dev->dev_private;
1066	u32 clock_cntl_index, mclk_cntl, rbbm_soft_reset;
1067	DRM_DEBUG("\n");
1068
1069	radeon_do_pixcache_flush(dev_priv);
1070
1071	clock_cntl_index = RADEON_READ(RADEON_CLOCK_CNTL_INDEX);
1072	mclk_cntl = RADEON_READ_PLL(dev, RADEON_MCLK_CNTL);
1073
1074	RADEON_WRITE_PLL(RADEON_MCLK_CNTL, (mclk_cntl |
1075					    RADEON_FORCEON_MCLKA |
1076					    RADEON_FORCEON_MCLKB |
1077					    RADEON_FORCEON_YCLKA |
1078					    RADEON_FORCEON_YCLKB |
1079					    RADEON_FORCEON_MC |
1080					    RADEON_FORCEON_AIC));
1081
1082	rbbm_soft_reset = RADEON_READ(RADEON_RBBM_SOFT_RESET);
1083
1084	RADEON_WRITE(RADEON_RBBM_SOFT_RESET, (rbbm_soft_reset |
1085					      RADEON_SOFT_RESET_CP |
1086					      RADEON_SOFT_RESET_HI |
1087					      RADEON_SOFT_RESET_SE |
1088					      RADEON_SOFT_RESET_RE |
1089					      RADEON_SOFT_RESET_PP |
1090					      RADEON_SOFT_RESET_E2 |
1091					      RADEON_SOFT_RESET_RB));
1092	RADEON_READ(RADEON_RBBM_SOFT_RESET);
1093	RADEON_WRITE(RADEON_RBBM_SOFT_RESET, (rbbm_soft_reset &
1094					      ~(RADEON_SOFT_RESET_CP |
1095						RADEON_SOFT_RESET_HI |
1096						RADEON_SOFT_RESET_SE |
1097						RADEON_SOFT_RESET_RE |
1098						RADEON_SOFT_RESET_PP |
1099						RADEON_SOFT_RESET_E2 |
1100						RADEON_SOFT_RESET_RB)));
1101	RADEON_READ(RADEON_RBBM_SOFT_RESET);
1102
1103	RADEON_WRITE_PLL(RADEON_MCLK_CNTL, mclk_cntl);
1104	RADEON_WRITE(RADEON_CLOCK_CNTL_INDEX, clock_cntl_index);
1105	RADEON_WRITE(RADEON_RBBM_SOFT_RESET, rbbm_soft_reset);
1106
1107	/* Reset the CP ring */
1108	radeon_do_cp_reset(dev_priv);
1109
1110	/* The CP is no longer running after an engine reset */
1111	dev_priv->cp_running = 0;
1112
1113	/* Reset any pending vertex, indirect buffers */
1114	radeon_freelist_reset(dev);
1115
1116	return 0;
1117}
1118
1119static void radeon_cp_init_ring_buffer(drm_device_t * dev,
1120				       drm_radeon_private_t * dev_priv)
1121{
1122	u32 ring_start, cur_read_ptr;
1123	u32 tmp;
1124
1125	/* Initialize the memory controller. With new memory map, the fb location
1126	 * is not changed, it should have been properly initialized already. Part
1127	 * of the problem is that the code below is bogus, assuming the GART is
1128	 * always appended to the fb which is not necessarily the case
1129	 */
1130	if (!dev_priv->new_memmap)
1131		RADEON_WRITE(RADEON_MC_FB_LOCATION,
1132			     ((dev_priv->gart_vm_start - 1) & 0xffff0000)
1133			     | (dev_priv->fb_location >> 16));
1134
1135#if __OS_HAS_AGP
1136	if (dev_priv->flags & CHIP_IS_AGP) {
1137		RADEON_WRITE(RADEON_AGP_BASE, (unsigned int)dev->agp->base);
1138		RADEON_WRITE(RADEON_MC_AGP_LOCATION,
1139			     (((dev_priv->gart_vm_start - 1 +
1140				dev_priv->gart_size) & 0xffff0000) |
1141			      (dev_priv->gart_vm_start >> 16)));
1142
1143		ring_start = (dev_priv->cp_ring->offset
1144			      - dev->agp->base
1145			      + dev_priv->gart_vm_start);
1146	} else
1147#endif
1148		ring_start = (dev_priv->cp_ring->offset
1149			      - (unsigned long)dev->sg->virtual
1150			      + dev_priv->gart_vm_start);
1151
1152	RADEON_WRITE(RADEON_CP_RB_BASE, ring_start);
1153
1154	/* Set the write pointer delay */
1155	RADEON_WRITE(RADEON_CP_RB_WPTR_DELAY, 0);
1156
1157	/* Initialize the ring buffer's read and write pointers */
1158	cur_read_ptr = RADEON_READ(RADEON_CP_RB_RPTR);
1159	RADEON_WRITE(RADEON_CP_RB_WPTR, cur_read_ptr);
1160	SET_RING_HEAD(dev_priv, cur_read_ptr);
1161	dev_priv->ring.tail = cur_read_ptr;
1162
1163#if __OS_HAS_AGP
1164	if (dev_priv->flags & CHIP_IS_AGP) {
1165		RADEON_WRITE(RADEON_CP_RB_RPTR_ADDR,
1166			     dev_priv->ring_rptr->offset
1167			     - dev->agp->base + dev_priv->gart_vm_start);
1168	} else
1169#endif
1170	{
1171		drm_sg_mem_t *entry = dev->sg;
1172		unsigned long tmp_ofs, page_ofs;
1173
1174		tmp_ofs = dev_priv->ring_rptr->offset -
1175				(unsigned long)dev->sg->virtual;
1176		page_ofs = tmp_ofs >> PAGE_SHIFT;
1177
1178		RADEON_WRITE(RADEON_CP_RB_RPTR_ADDR, entry->busaddr[page_ofs]);
1179		DRM_DEBUG("ring rptr: offset=0x%08lx handle=0x%08lx\n",
1180			  (unsigned long)entry->busaddr[page_ofs],
1181			  entry->handle + tmp_ofs);
1182	}
1183
1184	/* Set ring buffer size */
1185#ifdef __BIG_ENDIAN
1186	RADEON_WRITE(RADEON_CP_RB_CNTL,
1187		     dev_priv->ring.size_l2qw | RADEON_BUF_SWAP_32BIT);
1188#else
1189	RADEON_WRITE(RADEON_CP_RB_CNTL, dev_priv->ring.size_l2qw);
1190#endif
1191
1192	/* Start with assuming that writeback doesn't work */
1193	dev_priv->writeback_works = 0;
1194
1195	/* Initialize the scratch register pointer.  This will cause
1196	 * the scratch register values to be written out to memory
1197	 * whenever they are updated.
1198	 *
1199	 * We simply put this behind the ring read pointer, this works
1200	 * with PCI GART as well as (whatever kind of) AGP GART
1201	 */
1202	RADEON_WRITE(RADEON_SCRATCH_ADDR, RADEON_READ(RADEON_CP_RB_RPTR_ADDR)
1203		     + RADEON_SCRATCH_REG_OFFSET);
1204
1205	dev_priv->scratch = ((__volatile__ u32 *)
1206			     dev_priv->ring_rptr->handle +
1207			     (RADEON_SCRATCH_REG_OFFSET / sizeof(u32)));
1208
1209	RADEON_WRITE(RADEON_SCRATCH_UMSK, 0x7);
1210
1211	/* Turn on bus mastering */
1212	tmp = RADEON_READ(RADEON_BUS_CNTL) & ~RADEON_BUS_MASTER_DIS;
1213	RADEON_WRITE(RADEON_BUS_CNTL, tmp);
1214
1215	dev_priv->sarea_priv->last_frame = dev_priv->scratch[0] = 0;
1216	RADEON_WRITE(RADEON_LAST_FRAME_REG, dev_priv->sarea_priv->last_frame);
1217
1218	dev_priv->sarea_priv->last_dispatch = dev_priv->scratch[1] = 0;
1219	RADEON_WRITE(RADEON_LAST_DISPATCH_REG,
1220		     dev_priv->sarea_priv->last_dispatch);
1221
1222	dev_priv->sarea_priv->last_clear = dev_priv->scratch[2] = 0;
1223	RADEON_WRITE(RADEON_LAST_CLEAR_REG, dev_priv->sarea_priv->last_clear);
1224
1225	radeon_do_wait_for_idle(dev_priv);
1226
1227	/* Sync everything up */
1228	RADEON_WRITE(RADEON_ISYNC_CNTL,
1229		     (RADEON_ISYNC_ANY2D_IDLE3D |
1230		      RADEON_ISYNC_ANY3D_IDLE2D |
1231		      RADEON_ISYNC_WAIT_IDLEGUI |
1232		      RADEON_ISYNC_CPSCRATCH_IDLEGUI));
1233
1234}
1235
1236static void radeon_test_writeback(drm_radeon_private_t * dev_priv)
1237{
1238	u32 tmp;
1239
1240	/* Writeback doesn't seem to work everywhere, test it here and possibly
1241	 * enable it if it appears to work
1242	 */
1243	DRM_WRITE32(dev_priv->ring_rptr, RADEON_SCRATCHOFF(1), 0);
1244	RADEON_WRITE(RADEON_SCRATCH_REG1, 0xdeadbeef);
1245
1246	for (tmp = 0; tmp < dev_priv->usec_timeout; tmp++) {
1247		if (DRM_READ32(dev_priv->ring_rptr, RADEON_SCRATCHOFF(1)) ==
1248		    0xdeadbeef)
1249			break;
1250		DRM_UDELAY(1);
1251	}
1252
1253	if (tmp < dev_priv->usec_timeout) {
1254		dev_priv->writeback_works = 1;
1255		DRM_INFO("writeback test succeeded in %d usecs\n", tmp);
1256	} else {
1257		dev_priv->writeback_works = 0;
1258		DRM_INFO("writeback test failed\n");
1259	}
1260	if (radeon_no_wb == 1) {
1261		dev_priv->writeback_works = 0;
1262		DRM_INFO("writeback forced off\n");
1263	}
1264
1265	if (!dev_priv->writeback_works) {
1266		/* Disable writeback to avoid unnecessary bus master transfers */
1267		RADEON_WRITE(RADEON_CP_RB_CNTL, RADEON_READ(RADEON_CP_RB_CNTL) | RADEON_RB_NO_UPDATE);
1268		RADEON_WRITE(RADEON_SCRATCH_UMSK, 0);
1269	}
1270}
1271
1272/* Enable or disable PCI-E GART on the chip */
1273static void radeon_set_pciegart(drm_radeon_private_t * dev_priv, int on)
1274{
1275	u32 tmp = RADEON_READ_PCIE(dev_priv, RADEON_PCIE_TX_GART_CNTL);
1276	if (on) {
1277
1278		DRM_DEBUG("programming pcie %08X %08lX %08X\n",
1279			  dev_priv->gart_vm_start,
1280			  (long)dev_priv->gart_info.bus_addr,
1281			  dev_priv->gart_size);
1282		RADEON_WRITE_PCIE(RADEON_PCIE_TX_DISCARD_RD_ADDR_LO,
1283				  dev_priv->gart_vm_start);
1284		RADEON_WRITE_PCIE(RADEON_PCIE_TX_GART_BASE,
1285				  dev_priv->gart_info.bus_addr);
1286		RADEON_WRITE_PCIE(RADEON_PCIE_TX_GART_START_LO,
1287				  dev_priv->gart_vm_start);
1288		RADEON_WRITE_PCIE(RADEON_PCIE_TX_GART_END_LO,
1289				  dev_priv->gart_vm_start +
1290				  dev_priv->gart_size - 1);
1291
1292		RADEON_WRITE(RADEON_MC_AGP_LOCATION, 0xffffffc0);	/* ?? */
1293
1294		RADEON_WRITE_PCIE(RADEON_PCIE_TX_GART_CNTL,
1295				  RADEON_PCIE_TX_GART_EN);
1296	} else {
1297		RADEON_WRITE_PCIE(RADEON_PCIE_TX_GART_CNTL,
1298				  tmp & ~RADEON_PCIE_TX_GART_EN);
1299	}
1300}
1301
1302/* Enable or disable PCI GART on the chip */
1303static void radeon_set_pcigart(drm_radeon_private_t * dev_priv, int on)
1304{
1305	u32 tmp;
1306
1307	if (dev_priv->flags & CHIP_IS_PCIE) {
1308		radeon_set_pciegart(dev_priv, on);
1309		return;
1310	}
1311
1312 	tmp = RADEON_READ(RADEON_AIC_CNTL);
1313
1314	if (on) {
1315		RADEON_WRITE(RADEON_AIC_CNTL,
1316			     tmp | RADEON_PCIGART_TRANSLATE_EN);
1317
1318		/* set PCI GART page-table base address
1319		 */
1320		RADEON_WRITE(RADEON_AIC_PT_BASE, dev_priv->gart_info.bus_addr);
1321
1322		/* set address range for PCI address translate
1323		 */
1324		RADEON_WRITE(RADEON_AIC_LO_ADDR, dev_priv->gart_vm_start);
1325		RADEON_WRITE(RADEON_AIC_HI_ADDR, dev_priv->gart_vm_start
1326			     + dev_priv->gart_size - 1);
1327
1328		/* Turn off AGP aperture -- is this required for PCI GART?
1329		 */
1330		RADEON_WRITE(RADEON_MC_AGP_LOCATION, 0xffffffc0);	/* ?? */
1331		RADEON_WRITE(RADEON_AGP_COMMAND, 0);	/* clear AGP_COMMAND */
1332	} else {
1333		RADEON_WRITE(RADEON_AIC_CNTL,
1334			     tmp & ~RADEON_PCIGART_TRANSLATE_EN);
1335	}
1336}
1337
1338static int radeon_do_init_cp(drm_device_t * dev, drm_radeon_init_t * init)
1339{
1340	drm_radeon_private_t *dev_priv = dev->dev_private;
1341
1342	DRM_DEBUG("\n");
1343
1344	/* if we require new memory map but we don't have it fail */
1345	if ((dev_priv->flags & CHIP_NEW_MEMMAP) && !dev_priv->new_memmap)
1346	{
1347		DRM_ERROR("Cannot initialise DRM on this card\nThis card requires a new X.org DDX for 3D\n");
1348		radeon_do_cleanup_cp(dev);
1349		return DRM_ERR(EINVAL);
1350	}
1351
1352	if (init->is_pci && (dev_priv->flags & CHIP_IS_AGP))
1353	{
1354		DRM_DEBUG("Forcing AGP card to PCI mode\n");
1355		dev_priv->flags &= ~CHIP_IS_AGP;
1356	}
1357	else if (!(dev_priv->flags & (CHIP_IS_AGP | CHIP_IS_PCI | CHIP_IS_PCIE))
1358		 && !init->is_pci)
1359	{
1360		DRM_DEBUG("Restoring AGP flag\n");
1361		dev_priv->flags |= CHIP_IS_AGP;
1362	}
1363
1364	if ((!(dev_priv->flags & CHIP_IS_AGP)) && !dev->sg) {
1365		DRM_ERROR("PCI GART memory not allocated!\n");
1366		radeon_do_cleanup_cp(dev);
1367		return DRM_ERR(EINVAL);
1368	}
1369
1370	dev_priv->usec_timeout = init->usec_timeout;
1371	if (dev_priv->usec_timeout < 1 ||
1372	    dev_priv->usec_timeout > RADEON_MAX_USEC_TIMEOUT) {
1373		DRM_DEBUG("TIMEOUT problem!\n");
1374		radeon_do_cleanup_cp(dev);
1375		return DRM_ERR(EINVAL);
1376	}
1377
1378	switch(init->func) {
1379	case RADEON_INIT_R200_CP:
1380		dev_priv->microcode_version = UCODE_R200;
1381		break;
1382	case RADEON_INIT_R300_CP:
1383		dev_priv->microcode_version = UCODE_R300;
1384		break;
1385	default:
1386		dev_priv->microcode_version = UCODE_R100;
1387	}
1388
1389	dev_priv->do_boxes = 0;
1390	dev_priv->cp_mode = init->cp_mode;
1391
1392	/* We don't support anything other than bus-mastering ring mode,
1393	 * but the ring can be in either AGP or PCI space for the ring
1394	 * read pointer.
1395	 */
1396	if ((init->cp_mode != RADEON_CSQ_PRIBM_INDDIS) &&
1397	    (init->cp_mode != RADEON_CSQ_PRIBM_INDBM)) {
1398		DRM_DEBUG("BAD cp_mode (%x)!\n", init->cp_mode);
1399		radeon_do_cleanup_cp(dev);
1400		return DRM_ERR(EINVAL);
1401	}
1402
1403	switch (init->fb_bpp) {
1404	case 16:
1405		dev_priv->color_fmt = RADEON_COLOR_FORMAT_RGB565;
1406		break;
1407	case 32:
1408	default:
1409		dev_priv->color_fmt = RADEON_COLOR_FORMAT_ARGB8888;
1410		break;
1411	}
1412	dev_priv->front_offset = init->front_offset;
1413	dev_priv->front_pitch = init->front_pitch;
1414	dev_priv->back_offset = init->back_offset;
1415	dev_priv->back_pitch = init->back_pitch;
1416
1417	switch (init->depth_bpp) {
1418	case 16:
1419		dev_priv->depth_fmt = RADEON_DEPTH_FORMAT_16BIT_INT_Z;
1420		break;
1421	case 32:
1422	default:
1423		dev_priv->depth_fmt = RADEON_DEPTH_FORMAT_24BIT_INT_Z;
1424		break;
1425	}
1426	dev_priv->depth_offset = init->depth_offset;
1427	dev_priv->depth_pitch = init->depth_pitch;
1428
1429	/* Hardware state for depth clears.  Remove this if/when we no
1430	 * longer clear the depth buffer with a 3D rectangle.  Hard-code
1431	 * all values to prevent unwanted 3D state from slipping through
1432	 * and screwing with the clear operation.
1433	 */
1434	dev_priv->depth_clear.rb3d_cntl = (RADEON_PLANE_MASK_ENABLE |
1435					   (dev_priv->color_fmt << 10) |
1436					   (dev_priv->microcode_version ==
1437					    UCODE_R100 ? RADEON_ZBLOCK16 : 0));
1438
1439	dev_priv->depth_clear.rb3d_zstencilcntl =
1440	    (dev_priv->depth_fmt |
1441	     RADEON_Z_TEST_ALWAYS |
1442	     RADEON_STENCIL_TEST_ALWAYS |
1443	     RADEON_STENCIL_S_FAIL_REPLACE |
1444	     RADEON_STENCIL_ZPASS_REPLACE |
1445	     RADEON_STENCIL_ZFAIL_REPLACE | RADEON_Z_WRITE_ENABLE);
1446
1447	dev_priv->depth_clear.se_cntl = (RADEON_FFACE_CULL_CW |
1448					 RADEON_BFACE_SOLID |
1449					 RADEON_FFACE_SOLID |
1450					 RADEON_FLAT_SHADE_VTX_LAST |
1451					 RADEON_DIFFUSE_SHADE_FLAT |
1452					 RADEON_ALPHA_SHADE_FLAT |
1453					 RADEON_SPECULAR_SHADE_FLAT |
1454					 RADEON_FOG_SHADE_FLAT |
1455					 RADEON_VTX_PIX_CENTER_OGL |
1456					 RADEON_ROUND_MODE_TRUNC |
1457					 RADEON_ROUND_PREC_8TH_PIX);
1458
1459	DRM_GETSAREA();
1460
1461	dev_priv->ring_offset = init->ring_offset;
1462	dev_priv->ring_rptr_offset = init->ring_rptr_offset;
1463	dev_priv->buffers_offset = init->buffers_offset;
1464	dev_priv->gart_textures_offset = init->gart_textures_offset;
1465
1466	if (!dev_priv->sarea) {
1467		DRM_ERROR("could not find sarea!\n");
1468		radeon_do_cleanup_cp(dev);
1469		return DRM_ERR(EINVAL);
1470	}
1471
1472	dev_priv->cp_ring = drm_core_findmap(dev, init->ring_offset);
1473	if (!dev_priv->cp_ring) {
1474		DRM_ERROR("could not find cp ring region!\n");
1475		radeon_do_cleanup_cp(dev);
1476		return DRM_ERR(EINVAL);
1477	}
1478	dev_priv->ring_rptr = drm_core_findmap(dev, init->ring_rptr_offset);
1479	if (!dev_priv->ring_rptr) {
1480		DRM_ERROR("could not find ring read pointer!\n");
1481		radeon_do_cleanup_cp(dev);
1482		return DRM_ERR(EINVAL);
1483	}
1484	dev->agp_buffer_token = init->buffers_offset;
1485	dev->agp_buffer_map = drm_core_findmap(dev, init->buffers_offset);
1486	if (!dev->agp_buffer_map) {
1487		DRM_ERROR("could not find dma buffer region!\n");
1488		radeon_do_cleanup_cp(dev);
1489		return DRM_ERR(EINVAL);
1490	}
1491
1492	if (init->gart_textures_offset) {
1493		dev_priv->gart_textures =
1494		    drm_core_findmap(dev, init->gart_textures_offset);
1495		if (!dev_priv->gart_textures) {
1496			DRM_ERROR("could not find GART texture region!\n");
1497			radeon_do_cleanup_cp(dev);
1498			return DRM_ERR(EINVAL);
1499		}
1500	}
1501
1502	dev_priv->sarea_priv =
1503	    (drm_radeon_sarea_t *) ((u8 *) dev_priv->sarea->handle +
1504				    init->sarea_priv_offset);
1505
1506#if __OS_HAS_AGP
1507	if (dev_priv->flags & CHIP_IS_AGP) {
1508		drm_core_ioremap(dev_priv->cp_ring, dev);
1509		drm_core_ioremap(dev_priv->ring_rptr, dev);
1510		drm_core_ioremap(dev->agp_buffer_map, dev);
1511		if (!dev_priv->cp_ring->handle ||
1512		    !dev_priv->ring_rptr->handle ||
1513		    !dev->agp_buffer_map->handle) {
1514			DRM_ERROR("could not find ioremap agp regions!\n");
1515			radeon_do_cleanup_cp(dev);
1516			return DRM_ERR(EINVAL);
1517		}
1518	} else
1519#endif
1520	{
1521		dev_priv->cp_ring->handle = (void *)dev_priv->cp_ring->offset;
1522		dev_priv->ring_rptr->handle =
1523		    (void *)dev_priv->ring_rptr->offset;
1524		dev->agp_buffer_map->handle =
1525		    (void *)dev->agp_buffer_map->offset;
1526
1527		DRM_DEBUG("dev_priv->cp_ring->handle %p\n",
1528			  dev_priv->cp_ring->handle);
1529		DRM_DEBUG("dev_priv->ring_rptr->handle %p\n",
1530			  dev_priv->ring_rptr->handle);
1531		DRM_DEBUG("dev->agp_buffer_map->handle %p\n",
1532			  dev->agp_buffer_map->handle);
1533	}
1534
1535	dev_priv->fb_location = (RADEON_READ(RADEON_MC_FB_LOCATION)
1536				 & 0xffff) << 16;
1537	dev_priv->fb_size =
1538		((RADEON_READ(RADEON_MC_FB_LOCATION) & 0xffff0000u) + 0x10000)
1539		- dev_priv->fb_location;
1540
1541	dev_priv->front_pitch_offset = (((dev_priv->front_pitch / 64) << 22) |
1542					((dev_priv->front_offset
1543					  + dev_priv->fb_location) >> 10));
1544
1545	dev_priv->back_pitch_offset = (((dev_priv->back_pitch / 64) << 22) |
1546				       ((dev_priv->back_offset
1547					 + dev_priv->fb_location) >> 10));
1548
1549	dev_priv->depth_pitch_offset = (((dev_priv->depth_pitch / 64) << 22) |
1550					((dev_priv->depth_offset
1551					  + dev_priv->fb_location) >> 10));
1552
1553	dev_priv->gart_size = init->gart_size;
1554
1555	/* New let's set the memory map ... */
1556	if (dev_priv->new_memmap) {
1557		u32 base = 0;
1558
1559		DRM_INFO("Setting GART location based on new memory map\n");
1560
1561		/* If using AGP, try to locate the AGP aperture at the same
1562		 * location in the card and on the bus, though we have to
1563		 * align it down.
1564		 */
1565#if __OS_HAS_AGP
1566		if (dev_priv->flags & CHIP_IS_AGP) {
1567			base = dev->agp->base;
1568			/* Check if valid */
1569			if ((base + dev_priv->gart_size) > dev_priv->fb_location &&
1570			    base < (dev_priv->fb_location + dev_priv->fb_size)) {
1571				DRM_INFO("Can't use AGP base @0x%08lx, won't fit\n",
1572					 dev->agp->base);
1573				base = 0;
1574			}
1575		}
1576#endif
1577		/* If not or if AGP is at 0 (Macs), try to put it elsewhere */
1578		if (base == 0) {
1579			base = dev_priv->fb_location + dev_priv->fb_size;
1580			if (((base + dev_priv->gart_size) & 0xfffffffful)
1581			    < base)
1582				base = dev_priv->fb_location
1583					- dev_priv->gart_size;
1584		}
1585		dev_priv->gart_vm_start = base & 0xffc00000u;
1586		if (dev_priv->gart_vm_start != base)
1587			DRM_INFO("GART aligned down from 0x%08x to 0x%08x\n",
1588				 base, dev_priv->gart_vm_start);
1589	} else {
1590		DRM_INFO("Setting GART location based on old memory map\n");
1591		dev_priv->gart_vm_start = dev_priv->fb_location +
1592			RADEON_READ(RADEON_CONFIG_APER_SIZE);
1593	}
1594
1595#if __OS_HAS_AGP
1596	if (dev_priv->flags & CHIP_IS_AGP)
1597		dev_priv->gart_buffers_offset = (dev->agp_buffer_map->offset
1598						 - dev->agp->base
1599						 + dev_priv->gart_vm_start);
1600	else
1601#endif
1602		dev_priv->gart_buffers_offset = (dev->agp_buffer_map->offset
1603					- (unsigned long)dev->sg->virtual
1604					+ dev_priv->gart_vm_start);
1605
1606	DRM_DEBUG("dev_priv->gart_size %d\n", dev_priv->gart_size);
1607	DRM_DEBUG("dev_priv->gart_vm_start 0x%x\n", dev_priv->gart_vm_start);
1608	DRM_DEBUG("dev_priv->gart_buffers_offset 0x%lx\n",
1609		  dev_priv->gart_buffers_offset);
1610
1611	dev_priv->ring.start = (u32 *) dev_priv->cp_ring->handle;
1612	dev_priv->ring.end = ((u32 *) dev_priv->cp_ring->handle
1613			      + init->ring_size / sizeof(u32));
1614	dev_priv->ring.size = init->ring_size;
1615	dev_priv->ring.size_l2qw = drm_order(init->ring_size / 8);
1616
1617	dev_priv->ring.tail_mask = (dev_priv->ring.size / sizeof(u32)) - 1;
1618
1619	dev_priv->ring.high_mark = RADEON_RING_HIGH_MARK;
1620
1621#if __OS_HAS_AGP
1622	if (dev_priv->flags & CHIP_IS_AGP) {
1623		/* Turn off PCI GART */
1624		radeon_set_pcigart(dev_priv, 0);
1625	} else
1626#endif
1627	{
1628		/* if we have an offset set from userspace */
1629		if (dev_priv->pcigart_offset) {
1630			dev_priv->gart_info.bus_addr =
1631			    dev_priv->pcigart_offset + dev_priv->fb_location;
1632			dev_priv->gart_info.mapping.offset =
1633			    dev_priv->gart_info.bus_addr;
1634			dev_priv->gart_info.mapping.size =
1635			    RADEON_PCIGART_TABLE_SIZE;
1636
1637			drm_core_ioremap(&dev_priv->gart_info.mapping, dev);
1638			dev_priv->gart_info.addr =
1639			    dev_priv->gart_info.mapping.handle;
1640
1641			dev_priv->gart_info.is_pcie =
1642			    !!(dev_priv->flags & CHIP_IS_PCIE);
1643			dev_priv->gart_info.gart_table_location =
1644			    DRM_ATI_GART_FB;
1645
1646			DRM_DEBUG("Setting phys_pci_gart to %p %08lX\n",
1647				  dev_priv->gart_info.addr,
1648				  dev_priv->pcigart_offset);
1649		} else {
1650			dev_priv->gart_info.gart_table_location =
1651			    DRM_ATI_GART_MAIN;
1652			dev_priv->gart_info.addr = NULL;
1653			dev_priv->gart_info.bus_addr = 0;
1654			if (dev_priv->flags & CHIP_IS_PCIE) {
1655				DRM_ERROR
1656				    ("Cannot use PCI Express without GART in FB memory\n");
1657				radeon_do_cleanup_cp(dev);
1658				return DRM_ERR(EINVAL);
1659			}
1660		}
1661
1662		if (!drm_ati_pcigart_init(dev, &dev_priv->gart_info)) {
1663			DRM_ERROR("failed to init PCI GART!\n");
1664			radeon_do_cleanup_cp(dev);
1665			return DRM_ERR(ENOMEM);
1666		}
1667
1668		/* Turn on PCI GART */
1669		radeon_set_pcigart(dev_priv, 1);
1670	}
1671
1672	radeon_cp_load_microcode(dev_priv);
1673	radeon_cp_init_ring_buffer(dev, dev_priv);
1674
1675	dev_priv->last_buf = 0;
1676
1677	radeon_do_engine_reset(dev);
1678	radeon_test_writeback(dev_priv);
1679
1680	return 0;
1681}
1682
1683static int radeon_do_cleanup_cp(drm_device_t * dev)
1684{
1685	drm_radeon_private_t *dev_priv = dev->dev_private;
1686	DRM_DEBUG("\n");
1687
1688	/* Make sure interrupts are disabled here because the uninstall ioctl
1689	 * may not have been called from userspace and after dev_private
1690	 * is freed, it's too late.
1691	 */
1692	if (dev->irq_enabled)
1693		drm_irq_uninstall(dev);
1694
1695#if __OS_HAS_AGP
1696	if (dev_priv->flags & CHIP_IS_AGP) {
1697		if (dev_priv->cp_ring != NULL) {
1698			drm_core_ioremapfree(dev_priv->cp_ring, dev);
1699			dev_priv->cp_ring = NULL;
1700		}
1701		if (dev_priv->ring_rptr != NULL) {
1702			drm_core_ioremapfree(dev_priv->ring_rptr, dev);
1703			dev_priv->ring_rptr = NULL;
1704		}
1705		if (dev->agp_buffer_map != NULL) {
1706			drm_core_ioremapfree(dev->agp_buffer_map, dev);
1707			dev->agp_buffer_map = NULL;
1708		}
1709	} else
1710#endif
1711	{
1712
1713		if (dev_priv->gart_info.bus_addr) {
1714			/* Turn off PCI GART */
1715			radeon_set_pcigart(dev_priv, 0);
1716			if (!drm_ati_pcigart_cleanup(dev, &dev_priv->gart_info))
1717				DRM_ERROR("failed to cleanup PCI GART!\n");
1718		}
1719
1720		if (dev_priv->gart_info.gart_table_location == DRM_ATI_GART_FB)
1721		{
1722			drm_core_ioremapfree(&dev_priv->gart_info.mapping, dev);
1723			dev_priv->gart_info.addr = 0;
1724		}
1725	}
1726	/* only clear to the start of flags */
1727	memset(dev_priv, 0, offsetof(drm_radeon_private_t, flags));
1728
1729	return 0;
1730}
1731
1732/* This code will reinit the Radeon CP hardware after a resume from disc.
1733 * AFAIK, it would be very difficult to pickle the state at suspend time, so
1734 * here we make sure that all Radeon hardware initialisation is re-done without
1735 * affecting running applications.
1736 *
1737 * Charl P. Botha <http://cpbotha.net>
1738 */
1739static int radeon_do_resume_cp(drm_device_t * dev)
1740{
1741	drm_radeon_private_t *dev_priv = dev->dev_private;
1742
1743	if (!dev_priv) {
1744		DRM_ERROR("Called with no initialization\n");
1745		return DRM_ERR(EINVAL);
1746	}
1747
1748	DRM_DEBUG("Starting radeon_do_resume_cp()\n");
1749
1750#if __OS_HAS_AGP
1751	if (dev_priv->flags & CHIP_IS_AGP) {
1752		/* Turn off PCI GART */
1753		radeon_set_pcigart(dev_priv, 0);
1754	} else
1755#endif
1756	{
1757		/* Turn on PCI GART */
1758		radeon_set_pcigart(dev_priv, 1);
1759	}
1760
1761	radeon_cp_load_microcode(dev_priv);
1762	radeon_cp_init_ring_buffer(dev, dev_priv);
1763
1764	radeon_do_engine_reset(dev);
1765
1766	DRM_DEBUG("radeon_do_resume_cp() complete\n");
1767
1768	return 0;
1769}
1770
1771int radeon_cp_init(DRM_IOCTL_ARGS)
1772{
1773	DRM_DEVICE;
1774	drm_radeon_init_t init;
1775
1776	LOCK_TEST_WITH_RETURN(dev, filp);
1777
1778	DRM_COPY_FROM_USER_IOCTL(init, (drm_radeon_init_t __user *) data,
1779				 sizeof(init));
1780
1781	if (init.func == RADEON_INIT_R300_CP)
1782		r300_init_reg_flags();
1783
1784	switch (init.func) {
1785	case RADEON_INIT_CP:
1786	case RADEON_INIT_R200_CP:
1787	case RADEON_INIT_R300_CP:
1788		return radeon_do_init_cp(dev, &init);
1789	case RADEON_CLEANUP_CP:
1790		return radeon_do_cleanup_cp(dev);
1791	}
1792
1793	return DRM_ERR(EINVAL);
1794}
1795
1796int radeon_cp_start(DRM_IOCTL_ARGS)
1797{
1798	DRM_DEVICE;
1799	drm_radeon_private_t *dev_priv = dev->dev_private;
1800	DRM_DEBUG("\n");
1801
1802	LOCK_TEST_WITH_RETURN(dev, filp);
1803
1804	if (dev_priv->cp_running) {
1805		DRM_DEBUG("%s while CP running\n", __FUNCTION__);
1806		return 0;
1807	}
1808	if (dev_priv->cp_mode == RADEON_CSQ_PRIDIS_INDDIS) {
1809		DRM_DEBUG("%s called with bogus CP mode (%d)\n",
1810			  __FUNCTION__, dev_priv->cp_mode);
1811		return 0;
1812	}
1813
1814	radeon_do_cp_start(dev_priv);
1815
1816	return 0;
1817}
1818
1819/* Stop the CP.  The engine must have been idled before calling this
1820 * routine.
1821 */
1822int radeon_cp_stop(DRM_IOCTL_ARGS)
1823{
1824	DRM_DEVICE;
1825	drm_radeon_private_t *dev_priv = dev->dev_private;
1826	drm_radeon_cp_stop_t stop;
1827	int ret;
1828	DRM_DEBUG("\n");
1829
1830	LOCK_TEST_WITH_RETURN(dev, filp);
1831
1832	DRM_COPY_FROM_USER_IOCTL(stop, (drm_radeon_cp_stop_t __user *) data,
1833				 sizeof(stop));
1834
1835	if (!dev_priv->cp_running)
1836		return 0;
1837
1838	/* Flush any pending CP commands.  This ensures any outstanding
1839	 * commands are exectuted by the engine before we turn it off.
1840	 */
1841	if (stop.flush) {
1842		radeon_do_cp_flush(dev_priv);
1843	}
1844
1845	/* If we fail to make the engine go idle, we return an error
1846	 * code so that the DRM ioctl wrapper can try again.
1847	 */
1848	if (stop.idle) {
1849		ret = radeon_do_cp_idle(dev_priv);
1850		if (ret)
1851			return ret;
1852	}
1853
1854	/* Finally, we can turn off the CP.  If the engine isn't idle,
1855	 * we will get some dropped triangles as they won't be fully
1856	 * rendered before the CP is shut down.
1857	 */
1858	radeon_do_cp_stop(dev_priv);
1859
1860	/* Reset the engine */
1861	radeon_do_engine_reset(dev);
1862
1863	return 0;
1864}
1865
1866void radeon_do_release(drm_device_t * dev)
1867{
1868	drm_radeon_private_t *dev_priv = dev->dev_private;
1869	int i, ret;
1870
1871	if (dev_priv) {
1872		if (dev_priv->cp_running) {
1873			/* Stop the cp */
1874			while ((ret = radeon_do_cp_idle(dev_priv)) != 0) {
1875				DRM_DEBUG("radeon_do_cp_idle %d\n", ret);
1876#ifdef __linux__
1877				schedule();
1878#else
1879#if defined(__FreeBSD__) && __FreeBSD_version > 500000
1880				msleep(&ret, &dev->dev_lock, PZERO, "rdnrel",
1881				       1);
1882#else
1883				tsleep(&ret, PZERO, "rdnrel", 1);
1884#endif
1885#endif
1886			}
1887			radeon_do_cp_stop(dev_priv);
1888			radeon_do_engine_reset(dev);
1889		}
1890
1891		/* Disable *all* interrupts */
1892		if (dev_priv->mmio)	/* remove this after permanent addmaps */
1893			RADEON_WRITE(RADEON_GEN_INT_CNTL, 0);
1894
1895		if (dev_priv->mmio) {	/* remove all surfaces */
1896			for (i = 0; i < RADEON_MAX_SURFACES; i++) {
1897				RADEON_WRITE(RADEON_SURFACE0_INFO + 16 * i, 0);
1898				RADEON_WRITE(RADEON_SURFACE0_LOWER_BOUND +
1899					     16 * i, 0);
1900				RADEON_WRITE(RADEON_SURFACE0_UPPER_BOUND +
1901					     16 * i, 0);
1902			}
1903		}
1904
1905		/* Free memory heap structures */
1906		radeon_mem_takedown(&(dev_priv->gart_heap));
1907		radeon_mem_takedown(&(dev_priv->fb_heap));
1908
1909		/* deallocate kernel resources */
1910		radeon_do_cleanup_cp(dev);
1911	}
1912}
1913
1914/* Just reset the CP ring.  Called as part of an X Server engine reset.
1915 */
1916int radeon_cp_reset(DRM_IOCTL_ARGS)
1917{
1918	DRM_DEVICE;
1919	drm_radeon_private_t *dev_priv = dev->dev_private;
1920	DRM_DEBUG("\n");
1921
1922	LOCK_TEST_WITH_RETURN(dev, filp);
1923
1924	if (!dev_priv) {
1925		DRM_DEBUG("%s called before init done\n", __FUNCTION__);
1926		return DRM_ERR(EINVAL);
1927	}
1928
1929	radeon_do_cp_reset(dev_priv);
1930
1931	/* The CP is no longer running after an engine reset */
1932	dev_priv->cp_running = 0;
1933
1934	return 0;
1935}
1936
1937int radeon_cp_idle(DRM_IOCTL_ARGS)
1938{
1939	DRM_DEVICE;
1940	drm_radeon_private_t *dev_priv = dev->dev_private;
1941	DRM_DEBUG("\n");
1942
1943	LOCK_TEST_WITH_RETURN(dev, filp);
1944
1945	return radeon_do_cp_idle(dev_priv);
1946}
1947
1948/* Added by Charl P. Botha to call radeon_do_resume_cp().
1949 */
1950int radeon_cp_resume(DRM_IOCTL_ARGS)
1951{
1952	DRM_DEVICE;
1953
1954	return radeon_do_resume_cp(dev);
1955}
1956
1957int radeon_engine_reset(DRM_IOCTL_ARGS)
1958{
1959	DRM_DEVICE;
1960	DRM_DEBUG("\n");
1961
1962	LOCK_TEST_WITH_RETURN(dev, filp);
1963
1964	return radeon_do_engine_reset(dev);
1965}
1966
1967/* ================================================================
1968 * Fullscreen mode
1969 */
1970
1971/* KW: Deprecated to say the least:
1972 */
1973int radeon_fullscreen(DRM_IOCTL_ARGS)
1974{
1975	return 0;
1976}
1977
1978/* ================================================================
1979 * Freelist management
1980 */
1981
1982/* Original comment: FIXME: ROTATE_BUFS is a hack to cycle through
1983 *   bufs until freelist code is used.  Note this hides a problem with
1984 *   the scratch register * (used to keep track of last buffer
1985 *   completed) being written to before * the last buffer has actually
1986 *   completed rendering.
1987 *
1988 * KW:  It's also a good way to find free buffers quickly.
1989 *
1990 * KW: Ideally this loop wouldn't exist, and freelist_get wouldn't
1991 * sleep.  However, bugs in older versions of radeon_accel.c mean that
1992 * we essentially have to do this, else old clients will break.
1993 *
1994 * However, it does leave open a potential deadlock where all the
1995 * buffers are held by other clients, which can't release them because
1996 * they can't get the lock.
1997 */
1998
1999drm_buf_t *radeon_freelist_get(drm_device_t * dev)
2000{
2001	drm_device_dma_t *dma = dev->dma;
2002	drm_radeon_private_t *dev_priv = dev->dev_private;
2003	drm_radeon_buf_priv_t *buf_priv;
2004	drm_buf_t *buf;
2005	int i, t;
2006	int start;
2007
2008	if (++dev_priv->last_buf >= dma->buf_count)
2009		dev_priv->last_buf = 0;
2010
2011	start = dev_priv->last_buf;
2012
2013	for (t = 0; t < dev_priv->usec_timeout; t++) {
2014		u32 done_age = GET_SCRATCH(1);
2015		DRM_DEBUG("done_age = %d\n", done_age);
2016		for (i = start; i < dma->buf_count; i++) {
2017			buf = dma->buflist[i];
2018			buf_priv = buf->dev_private;
2019			if (buf->filp == 0 || (buf->pending &&
2020					       buf_priv->age <= done_age)) {
2021				dev_priv->stats.requested_bufs++;
2022				buf->pending = 0;
2023				return buf;
2024			}
2025			start = 0;
2026		}
2027
2028		if (t) {
2029			DRM_UDELAY(1);
2030			dev_priv->stats.freelist_loops++;
2031		}
2032	}
2033
2034	DRM_DEBUG("returning NULL!\n");
2035	return NULL;
2036}
2037
2038#if 0
2039drm_buf_t *radeon_freelist_get(drm_device_t * dev)
2040{
2041	drm_device_dma_t *dma = dev->dma;
2042	drm_radeon_private_t *dev_priv = dev->dev_private;
2043	drm_radeon_buf_priv_t *buf_priv;
2044	drm_buf_t *buf;
2045	int i, t;
2046	int start;
2047	u32 done_age = DRM_READ32(dev_priv->ring_rptr, RADEON_SCRATCHOFF(1));
2048
2049	if (++dev_priv->last_buf >= dma->buf_count)
2050		dev_priv->last_buf = 0;
2051
2052	start = dev_priv->last_buf;
2053	dev_priv->stats.freelist_loops++;
2054
2055	for (t = 0; t < 2; t++) {
2056		for (i = start; i < dma->buf_count; i++) {
2057			buf = dma->buflist[i];
2058			buf_priv = buf->dev_private;
2059			if (buf->filp == 0 || (buf->pending &&
2060					       buf_priv->age <= done_age)) {
2061				dev_priv->stats.requested_bufs++;
2062				buf->pending = 0;
2063				return buf;
2064			}
2065		}
2066		start = 0;
2067	}
2068
2069	return NULL;
2070}
2071#endif
2072
2073void radeon_freelist_reset(drm_device_t * dev)
2074{
2075	drm_device_dma_t *dma = dev->dma;
2076	drm_radeon_private_t *dev_priv = dev->dev_private;
2077	int i;
2078
2079	dev_priv->last_buf = 0;
2080	for (i = 0; i < dma->buf_count; i++) {
2081		drm_buf_t *buf = dma->buflist[i];
2082		drm_radeon_buf_priv_t *buf_priv = buf->dev_private;
2083		buf_priv->age = 0;
2084	}
2085}
2086
2087/* ================================================================
2088 * CP command submission
2089 */
2090
2091int radeon_wait_ring(drm_radeon_private_t * dev_priv, int n)
2092{
2093	drm_radeon_ring_buffer_t *ring = &dev_priv->ring;
2094	int i;
2095	u32 last_head = GET_RING_HEAD(dev_priv);
2096
2097	for (i = 0; i < dev_priv->usec_timeout; i++) {
2098		u32 head = GET_RING_HEAD(dev_priv);
2099
2100		ring->space = (head - ring->tail) * sizeof(u32);
2101		if (ring->space <= 0)
2102			ring->space += ring->size;
2103		if (ring->space > n)
2104			return 0;
2105
2106		dev_priv->stats.boxes |= RADEON_BOX_WAIT_IDLE;
2107
2108		if (head != last_head)
2109			i = 0;
2110		last_head = head;
2111
2112		DRM_UDELAY(1);
2113	}
2114
2115	/* FIXME: This return value is ignored in the BEGIN_RING macro! */
2116#if RADEON_FIFO_DEBUG
2117	radeon_status(dev_priv);
2118	DRM_ERROR("failed!\n");
2119#endif
2120	return DRM_ERR(EBUSY);
2121}
2122
2123static int radeon_cp_get_buffers(DRMFILE filp, drm_device_t * dev,
2124				 drm_dma_t * d)
2125{
2126	int i;
2127	drm_buf_t *buf;
2128
2129	for (i = d->granted_count; i < d->request_count; i++) {
2130		buf = radeon_freelist_get(dev);
2131		if (!buf)
2132			return DRM_ERR(EBUSY);	/* NOTE: broken client */
2133
2134		buf->filp = filp;
2135
2136		if (DRM_COPY_TO_USER(&d->request_indices[i], &buf->idx,
2137				     sizeof(buf->idx)))
2138			return DRM_ERR(EFAULT);
2139		if (DRM_COPY_TO_USER(&d->request_sizes[i], &buf->total,
2140				     sizeof(buf->total)))
2141			return DRM_ERR(EFAULT);
2142
2143		d->granted_count++;
2144	}
2145	return 0;
2146}
2147
2148int radeon_cp_buffers(DRM_IOCTL_ARGS)
2149{
2150	DRM_DEVICE;
2151	drm_device_dma_t *dma = dev->dma;
2152	int ret = 0;
2153	drm_dma_t __user *argp = (void __user *)data;
2154	drm_dma_t d;
2155
2156	LOCK_TEST_WITH_RETURN(dev, filp);
2157
2158	DRM_COPY_FROM_USER_IOCTL(d, argp, sizeof(d));
2159
2160	/* Please don't send us buffers.
2161	 */
2162	if (d.send_count != 0) {
2163		DRM_ERROR("Process %d trying to send %d buffers via drmDMA\n",
2164			  DRM_CURRENTPID, d.send_count);
2165		return DRM_ERR(EINVAL);
2166	}
2167
2168	/* We'll send you buffers.
2169	 */
2170	if (d.request_count < 0 || d.request_count > dma->buf_count) {
2171		DRM_ERROR("Process %d trying to get %d buffers (of %d max)\n",
2172			  DRM_CURRENTPID, d.request_count, dma->buf_count);
2173		return DRM_ERR(EINVAL);
2174	}
2175
2176	d.granted_count = 0;
2177
2178	if (d.request_count) {
2179		ret = radeon_cp_get_buffers(filp, dev, &d);
2180	}
2181
2182	DRM_COPY_TO_USER_IOCTL(argp, d, sizeof(d));
2183
2184	return ret;
2185}
2186
2187int radeon_driver_load(struct drm_device *dev, unsigned long flags)
2188{
2189	drm_radeon_private_t *dev_priv;
2190	int ret = 0;
2191
2192	dev_priv = drm_alloc(sizeof(drm_radeon_private_t), DRM_MEM_DRIVER);
2193	if (dev_priv == NULL)
2194		return DRM_ERR(ENOMEM);
2195
2196	memset(dev_priv, 0, sizeof(drm_radeon_private_t));
2197	dev->dev_private = (void *)dev_priv;
2198	dev_priv->flags = flags;
2199
2200	switch (flags & CHIP_FAMILY_MASK) {
2201	case CHIP_R100:
2202	case CHIP_RV200:
2203	case CHIP_R200:
2204	case CHIP_R300:
2205	case CHIP_R350:
2206	case CHIP_R420:
2207	case CHIP_RV410:
2208		dev_priv->flags |= CHIP_HAS_HIERZ;
2209		break;
2210	default:
2211		/* all other chips have no hierarchical z buffer */
2212		break;
2213	}
2214
2215	if (drm_device_is_agp(dev))
2216		dev_priv->flags |= CHIP_IS_AGP;
2217	else if (drm_device_is_pcie(dev))
2218		dev_priv->flags |= CHIP_IS_PCIE;
2219	else
2220		dev_priv->flags |= CHIP_IS_PCI;
2221
2222	DRM_DEBUG("%s card detected\n",
2223		  ((dev_priv->flags & CHIP_IS_AGP) ? "AGP" : (((dev_priv->flags & CHIP_IS_PCIE) ? "PCIE" : "PCI"))));
2224	return ret;
2225}
2226
2227/* Create mappings for registers and framebuffer so userland doesn't necessarily
2228 * have to find them.
2229 */
2230int radeon_driver_firstopen(struct drm_device *dev)
2231{
2232	int ret;
2233	drm_local_map_t *map;
2234	drm_radeon_private_t *dev_priv = dev->dev_private;
2235
2236	ret = drm_addmap(dev, drm_get_resource_start(dev, 2),
2237			 drm_get_resource_len(dev, 2), _DRM_REGISTERS,
2238			 _DRM_READ_ONLY, &dev_priv->mmio);
2239	if (ret != 0)
2240		return ret;
2241
2242	ret = drm_addmap(dev, drm_get_resource_start(dev, 0),
2243			 drm_get_resource_len(dev, 0), _DRM_FRAME_BUFFER,
2244			 _DRM_WRITE_COMBINING, &map);
2245	if (ret != 0)
2246		return ret;
2247
2248	return 0;
2249}
2250
2251int radeon_driver_unload(struct drm_device *dev)
2252{
2253	drm_radeon_private_t *dev_priv = dev->dev_private;
2254
2255	DRM_DEBUG("\n");
2256	drm_free(dev_priv, sizeof(*dev_priv), DRM_MEM_DRIVER);
2257
2258	dev->dev_private = NULL;
2259	return 0;
2260}
2261