1 /*
2 * Copyright 2010 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 * Authors: Alex Deucher
23 */
24 #include <linux/firmware.h>
25 #include <linux/slab.h>
26 #include <drm/drmP.h>
27 #include "radeon.h"
28 #include "radeon_asic.h"
29 #include "radeon_audio.h"
30 #include <drm/radeon_drm.h>
31 #include "evergreend.h"
32 #include "atom.h"
33 #include "avivod.h"
34 #include "evergreen_reg.h"
35 #include "evergreen_blit_shaders.h"
36 #include "radeon_ucode.h"
37
38 static const u32 crtc_offsets[6] =
39 {
40 EVERGREEN_CRTC0_REGISTER_OFFSET,
41 EVERGREEN_CRTC1_REGISTER_OFFSET,
42 EVERGREEN_CRTC2_REGISTER_OFFSET,
43 EVERGREEN_CRTC3_REGISTER_OFFSET,
44 EVERGREEN_CRTC4_REGISTER_OFFSET,
45 EVERGREEN_CRTC5_REGISTER_OFFSET
46 };
47
48 #include "clearstate_evergreen.h"
49
50 static const u32 sumo_rlc_save_restore_register_list[] =
51 {
52 0x98fc,
53 0x9830,
54 0x9834,
55 0x9838,
56 0x9870,
57 0x9874,
58 0x8a14,
59 0x8b24,
60 0x8bcc,
61 0x8b10,
62 0x8d00,
63 0x8d04,
64 0x8c00,
65 0x8c04,
66 0x8c08,
67 0x8c0c,
68 0x8d8c,
69 0x8c20,
70 0x8c24,
71 0x8c28,
72 0x8c18,
73 0x8c1c,
74 0x8cf0,
75 0x8e2c,
76 0x8e38,
77 0x8c30,
78 0x9508,
79 0x9688,
80 0x9608,
81 0x960c,
82 0x9610,
83 0x9614,
84 0x88c4,
85 0x88d4,
86 0xa008,
87 0x900c,
88 0x9100,
89 0x913c,
90 0x98f8,
91 0x98f4,
92 0x9b7c,
93 0x3f8c,
94 0x8950,
95 0x8954,
96 0x8a18,
97 0x8b28,
98 0x9144,
99 0x9148,
100 0x914c,
101 0x3f90,
102 0x3f94,
103 0x915c,
104 0x9160,
105 0x9178,
106 0x917c,
107 0x9180,
108 0x918c,
109 0x9190,
110 0x9194,
111 0x9198,
112 0x919c,
113 0x91a8,
114 0x91ac,
115 0x91b0,
116 0x91b4,
117 0x91b8,
118 0x91c4,
119 0x91c8,
120 0x91cc,
121 0x91d0,
122 0x91d4,
123 0x91e0,
124 0x91e4,
125 0x91ec,
126 0x91f0,
127 0x91f4,
128 0x9200,
129 0x9204,
130 0x929c,
131 0x9150,
132 0x802c,
133 };
134
135 static void evergreen_gpu_init(struct radeon_device *rdev);
136 void evergreen_fini(struct radeon_device *rdev);
137 void evergreen_pcie_gen2_enable(struct radeon_device *rdev);
138 void evergreen_program_aspm(struct radeon_device *rdev);
139 extern void cayman_cp_int_cntl_setup(struct radeon_device *rdev,
140 int ring, u32 cp_int_cntl);
141 extern void cayman_vm_decode_fault(struct radeon_device *rdev,
142 u32 status, u32 addr);
143 void cik_init_cp_pg_table(struct radeon_device *rdev);
144
145 extern u32 si_get_csb_size(struct radeon_device *rdev);
146 extern void si_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer);
147 extern u32 cik_get_csb_size(struct radeon_device *rdev);
148 extern void cik_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer);
149 extern void rv770_set_clk_bypass_mode(struct radeon_device *rdev);
150
151 static const u32 evergreen_golden_registers[] =
152 {
153 0x3f90, 0xffff0000, 0xff000000,
154 0x9148, 0xffff0000, 0xff000000,
155 0x3f94, 0xffff0000, 0xff000000,
156 0x914c, 0xffff0000, 0xff000000,
157 0x9b7c, 0xffffffff, 0x00000000,
158 0x8a14, 0xffffffff, 0x00000007,
159 0x8b10, 0xffffffff, 0x00000000,
160 0x960c, 0xffffffff, 0x54763210,
161 0x88c4, 0xffffffff, 0x000000c2,
162 0x88d4, 0xffffffff, 0x00000010,
163 0x8974, 0xffffffff, 0x00000000,
164 0xc78, 0x00000080, 0x00000080,
165 0x5eb4, 0xffffffff, 0x00000002,
166 0x5e78, 0xffffffff, 0x001000f0,
167 0x6104, 0x01000300, 0x00000000,
168 0x5bc0, 0x00300000, 0x00000000,
169 0x7030, 0xffffffff, 0x00000011,
170 0x7c30, 0xffffffff, 0x00000011,
171 0x10830, 0xffffffff, 0x00000011,
172 0x11430, 0xffffffff, 0x00000011,
173 0x12030, 0xffffffff, 0x00000011,
174 0x12c30, 0xffffffff, 0x00000011,
175 0xd02c, 0xffffffff, 0x08421000,
176 0x240c, 0xffffffff, 0x00000380,
177 0x8b24, 0xffffffff, 0x00ff0fff,
178 0x28a4c, 0x06000000, 0x06000000,
179 0x10c, 0x00000001, 0x00000001,
180 0x8d00, 0xffffffff, 0x100e4848,
181 0x8d04, 0xffffffff, 0x00164745,
182 0x8c00, 0xffffffff, 0xe4000003,
183 0x8c04, 0xffffffff, 0x40600060,
184 0x8c08, 0xffffffff, 0x001c001c,
185 0x8cf0, 0xffffffff, 0x08e00620,
186 0x8c20, 0xffffffff, 0x00800080,
187 0x8c24, 0xffffffff, 0x00800080,
188 0x8c18, 0xffffffff, 0x20202078,
189 0x8c1c, 0xffffffff, 0x00001010,
190 0x28350, 0xffffffff, 0x00000000,
191 0xa008, 0xffffffff, 0x00010000,
192 0x5c4, 0xffffffff, 0x00000001,
193 0x9508, 0xffffffff, 0x00000002,
194 0x913c, 0x0000000f, 0x0000000a
195 };
196
197 static const u32 evergreen_golden_registers2[] =
198 {
199 0x2f4c, 0xffffffff, 0x00000000,
200 0x54f4, 0xffffffff, 0x00000000,
201 0x54f0, 0xffffffff, 0x00000000,
202 0x5498, 0xffffffff, 0x00000000,
203 0x549c, 0xffffffff, 0x00000000,
204 0x5494, 0xffffffff, 0x00000000,
205 0x53cc, 0xffffffff, 0x00000000,
206 0x53c8, 0xffffffff, 0x00000000,
207 0x53c4, 0xffffffff, 0x00000000,
208 0x53c0, 0xffffffff, 0x00000000,
209 0x53bc, 0xffffffff, 0x00000000,
210 0x53b8, 0xffffffff, 0x00000000,
211 0x53b4, 0xffffffff, 0x00000000,
212 0x53b0, 0xffffffff, 0x00000000
213 };
214
215 static const u32 cypress_mgcg_init[] =
216 {
217 0x802c, 0xffffffff, 0xc0000000,
218 0x5448, 0xffffffff, 0x00000100,
219 0x55e4, 0xffffffff, 0x00000100,
220 0x160c, 0xffffffff, 0x00000100,
221 0x5644, 0xffffffff, 0x00000100,
222 0xc164, 0xffffffff, 0x00000100,
223 0x8a18, 0xffffffff, 0x00000100,
224 0x897c, 0xffffffff, 0x06000100,
225 0x8b28, 0xffffffff, 0x00000100,
226 0x9144, 0xffffffff, 0x00000100,
227 0x9a60, 0xffffffff, 0x00000100,
228 0x9868, 0xffffffff, 0x00000100,
229 0x8d58, 0xffffffff, 0x00000100,
230 0x9510, 0xffffffff, 0x00000100,
231 0x949c, 0xffffffff, 0x00000100,
232 0x9654, 0xffffffff, 0x00000100,
233 0x9030, 0xffffffff, 0x00000100,
234 0x9034, 0xffffffff, 0x00000100,
235 0x9038, 0xffffffff, 0x00000100,
236 0x903c, 0xffffffff, 0x00000100,
237 0x9040, 0xffffffff, 0x00000100,
238 0xa200, 0xffffffff, 0x00000100,
239 0xa204, 0xffffffff, 0x00000100,
240 0xa208, 0xffffffff, 0x00000100,
241 0xa20c, 0xffffffff, 0x00000100,
242 0x971c, 0xffffffff, 0x00000100,
243 0x977c, 0xffffffff, 0x00000100,
244 0x3f80, 0xffffffff, 0x00000100,
245 0xa210, 0xffffffff, 0x00000100,
246 0xa214, 0xffffffff, 0x00000100,
247 0x4d8, 0xffffffff, 0x00000100,
248 0x9784, 0xffffffff, 0x00000100,
249 0x9698, 0xffffffff, 0x00000100,
250 0x4d4, 0xffffffff, 0x00000200,
251 0x30cc, 0xffffffff, 0x00000100,
252 0xd0c0, 0xffffffff, 0xff000100,
253 0x802c, 0xffffffff, 0x40000000,
254 0x915c, 0xffffffff, 0x00010000,
255 0x9160, 0xffffffff, 0x00030002,
256 0x9178, 0xffffffff, 0x00070000,
257 0x917c, 0xffffffff, 0x00030002,
258 0x9180, 0xffffffff, 0x00050004,
259 0x918c, 0xffffffff, 0x00010006,
260 0x9190, 0xffffffff, 0x00090008,
261 0x9194, 0xffffffff, 0x00070000,
262 0x9198, 0xffffffff, 0x00030002,
263 0x919c, 0xffffffff, 0x00050004,
264 0x91a8, 0xffffffff, 0x00010006,
265 0x91ac, 0xffffffff, 0x00090008,
266 0x91b0, 0xffffffff, 0x00070000,
267 0x91b4, 0xffffffff, 0x00030002,
268 0x91b8, 0xffffffff, 0x00050004,
269 0x91c4, 0xffffffff, 0x00010006,
270 0x91c8, 0xffffffff, 0x00090008,
271 0x91cc, 0xffffffff, 0x00070000,
272 0x91d0, 0xffffffff, 0x00030002,
273 0x91d4, 0xffffffff, 0x00050004,
274 0x91e0, 0xffffffff, 0x00010006,
275 0x91e4, 0xffffffff, 0x00090008,
276 0x91e8, 0xffffffff, 0x00000000,
277 0x91ec, 0xffffffff, 0x00070000,
278 0x91f0, 0xffffffff, 0x00030002,
279 0x91f4, 0xffffffff, 0x00050004,
280 0x9200, 0xffffffff, 0x00010006,
281 0x9204, 0xffffffff, 0x00090008,
282 0x9208, 0xffffffff, 0x00070000,
283 0x920c, 0xffffffff, 0x00030002,
284 0x9210, 0xffffffff, 0x00050004,
285 0x921c, 0xffffffff, 0x00010006,
286 0x9220, 0xffffffff, 0x00090008,
287 0x9224, 0xffffffff, 0x00070000,
288 0x9228, 0xffffffff, 0x00030002,
289 0x922c, 0xffffffff, 0x00050004,
290 0x9238, 0xffffffff, 0x00010006,
291 0x923c, 0xffffffff, 0x00090008,
292 0x9240, 0xffffffff, 0x00070000,
293 0x9244, 0xffffffff, 0x00030002,
294 0x9248, 0xffffffff, 0x00050004,
295 0x9254, 0xffffffff, 0x00010006,
296 0x9258, 0xffffffff, 0x00090008,
297 0x925c, 0xffffffff, 0x00070000,
298 0x9260, 0xffffffff, 0x00030002,
299 0x9264, 0xffffffff, 0x00050004,
300 0x9270, 0xffffffff, 0x00010006,
301 0x9274, 0xffffffff, 0x00090008,
302 0x9278, 0xffffffff, 0x00070000,
303 0x927c, 0xffffffff, 0x00030002,
304 0x9280, 0xffffffff, 0x00050004,
305 0x928c, 0xffffffff, 0x00010006,
306 0x9290, 0xffffffff, 0x00090008,
307 0x9294, 0xffffffff, 0x00000000,
308 0x929c, 0xffffffff, 0x00000001,
309 0x802c, 0xffffffff, 0x40010000,
310 0x915c, 0xffffffff, 0x00010000,
311 0x9160, 0xffffffff, 0x00030002,
312 0x9178, 0xffffffff, 0x00070000,
313 0x917c, 0xffffffff, 0x00030002,
314 0x9180, 0xffffffff, 0x00050004,
315 0x918c, 0xffffffff, 0x00010006,
316 0x9190, 0xffffffff, 0x00090008,
317 0x9194, 0xffffffff, 0x00070000,
318 0x9198, 0xffffffff, 0x00030002,
319 0x919c, 0xffffffff, 0x00050004,
320 0x91a8, 0xffffffff, 0x00010006,
321 0x91ac, 0xffffffff, 0x00090008,
322 0x91b0, 0xffffffff, 0x00070000,
323 0x91b4, 0xffffffff, 0x00030002,
324 0x91b8, 0xffffffff, 0x00050004,
325 0x91c4, 0xffffffff, 0x00010006,
326 0x91c8, 0xffffffff, 0x00090008,
327 0x91cc, 0xffffffff, 0x00070000,
328 0x91d0, 0xffffffff, 0x00030002,
329 0x91d4, 0xffffffff, 0x00050004,
330 0x91e0, 0xffffffff, 0x00010006,
331 0x91e4, 0xffffffff, 0x00090008,
332 0x91e8, 0xffffffff, 0x00000000,
333 0x91ec, 0xffffffff, 0x00070000,
334 0x91f0, 0xffffffff, 0x00030002,
335 0x91f4, 0xffffffff, 0x00050004,
336 0x9200, 0xffffffff, 0x00010006,
337 0x9204, 0xffffffff, 0x00090008,
338 0x9208, 0xffffffff, 0x00070000,
339 0x920c, 0xffffffff, 0x00030002,
340 0x9210, 0xffffffff, 0x00050004,
341 0x921c, 0xffffffff, 0x00010006,
342 0x9220, 0xffffffff, 0x00090008,
343 0x9224, 0xffffffff, 0x00070000,
344 0x9228, 0xffffffff, 0x00030002,
345 0x922c, 0xffffffff, 0x00050004,
346 0x9238, 0xffffffff, 0x00010006,
347 0x923c, 0xffffffff, 0x00090008,
348 0x9240, 0xffffffff, 0x00070000,
349 0x9244, 0xffffffff, 0x00030002,
350 0x9248, 0xffffffff, 0x00050004,
351 0x9254, 0xffffffff, 0x00010006,
352 0x9258, 0xffffffff, 0x00090008,
353 0x925c, 0xffffffff, 0x00070000,
354 0x9260, 0xffffffff, 0x00030002,
355 0x9264, 0xffffffff, 0x00050004,
356 0x9270, 0xffffffff, 0x00010006,
357 0x9274, 0xffffffff, 0x00090008,
358 0x9278, 0xffffffff, 0x00070000,
359 0x927c, 0xffffffff, 0x00030002,
360 0x9280, 0xffffffff, 0x00050004,
361 0x928c, 0xffffffff, 0x00010006,
362 0x9290, 0xffffffff, 0x00090008,
363 0x9294, 0xffffffff, 0x00000000,
364 0x929c, 0xffffffff, 0x00000001,
365 0x802c, 0xffffffff, 0xc0000000
366 };
367
368 static const u32 redwood_mgcg_init[] =
369 {
370 0x802c, 0xffffffff, 0xc0000000,
371 0x5448, 0xffffffff, 0x00000100,
372 0x55e4, 0xffffffff, 0x00000100,
373 0x160c, 0xffffffff, 0x00000100,
374 0x5644, 0xffffffff, 0x00000100,
375 0xc164, 0xffffffff, 0x00000100,
376 0x8a18, 0xffffffff, 0x00000100,
377 0x897c, 0xffffffff, 0x06000100,
378 0x8b28, 0xffffffff, 0x00000100,
379 0x9144, 0xffffffff, 0x00000100,
380 0x9a60, 0xffffffff, 0x00000100,
381 0x9868, 0xffffffff, 0x00000100,
382 0x8d58, 0xffffffff, 0x00000100,
383 0x9510, 0xffffffff, 0x00000100,
384 0x949c, 0xffffffff, 0x00000100,
385 0x9654, 0xffffffff, 0x00000100,
386 0x9030, 0xffffffff, 0x00000100,
387 0x9034, 0xffffffff, 0x00000100,
388 0x9038, 0xffffffff, 0x00000100,
389 0x903c, 0xffffffff, 0x00000100,
390 0x9040, 0xffffffff, 0x00000100,
391 0xa200, 0xffffffff, 0x00000100,
392 0xa204, 0xffffffff, 0x00000100,
393 0xa208, 0xffffffff, 0x00000100,
394 0xa20c, 0xffffffff, 0x00000100,
395 0x971c, 0xffffffff, 0x00000100,
396 0x977c, 0xffffffff, 0x00000100,
397 0x3f80, 0xffffffff, 0x00000100,
398 0xa210, 0xffffffff, 0x00000100,
399 0xa214, 0xffffffff, 0x00000100,
400 0x4d8, 0xffffffff, 0x00000100,
401 0x9784, 0xffffffff, 0x00000100,
402 0x9698, 0xffffffff, 0x00000100,
403 0x4d4, 0xffffffff, 0x00000200,
404 0x30cc, 0xffffffff, 0x00000100,
405 0xd0c0, 0xffffffff, 0xff000100,
406 0x802c, 0xffffffff, 0x40000000,
407 0x915c, 0xffffffff, 0x00010000,
408 0x9160, 0xffffffff, 0x00030002,
409 0x9178, 0xffffffff, 0x00070000,
410 0x917c, 0xffffffff, 0x00030002,
411 0x9180, 0xffffffff, 0x00050004,
412 0x918c, 0xffffffff, 0x00010006,
413 0x9190, 0xffffffff, 0x00090008,
414 0x9194, 0xffffffff, 0x00070000,
415 0x9198, 0xffffffff, 0x00030002,
416 0x919c, 0xffffffff, 0x00050004,
417 0x91a8, 0xffffffff, 0x00010006,
418 0x91ac, 0xffffffff, 0x00090008,
419 0x91b0, 0xffffffff, 0x00070000,
420 0x91b4, 0xffffffff, 0x00030002,
421 0x91b8, 0xffffffff, 0x00050004,
422 0x91c4, 0xffffffff, 0x00010006,
423 0x91c8, 0xffffffff, 0x00090008,
424 0x91cc, 0xffffffff, 0x00070000,
425 0x91d0, 0xffffffff, 0x00030002,
426 0x91d4, 0xffffffff, 0x00050004,
427 0x91e0, 0xffffffff, 0x00010006,
428 0x91e4, 0xffffffff, 0x00090008,
429 0x91e8, 0xffffffff, 0x00000000,
430 0x91ec, 0xffffffff, 0x00070000,
431 0x91f0, 0xffffffff, 0x00030002,
432 0x91f4, 0xffffffff, 0x00050004,
433 0x9200, 0xffffffff, 0x00010006,
434 0x9204, 0xffffffff, 0x00090008,
435 0x9294, 0xffffffff, 0x00000000,
436 0x929c, 0xffffffff, 0x00000001,
437 0x802c, 0xffffffff, 0xc0000000
438 };
439
440 static const u32 cedar_golden_registers[] =
441 {
442 0x3f90, 0xffff0000, 0xff000000,
443 0x9148, 0xffff0000, 0xff000000,
444 0x3f94, 0xffff0000, 0xff000000,
445 0x914c, 0xffff0000, 0xff000000,
446 0x9b7c, 0xffffffff, 0x00000000,
447 0x8a14, 0xffffffff, 0x00000007,
448 0x8b10, 0xffffffff, 0x00000000,
449 0x960c, 0xffffffff, 0x54763210,
450 0x88c4, 0xffffffff, 0x000000c2,
451 0x88d4, 0xffffffff, 0x00000000,
452 0x8974, 0xffffffff, 0x00000000,
453 0xc78, 0x00000080, 0x00000080,
454 0x5eb4, 0xffffffff, 0x00000002,
455 0x5e78, 0xffffffff, 0x001000f0,
456 0x6104, 0x01000300, 0x00000000,
457 0x5bc0, 0x00300000, 0x00000000,
458 0x7030, 0xffffffff, 0x00000011,
459 0x7c30, 0xffffffff, 0x00000011,
460 0x10830, 0xffffffff, 0x00000011,
461 0x11430, 0xffffffff, 0x00000011,
462 0xd02c, 0xffffffff, 0x08421000,
463 0x240c, 0xffffffff, 0x00000380,
464 0x8b24, 0xffffffff, 0x00ff0fff,
465 0x28a4c, 0x06000000, 0x06000000,
466 0x10c, 0x00000001, 0x00000001,
467 0x8d00, 0xffffffff, 0x100e4848,
468 0x8d04, 0xffffffff, 0x00164745,
469 0x8c00, 0xffffffff, 0xe4000003,
470 0x8c04, 0xffffffff, 0x40600060,
471 0x8c08, 0xffffffff, 0x001c001c,
472 0x8cf0, 0xffffffff, 0x08e00410,
473 0x8c20, 0xffffffff, 0x00800080,
474 0x8c24, 0xffffffff, 0x00800080,
475 0x8c18, 0xffffffff, 0x20202078,
476 0x8c1c, 0xffffffff, 0x00001010,
477 0x28350, 0xffffffff, 0x00000000,
478 0xa008, 0xffffffff, 0x00010000,
479 0x5c4, 0xffffffff, 0x00000001,
480 0x9508, 0xffffffff, 0x00000002
481 };
482
483 static const u32 cedar_mgcg_init[] =
484 {
485 0x802c, 0xffffffff, 0xc0000000,
486 0x5448, 0xffffffff, 0x00000100,
487 0x55e4, 0xffffffff, 0x00000100,
488 0x160c, 0xffffffff, 0x00000100,
489 0x5644, 0xffffffff, 0x00000100,
490 0xc164, 0xffffffff, 0x00000100,
491 0x8a18, 0xffffffff, 0x00000100,
492 0x897c, 0xffffffff, 0x06000100,
493 0x8b28, 0xffffffff, 0x00000100,
494 0x9144, 0xffffffff, 0x00000100,
495 0x9a60, 0xffffffff, 0x00000100,
496 0x9868, 0xffffffff, 0x00000100,
497 0x8d58, 0xffffffff, 0x00000100,
498 0x9510, 0xffffffff, 0x00000100,
499 0x949c, 0xffffffff, 0x00000100,
500 0x9654, 0xffffffff, 0x00000100,
501 0x9030, 0xffffffff, 0x00000100,
502 0x9034, 0xffffffff, 0x00000100,
503 0x9038, 0xffffffff, 0x00000100,
504 0x903c, 0xffffffff, 0x00000100,
505 0x9040, 0xffffffff, 0x00000100,
506 0xa200, 0xffffffff, 0x00000100,
507 0xa204, 0xffffffff, 0x00000100,
508 0xa208, 0xffffffff, 0x00000100,
509 0xa20c, 0xffffffff, 0x00000100,
510 0x971c, 0xffffffff, 0x00000100,
511 0x977c, 0xffffffff, 0x00000100,
512 0x3f80, 0xffffffff, 0x00000100,
513 0xa210, 0xffffffff, 0x00000100,
514 0xa214, 0xffffffff, 0x00000100,
515 0x4d8, 0xffffffff, 0x00000100,
516 0x9784, 0xffffffff, 0x00000100,
517 0x9698, 0xffffffff, 0x00000100,
518 0x4d4, 0xffffffff, 0x00000200,
519 0x30cc, 0xffffffff, 0x00000100,
520 0xd0c0, 0xffffffff, 0xff000100,
521 0x802c, 0xffffffff, 0x40000000,
522 0x915c, 0xffffffff, 0x00010000,
523 0x9178, 0xffffffff, 0x00050000,
524 0x917c, 0xffffffff, 0x00030002,
525 0x918c, 0xffffffff, 0x00010004,
526 0x9190, 0xffffffff, 0x00070006,
527 0x9194, 0xffffffff, 0x00050000,
528 0x9198, 0xffffffff, 0x00030002,
529 0x91a8, 0xffffffff, 0x00010004,
530 0x91ac, 0xffffffff, 0x00070006,
531 0x91e8, 0xffffffff, 0x00000000,
532 0x9294, 0xffffffff, 0x00000000,
533 0x929c, 0xffffffff, 0x00000001,
534 0x802c, 0xffffffff, 0xc0000000
535 };
536
537 static const u32 juniper_mgcg_init[] =
538 {
539 0x802c, 0xffffffff, 0xc0000000,
540 0x5448, 0xffffffff, 0x00000100,
541 0x55e4, 0xffffffff, 0x00000100,
542 0x160c, 0xffffffff, 0x00000100,
543 0x5644, 0xffffffff, 0x00000100,
544 0xc164, 0xffffffff, 0x00000100,
545 0x8a18, 0xffffffff, 0x00000100,
546 0x897c, 0xffffffff, 0x06000100,
547 0x8b28, 0xffffffff, 0x00000100,
548 0x9144, 0xffffffff, 0x00000100,
549 0x9a60, 0xffffffff, 0x00000100,
550 0x9868, 0xffffffff, 0x00000100,
551 0x8d58, 0xffffffff, 0x00000100,
552 0x9510, 0xffffffff, 0x00000100,
553 0x949c, 0xffffffff, 0x00000100,
554 0x9654, 0xffffffff, 0x00000100,
555 0x9030, 0xffffffff, 0x00000100,
556 0x9034, 0xffffffff, 0x00000100,
557 0x9038, 0xffffffff, 0x00000100,
558 0x903c, 0xffffffff, 0x00000100,
559 0x9040, 0xffffffff, 0x00000100,
560 0xa200, 0xffffffff, 0x00000100,
561 0xa204, 0xffffffff, 0x00000100,
562 0xa208, 0xffffffff, 0x00000100,
563 0xa20c, 0xffffffff, 0x00000100,
564 0x971c, 0xffffffff, 0x00000100,
565 0xd0c0, 0xffffffff, 0xff000100,
566 0x802c, 0xffffffff, 0x40000000,
567 0x915c, 0xffffffff, 0x00010000,
568 0x9160, 0xffffffff, 0x00030002,
569 0x9178, 0xffffffff, 0x00070000,
570 0x917c, 0xffffffff, 0x00030002,
571 0x9180, 0xffffffff, 0x00050004,
572 0x918c, 0xffffffff, 0x00010006,
573 0x9190, 0xffffffff, 0x00090008,
574 0x9194, 0xffffffff, 0x00070000,
575 0x9198, 0xffffffff, 0x00030002,
576 0x919c, 0xffffffff, 0x00050004,
577 0x91a8, 0xffffffff, 0x00010006,
578 0x91ac, 0xffffffff, 0x00090008,
579 0x91b0, 0xffffffff, 0x00070000,
580 0x91b4, 0xffffffff, 0x00030002,
581 0x91b8, 0xffffffff, 0x00050004,
582 0x91c4, 0xffffffff, 0x00010006,
583 0x91c8, 0xffffffff, 0x00090008,
584 0x91cc, 0xffffffff, 0x00070000,
585 0x91d0, 0xffffffff, 0x00030002,
586 0x91d4, 0xffffffff, 0x00050004,
587 0x91e0, 0xffffffff, 0x00010006,
588 0x91e4, 0xffffffff, 0x00090008,
589 0x91e8, 0xffffffff, 0x00000000,
590 0x91ec, 0xffffffff, 0x00070000,
591 0x91f0, 0xffffffff, 0x00030002,
592 0x91f4, 0xffffffff, 0x00050004,
593 0x9200, 0xffffffff, 0x00010006,
594 0x9204, 0xffffffff, 0x00090008,
595 0x9208, 0xffffffff, 0x00070000,
596 0x920c, 0xffffffff, 0x00030002,
597 0x9210, 0xffffffff, 0x00050004,
598 0x921c, 0xffffffff, 0x00010006,
599 0x9220, 0xffffffff, 0x00090008,
600 0x9224, 0xffffffff, 0x00070000,
601 0x9228, 0xffffffff, 0x00030002,
602 0x922c, 0xffffffff, 0x00050004,
603 0x9238, 0xffffffff, 0x00010006,
604 0x923c, 0xffffffff, 0x00090008,
605 0x9240, 0xffffffff, 0x00070000,
606 0x9244, 0xffffffff, 0x00030002,
607 0x9248, 0xffffffff, 0x00050004,
608 0x9254, 0xffffffff, 0x00010006,
609 0x9258, 0xffffffff, 0x00090008,
610 0x925c, 0xffffffff, 0x00070000,
611 0x9260, 0xffffffff, 0x00030002,
612 0x9264, 0xffffffff, 0x00050004,
613 0x9270, 0xffffffff, 0x00010006,
614 0x9274, 0xffffffff, 0x00090008,
615 0x9278, 0xffffffff, 0x00070000,
616 0x927c, 0xffffffff, 0x00030002,
617 0x9280, 0xffffffff, 0x00050004,
618 0x928c, 0xffffffff, 0x00010006,
619 0x9290, 0xffffffff, 0x00090008,
620 0x9294, 0xffffffff, 0x00000000,
621 0x929c, 0xffffffff, 0x00000001,
622 0x802c, 0xffffffff, 0xc0000000,
623 0x977c, 0xffffffff, 0x00000100,
624 0x3f80, 0xffffffff, 0x00000100,
625 0xa210, 0xffffffff, 0x00000100,
626 0xa214, 0xffffffff, 0x00000100,
627 0x4d8, 0xffffffff, 0x00000100,
628 0x9784, 0xffffffff, 0x00000100,
629 0x9698, 0xffffffff, 0x00000100,
630 0x4d4, 0xffffffff, 0x00000200,
631 0x30cc, 0xffffffff, 0x00000100,
632 0x802c, 0xffffffff, 0xc0000000
633 };
634
635 static const u32 supersumo_golden_registers[] =
636 {
637 0x5eb4, 0xffffffff, 0x00000002,
638 0x5c4, 0xffffffff, 0x00000001,
639 0x7030, 0xffffffff, 0x00000011,
640 0x7c30, 0xffffffff, 0x00000011,
641 0x6104, 0x01000300, 0x00000000,
642 0x5bc0, 0x00300000, 0x00000000,
643 0x8c04, 0xffffffff, 0x40600060,
644 0x8c08, 0xffffffff, 0x001c001c,
645 0x8c20, 0xffffffff, 0x00800080,
646 0x8c24, 0xffffffff, 0x00800080,
647 0x8c18, 0xffffffff, 0x20202078,
648 0x8c1c, 0xffffffff, 0x00001010,
649 0x918c, 0xffffffff, 0x00010006,
650 0x91a8, 0xffffffff, 0x00010006,
651 0x91c4, 0xffffffff, 0x00010006,
652 0x91e0, 0xffffffff, 0x00010006,
653 0x9200, 0xffffffff, 0x00010006,
654 0x9150, 0xffffffff, 0x6e944040,
655 0x917c, 0xffffffff, 0x00030002,
656 0x9180, 0xffffffff, 0x00050004,
657 0x9198, 0xffffffff, 0x00030002,
658 0x919c, 0xffffffff, 0x00050004,
659 0x91b4, 0xffffffff, 0x00030002,
660 0x91b8, 0xffffffff, 0x00050004,
661 0x91d0, 0xffffffff, 0x00030002,
662 0x91d4, 0xffffffff, 0x00050004,
663 0x91f0, 0xffffffff, 0x00030002,
664 0x91f4, 0xffffffff, 0x00050004,
665 0x915c, 0xffffffff, 0x00010000,
666 0x9160, 0xffffffff, 0x00030002,
667 0x3f90, 0xffff0000, 0xff000000,
668 0x9178, 0xffffffff, 0x00070000,
669 0x9194, 0xffffffff, 0x00070000,
670 0x91b0, 0xffffffff, 0x00070000,
671 0x91cc, 0xffffffff, 0x00070000,
672 0x91ec, 0xffffffff, 0x00070000,
673 0x9148, 0xffff0000, 0xff000000,
674 0x9190, 0xffffffff, 0x00090008,
675 0x91ac, 0xffffffff, 0x00090008,
676 0x91c8, 0xffffffff, 0x00090008,
677 0x91e4, 0xffffffff, 0x00090008,
678 0x9204, 0xffffffff, 0x00090008,
679 0x3f94, 0xffff0000, 0xff000000,
680 0x914c, 0xffff0000, 0xff000000,
681 0x929c, 0xffffffff, 0x00000001,
682 0x8a18, 0xffffffff, 0x00000100,
683 0x8b28, 0xffffffff, 0x00000100,
684 0x9144, 0xffffffff, 0x00000100,
685 0x5644, 0xffffffff, 0x00000100,
686 0x9b7c, 0xffffffff, 0x00000000,
687 0x8030, 0xffffffff, 0x0000100a,
688 0x8a14, 0xffffffff, 0x00000007,
689 0x8b24, 0xffffffff, 0x00ff0fff,
690 0x8b10, 0xffffffff, 0x00000000,
691 0x28a4c, 0x06000000, 0x06000000,
692 0x4d8, 0xffffffff, 0x00000100,
693 0x913c, 0xffff000f, 0x0100000a,
694 0x960c, 0xffffffff, 0x54763210,
695 0x88c4, 0xffffffff, 0x000000c2,
696 0x88d4, 0xffffffff, 0x00000010,
697 0x8974, 0xffffffff, 0x00000000,
698 0xc78, 0x00000080, 0x00000080,
699 0x5e78, 0xffffffff, 0x001000f0,
700 0xd02c, 0xffffffff, 0x08421000,
701 0xa008, 0xffffffff, 0x00010000,
702 0x8d00, 0xffffffff, 0x100e4848,
703 0x8d04, 0xffffffff, 0x00164745,
704 0x8c00, 0xffffffff, 0xe4000003,
705 0x8cf0, 0x1fffffff, 0x08e00620,
706 0x28350, 0xffffffff, 0x00000000,
707 0x9508, 0xffffffff, 0x00000002
708 };
709
710 static const u32 sumo_golden_registers[] =
711 {
712 0x900c, 0x00ffffff, 0x0017071f,
713 0x8c18, 0xffffffff, 0x10101060,
714 0x8c1c, 0xffffffff, 0x00001010,
715 0x8c30, 0x0000000f, 0x00000005,
716 0x9688, 0x0000000f, 0x00000007
717 };
718
719 static const u32 wrestler_golden_registers[] =
720 {
721 0x5eb4, 0xffffffff, 0x00000002,
722 0x5c4, 0xffffffff, 0x00000001,
723 0x7030, 0xffffffff, 0x00000011,
724 0x7c30, 0xffffffff, 0x00000011,
725 0x6104, 0x01000300, 0x00000000,
726 0x5bc0, 0x00300000, 0x00000000,
727 0x918c, 0xffffffff, 0x00010006,
728 0x91a8, 0xffffffff, 0x00010006,
729 0x9150, 0xffffffff, 0x6e944040,
730 0x917c, 0xffffffff, 0x00030002,
731 0x9198, 0xffffffff, 0x00030002,
732 0x915c, 0xffffffff, 0x00010000,
733 0x3f90, 0xffff0000, 0xff000000,
734 0x9178, 0xffffffff, 0x00070000,
735 0x9194, 0xffffffff, 0x00070000,
736 0x9148, 0xffff0000, 0xff000000,
737 0x9190, 0xffffffff, 0x00090008,
738 0x91ac, 0xffffffff, 0x00090008,
739 0x3f94, 0xffff0000, 0xff000000,
740 0x914c, 0xffff0000, 0xff000000,
741 0x929c, 0xffffffff, 0x00000001,
742 0x8a18, 0xffffffff, 0x00000100,
743 0x8b28, 0xffffffff, 0x00000100,
744 0x9144, 0xffffffff, 0x00000100,
745 0x9b7c, 0xffffffff, 0x00000000,
746 0x8030, 0xffffffff, 0x0000100a,
747 0x8a14, 0xffffffff, 0x00000001,
748 0x8b24, 0xffffffff, 0x00ff0fff,
749 0x8b10, 0xffffffff, 0x00000000,
750 0x28a4c, 0x06000000, 0x06000000,
751 0x4d8, 0xffffffff, 0x00000100,
752 0x913c, 0xffff000f, 0x0100000a,
753 0x960c, 0xffffffff, 0x54763210,
754 0x88c4, 0xffffffff, 0x000000c2,
755 0x88d4, 0xffffffff, 0x00000010,
756 0x8974, 0xffffffff, 0x00000000,
757 0xc78, 0x00000080, 0x00000080,
758 0x5e78, 0xffffffff, 0x001000f0,
759 0xd02c, 0xffffffff, 0x08421000,
760 0xa008, 0xffffffff, 0x00010000,
761 0x8d00, 0xffffffff, 0x100e4848,
762 0x8d04, 0xffffffff, 0x00164745,
763 0x8c00, 0xffffffff, 0xe4000003,
764 0x8cf0, 0x1fffffff, 0x08e00410,
765 0x28350, 0xffffffff, 0x00000000,
766 0x9508, 0xffffffff, 0x00000002,
767 0x900c, 0xffffffff, 0x0017071f,
768 0x8c18, 0xffffffff, 0x10101060,
769 0x8c1c, 0xffffffff, 0x00001010
770 };
771
772 static const u32 barts_golden_registers[] =
773 {
774 0x5eb4, 0xffffffff, 0x00000002,
775 0x5e78, 0x8f311ff1, 0x001000f0,
776 0x3f90, 0xffff0000, 0xff000000,
777 0x9148, 0xffff0000, 0xff000000,
778 0x3f94, 0xffff0000, 0xff000000,
779 0x914c, 0xffff0000, 0xff000000,
780 0xc78, 0x00000080, 0x00000080,
781 0xbd4, 0x70073777, 0x00010001,
782 0xd02c, 0xbfffff1f, 0x08421000,
783 0xd0b8, 0x03773777, 0x02011003,
784 0x5bc0, 0x00200000, 0x50100000,
785 0x98f8, 0x33773777, 0x02011003,
786 0x98fc, 0xffffffff, 0x76543210,
787 0x7030, 0x31000311, 0x00000011,
788 0x2f48, 0x00000007, 0x02011003,
789 0x6b28, 0x00000010, 0x00000012,
790 0x7728, 0x00000010, 0x00000012,
791 0x10328, 0x00000010, 0x00000012,
792 0x10f28, 0x00000010, 0x00000012,
793 0x11b28, 0x00000010, 0x00000012,
794 0x12728, 0x00000010, 0x00000012,
795 0x240c, 0x000007ff, 0x00000380,
796 0x8a14, 0xf000001f, 0x00000007,
797 0x8b24, 0x3fff3fff, 0x00ff0fff,
798 0x8b10, 0x0000ff0f, 0x00000000,
799 0x28a4c, 0x07ffffff, 0x06000000,
800 0x10c, 0x00000001, 0x00010003,
801 0xa02c, 0xffffffff, 0x0000009b,
802 0x913c, 0x0000000f, 0x0100000a,
803 0x8d00, 0xffff7f7f, 0x100e4848,
804 0x8d04, 0x00ffffff, 0x00164745,
805 0x8c00, 0xfffc0003, 0xe4000003,
806 0x8c04, 0xf8ff00ff, 0x40600060,
807 0x8c08, 0x00ff00ff, 0x001c001c,
808 0x8cf0, 0x1fff1fff, 0x08e00620,
809 0x8c20, 0x0fff0fff, 0x00800080,
810 0x8c24, 0x0fff0fff, 0x00800080,
811 0x8c18, 0xffffffff, 0x20202078,
812 0x8c1c, 0x0000ffff, 0x00001010,
813 0x28350, 0x00000f01, 0x00000000,
814 0x9508, 0x3700001f, 0x00000002,
815 0x960c, 0xffffffff, 0x54763210,
816 0x88c4, 0x001f3ae3, 0x000000c2,
817 0x88d4, 0x0000001f, 0x00000010,
818 0x8974, 0xffffffff, 0x00000000
819 };
820
821 static const u32 turks_golden_registers[] =
822 {
823 0x5eb4, 0xffffffff, 0x00000002,
824 0x5e78, 0x8f311ff1, 0x001000f0,
825 0x8c8, 0x00003000, 0x00001070,
826 0x8cc, 0x000fffff, 0x00040035,
827 0x3f90, 0xffff0000, 0xfff00000,
828 0x9148, 0xffff0000, 0xfff00000,
829 0x3f94, 0xffff0000, 0xfff00000,
830 0x914c, 0xffff0000, 0xfff00000,
831 0xc78, 0x00000080, 0x00000080,
832 0xbd4, 0x00073007, 0x00010002,
833 0xd02c, 0xbfffff1f, 0x08421000,
834 0xd0b8, 0x03773777, 0x02010002,
835 0x5bc0, 0x00200000, 0x50100000,
836 0x98f8, 0x33773777, 0x00010002,
837 0x98fc, 0xffffffff, 0x33221100,
838 0x7030, 0x31000311, 0x00000011,
839 0x2f48, 0x33773777, 0x00010002,
840 0x6b28, 0x00000010, 0x00000012,
841 0x7728, 0x00000010, 0x00000012,
842 0x10328, 0x00000010, 0x00000012,
843 0x10f28, 0x00000010, 0x00000012,
844 0x11b28, 0x00000010, 0x00000012,
845 0x12728, 0x00000010, 0x00000012,
846 0x240c, 0x000007ff, 0x00000380,
847 0x8a14, 0xf000001f, 0x00000007,
848 0x8b24, 0x3fff3fff, 0x00ff0fff,
849 0x8b10, 0x0000ff0f, 0x00000000,
850 0x28a4c, 0x07ffffff, 0x06000000,
851 0x10c, 0x00000001, 0x00010003,
852 0xa02c, 0xffffffff, 0x0000009b,
853 0x913c, 0x0000000f, 0x0100000a,
854 0x8d00, 0xffff7f7f, 0x100e4848,
855 0x8d04, 0x00ffffff, 0x00164745,
856 0x8c00, 0xfffc0003, 0xe4000003,
857 0x8c04, 0xf8ff00ff, 0x40600060,
858 0x8c08, 0x00ff00ff, 0x001c001c,
859 0x8cf0, 0x1fff1fff, 0x08e00410,
860 0x8c20, 0x0fff0fff, 0x00800080,
861 0x8c24, 0x0fff0fff, 0x00800080,
862 0x8c18, 0xffffffff, 0x20202078,
863 0x8c1c, 0x0000ffff, 0x00001010,
864 0x28350, 0x00000f01, 0x00000000,
865 0x9508, 0x3700001f, 0x00000002,
866 0x960c, 0xffffffff, 0x54763210,
867 0x88c4, 0x001f3ae3, 0x000000c2,
868 0x88d4, 0x0000001f, 0x00000010,
869 0x8974, 0xffffffff, 0x00000000
870 };
871
872 static const u32 caicos_golden_registers[] =
873 {
874 0x5eb4, 0xffffffff, 0x00000002,
875 0x5e78, 0x8f311ff1, 0x001000f0,
876 0x8c8, 0x00003420, 0x00001450,
877 0x8cc, 0x000fffff, 0x00040035,
878 0x3f90, 0xffff0000, 0xfffc0000,
879 0x9148, 0xffff0000, 0xfffc0000,
880 0x3f94, 0xffff0000, 0xfffc0000,
881 0x914c, 0xffff0000, 0xfffc0000,
882 0xc78, 0x00000080, 0x00000080,
883 0xbd4, 0x00073007, 0x00010001,
884 0xd02c, 0xbfffff1f, 0x08421000,
885 0xd0b8, 0x03773777, 0x02010001,
886 0x5bc0, 0x00200000, 0x50100000,
887 0x98f8, 0x33773777, 0x02010001,
888 0x98fc, 0xffffffff, 0x33221100,
889 0x7030, 0x31000311, 0x00000011,
890 0x2f48, 0x33773777, 0x02010001,
891 0x6b28, 0x00000010, 0x00000012,
892 0x7728, 0x00000010, 0x00000012,
893 0x10328, 0x00000010, 0x00000012,
894 0x10f28, 0x00000010, 0x00000012,
895 0x11b28, 0x00000010, 0x00000012,
896 0x12728, 0x00000010, 0x00000012,
897 0x240c, 0x000007ff, 0x00000380,
898 0x8a14, 0xf000001f, 0x00000001,
899 0x8b24, 0x3fff3fff, 0x00ff0fff,
900 0x8b10, 0x0000ff0f, 0x00000000,
901 0x28a4c, 0x07ffffff, 0x06000000,
902 0x10c, 0x00000001, 0x00010003,
903 0xa02c, 0xffffffff, 0x0000009b,
904 0x913c, 0x0000000f, 0x0100000a,
905 0x8d00, 0xffff7f7f, 0x100e4848,
906 0x8d04, 0x00ffffff, 0x00164745,
907 0x8c00, 0xfffc0003, 0xe4000003,
908 0x8c04, 0xf8ff00ff, 0x40600060,
909 0x8c08, 0x00ff00ff, 0x001c001c,
910 0x8cf0, 0x1fff1fff, 0x08e00410,
911 0x8c20, 0x0fff0fff, 0x00800080,
912 0x8c24, 0x0fff0fff, 0x00800080,
913 0x8c18, 0xffffffff, 0x20202078,
914 0x8c1c, 0x0000ffff, 0x00001010,
915 0x28350, 0x00000f01, 0x00000000,
916 0x9508, 0x3700001f, 0x00000002,
917 0x960c, 0xffffffff, 0x54763210,
918 0x88c4, 0x001f3ae3, 0x000000c2,
919 0x88d4, 0x0000001f, 0x00000010,
920 0x8974, 0xffffffff, 0x00000000
921 };
922
evergreen_init_golden_registers(struct radeon_device * rdev)923 static void evergreen_init_golden_registers(struct radeon_device *rdev)
924 {
925 switch (rdev->family) {
926 case CHIP_CYPRESS:
927 case CHIP_HEMLOCK:
928 radeon_program_register_sequence(rdev,
929 evergreen_golden_registers,
930 (const u32)ARRAY_SIZE(evergreen_golden_registers));
931 radeon_program_register_sequence(rdev,
932 evergreen_golden_registers2,
933 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
934 radeon_program_register_sequence(rdev,
935 cypress_mgcg_init,
936 (const u32)ARRAY_SIZE(cypress_mgcg_init));
937 break;
938 case CHIP_JUNIPER:
939 radeon_program_register_sequence(rdev,
940 evergreen_golden_registers,
941 (const u32)ARRAY_SIZE(evergreen_golden_registers));
942 radeon_program_register_sequence(rdev,
943 evergreen_golden_registers2,
944 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
945 radeon_program_register_sequence(rdev,
946 juniper_mgcg_init,
947 (const u32)ARRAY_SIZE(juniper_mgcg_init));
948 break;
949 case CHIP_REDWOOD:
950 radeon_program_register_sequence(rdev,
951 evergreen_golden_registers,
952 (const u32)ARRAY_SIZE(evergreen_golden_registers));
953 radeon_program_register_sequence(rdev,
954 evergreen_golden_registers2,
955 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
956 radeon_program_register_sequence(rdev,
957 redwood_mgcg_init,
958 (const u32)ARRAY_SIZE(redwood_mgcg_init));
959 break;
960 case CHIP_CEDAR:
961 radeon_program_register_sequence(rdev,
962 cedar_golden_registers,
963 (const u32)ARRAY_SIZE(cedar_golden_registers));
964 radeon_program_register_sequence(rdev,
965 evergreen_golden_registers2,
966 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
967 radeon_program_register_sequence(rdev,
968 cedar_mgcg_init,
969 (const u32)ARRAY_SIZE(cedar_mgcg_init));
970 break;
971 case CHIP_PALM:
972 radeon_program_register_sequence(rdev,
973 wrestler_golden_registers,
974 (const u32)ARRAY_SIZE(wrestler_golden_registers));
975 break;
976 case CHIP_SUMO:
977 radeon_program_register_sequence(rdev,
978 supersumo_golden_registers,
979 (const u32)ARRAY_SIZE(supersumo_golden_registers));
980 break;
981 case CHIP_SUMO2:
982 radeon_program_register_sequence(rdev,
983 supersumo_golden_registers,
984 (const u32)ARRAY_SIZE(supersumo_golden_registers));
985 radeon_program_register_sequence(rdev,
986 sumo_golden_registers,
987 (const u32)ARRAY_SIZE(sumo_golden_registers));
988 break;
989 case CHIP_BARTS:
990 radeon_program_register_sequence(rdev,
991 barts_golden_registers,
992 (const u32)ARRAY_SIZE(barts_golden_registers));
993 break;
994 case CHIP_TURKS:
995 radeon_program_register_sequence(rdev,
996 turks_golden_registers,
997 (const u32)ARRAY_SIZE(turks_golden_registers));
998 break;
999 case CHIP_CAICOS:
1000 radeon_program_register_sequence(rdev,
1001 caicos_golden_registers,
1002 (const u32)ARRAY_SIZE(caicos_golden_registers));
1003 break;
1004 default:
1005 break;
1006 }
1007 }
1008
1009 /**
1010 * evergreen_get_allowed_info_register - fetch the register for the info ioctl
1011 *
1012 * @rdev: radeon_device pointer
1013 * @reg: register offset in bytes
1014 * @val: register value
1015 *
1016 * Returns 0 for success or -EINVAL for an invalid register
1017 *
1018 */
evergreen_get_allowed_info_register(struct radeon_device * rdev,u32 reg,u32 * val)1019 int evergreen_get_allowed_info_register(struct radeon_device *rdev,
1020 u32 reg, u32 *val)
1021 {
1022 switch (reg) {
1023 case GRBM_STATUS:
1024 case GRBM_STATUS_SE0:
1025 case GRBM_STATUS_SE1:
1026 case SRBM_STATUS:
1027 case SRBM_STATUS2:
1028 case DMA_STATUS_REG:
1029 case UVD_STATUS:
1030 *val = RREG32(reg);
1031 return 0;
1032 default:
1033 return -EINVAL;
1034 }
1035 }
1036
evergreen_tiling_fields(unsigned tiling_flags,unsigned * bankw,unsigned * bankh,unsigned * mtaspect,unsigned * tile_split)1037 void evergreen_tiling_fields(unsigned tiling_flags, unsigned *bankw,
1038 unsigned *bankh, unsigned *mtaspect,
1039 unsigned *tile_split)
1040 {
1041 *bankw = (tiling_flags >> RADEON_TILING_EG_BANKW_SHIFT) & RADEON_TILING_EG_BANKW_MASK;
1042 *bankh = (tiling_flags >> RADEON_TILING_EG_BANKH_SHIFT) & RADEON_TILING_EG_BANKH_MASK;
1043 *mtaspect = (tiling_flags >> RADEON_TILING_EG_MACRO_TILE_ASPECT_SHIFT) & RADEON_TILING_EG_MACRO_TILE_ASPECT_MASK;
1044 *tile_split = (tiling_flags >> RADEON_TILING_EG_TILE_SPLIT_SHIFT) & RADEON_TILING_EG_TILE_SPLIT_MASK;
1045 switch (*bankw) {
1046 default:
1047 case 1: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_1; break;
1048 case 2: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_2; break;
1049 case 4: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_4; break;
1050 case 8: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_8; break;
1051 }
1052 switch (*bankh) {
1053 default:
1054 case 1: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_1; break;
1055 case 2: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_2; break;
1056 case 4: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_4; break;
1057 case 8: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_8; break;
1058 }
1059 switch (*mtaspect) {
1060 default:
1061 case 1: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_1; break;
1062 case 2: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_2; break;
1063 case 4: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_4; break;
1064 case 8: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_8; break;
1065 }
1066 }
1067
sumo_set_uvd_clock(struct radeon_device * rdev,u32 clock,u32 cntl_reg,u32 status_reg)1068 static int sumo_set_uvd_clock(struct radeon_device *rdev, u32 clock,
1069 u32 cntl_reg, u32 status_reg)
1070 {
1071 int r, i;
1072 struct atom_clock_dividers dividers;
1073
1074 r = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM,
1075 clock, false, ÷rs);
1076 if (r)
1077 return r;
1078
1079 WREG32_P(cntl_reg, dividers.post_div, ~(DCLK_DIR_CNTL_EN|DCLK_DIVIDER_MASK));
1080
1081 for (i = 0; i < 100; i++) {
1082 if (RREG32(status_reg) & DCLK_STATUS)
1083 break;
1084 mdelay(10);
1085 }
1086 if (i == 100)
1087 return -ETIMEDOUT;
1088
1089 return 0;
1090 }
1091
sumo_set_uvd_clocks(struct radeon_device * rdev,u32 vclk,u32 dclk)1092 int sumo_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
1093 {
1094 int r = 0;
1095 u32 cg_scratch = RREG32(CG_SCRATCH1);
1096
1097 r = sumo_set_uvd_clock(rdev, vclk, CG_VCLK_CNTL, CG_VCLK_STATUS);
1098 if (r)
1099 goto done;
1100 cg_scratch &= 0xffff0000;
1101 cg_scratch |= vclk / 100; /* Mhz */
1102
1103 r = sumo_set_uvd_clock(rdev, dclk, CG_DCLK_CNTL, CG_DCLK_STATUS);
1104 if (r)
1105 goto done;
1106 cg_scratch &= 0x0000ffff;
1107 cg_scratch |= (dclk / 100) << 16; /* Mhz */
1108
1109 done:
1110 WREG32(CG_SCRATCH1, cg_scratch);
1111
1112 return r;
1113 }
1114
evergreen_set_uvd_clocks(struct radeon_device * rdev,u32 vclk,u32 dclk)1115 int evergreen_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
1116 {
1117 /* start off with something large */
1118 unsigned fb_div = 0, vclk_div = 0, dclk_div = 0;
1119 int r;
1120
1121 /* bypass vclk and dclk with bclk */
1122 WREG32_P(CG_UPLL_FUNC_CNTL_2,
1123 VCLK_SRC_SEL(1) | DCLK_SRC_SEL(1),
1124 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
1125
1126 /* put PLL in bypass mode */
1127 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_BYPASS_EN_MASK, ~UPLL_BYPASS_EN_MASK);
1128
1129 if (!vclk || !dclk) {
1130 /* keep the Bypass mode, put PLL to sleep */
1131 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
1132 return 0;
1133 }
1134
1135 r = radeon_uvd_calc_upll_dividers(rdev, vclk, dclk, 125000, 250000,
1136 16384, 0x03FFFFFF, 0, 128, 5,
1137 &fb_div, &vclk_div, &dclk_div);
1138 if (r)
1139 return r;
1140
1141 /* set VCO_MODE to 1 */
1142 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_VCO_MODE_MASK, ~UPLL_VCO_MODE_MASK);
1143
1144 /* toggle UPLL_SLEEP to 1 then back to 0 */
1145 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
1146 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_SLEEP_MASK);
1147
1148 /* deassert UPLL_RESET */
1149 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
1150
1151 mdelay(1);
1152
1153 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
1154 if (r)
1155 return r;
1156
1157 /* assert UPLL_RESET again */
1158 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_RESET_MASK, ~UPLL_RESET_MASK);
1159
1160 /* disable spread spectrum. */
1161 WREG32_P(CG_UPLL_SPREAD_SPECTRUM, 0, ~SSEN_MASK);
1162
1163 /* set feedback divider */
1164 WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(fb_div), ~UPLL_FB_DIV_MASK);
1165
1166 /* set ref divider to 0 */
1167 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_REF_DIV_MASK);
1168
1169 if (fb_div < 307200)
1170 WREG32_P(CG_UPLL_FUNC_CNTL_4, 0, ~UPLL_SPARE_ISPARE9);
1171 else
1172 WREG32_P(CG_UPLL_FUNC_CNTL_4, UPLL_SPARE_ISPARE9, ~UPLL_SPARE_ISPARE9);
1173
1174 /* set PDIV_A and PDIV_B */
1175 WREG32_P(CG_UPLL_FUNC_CNTL_2,
1176 UPLL_PDIV_A(vclk_div) | UPLL_PDIV_B(dclk_div),
1177 ~(UPLL_PDIV_A_MASK | UPLL_PDIV_B_MASK));
1178
1179 /* give the PLL some time to settle */
1180 mdelay(15);
1181
1182 /* deassert PLL_RESET */
1183 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
1184
1185 mdelay(15);
1186
1187 /* switch from bypass mode to normal mode */
1188 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_BYPASS_EN_MASK);
1189
1190 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
1191 if (r)
1192 return r;
1193
1194 /* switch VCLK and DCLK selection */
1195 WREG32_P(CG_UPLL_FUNC_CNTL_2,
1196 VCLK_SRC_SEL(2) | DCLK_SRC_SEL(2),
1197 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
1198
1199 mdelay(100);
1200
1201 return 0;
1202 }
1203
evergreen_fix_pci_max_read_req_size(struct radeon_device * rdev)1204 void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev)
1205 {
1206 int readrq;
1207 u16 v;
1208
1209 readrq = pcie_get_readrq(rdev->pdev);
1210 v = ffs(readrq) - 8;
1211 /* if bios or OS sets MAX_READ_REQUEST_SIZE to an invalid value, fix it
1212 * to avoid hangs or perfomance issues
1213 */
1214 if ((v == 0) || (v == 6) || (v == 7))
1215 pcie_set_readrq(rdev->pdev, 512);
1216 }
1217
dce4_program_fmt(struct drm_encoder * encoder)1218 void dce4_program_fmt(struct drm_encoder *encoder)
1219 {
1220 struct drm_device *dev = encoder->dev;
1221 struct radeon_device *rdev = dev->dev_private;
1222 struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder);
1223 struct radeon_crtc *radeon_crtc = to_radeon_crtc(encoder->crtc);
1224 struct drm_connector *connector = radeon_get_connector_for_encoder(encoder);
1225 int bpc = 0;
1226 u32 tmp = 0;
1227 enum radeon_connector_dither dither = RADEON_FMT_DITHER_DISABLE;
1228
1229 if (connector) {
1230 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
1231 bpc = radeon_get_monitor_bpc(connector);
1232 dither = radeon_connector->dither;
1233 }
1234
1235 /* LVDS/eDP FMT is set up by atom */
1236 if (radeon_encoder->devices & ATOM_DEVICE_LCD_SUPPORT)
1237 return;
1238
1239 /* not needed for analog */
1240 if ((radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC1) ||
1241 (radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC2))
1242 return;
1243
1244 if (bpc == 0)
1245 return;
1246
1247 switch (bpc) {
1248 case 6:
1249 if (dither == RADEON_FMT_DITHER_ENABLE)
1250 /* XXX sort out optimal dither settings */
1251 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
1252 FMT_SPATIAL_DITHER_EN);
1253 else
1254 tmp |= FMT_TRUNCATE_EN;
1255 break;
1256 case 8:
1257 if (dither == RADEON_FMT_DITHER_ENABLE)
1258 /* XXX sort out optimal dither settings */
1259 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
1260 FMT_RGB_RANDOM_ENABLE |
1261 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH);
1262 else
1263 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH);
1264 break;
1265 case 10:
1266 default:
1267 /* not needed */
1268 break;
1269 }
1270
1271 WREG32(FMT_BIT_DEPTH_CONTROL + radeon_crtc->crtc_offset, tmp);
1272 }
1273
dce4_is_in_vblank(struct radeon_device * rdev,int crtc)1274 static bool dce4_is_in_vblank(struct radeon_device *rdev, int crtc)
1275 {
1276 if (RREG32(EVERGREEN_CRTC_STATUS + crtc_offsets[crtc]) & EVERGREEN_CRTC_V_BLANK)
1277 return true;
1278 else
1279 return false;
1280 }
1281
dce4_is_counter_moving(struct radeon_device * rdev,int crtc)1282 static bool dce4_is_counter_moving(struct radeon_device *rdev, int crtc)
1283 {
1284 u32 pos1, pos2;
1285
1286 pos1 = RREG32(EVERGREEN_CRTC_STATUS_POSITION + crtc_offsets[crtc]);
1287 pos2 = RREG32(EVERGREEN_CRTC_STATUS_POSITION + crtc_offsets[crtc]);
1288
1289 if (pos1 != pos2)
1290 return true;
1291 else
1292 return false;
1293 }
1294
1295 /**
1296 * dce4_wait_for_vblank - vblank wait asic callback.
1297 *
1298 * @rdev: radeon_device pointer
1299 * @crtc: crtc to wait for vblank on
1300 *
1301 * Wait for vblank on the requested crtc (evergreen+).
1302 */
dce4_wait_for_vblank(struct radeon_device * rdev,int crtc)1303 void dce4_wait_for_vblank(struct radeon_device *rdev, int crtc)
1304 {
1305 unsigned i = 0;
1306
1307 if (crtc >= rdev->num_crtc)
1308 return;
1309
1310 if (!(RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[crtc]) & EVERGREEN_CRTC_MASTER_EN))
1311 return;
1312
1313 /* depending on when we hit vblank, we may be close to active; if so,
1314 * wait for another frame.
1315 */
1316 while (dce4_is_in_vblank(rdev, crtc)) {
1317 if (i++ % 100 == 0) {
1318 if (!dce4_is_counter_moving(rdev, crtc))
1319 break;
1320 }
1321 }
1322
1323 while (!dce4_is_in_vblank(rdev, crtc)) {
1324 if (i++ % 100 == 0) {
1325 if (!dce4_is_counter_moving(rdev, crtc))
1326 break;
1327 }
1328 }
1329 }
1330
1331 /**
1332 * evergreen_page_flip - pageflip callback.
1333 *
1334 * @rdev: radeon_device pointer
1335 * @crtc_id: crtc to cleanup pageflip on
1336 * @crtc_base: new address of the crtc (GPU MC address)
1337 *
1338 * Does the actual pageflip (evergreen+).
1339 * During vblank we take the crtc lock and wait for the update_pending
1340 * bit to go high, when it does, we release the lock, and allow the
1341 * double buffered update to take place.
1342 * Returns the current update pending status.
1343 */
evergreen_page_flip(struct radeon_device * rdev,int crtc_id,u64 crtc_base)1344 void evergreen_page_flip(struct radeon_device *rdev, int crtc_id, u64 crtc_base)
1345 {
1346 struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id];
1347 u32 tmp = RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset);
1348 int i;
1349
1350 /* Lock the graphics update lock */
1351 tmp |= EVERGREEN_GRPH_UPDATE_LOCK;
1352 WREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset, tmp);
1353
1354 /* update the scanout addresses */
1355 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + radeon_crtc->crtc_offset,
1356 upper_32_bits(crtc_base));
1357 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
1358 (u32)crtc_base);
1359
1360 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + radeon_crtc->crtc_offset,
1361 upper_32_bits(crtc_base));
1362 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
1363 (u32)crtc_base);
1364
1365 /* Wait for update_pending to go high. */
1366 for (i = 0; i < rdev->usec_timeout; i++) {
1367 if (RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset) & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING)
1368 break;
1369 udelay(1);
1370 }
1371 DRM_DEBUG("Update pending now high. Unlocking vupdate_lock.\n");
1372
1373 /* Unlock the lock, so double-buffering can take place inside vblank */
1374 tmp &= ~EVERGREEN_GRPH_UPDATE_LOCK;
1375 WREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset, tmp);
1376 }
1377
1378 /**
1379 * evergreen_page_flip_pending - check if page flip is still pending
1380 *
1381 * @rdev: radeon_device pointer
1382 * @crtc_id: crtc to check
1383 *
1384 * Returns the current update pending status.
1385 */
evergreen_page_flip_pending(struct radeon_device * rdev,int crtc_id)1386 bool evergreen_page_flip_pending(struct radeon_device *rdev, int crtc_id)
1387 {
1388 struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id];
1389
1390 /* Return current update_pending status: */
1391 return !!(RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset) &
1392 EVERGREEN_GRPH_SURFACE_UPDATE_PENDING);
1393 }
1394
1395 /* get temperature in millidegrees */
evergreen_get_temp(struct radeon_device * rdev)1396 int evergreen_get_temp(struct radeon_device *rdev)
1397 {
1398 u32 temp, toffset;
1399 int actual_temp = 0;
1400
1401 if (rdev->family == CHIP_JUNIPER) {
1402 toffset = (RREG32(CG_THERMAL_CTRL) & TOFFSET_MASK) >>
1403 TOFFSET_SHIFT;
1404 temp = (RREG32(CG_TS0_STATUS) & TS0_ADC_DOUT_MASK) >>
1405 TS0_ADC_DOUT_SHIFT;
1406
1407 if (toffset & 0x100)
1408 actual_temp = temp / 2 - (0x200 - toffset);
1409 else
1410 actual_temp = temp / 2 + toffset;
1411
1412 actual_temp = actual_temp * 1000;
1413
1414 } else {
1415 temp = (RREG32(CG_MULT_THERMAL_STATUS) & ASIC_T_MASK) >>
1416 ASIC_T_SHIFT;
1417
1418 if (temp & 0x400)
1419 actual_temp = -256;
1420 else if (temp & 0x200)
1421 actual_temp = 255;
1422 else if (temp & 0x100) {
1423 actual_temp = temp & 0x1ff;
1424 actual_temp |= ~0x1ff;
1425 } else
1426 actual_temp = temp & 0xff;
1427
1428 actual_temp = (actual_temp * 1000) / 2;
1429 }
1430
1431 return actual_temp;
1432 }
1433
sumo_get_temp(struct radeon_device * rdev)1434 int sumo_get_temp(struct radeon_device *rdev)
1435 {
1436 u32 temp = RREG32(CG_THERMAL_STATUS) & 0xff;
1437 int actual_temp = temp - 49;
1438
1439 return actual_temp * 1000;
1440 }
1441
1442 /**
1443 * sumo_pm_init_profile - Initialize power profiles callback.
1444 *
1445 * @rdev: radeon_device pointer
1446 *
1447 * Initialize the power states used in profile mode
1448 * (sumo, trinity, SI).
1449 * Used for profile mode only.
1450 */
sumo_pm_init_profile(struct radeon_device * rdev)1451 void sumo_pm_init_profile(struct radeon_device *rdev)
1452 {
1453 int idx;
1454
1455 /* default */
1456 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
1457 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
1458 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
1459 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 0;
1460
1461 /* low,mid sh/mh */
1462 if (rdev->flags & RADEON_IS_MOBILITY)
1463 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 0);
1464 else
1465 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1466
1467 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = idx;
1468 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = idx;
1469 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
1470 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
1471
1472 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = idx;
1473 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = idx;
1474 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
1475 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
1476
1477 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = idx;
1478 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = idx;
1479 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
1480 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 0;
1481
1482 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = idx;
1483 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = idx;
1484 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
1485 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 0;
1486
1487 /* high sh/mh */
1488 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1489 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = idx;
1490 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = idx;
1491 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
1492 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx =
1493 rdev->pm.power_state[idx].num_clock_modes - 1;
1494
1495 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = idx;
1496 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = idx;
1497 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
1498 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx =
1499 rdev->pm.power_state[idx].num_clock_modes - 1;
1500 }
1501
1502 /**
1503 * btc_pm_init_profile - Initialize power profiles callback.
1504 *
1505 * @rdev: radeon_device pointer
1506 *
1507 * Initialize the power states used in profile mode
1508 * (BTC, cayman).
1509 * Used for profile mode only.
1510 */
btc_pm_init_profile(struct radeon_device * rdev)1511 void btc_pm_init_profile(struct radeon_device *rdev)
1512 {
1513 int idx;
1514
1515 /* default */
1516 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
1517 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
1518 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
1519 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 2;
1520 /* starting with BTC, there is one state that is used for both
1521 * MH and SH. Difference is that we always use the high clock index for
1522 * mclk.
1523 */
1524 if (rdev->flags & RADEON_IS_MOBILITY)
1525 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 0);
1526 else
1527 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1528 /* low sh */
1529 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = idx;
1530 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = idx;
1531 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
1532 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
1533 /* mid sh */
1534 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = idx;
1535 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = idx;
1536 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
1537 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 1;
1538 /* high sh */
1539 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = idx;
1540 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = idx;
1541 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
1542 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx = 2;
1543 /* low mh */
1544 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = idx;
1545 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = idx;
1546 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
1547 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
1548 /* mid mh */
1549 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = idx;
1550 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = idx;
1551 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
1552 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 1;
1553 /* high mh */
1554 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = idx;
1555 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = idx;
1556 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
1557 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx = 2;
1558 }
1559
1560 /**
1561 * evergreen_pm_misc - set additional pm hw parameters callback.
1562 *
1563 * @rdev: radeon_device pointer
1564 *
1565 * Set non-clock parameters associated with a power state
1566 * (voltage, etc.) (evergreen+).
1567 */
evergreen_pm_misc(struct radeon_device * rdev)1568 void evergreen_pm_misc(struct radeon_device *rdev)
1569 {
1570 int req_ps_idx = rdev->pm.requested_power_state_index;
1571 int req_cm_idx = rdev->pm.requested_clock_mode_index;
1572 struct radeon_power_state *ps = &rdev->pm.power_state[req_ps_idx];
1573 struct radeon_voltage *voltage = &ps->clock_info[req_cm_idx].voltage;
1574
1575 if (voltage->type == VOLTAGE_SW) {
1576 /* 0xff0x are flags rather then an actual voltage */
1577 if ((voltage->voltage & 0xff00) == 0xff00)
1578 return;
1579 if (voltage->voltage && (voltage->voltage != rdev->pm.current_vddc)) {
1580 radeon_atom_set_voltage(rdev, voltage->voltage, SET_VOLTAGE_TYPE_ASIC_VDDC);
1581 rdev->pm.current_vddc = voltage->voltage;
1582 DRM_DEBUG("Setting: vddc: %d\n", voltage->voltage);
1583 }
1584
1585 /* starting with BTC, there is one state that is used for both
1586 * MH and SH. Difference is that we always use the high clock index for
1587 * mclk and vddci.
1588 */
1589 if ((rdev->pm.pm_method == PM_METHOD_PROFILE) &&
1590 (rdev->family >= CHIP_BARTS) &&
1591 rdev->pm.active_crtc_count &&
1592 ((rdev->pm.profile_index == PM_PROFILE_MID_MH_IDX) ||
1593 (rdev->pm.profile_index == PM_PROFILE_LOW_MH_IDX)))
1594 voltage = &rdev->pm.power_state[req_ps_idx].
1595 clock_info[rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx].voltage;
1596
1597 /* 0xff0x are flags rather then an actual voltage */
1598 if ((voltage->vddci & 0xff00) == 0xff00)
1599 return;
1600 if (voltage->vddci && (voltage->vddci != rdev->pm.current_vddci)) {
1601 radeon_atom_set_voltage(rdev, voltage->vddci, SET_VOLTAGE_TYPE_ASIC_VDDCI);
1602 rdev->pm.current_vddci = voltage->vddci;
1603 DRM_DEBUG("Setting: vddci: %d\n", voltage->vddci);
1604 }
1605 }
1606 }
1607
1608 /**
1609 * evergreen_pm_prepare - pre-power state change callback.
1610 *
1611 * @rdev: radeon_device pointer
1612 *
1613 * Prepare for a power state change (evergreen+).
1614 */
evergreen_pm_prepare(struct radeon_device * rdev)1615 void evergreen_pm_prepare(struct radeon_device *rdev)
1616 {
1617 struct drm_device *ddev = rdev->ddev;
1618 struct drm_crtc *crtc;
1619 struct radeon_crtc *radeon_crtc;
1620 u32 tmp;
1621
1622 /* disable any active CRTCs */
1623 list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) {
1624 radeon_crtc = to_radeon_crtc(crtc);
1625 if (radeon_crtc->enabled) {
1626 tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset);
1627 tmp |= EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
1628 WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp);
1629 }
1630 }
1631 }
1632
1633 /**
1634 * evergreen_pm_finish - post-power state change callback.
1635 *
1636 * @rdev: radeon_device pointer
1637 *
1638 * Clean up after a power state change (evergreen+).
1639 */
evergreen_pm_finish(struct radeon_device * rdev)1640 void evergreen_pm_finish(struct radeon_device *rdev)
1641 {
1642 struct drm_device *ddev = rdev->ddev;
1643 struct drm_crtc *crtc;
1644 struct radeon_crtc *radeon_crtc;
1645 u32 tmp;
1646
1647 /* enable any active CRTCs */
1648 list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) {
1649 radeon_crtc = to_radeon_crtc(crtc);
1650 if (radeon_crtc->enabled) {
1651 tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset);
1652 tmp &= ~EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
1653 WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp);
1654 }
1655 }
1656 }
1657
1658 /**
1659 * evergreen_hpd_sense - hpd sense callback.
1660 *
1661 * @rdev: radeon_device pointer
1662 * @hpd: hpd (hotplug detect) pin
1663 *
1664 * Checks if a digital monitor is connected (evergreen+).
1665 * Returns true if connected, false if not connected.
1666 */
evergreen_hpd_sense(struct radeon_device * rdev,enum radeon_hpd_id hpd)1667 bool evergreen_hpd_sense(struct radeon_device *rdev, enum radeon_hpd_id hpd)
1668 {
1669 bool connected = false;
1670
1671 switch (hpd) {
1672 case RADEON_HPD_1:
1673 if (RREG32(DC_HPD1_INT_STATUS) & DC_HPDx_SENSE)
1674 connected = true;
1675 break;
1676 case RADEON_HPD_2:
1677 if (RREG32(DC_HPD2_INT_STATUS) & DC_HPDx_SENSE)
1678 connected = true;
1679 break;
1680 case RADEON_HPD_3:
1681 if (RREG32(DC_HPD3_INT_STATUS) & DC_HPDx_SENSE)
1682 connected = true;
1683 break;
1684 case RADEON_HPD_4:
1685 if (RREG32(DC_HPD4_INT_STATUS) & DC_HPDx_SENSE)
1686 connected = true;
1687 break;
1688 case RADEON_HPD_5:
1689 if (RREG32(DC_HPD5_INT_STATUS) & DC_HPDx_SENSE)
1690 connected = true;
1691 break;
1692 case RADEON_HPD_6:
1693 if (RREG32(DC_HPD6_INT_STATUS) & DC_HPDx_SENSE)
1694 connected = true;
1695 break;
1696 default:
1697 break;
1698 }
1699
1700 return connected;
1701 }
1702
1703 /**
1704 * evergreen_hpd_set_polarity - hpd set polarity callback.
1705 *
1706 * @rdev: radeon_device pointer
1707 * @hpd: hpd (hotplug detect) pin
1708 *
1709 * Set the polarity of the hpd pin (evergreen+).
1710 */
evergreen_hpd_set_polarity(struct radeon_device * rdev,enum radeon_hpd_id hpd)1711 void evergreen_hpd_set_polarity(struct radeon_device *rdev,
1712 enum radeon_hpd_id hpd)
1713 {
1714 u32 tmp;
1715 bool connected = evergreen_hpd_sense(rdev, hpd);
1716
1717 switch (hpd) {
1718 case RADEON_HPD_1:
1719 tmp = RREG32(DC_HPD1_INT_CONTROL);
1720 if (connected)
1721 tmp &= ~DC_HPDx_INT_POLARITY;
1722 else
1723 tmp |= DC_HPDx_INT_POLARITY;
1724 WREG32(DC_HPD1_INT_CONTROL, tmp);
1725 break;
1726 case RADEON_HPD_2:
1727 tmp = RREG32(DC_HPD2_INT_CONTROL);
1728 if (connected)
1729 tmp &= ~DC_HPDx_INT_POLARITY;
1730 else
1731 tmp |= DC_HPDx_INT_POLARITY;
1732 WREG32(DC_HPD2_INT_CONTROL, tmp);
1733 break;
1734 case RADEON_HPD_3:
1735 tmp = RREG32(DC_HPD3_INT_CONTROL);
1736 if (connected)
1737 tmp &= ~DC_HPDx_INT_POLARITY;
1738 else
1739 tmp |= DC_HPDx_INT_POLARITY;
1740 WREG32(DC_HPD3_INT_CONTROL, tmp);
1741 break;
1742 case RADEON_HPD_4:
1743 tmp = RREG32(DC_HPD4_INT_CONTROL);
1744 if (connected)
1745 tmp &= ~DC_HPDx_INT_POLARITY;
1746 else
1747 tmp |= DC_HPDx_INT_POLARITY;
1748 WREG32(DC_HPD4_INT_CONTROL, tmp);
1749 break;
1750 case RADEON_HPD_5:
1751 tmp = RREG32(DC_HPD5_INT_CONTROL);
1752 if (connected)
1753 tmp &= ~DC_HPDx_INT_POLARITY;
1754 else
1755 tmp |= DC_HPDx_INT_POLARITY;
1756 WREG32(DC_HPD5_INT_CONTROL, tmp);
1757 break;
1758 case RADEON_HPD_6:
1759 tmp = RREG32(DC_HPD6_INT_CONTROL);
1760 if (connected)
1761 tmp &= ~DC_HPDx_INT_POLARITY;
1762 else
1763 tmp |= DC_HPDx_INT_POLARITY;
1764 WREG32(DC_HPD6_INT_CONTROL, tmp);
1765 break;
1766 default:
1767 break;
1768 }
1769 }
1770
1771 /**
1772 * evergreen_hpd_init - hpd setup callback.
1773 *
1774 * @rdev: radeon_device pointer
1775 *
1776 * Setup the hpd pins used by the card (evergreen+).
1777 * Enable the pin, set the polarity, and enable the hpd interrupts.
1778 */
evergreen_hpd_init(struct radeon_device * rdev)1779 void evergreen_hpd_init(struct radeon_device *rdev)
1780 {
1781 struct drm_device *dev = rdev->ddev;
1782 struct drm_connector *connector;
1783 unsigned enabled = 0;
1784 u32 tmp = DC_HPDx_CONNECTION_TIMER(0x9c4) |
1785 DC_HPDx_RX_INT_TIMER(0xfa) | DC_HPDx_EN;
1786
1787 list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
1788 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
1789
1790 if (connector->connector_type == DRM_MODE_CONNECTOR_eDP ||
1791 connector->connector_type == DRM_MODE_CONNECTOR_LVDS) {
1792 /* don't try to enable hpd on eDP or LVDS avoid breaking the
1793 * aux dp channel on imac and help (but not completely fix)
1794 * https://bugzilla.redhat.com/show_bug.cgi?id=726143
1795 * also avoid interrupt storms during dpms.
1796 */
1797 continue;
1798 }
1799 switch (radeon_connector->hpd.hpd) {
1800 case RADEON_HPD_1:
1801 WREG32(DC_HPD1_CONTROL, tmp);
1802 break;
1803 case RADEON_HPD_2:
1804 WREG32(DC_HPD2_CONTROL, tmp);
1805 break;
1806 case RADEON_HPD_3:
1807 WREG32(DC_HPD3_CONTROL, tmp);
1808 break;
1809 case RADEON_HPD_4:
1810 WREG32(DC_HPD4_CONTROL, tmp);
1811 break;
1812 case RADEON_HPD_5:
1813 WREG32(DC_HPD5_CONTROL, tmp);
1814 break;
1815 case RADEON_HPD_6:
1816 WREG32(DC_HPD6_CONTROL, tmp);
1817 break;
1818 default:
1819 break;
1820 }
1821 radeon_hpd_set_polarity(rdev, radeon_connector->hpd.hpd);
1822 enabled |= 1 << radeon_connector->hpd.hpd;
1823 }
1824 radeon_irq_kms_enable_hpd(rdev, enabled);
1825 }
1826
1827 /**
1828 * evergreen_hpd_fini - hpd tear down callback.
1829 *
1830 * @rdev: radeon_device pointer
1831 *
1832 * Tear down the hpd pins used by the card (evergreen+).
1833 * Disable the hpd interrupts.
1834 */
evergreen_hpd_fini(struct radeon_device * rdev)1835 void evergreen_hpd_fini(struct radeon_device *rdev)
1836 {
1837 struct drm_device *dev = rdev->ddev;
1838 struct drm_connector *connector;
1839 unsigned disabled = 0;
1840
1841 list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
1842 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
1843 switch (radeon_connector->hpd.hpd) {
1844 case RADEON_HPD_1:
1845 WREG32(DC_HPD1_CONTROL, 0);
1846 break;
1847 case RADEON_HPD_2:
1848 WREG32(DC_HPD2_CONTROL, 0);
1849 break;
1850 case RADEON_HPD_3:
1851 WREG32(DC_HPD3_CONTROL, 0);
1852 break;
1853 case RADEON_HPD_4:
1854 WREG32(DC_HPD4_CONTROL, 0);
1855 break;
1856 case RADEON_HPD_5:
1857 WREG32(DC_HPD5_CONTROL, 0);
1858 break;
1859 case RADEON_HPD_6:
1860 WREG32(DC_HPD6_CONTROL, 0);
1861 break;
1862 default:
1863 break;
1864 }
1865 disabled |= 1 << radeon_connector->hpd.hpd;
1866 }
1867 radeon_irq_kms_disable_hpd(rdev, disabled);
1868 }
1869
1870 /* watermark setup */
1871
evergreen_line_buffer_adjust(struct radeon_device * rdev,struct radeon_crtc * radeon_crtc,struct drm_display_mode * mode,struct drm_display_mode * other_mode)1872 static u32 evergreen_line_buffer_adjust(struct radeon_device *rdev,
1873 struct radeon_crtc *radeon_crtc,
1874 struct drm_display_mode *mode,
1875 struct drm_display_mode *other_mode)
1876 {
1877 u32 tmp, buffer_alloc, i;
1878 u32 pipe_offset = radeon_crtc->crtc_id * 0x20;
1879 /*
1880 * Line Buffer Setup
1881 * There are 3 line buffers, each one shared by 2 display controllers.
1882 * DC_LB_MEMORY_SPLIT controls how that line buffer is shared between
1883 * the display controllers. The paritioning is done via one of four
1884 * preset allocations specified in bits 2:0:
1885 * first display controller
1886 * 0 - first half of lb (3840 * 2)
1887 * 1 - first 3/4 of lb (5760 * 2)
1888 * 2 - whole lb (7680 * 2), other crtc must be disabled
1889 * 3 - first 1/4 of lb (1920 * 2)
1890 * second display controller
1891 * 4 - second half of lb (3840 * 2)
1892 * 5 - second 3/4 of lb (5760 * 2)
1893 * 6 - whole lb (7680 * 2), other crtc must be disabled
1894 * 7 - last 1/4 of lb (1920 * 2)
1895 */
1896 /* this can get tricky if we have two large displays on a paired group
1897 * of crtcs. Ideally for multiple large displays we'd assign them to
1898 * non-linked crtcs for maximum line buffer allocation.
1899 */
1900 if (radeon_crtc->base.enabled && mode) {
1901 if (other_mode) {
1902 tmp = 0; /* 1/2 */
1903 buffer_alloc = 1;
1904 } else {
1905 tmp = 2; /* whole */
1906 buffer_alloc = 2;
1907 }
1908 } else {
1909 tmp = 0;
1910 buffer_alloc = 0;
1911 }
1912
1913 /* second controller of the pair uses second half of the lb */
1914 if (radeon_crtc->crtc_id % 2)
1915 tmp += 4;
1916 WREG32(DC_LB_MEMORY_SPLIT + radeon_crtc->crtc_offset, tmp);
1917
1918 if (ASIC_IS_DCE41(rdev) || ASIC_IS_DCE5(rdev)) {
1919 WREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset,
1920 DMIF_BUFFERS_ALLOCATED(buffer_alloc));
1921 for (i = 0; i < rdev->usec_timeout; i++) {
1922 if (RREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset) &
1923 DMIF_BUFFERS_ALLOCATED_COMPLETED)
1924 break;
1925 udelay(1);
1926 }
1927 }
1928
1929 if (radeon_crtc->base.enabled && mode) {
1930 switch (tmp) {
1931 case 0:
1932 case 4:
1933 default:
1934 if (ASIC_IS_DCE5(rdev))
1935 return 4096 * 2;
1936 else
1937 return 3840 * 2;
1938 case 1:
1939 case 5:
1940 if (ASIC_IS_DCE5(rdev))
1941 return 6144 * 2;
1942 else
1943 return 5760 * 2;
1944 case 2:
1945 case 6:
1946 if (ASIC_IS_DCE5(rdev))
1947 return 8192 * 2;
1948 else
1949 return 7680 * 2;
1950 case 3:
1951 case 7:
1952 if (ASIC_IS_DCE5(rdev))
1953 return 2048 * 2;
1954 else
1955 return 1920 * 2;
1956 }
1957 }
1958
1959 /* controller not enabled, so no lb used */
1960 return 0;
1961 }
1962
evergreen_get_number_of_dram_channels(struct radeon_device * rdev)1963 u32 evergreen_get_number_of_dram_channels(struct radeon_device *rdev)
1964 {
1965 u32 tmp = RREG32(MC_SHARED_CHMAP);
1966
1967 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
1968 case 0:
1969 default:
1970 return 1;
1971 case 1:
1972 return 2;
1973 case 2:
1974 return 4;
1975 case 3:
1976 return 8;
1977 }
1978 }
1979
1980 struct evergreen_wm_params {
1981 u32 dram_channels; /* number of dram channels */
1982 u32 yclk; /* bandwidth per dram data pin in kHz */
1983 u32 sclk; /* engine clock in kHz */
1984 u32 disp_clk; /* display clock in kHz */
1985 u32 src_width; /* viewport width */
1986 u32 active_time; /* active display time in ns */
1987 u32 blank_time; /* blank time in ns */
1988 bool interlaced; /* mode is interlaced */
1989 fixed20_12 vsc; /* vertical scale ratio */
1990 u32 num_heads; /* number of active crtcs */
1991 u32 bytes_per_pixel; /* bytes per pixel display + overlay */
1992 u32 lb_size; /* line buffer allocated to pipe */
1993 u32 vtaps; /* vertical scaler taps */
1994 };
1995
evergreen_dram_bandwidth(struct evergreen_wm_params * wm)1996 static u32 evergreen_dram_bandwidth(struct evergreen_wm_params *wm)
1997 {
1998 /* Calculate DRAM Bandwidth and the part allocated to display. */
1999 fixed20_12 dram_efficiency; /* 0.7 */
2000 fixed20_12 yclk, dram_channels, bandwidth;
2001 fixed20_12 a;
2002
2003 a.full = dfixed_const(1000);
2004 yclk.full = dfixed_const(wm->yclk);
2005 yclk.full = dfixed_div(yclk, a);
2006 dram_channels.full = dfixed_const(wm->dram_channels * 4);
2007 a.full = dfixed_const(10);
2008 dram_efficiency.full = dfixed_const(7);
2009 dram_efficiency.full = dfixed_div(dram_efficiency, a);
2010 bandwidth.full = dfixed_mul(dram_channels, yclk);
2011 bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
2012
2013 return dfixed_trunc(bandwidth);
2014 }
2015
evergreen_dram_bandwidth_for_display(struct evergreen_wm_params * wm)2016 static u32 evergreen_dram_bandwidth_for_display(struct evergreen_wm_params *wm)
2017 {
2018 /* Calculate DRAM Bandwidth and the part allocated to display. */
2019 fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
2020 fixed20_12 yclk, dram_channels, bandwidth;
2021 fixed20_12 a;
2022
2023 a.full = dfixed_const(1000);
2024 yclk.full = dfixed_const(wm->yclk);
2025 yclk.full = dfixed_div(yclk, a);
2026 dram_channels.full = dfixed_const(wm->dram_channels * 4);
2027 a.full = dfixed_const(10);
2028 disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
2029 disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
2030 bandwidth.full = dfixed_mul(dram_channels, yclk);
2031 bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
2032
2033 return dfixed_trunc(bandwidth);
2034 }
2035
evergreen_data_return_bandwidth(struct evergreen_wm_params * wm)2036 static u32 evergreen_data_return_bandwidth(struct evergreen_wm_params *wm)
2037 {
2038 /* Calculate the display Data return Bandwidth */
2039 fixed20_12 return_efficiency; /* 0.8 */
2040 fixed20_12 sclk, bandwidth;
2041 fixed20_12 a;
2042
2043 a.full = dfixed_const(1000);
2044 sclk.full = dfixed_const(wm->sclk);
2045 sclk.full = dfixed_div(sclk, a);
2046 a.full = dfixed_const(10);
2047 return_efficiency.full = dfixed_const(8);
2048 return_efficiency.full = dfixed_div(return_efficiency, a);
2049 a.full = dfixed_const(32);
2050 bandwidth.full = dfixed_mul(a, sclk);
2051 bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
2052
2053 return dfixed_trunc(bandwidth);
2054 }
2055
evergreen_dmif_request_bandwidth(struct evergreen_wm_params * wm)2056 static u32 evergreen_dmif_request_bandwidth(struct evergreen_wm_params *wm)
2057 {
2058 /* Calculate the DMIF Request Bandwidth */
2059 fixed20_12 disp_clk_request_efficiency; /* 0.8 */
2060 fixed20_12 disp_clk, bandwidth;
2061 fixed20_12 a;
2062
2063 a.full = dfixed_const(1000);
2064 disp_clk.full = dfixed_const(wm->disp_clk);
2065 disp_clk.full = dfixed_div(disp_clk, a);
2066 a.full = dfixed_const(10);
2067 disp_clk_request_efficiency.full = dfixed_const(8);
2068 disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
2069 a.full = dfixed_const(32);
2070 bandwidth.full = dfixed_mul(a, disp_clk);
2071 bandwidth.full = dfixed_mul(bandwidth, disp_clk_request_efficiency);
2072
2073 return dfixed_trunc(bandwidth);
2074 }
2075
evergreen_available_bandwidth(struct evergreen_wm_params * wm)2076 static u32 evergreen_available_bandwidth(struct evergreen_wm_params *wm)
2077 {
2078 /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
2079 u32 dram_bandwidth = evergreen_dram_bandwidth(wm);
2080 u32 data_return_bandwidth = evergreen_data_return_bandwidth(wm);
2081 u32 dmif_req_bandwidth = evergreen_dmif_request_bandwidth(wm);
2082
2083 return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
2084 }
2085
evergreen_average_bandwidth(struct evergreen_wm_params * wm)2086 static u32 evergreen_average_bandwidth(struct evergreen_wm_params *wm)
2087 {
2088 /* Calculate the display mode Average Bandwidth
2089 * DisplayMode should contain the source and destination dimensions,
2090 * timing, etc.
2091 */
2092 fixed20_12 bpp;
2093 fixed20_12 line_time;
2094 fixed20_12 src_width;
2095 fixed20_12 bandwidth;
2096 fixed20_12 a;
2097
2098 a.full = dfixed_const(1000);
2099 line_time.full = dfixed_const(wm->active_time + wm->blank_time);
2100 line_time.full = dfixed_div(line_time, a);
2101 bpp.full = dfixed_const(wm->bytes_per_pixel);
2102 src_width.full = dfixed_const(wm->src_width);
2103 bandwidth.full = dfixed_mul(src_width, bpp);
2104 bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
2105 bandwidth.full = dfixed_div(bandwidth, line_time);
2106
2107 return dfixed_trunc(bandwidth);
2108 }
2109
evergreen_latency_watermark(struct evergreen_wm_params * wm)2110 static u32 evergreen_latency_watermark(struct evergreen_wm_params *wm)
2111 {
2112 /* First calcualte the latency in ns */
2113 u32 mc_latency = 2000; /* 2000 ns. */
2114 u32 available_bandwidth = evergreen_available_bandwidth(wm);
2115 u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
2116 u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
2117 u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
2118 u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
2119 (wm->num_heads * cursor_line_pair_return_time);
2120 u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
2121 u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
2122 fixed20_12 a, b, c;
2123
2124 if (wm->num_heads == 0)
2125 return 0;
2126
2127 a.full = dfixed_const(2);
2128 b.full = dfixed_const(1);
2129 if ((wm->vsc.full > a.full) ||
2130 ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
2131 (wm->vtaps >= 5) ||
2132 ((wm->vsc.full >= a.full) && wm->interlaced))
2133 max_src_lines_per_dst_line = 4;
2134 else
2135 max_src_lines_per_dst_line = 2;
2136
2137 a.full = dfixed_const(available_bandwidth);
2138 b.full = dfixed_const(wm->num_heads);
2139 a.full = dfixed_div(a, b);
2140
2141 b.full = dfixed_const(1000);
2142 c.full = dfixed_const(wm->disp_clk);
2143 b.full = dfixed_div(c, b);
2144 c.full = dfixed_const(wm->bytes_per_pixel);
2145 b.full = dfixed_mul(b, c);
2146
2147 lb_fill_bw = min(dfixed_trunc(a), dfixed_trunc(b));
2148
2149 a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
2150 b.full = dfixed_const(1000);
2151 c.full = dfixed_const(lb_fill_bw);
2152 b.full = dfixed_div(c, b);
2153 a.full = dfixed_div(a, b);
2154 line_fill_time = dfixed_trunc(a);
2155
2156 if (line_fill_time < wm->active_time)
2157 return latency;
2158 else
2159 return latency + (line_fill_time - wm->active_time);
2160
2161 }
2162
evergreen_average_bandwidth_vs_dram_bandwidth_for_display(struct evergreen_wm_params * wm)2163 static bool evergreen_average_bandwidth_vs_dram_bandwidth_for_display(struct evergreen_wm_params *wm)
2164 {
2165 if (evergreen_average_bandwidth(wm) <=
2166 (evergreen_dram_bandwidth_for_display(wm) / wm->num_heads))
2167 return true;
2168 else
2169 return false;
2170 };
2171
evergreen_average_bandwidth_vs_available_bandwidth(struct evergreen_wm_params * wm)2172 static bool evergreen_average_bandwidth_vs_available_bandwidth(struct evergreen_wm_params *wm)
2173 {
2174 if (evergreen_average_bandwidth(wm) <=
2175 (evergreen_available_bandwidth(wm) / wm->num_heads))
2176 return true;
2177 else
2178 return false;
2179 };
2180
evergreen_check_latency_hiding(struct evergreen_wm_params * wm)2181 static bool evergreen_check_latency_hiding(struct evergreen_wm_params *wm)
2182 {
2183 u32 lb_partitions = wm->lb_size / wm->src_width;
2184 u32 line_time = wm->active_time + wm->blank_time;
2185 u32 latency_tolerant_lines;
2186 u32 latency_hiding;
2187 fixed20_12 a;
2188
2189 a.full = dfixed_const(1);
2190 if (wm->vsc.full > a.full)
2191 latency_tolerant_lines = 1;
2192 else {
2193 if (lb_partitions <= (wm->vtaps + 1))
2194 latency_tolerant_lines = 1;
2195 else
2196 latency_tolerant_lines = 2;
2197 }
2198
2199 latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
2200
2201 if (evergreen_latency_watermark(wm) <= latency_hiding)
2202 return true;
2203 else
2204 return false;
2205 }
2206
evergreen_program_watermarks(struct radeon_device * rdev,struct radeon_crtc * radeon_crtc,u32 lb_size,u32 num_heads)2207 static void evergreen_program_watermarks(struct radeon_device *rdev,
2208 struct radeon_crtc *radeon_crtc,
2209 u32 lb_size, u32 num_heads)
2210 {
2211 struct drm_display_mode *mode = &radeon_crtc->base.mode;
2212 struct evergreen_wm_params wm_low, wm_high;
2213 u32 dram_channels;
2214 u32 pixel_period;
2215 u32 line_time = 0;
2216 u32 latency_watermark_a = 0, latency_watermark_b = 0;
2217 u32 priority_a_mark = 0, priority_b_mark = 0;
2218 u32 priority_a_cnt = PRIORITY_OFF;
2219 u32 priority_b_cnt = PRIORITY_OFF;
2220 u32 pipe_offset = radeon_crtc->crtc_id * 16;
2221 u32 tmp, arb_control3;
2222 fixed20_12 a, b, c;
2223
2224 if (radeon_crtc->base.enabled && num_heads && mode) {
2225 pixel_period = 1000000 / (u32)mode->clock;
2226 line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535);
2227 priority_a_cnt = 0;
2228 priority_b_cnt = 0;
2229 dram_channels = evergreen_get_number_of_dram_channels(rdev);
2230
2231 /* watermark for high clocks */
2232 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2233 wm_high.yclk =
2234 radeon_dpm_get_mclk(rdev, false) * 10;
2235 wm_high.sclk =
2236 radeon_dpm_get_sclk(rdev, false) * 10;
2237 } else {
2238 wm_high.yclk = rdev->pm.current_mclk * 10;
2239 wm_high.sclk = rdev->pm.current_sclk * 10;
2240 }
2241
2242 wm_high.disp_clk = mode->clock;
2243 wm_high.src_width = mode->crtc_hdisplay;
2244 wm_high.active_time = mode->crtc_hdisplay * pixel_period;
2245 wm_high.blank_time = line_time - wm_high.active_time;
2246 wm_high.interlaced = false;
2247 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2248 wm_high.interlaced = true;
2249 wm_high.vsc = radeon_crtc->vsc;
2250 wm_high.vtaps = 1;
2251 if (radeon_crtc->rmx_type != RMX_OFF)
2252 wm_high.vtaps = 2;
2253 wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */
2254 wm_high.lb_size = lb_size;
2255 wm_high.dram_channels = dram_channels;
2256 wm_high.num_heads = num_heads;
2257
2258 /* watermark for low clocks */
2259 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2260 wm_low.yclk =
2261 radeon_dpm_get_mclk(rdev, true) * 10;
2262 wm_low.sclk =
2263 radeon_dpm_get_sclk(rdev, true) * 10;
2264 } else {
2265 wm_low.yclk = rdev->pm.current_mclk * 10;
2266 wm_low.sclk = rdev->pm.current_sclk * 10;
2267 }
2268
2269 wm_low.disp_clk = mode->clock;
2270 wm_low.src_width = mode->crtc_hdisplay;
2271 wm_low.active_time = mode->crtc_hdisplay * pixel_period;
2272 wm_low.blank_time = line_time - wm_low.active_time;
2273 wm_low.interlaced = false;
2274 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2275 wm_low.interlaced = true;
2276 wm_low.vsc = radeon_crtc->vsc;
2277 wm_low.vtaps = 1;
2278 if (radeon_crtc->rmx_type != RMX_OFF)
2279 wm_low.vtaps = 2;
2280 wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */
2281 wm_low.lb_size = lb_size;
2282 wm_low.dram_channels = dram_channels;
2283 wm_low.num_heads = num_heads;
2284
2285 /* set for high clocks */
2286 latency_watermark_a = min(evergreen_latency_watermark(&wm_high), (u32)65535);
2287 /* set for low clocks */
2288 latency_watermark_b = min(evergreen_latency_watermark(&wm_low), (u32)65535);
2289
2290 /* possibly force display priority to high */
2291 /* should really do this at mode validation time... */
2292 if (!evergreen_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) ||
2293 !evergreen_average_bandwidth_vs_available_bandwidth(&wm_high) ||
2294 !evergreen_check_latency_hiding(&wm_high) ||
2295 (rdev->disp_priority == 2)) {
2296 DRM_DEBUG_KMS("force priority a to high\n");
2297 priority_a_cnt |= PRIORITY_ALWAYS_ON;
2298 }
2299 if (!evergreen_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) ||
2300 !evergreen_average_bandwidth_vs_available_bandwidth(&wm_low) ||
2301 !evergreen_check_latency_hiding(&wm_low) ||
2302 (rdev->disp_priority == 2)) {
2303 DRM_DEBUG_KMS("force priority b to high\n");
2304 priority_b_cnt |= PRIORITY_ALWAYS_ON;
2305 }
2306
2307 a.full = dfixed_const(1000);
2308 b.full = dfixed_const(mode->clock);
2309 b.full = dfixed_div(b, a);
2310 c.full = dfixed_const(latency_watermark_a);
2311 c.full = dfixed_mul(c, b);
2312 c.full = dfixed_mul(c, radeon_crtc->hsc);
2313 c.full = dfixed_div(c, a);
2314 a.full = dfixed_const(16);
2315 c.full = dfixed_div(c, a);
2316 priority_a_mark = dfixed_trunc(c);
2317 priority_a_cnt |= priority_a_mark & PRIORITY_MARK_MASK;
2318
2319 a.full = dfixed_const(1000);
2320 b.full = dfixed_const(mode->clock);
2321 b.full = dfixed_div(b, a);
2322 c.full = dfixed_const(latency_watermark_b);
2323 c.full = dfixed_mul(c, b);
2324 c.full = dfixed_mul(c, radeon_crtc->hsc);
2325 c.full = dfixed_div(c, a);
2326 a.full = dfixed_const(16);
2327 c.full = dfixed_div(c, a);
2328 priority_b_mark = dfixed_trunc(c);
2329 priority_b_cnt |= priority_b_mark & PRIORITY_MARK_MASK;
2330 }
2331
2332 /* select wm A */
2333 arb_control3 = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset);
2334 tmp = arb_control3;
2335 tmp &= ~LATENCY_WATERMARK_MASK(3);
2336 tmp |= LATENCY_WATERMARK_MASK(1);
2337 WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp);
2338 WREG32(PIPE0_LATENCY_CONTROL + pipe_offset,
2339 (LATENCY_LOW_WATERMARK(latency_watermark_a) |
2340 LATENCY_HIGH_WATERMARK(line_time)));
2341 /* select wm B */
2342 tmp = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset);
2343 tmp &= ~LATENCY_WATERMARK_MASK(3);
2344 tmp |= LATENCY_WATERMARK_MASK(2);
2345 WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp);
2346 WREG32(PIPE0_LATENCY_CONTROL + pipe_offset,
2347 (LATENCY_LOW_WATERMARK(latency_watermark_b) |
2348 LATENCY_HIGH_WATERMARK(line_time)));
2349 /* restore original selection */
2350 WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, arb_control3);
2351
2352 /* write the priority marks */
2353 WREG32(PRIORITY_A_CNT + radeon_crtc->crtc_offset, priority_a_cnt);
2354 WREG32(PRIORITY_B_CNT + radeon_crtc->crtc_offset, priority_b_cnt);
2355
2356 /* save values for DPM */
2357 radeon_crtc->line_time = line_time;
2358 radeon_crtc->wm_high = latency_watermark_a;
2359 radeon_crtc->wm_low = latency_watermark_b;
2360 }
2361
2362 /**
2363 * evergreen_bandwidth_update - update display watermarks callback.
2364 *
2365 * @rdev: radeon_device pointer
2366 *
2367 * Update the display watermarks based on the requested mode(s)
2368 * (evergreen+).
2369 */
evergreen_bandwidth_update(struct radeon_device * rdev)2370 void evergreen_bandwidth_update(struct radeon_device *rdev)
2371 {
2372 struct drm_display_mode *mode0 = NULL;
2373 struct drm_display_mode *mode1 = NULL;
2374 u32 num_heads = 0, lb_size;
2375 int i;
2376
2377 if (!rdev->mode_info.mode_config_initialized)
2378 return;
2379
2380 radeon_update_display_priority(rdev);
2381
2382 for (i = 0; i < rdev->num_crtc; i++) {
2383 if (rdev->mode_info.crtcs[i]->base.enabled)
2384 num_heads++;
2385 }
2386 for (i = 0; i < rdev->num_crtc; i += 2) {
2387 mode0 = &rdev->mode_info.crtcs[i]->base.mode;
2388 mode1 = &rdev->mode_info.crtcs[i+1]->base.mode;
2389 lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode0, mode1);
2390 evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
2391 lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i+1], mode1, mode0);
2392 evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i+1], lb_size, num_heads);
2393 }
2394 }
2395
2396 /**
2397 * evergreen_mc_wait_for_idle - wait for MC idle callback.
2398 *
2399 * @rdev: radeon_device pointer
2400 *
2401 * Wait for the MC (memory controller) to be idle.
2402 * (evergreen+).
2403 * Returns 0 if the MC is idle, -1 if not.
2404 */
evergreen_mc_wait_for_idle(struct radeon_device * rdev)2405 int evergreen_mc_wait_for_idle(struct radeon_device *rdev)
2406 {
2407 unsigned i;
2408 u32 tmp;
2409
2410 for (i = 0; i < rdev->usec_timeout; i++) {
2411 /* read MC_STATUS */
2412 tmp = RREG32(SRBM_STATUS) & 0x1F00;
2413 if (!tmp)
2414 return 0;
2415 udelay(1);
2416 }
2417 return -1;
2418 }
2419
2420 /*
2421 * GART
2422 */
evergreen_pcie_gart_tlb_flush(struct radeon_device * rdev)2423 void evergreen_pcie_gart_tlb_flush(struct radeon_device *rdev)
2424 {
2425 unsigned i;
2426 u32 tmp;
2427
2428 WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
2429
2430 WREG32(VM_CONTEXT0_REQUEST_RESPONSE, REQUEST_TYPE(1));
2431 for (i = 0; i < rdev->usec_timeout; i++) {
2432 /* read MC_STATUS */
2433 tmp = RREG32(VM_CONTEXT0_REQUEST_RESPONSE);
2434 tmp = (tmp & RESPONSE_TYPE_MASK) >> RESPONSE_TYPE_SHIFT;
2435 if (tmp == 2) {
2436 printk(KERN_WARNING "[drm] r600 flush TLB failed\n");
2437 return;
2438 }
2439 if (tmp) {
2440 return;
2441 }
2442 udelay(1);
2443 }
2444 }
2445
evergreen_pcie_gart_enable(struct radeon_device * rdev)2446 static int evergreen_pcie_gart_enable(struct radeon_device *rdev)
2447 {
2448 u32 tmp;
2449 int r;
2450
2451 if (rdev->gart.robj == NULL) {
2452 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
2453 return -EINVAL;
2454 }
2455 r = radeon_gart_table_vram_pin(rdev);
2456 if (r)
2457 return r;
2458 /* Setup L2 cache */
2459 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
2460 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
2461 EFFECTIVE_L2_QUEUE_SIZE(7));
2462 WREG32(VM_L2_CNTL2, 0);
2463 WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2464 /* Setup TLB control */
2465 tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
2466 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
2467 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
2468 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
2469 if (rdev->flags & RADEON_IS_IGP) {
2470 WREG32(FUS_MC_VM_MD_L1_TLB0_CNTL, tmp);
2471 WREG32(FUS_MC_VM_MD_L1_TLB1_CNTL, tmp);
2472 WREG32(FUS_MC_VM_MD_L1_TLB2_CNTL, tmp);
2473 } else {
2474 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2475 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2476 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2477 if ((rdev->family == CHIP_JUNIPER) ||
2478 (rdev->family == CHIP_CYPRESS) ||
2479 (rdev->family == CHIP_HEMLOCK) ||
2480 (rdev->family == CHIP_BARTS))
2481 WREG32(MC_VM_MD_L1_TLB3_CNTL, tmp);
2482 }
2483 WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2484 WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2485 WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2486 WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
2487 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
2488 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
2489 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
2490 WREG32(VM_CONTEXT0_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
2491 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT);
2492 WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
2493 (u32)(rdev->dummy_page.addr >> 12));
2494 WREG32(VM_CONTEXT1_CNTL, 0);
2495
2496 evergreen_pcie_gart_tlb_flush(rdev);
2497 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
2498 (unsigned)(rdev->mc.gtt_size >> 20),
2499 (unsigned long long)rdev->gart.table_addr);
2500 rdev->gart.ready = true;
2501 return 0;
2502 }
2503
evergreen_pcie_gart_disable(struct radeon_device * rdev)2504 static void evergreen_pcie_gart_disable(struct radeon_device *rdev)
2505 {
2506 u32 tmp;
2507
2508 /* Disable all tables */
2509 WREG32(VM_CONTEXT0_CNTL, 0);
2510 WREG32(VM_CONTEXT1_CNTL, 0);
2511
2512 /* Setup L2 cache */
2513 WREG32(VM_L2_CNTL, ENABLE_L2_FRAGMENT_PROCESSING |
2514 EFFECTIVE_L2_QUEUE_SIZE(7));
2515 WREG32(VM_L2_CNTL2, 0);
2516 WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2517 /* Setup TLB control */
2518 tmp = EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
2519 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2520 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2521 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2522 WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2523 WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2524 WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2525 WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
2526 radeon_gart_table_vram_unpin(rdev);
2527 }
2528
evergreen_pcie_gart_fini(struct radeon_device * rdev)2529 static void evergreen_pcie_gart_fini(struct radeon_device *rdev)
2530 {
2531 evergreen_pcie_gart_disable(rdev);
2532 radeon_gart_table_vram_free(rdev);
2533 radeon_gart_fini(rdev);
2534 }
2535
2536
evergreen_agp_enable(struct radeon_device * rdev)2537 static void evergreen_agp_enable(struct radeon_device *rdev)
2538 {
2539 u32 tmp;
2540
2541 /* Setup L2 cache */
2542 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
2543 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
2544 EFFECTIVE_L2_QUEUE_SIZE(7));
2545 WREG32(VM_L2_CNTL2, 0);
2546 WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2547 /* Setup TLB control */
2548 tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
2549 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
2550 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
2551 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
2552 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2553 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2554 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2555 WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2556 WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2557 WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2558 WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
2559 WREG32(VM_CONTEXT0_CNTL, 0);
2560 WREG32(VM_CONTEXT1_CNTL, 0);
2561 }
2562
2563 static const unsigned ni_dig_offsets[] =
2564 {
2565 NI_DIG0_REGISTER_OFFSET,
2566 NI_DIG1_REGISTER_OFFSET,
2567 NI_DIG2_REGISTER_OFFSET,
2568 NI_DIG3_REGISTER_OFFSET,
2569 NI_DIG4_REGISTER_OFFSET,
2570 NI_DIG5_REGISTER_OFFSET
2571 };
2572
2573 static const unsigned ni_tx_offsets[] =
2574 {
2575 NI_DCIO_UNIPHY0_UNIPHY_TX_CONTROL1,
2576 NI_DCIO_UNIPHY1_UNIPHY_TX_CONTROL1,
2577 NI_DCIO_UNIPHY2_UNIPHY_TX_CONTROL1,
2578 NI_DCIO_UNIPHY3_UNIPHY_TX_CONTROL1,
2579 NI_DCIO_UNIPHY4_UNIPHY_TX_CONTROL1,
2580 NI_DCIO_UNIPHY5_UNIPHY_TX_CONTROL1
2581 };
2582
2583 static const unsigned evergreen_dp_offsets[] =
2584 {
2585 EVERGREEN_DP0_REGISTER_OFFSET,
2586 EVERGREEN_DP1_REGISTER_OFFSET,
2587 EVERGREEN_DP2_REGISTER_OFFSET,
2588 EVERGREEN_DP3_REGISTER_OFFSET,
2589 EVERGREEN_DP4_REGISTER_OFFSET,
2590 EVERGREEN_DP5_REGISTER_OFFSET
2591 };
2592
2593
2594 /*
2595 * Assumption is that EVERGREEN_CRTC_MASTER_EN enable for requested crtc
2596 * We go from crtc to connector and it is not relible since it
2597 * should be an opposite direction .If crtc is enable then
2598 * find the dig_fe which selects this crtc and insure that it enable.
2599 * if such dig_fe is found then find dig_be which selects found dig_be and
2600 * insure that it enable and in DP_SST mode.
2601 * if UNIPHY_PLL_CONTROL1.enable then we should disconnect timing
2602 * from dp symbols clocks .
2603 */
evergreen_is_dp_sst_stream_enabled(struct radeon_device * rdev,unsigned crtc_id,unsigned * ret_dig_fe)2604 static bool evergreen_is_dp_sst_stream_enabled(struct radeon_device *rdev,
2605 unsigned crtc_id, unsigned *ret_dig_fe)
2606 {
2607 unsigned i;
2608 unsigned dig_fe;
2609 unsigned dig_be;
2610 unsigned dig_en_be;
2611 unsigned uniphy_pll;
2612 unsigned digs_fe_selected;
2613 unsigned dig_be_mode;
2614 unsigned dig_fe_mask;
2615 bool is_enabled = false;
2616 bool found_crtc = false;
2617
2618 /* loop through all running dig_fe to find selected crtc */
2619 for (i = 0; i < ARRAY_SIZE(ni_dig_offsets); i++) {
2620 dig_fe = RREG32(NI_DIG_FE_CNTL + ni_dig_offsets[i]);
2621 if (dig_fe & NI_DIG_FE_CNTL_SYMCLK_FE_ON &&
2622 crtc_id == NI_DIG_FE_CNTL_SOURCE_SELECT(dig_fe)) {
2623 /* found running pipe */
2624 found_crtc = true;
2625 dig_fe_mask = 1 << i;
2626 dig_fe = i;
2627 break;
2628 }
2629 }
2630
2631 if (found_crtc) {
2632 /* loop through all running dig_be to find selected dig_fe */
2633 for (i = 0; i < ARRAY_SIZE(ni_dig_offsets); i++) {
2634 dig_be = RREG32(NI_DIG_BE_CNTL + ni_dig_offsets[i]);
2635 /* if dig_fe_selected by dig_be? */
2636 digs_fe_selected = NI_DIG_BE_CNTL_FE_SOURCE_SELECT(dig_be);
2637 dig_be_mode = NI_DIG_FE_CNTL_MODE(dig_be);
2638 if (dig_fe_mask & digs_fe_selected &&
2639 /* if dig_be in sst mode? */
2640 dig_be_mode == NI_DIG_BE_DPSST) {
2641 dig_en_be = RREG32(NI_DIG_BE_EN_CNTL +
2642 ni_dig_offsets[i]);
2643 uniphy_pll = RREG32(NI_DCIO_UNIPHY0_PLL_CONTROL1 +
2644 ni_tx_offsets[i]);
2645 /* dig_be enable and tx is running */
2646 if (dig_en_be & NI_DIG_BE_EN_CNTL_ENABLE &&
2647 dig_en_be & NI_DIG_BE_EN_CNTL_SYMBCLK_ON &&
2648 uniphy_pll & NI_DCIO_UNIPHY0_PLL_CONTROL1_ENABLE) {
2649 is_enabled = true;
2650 *ret_dig_fe = dig_fe;
2651 break;
2652 }
2653 }
2654 }
2655 }
2656
2657 return is_enabled;
2658 }
2659
2660 /*
2661 * Blank dig when in dp sst mode
2662 * Dig ignores crtc timing
2663 */
evergreen_blank_dp_output(struct radeon_device * rdev,unsigned dig_fe)2664 static void evergreen_blank_dp_output(struct radeon_device *rdev,
2665 unsigned dig_fe)
2666 {
2667 unsigned stream_ctrl;
2668 unsigned fifo_ctrl;
2669 unsigned counter = 0;
2670
2671 if (dig_fe >= ARRAY_SIZE(evergreen_dp_offsets)) {
2672 DRM_ERROR("invalid dig_fe %d\n", dig_fe);
2673 return;
2674 }
2675
2676 stream_ctrl = RREG32(EVERGREEN_DP_VID_STREAM_CNTL +
2677 evergreen_dp_offsets[dig_fe]);
2678 if (!(stream_ctrl & EVERGREEN_DP_VID_STREAM_CNTL_ENABLE)) {
2679 DRM_ERROR("dig %d , should be enable\n", dig_fe);
2680 return;
2681 }
2682
2683 stream_ctrl &=~EVERGREEN_DP_VID_STREAM_CNTL_ENABLE;
2684 WREG32(EVERGREEN_DP_VID_STREAM_CNTL +
2685 evergreen_dp_offsets[dig_fe], stream_ctrl);
2686
2687 stream_ctrl = RREG32(EVERGREEN_DP_VID_STREAM_CNTL +
2688 evergreen_dp_offsets[dig_fe]);
2689 while (counter < 32 && stream_ctrl & EVERGREEN_DP_VID_STREAM_STATUS) {
2690 msleep(1);
2691 counter++;
2692 stream_ctrl = RREG32(EVERGREEN_DP_VID_STREAM_CNTL +
2693 evergreen_dp_offsets[dig_fe]);
2694 }
2695 if (counter >= 32 )
2696 DRM_ERROR("counter exceeds %d\n", counter);
2697
2698 fifo_ctrl = RREG32(EVERGREEN_DP_STEER_FIFO + evergreen_dp_offsets[dig_fe]);
2699 fifo_ctrl |= EVERGREEN_DP_STEER_FIFO_RESET;
2700 WREG32(EVERGREEN_DP_STEER_FIFO + evergreen_dp_offsets[dig_fe], fifo_ctrl);
2701
2702 }
2703
evergreen_mc_stop(struct radeon_device * rdev,struct evergreen_mc_save * save)2704 void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save)
2705 {
2706 u32 crtc_enabled, tmp, frame_count, blackout;
2707 int i, j;
2708 unsigned dig_fe;
2709
2710 if (!ASIC_IS_NODCE(rdev)) {
2711 save->vga_render_control = RREG32(VGA_RENDER_CONTROL);
2712 save->vga_hdp_control = RREG32(VGA_HDP_CONTROL);
2713
2714 /* disable VGA render */
2715 WREG32(VGA_RENDER_CONTROL, 0);
2716 }
2717 /* blank the display controllers */
2718 for (i = 0; i < rdev->num_crtc; i++) {
2719 crtc_enabled = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]) & EVERGREEN_CRTC_MASTER_EN;
2720 if (crtc_enabled) {
2721 save->crtc_enabled[i] = true;
2722 if (ASIC_IS_DCE6(rdev)) {
2723 tmp = RREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i]);
2724 if (!(tmp & EVERGREEN_CRTC_BLANK_DATA_EN)) {
2725 radeon_wait_for_vblank(rdev, i);
2726 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2727 tmp |= EVERGREEN_CRTC_BLANK_DATA_EN;
2728 WREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i], tmp);
2729 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2730 }
2731 } else {
2732 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2733 if (!(tmp & EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE)) {
2734 radeon_wait_for_vblank(rdev, i);
2735 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2736 tmp |= EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
2737 WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
2738 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2739 }
2740 }
2741 /* wait for the next frame */
2742 frame_count = radeon_get_vblank_counter(rdev, i);
2743 for (j = 0; j < rdev->usec_timeout; j++) {
2744 if (radeon_get_vblank_counter(rdev, i) != frame_count)
2745 break;
2746 udelay(1);
2747 }
2748 /*we should disable dig if it drives dp sst*/
2749 /*but we are in radeon_device_init and the topology is unknown*/
2750 /*and it is available after radeon_modeset_init*/
2751 /*the following method radeon_atom_encoder_dpms_dig*/
2752 /*does the job if we initialize it properly*/
2753 /*for now we do it this manually*/
2754 /**/
2755 if (ASIC_IS_DCE5(rdev) &&
2756 evergreen_is_dp_sst_stream_enabled(rdev, i ,&dig_fe))
2757 evergreen_blank_dp_output(rdev, dig_fe);
2758 /*we could remove 6 lines below*/
2759 /* XXX this is a hack to avoid strange behavior with EFI on certain systems */
2760 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2761 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2762 tmp &= ~EVERGREEN_CRTC_MASTER_EN;
2763 WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
2764 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2765 save->crtc_enabled[i] = false;
2766 /* ***** */
2767 } else {
2768 save->crtc_enabled[i] = false;
2769 }
2770 }
2771
2772 radeon_mc_wait_for_idle(rdev);
2773
2774 blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
2775 if ((blackout & BLACKOUT_MODE_MASK) != 1) {
2776 /* Block CPU access */
2777 WREG32(BIF_FB_EN, 0);
2778 /* blackout the MC */
2779 blackout &= ~BLACKOUT_MODE_MASK;
2780 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1);
2781 }
2782 /* wait for the MC to settle */
2783 udelay(100);
2784
2785 /* lock double buffered regs */
2786 for (i = 0; i < rdev->num_crtc; i++) {
2787 if (save->crtc_enabled[i]) {
2788 tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2789 if (!(tmp & EVERGREEN_GRPH_UPDATE_LOCK)) {
2790 tmp |= EVERGREEN_GRPH_UPDATE_LOCK;
2791 WREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i], tmp);
2792 }
2793 tmp = RREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i]);
2794 if (!(tmp & 1)) {
2795 tmp |= 1;
2796 WREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i], tmp);
2797 }
2798 }
2799 }
2800 }
2801
evergreen_mc_resume(struct radeon_device * rdev,struct evergreen_mc_save * save)2802 void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save)
2803 {
2804 u32 tmp, frame_count;
2805 int i, j;
2806
2807 /* update crtc base addresses */
2808 for (i = 0; i < rdev->num_crtc; i++) {
2809 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i],
2810 upper_32_bits(rdev->mc.vram_start));
2811 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i],
2812 upper_32_bits(rdev->mc.vram_start));
2813 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + crtc_offsets[i],
2814 (u32)rdev->mc.vram_start);
2815 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + crtc_offsets[i],
2816 (u32)rdev->mc.vram_start);
2817 }
2818
2819 if (!ASIC_IS_NODCE(rdev)) {
2820 WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS_HIGH, upper_32_bits(rdev->mc.vram_start));
2821 WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS, (u32)rdev->mc.vram_start);
2822 }
2823
2824 /* unlock regs and wait for update */
2825 for (i = 0; i < rdev->num_crtc; i++) {
2826 if (save->crtc_enabled[i]) {
2827 tmp = RREG32(EVERGREEN_MASTER_UPDATE_MODE + crtc_offsets[i]);
2828 if ((tmp & 0x7) != 3) {
2829 tmp &= ~0x7;
2830 tmp |= 0x3;
2831 WREG32(EVERGREEN_MASTER_UPDATE_MODE + crtc_offsets[i], tmp);
2832 }
2833 tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2834 if (tmp & EVERGREEN_GRPH_UPDATE_LOCK) {
2835 tmp &= ~EVERGREEN_GRPH_UPDATE_LOCK;
2836 WREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i], tmp);
2837 }
2838 tmp = RREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i]);
2839 if (tmp & 1) {
2840 tmp &= ~1;
2841 WREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i], tmp);
2842 }
2843 for (j = 0; j < rdev->usec_timeout; j++) {
2844 tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2845 if ((tmp & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING) == 0)
2846 break;
2847 udelay(1);
2848 }
2849 }
2850 }
2851
2852 /* unblackout the MC */
2853 tmp = RREG32(MC_SHARED_BLACKOUT_CNTL);
2854 tmp &= ~BLACKOUT_MODE_MASK;
2855 WREG32(MC_SHARED_BLACKOUT_CNTL, tmp);
2856 /* allow CPU access */
2857 WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
2858
2859 for (i = 0; i < rdev->num_crtc; i++) {
2860 if (save->crtc_enabled[i]) {
2861 if (ASIC_IS_DCE6(rdev)) {
2862 tmp = RREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i]);
2863 tmp &= ~EVERGREEN_CRTC_BLANK_DATA_EN;
2864 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2865 WREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i], tmp);
2866 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2867 } else {
2868 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2869 tmp &= ~EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
2870 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2871 WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
2872 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2873 }
2874 /* wait for the next frame */
2875 frame_count = radeon_get_vblank_counter(rdev, i);
2876 for (j = 0; j < rdev->usec_timeout; j++) {
2877 if (radeon_get_vblank_counter(rdev, i) != frame_count)
2878 break;
2879 udelay(1);
2880 }
2881 }
2882 }
2883 if (!ASIC_IS_NODCE(rdev)) {
2884 /* Unlock vga access */
2885 WREG32(VGA_HDP_CONTROL, save->vga_hdp_control);
2886 mdelay(1);
2887 WREG32(VGA_RENDER_CONTROL, save->vga_render_control);
2888 }
2889 }
2890
evergreen_mc_program(struct radeon_device * rdev)2891 void evergreen_mc_program(struct radeon_device *rdev)
2892 {
2893 struct evergreen_mc_save save;
2894 u32 tmp;
2895 int i, j;
2896
2897 /* Initialize HDP */
2898 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
2899 WREG32((0x2c14 + j), 0x00000000);
2900 WREG32((0x2c18 + j), 0x00000000);
2901 WREG32((0x2c1c + j), 0x00000000);
2902 WREG32((0x2c20 + j), 0x00000000);
2903 WREG32((0x2c24 + j), 0x00000000);
2904 }
2905 WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
2906
2907 evergreen_mc_stop(rdev, &save);
2908 if (evergreen_mc_wait_for_idle(rdev)) {
2909 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
2910 }
2911 /* Lockout access through VGA aperture*/
2912 WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
2913 /* Update configuration */
2914 if (rdev->flags & RADEON_IS_AGP) {
2915 if (rdev->mc.vram_start < rdev->mc.gtt_start) {
2916 /* VRAM before AGP */
2917 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2918 rdev->mc.vram_start >> 12);
2919 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2920 rdev->mc.gtt_end >> 12);
2921 } else {
2922 /* VRAM after AGP */
2923 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2924 rdev->mc.gtt_start >> 12);
2925 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2926 rdev->mc.vram_end >> 12);
2927 }
2928 } else {
2929 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2930 rdev->mc.vram_start >> 12);
2931 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2932 rdev->mc.vram_end >> 12);
2933 }
2934 WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR, rdev->vram_scratch.gpu_addr >> 12);
2935 /* llano/ontario only */
2936 if ((rdev->family == CHIP_PALM) ||
2937 (rdev->family == CHIP_SUMO) ||
2938 (rdev->family == CHIP_SUMO2)) {
2939 tmp = RREG32(MC_FUS_VM_FB_OFFSET) & 0x000FFFFF;
2940 tmp |= ((rdev->mc.vram_end >> 20) & 0xF) << 24;
2941 tmp |= ((rdev->mc.vram_start >> 20) & 0xF) << 20;
2942 WREG32(MC_FUS_VM_FB_OFFSET, tmp);
2943 }
2944 tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
2945 tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
2946 WREG32(MC_VM_FB_LOCATION, tmp);
2947 WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
2948 WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
2949 WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
2950 if (rdev->flags & RADEON_IS_AGP) {
2951 WREG32(MC_VM_AGP_TOP, rdev->mc.gtt_end >> 16);
2952 WREG32(MC_VM_AGP_BOT, rdev->mc.gtt_start >> 16);
2953 WREG32(MC_VM_AGP_BASE, rdev->mc.agp_base >> 22);
2954 } else {
2955 WREG32(MC_VM_AGP_BASE, 0);
2956 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
2957 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
2958 }
2959 if (evergreen_mc_wait_for_idle(rdev)) {
2960 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
2961 }
2962 evergreen_mc_resume(rdev, &save);
2963 /* we need to own VRAM, so turn off the VGA renderer here
2964 * to stop it overwriting our objects */
2965 rv515_vga_render_disable(rdev);
2966 }
2967
2968 /*
2969 * CP.
2970 */
evergreen_ring_ib_execute(struct radeon_device * rdev,struct radeon_ib * ib)2971 void evergreen_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
2972 {
2973 struct radeon_ring *ring = &rdev->ring[ib->ring];
2974 u32 next_rptr;
2975
2976 /* set to DX10/11 mode */
2977 radeon_ring_write(ring, PACKET3(PACKET3_MODE_CONTROL, 0));
2978 radeon_ring_write(ring, 1);
2979
2980 if (ring->rptr_save_reg) {
2981 next_rptr = ring->wptr + 3 + 4;
2982 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
2983 radeon_ring_write(ring, ((ring->rptr_save_reg -
2984 PACKET3_SET_CONFIG_REG_START) >> 2));
2985 radeon_ring_write(ring, next_rptr);
2986 } else if (rdev->wb.enabled) {
2987 next_rptr = ring->wptr + 5 + 4;
2988 radeon_ring_write(ring, PACKET3(PACKET3_MEM_WRITE, 3));
2989 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
2990 radeon_ring_write(ring, (upper_32_bits(ring->next_rptr_gpu_addr) & 0xff) | (1 << 18));
2991 radeon_ring_write(ring, next_rptr);
2992 radeon_ring_write(ring, 0);
2993 }
2994
2995 radeon_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
2996 radeon_ring_write(ring,
2997 #ifdef __BIG_ENDIAN
2998 (2 << 0) |
2999 #endif
3000 (ib->gpu_addr & 0xFFFFFFFC));
3001 radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFF);
3002 radeon_ring_write(ring, ib->length_dw);
3003 }
3004
3005
evergreen_cp_load_microcode(struct radeon_device * rdev)3006 static int evergreen_cp_load_microcode(struct radeon_device *rdev)
3007 {
3008 const __be32 *fw_data;
3009 int i;
3010
3011 if (!rdev->me_fw || !rdev->pfp_fw)
3012 return -EINVAL;
3013
3014 r700_cp_stop(rdev);
3015 WREG32(CP_RB_CNTL,
3016 #ifdef __BIG_ENDIAN
3017 BUF_SWAP_32BIT |
3018 #endif
3019 RB_NO_UPDATE | RB_BLKSZ(15) | RB_BUFSZ(3));
3020
3021 fw_data = (const __be32 *)rdev->pfp_fw->data;
3022 WREG32(CP_PFP_UCODE_ADDR, 0);
3023 for (i = 0; i < EVERGREEN_PFP_UCODE_SIZE; i++)
3024 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
3025 WREG32(CP_PFP_UCODE_ADDR, 0);
3026
3027 fw_data = (const __be32 *)rdev->me_fw->data;
3028 WREG32(CP_ME_RAM_WADDR, 0);
3029 for (i = 0; i < EVERGREEN_PM4_UCODE_SIZE; i++)
3030 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
3031
3032 WREG32(CP_PFP_UCODE_ADDR, 0);
3033 WREG32(CP_ME_RAM_WADDR, 0);
3034 WREG32(CP_ME_RAM_RADDR, 0);
3035 return 0;
3036 }
3037
evergreen_cp_start(struct radeon_device * rdev)3038 static int evergreen_cp_start(struct radeon_device *rdev)
3039 {
3040 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3041 int r, i;
3042 uint32_t cp_me;
3043
3044 r = radeon_ring_lock(rdev, ring, 7);
3045 if (r) {
3046 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
3047 return r;
3048 }
3049 radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
3050 radeon_ring_write(ring, 0x1);
3051 radeon_ring_write(ring, 0x0);
3052 radeon_ring_write(ring, rdev->config.evergreen.max_hw_contexts - 1);
3053 radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
3054 radeon_ring_write(ring, 0);
3055 radeon_ring_write(ring, 0);
3056 radeon_ring_unlock_commit(rdev, ring, false);
3057
3058 cp_me = 0xff;
3059 WREG32(CP_ME_CNTL, cp_me);
3060
3061 r = radeon_ring_lock(rdev, ring, evergreen_default_size + 19);
3062 if (r) {
3063 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
3064 return r;
3065 }
3066
3067 /* setup clear context state */
3068 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3069 radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
3070
3071 for (i = 0; i < evergreen_default_size; i++)
3072 radeon_ring_write(ring, evergreen_default_state[i]);
3073
3074 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3075 radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
3076
3077 /* set clear context state */
3078 radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
3079 radeon_ring_write(ring, 0);
3080
3081 /* SQ_VTX_BASE_VTX_LOC */
3082 radeon_ring_write(ring, 0xc0026f00);
3083 radeon_ring_write(ring, 0x00000000);
3084 radeon_ring_write(ring, 0x00000000);
3085 radeon_ring_write(ring, 0x00000000);
3086
3087 /* Clear consts */
3088 radeon_ring_write(ring, 0xc0036f00);
3089 radeon_ring_write(ring, 0x00000bc4);
3090 radeon_ring_write(ring, 0xffffffff);
3091 radeon_ring_write(ring, 0xffffffff);
3092 radeon_ring_write(ring, 0xffffffff);
3093
3094 radeon_ring_write(ring, 0xc0026900);
3095 radeon_ring_write(ring, 0x00000316);
3096 radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
3097 radeon_ring_write(ring, 0x00000010); /* */
3098
3099 radeon_ring_unlock_commit(rdev, ring, false);
3100
3101 return 0;
3102 }
3103
evergreen_cp_resume(struct radeon_device * rdev)3104 static int evergreen_cp_resume(struct radeon_device *rdev)
3105 {
3106 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3107 u32 tmp;
3108 u32 rb_bufsz;
3109 int r;
3110
3111 /* Reset cp; if cp is reset, then PA, SH, VGT also need to be reset */
3112 WREG32(GRBM_SOFT_RESET, (SOFT_RESET_CP |
3113 SOFT_RESET_PA |
3114 SOFT_RESET_SH |
3115 SOFT_RESET_VGT |
3116 SOFT_RESET_SPI |
3117 SOFT_RESET_SX));
3118 RREG32(GRBM_SOFT_RESET);
3119 mdelay(15);
3120 WREG32(GRBM_SOFT_RESET, 0);
3121 RREG32(GRBM_SOFT_RESET);
3122
3123 /* Set ring buffer size */
3124 rb_bufsz = order_base_2(ring->ring_size / 8);
3125 tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
3126 #ifdef __BIG_ENDIAN
3127 tmp |= BUF_SWAP_32BIT;
3128 #endif
3129 WREG32(CP_RB_CNTL, tmp);
3130 WREG32(CP_SEM_WAIT_TIMER, 0x0);
3131 WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
3132
3133 /* Set the write pointer delay */
3134 WREG32(CP_RB_WPTR_DELAY, 0);
3135
3136 /* Initialize the ring buffer's read and write pointers */
3137 WREG32(CP_RB_CNTL, tmp | RB_RPTR_WR_ENA);
3138 WREG32(CP_RB_RPTR_WR, 0);
3139 ring->wptr = 0;
3140 WREG32(CP_RB_WPTR, ring->wptr);
3141
3142 /* set the wb address whether it's enabled or not */
3143 WREG32(CP_RB_RPTR_ADDR,
3144 ((rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC));
3145 WREG32(CP_RB_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
3146 WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
3147
3148 if (rdev->wb.enabled)
3149 WREG32(SCRATCH_UMSK, 0xff);
3150 else {
3151 tmp |= RB_NO_UPDATE;
3152 WREG32(SCRATCH_UMSK, 0);
3153 }
3154
3155 mdelay(1);
3156 WREG32(CP_RB_CNTL, tmp);
3157
3158 WREG32(CP_RB_BASE, ring->gpu_addr >> 8);
3159 WREG32(CP_DEBUG, (1 << 27) | (1 << 28));
3160
3161 evergreen_cp_start(rdev);
3162 ring->ready = true;
3163 r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, ring);
3164 if (r) {
3165 ring->ready = false;
3166 return r;
3167 }
3168 return 0;
3169 }
3170
3171 /*
3172 * Core functions
3173 */
evergreen_gpu_init(struct radeon_device * rdev)3174 static void evergreen_gpu_init(struct radeon_device *rdev)
3175 {
3176 u32 gb_addr_config;
3177 u32 mc_shared_chmap, mc_arb_ramcfg;
3178 u32 sx_debug_1;
3179 u32 smx_dc_ctl0;
3180 u32 sq_config;
3181 u32 sq_lds_resource_mgmt;
3182 u32 sq_gpr_resource_mgmt_1;
3183 u32 sq_gpr_resource_mgmt_2;
3184 u32 sq_gpr_resource_mgmt_3;
3185 u32 sq_thread_resource_mgmt;
3186 u32 sq_thread_resource_mgmt_2;
3187 u32 sq_stack_resource_mgmt_1;
3188 u32 sq_stack_resource_mgmt_2;
3189 u32 sq_stack_resource_mgmt_3;
3190 u32 vgt_cache_invalidation;
3191 u32 hdp_host_path_cntl, tmp;
3192 u32 disabled_rb_mask;
3193 int i, j, ps_thread_count;
3194
3195 switch (rdev->family) {
3196 case CHIP_CYPRESS:
3197 case CHIP_HEMLOCK:
3198 rdev->config.evergreen.num_ses = 2;
3199 rdev->config.evergreen.max_pipes = 4;
3200 rdev->config.evergreen.max_tile_pipes = 8;
3201 rdev->config.evergreen.max_simds = 10;
3202 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
3203 rdev->config.evergreen.max_gprs = 256;
3204 rdev->config.evergreen.max_threads = 248;
3205 rdev->config.evergreen.max_gs_threads = 32;
3206 rdev->config.evergreen.max_stack_entries = 512;
3207 rdev->config.evergreen.sx_num_of_sets = 4;
3208 rdev->config.evergreen.sx_max_export_size = 256;
3209 rdev->config.evergreen.sx_max_export_pos_size = 64;
3210 rdev->config.evergreen.sx_max_export_smx_size = 192;
3211 rdev->config.evergreen.max_hw_contexts = 8;
3212 rdev->config.evergreen.sq_num_cf_insts = 2;
3213
3214 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3215 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3216 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3217 gb_addr_config = CYPRESS_GB_ADDR_CONFIG_GOLDEN;
3218 break;
3219 case CHIP_JUNIPER:
3220 rdev->config.evergreen.num_ses = 1;
3221 rdev->config.evergreen.max_pipes = 4;
3222 rdev->config.evergreen.max_tile_pipes = 4;
3223 rdev->config.evergreen.max_simds = 10;
3224 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
3225 rdev->config.evergreen.max_gprs = 256;
3226 rdev->config.evergreen.max_threads = 248;
3227 rdev->config.evergreen.max_gs_threads = 32;
3228 rdev->config.evergreen.max_stack_entries = 512;
3229 rdev->config.evergreen.sx_num_of_sets = 4;
3230 rdev->config.evergreen.sx_max_export_size = 256;
3231 rdev->config.evergreen.sx_max_export_pos_size = 64;
3232 rdev->config.evergreen.sx_max_export_smx_size = 192;
3233 rdev->config.evergreen.max_hw_contexts = 8;
3234 rdev->config.evergreen.sq_num_cf_insts = 2;
3235
3236 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3237 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3238 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3239 gb_addr_config = JUNIPER_GB_ADDR_CONFIG_GOLDEN;
3240 break;
3241 case CHIP_REDWOOD:
3242 rdev->config.evergreen.num_ses = 1;
3243 rdev->config.evergreen.max_pipes = 4;
3244 rdev->config.evergreen.max_tile_pipes = 4;
3245 rdev->config.evergreen.max_simds = 5;
3246 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3247 rdev->config.evergreen.max_gprs = 256;
3248 rdev->config.evergreen.max_threads = 248;
3249 rdev->config.evergreen.max_gs_threads = 32;
3250 rdev->config.evergreen.max_stack_entries = 256;
3251 rdev->config.evergreen.sx_num_of_sets = 4;
3252 rdev->config.evergreen.sx_max_export_size = 256;
3253 rdev->config.evergreen.sx_max_export_pos_size = 64;
3254 rdev->config.evergreen.sx_max_export_smx_size = 192;
3255 rdev->config.evergreen.max_hw_contexts = 8;
3256 rdev->config.evergreen.sq_num_cf_insts = 2;
3257
3258 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3259 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3260 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3261 gb_addr_config = REDWOOD_GB_ADDR_CONFIG_GOLDEN;
3262 break;
3263 case CHIP_CEDAR:
3264 default:
3265 rdev->config.evergreen.num_ses = 1;
3266 rdev->config.evergreen.max_pipes = 2;
3267 rdev->config.evergreen.max_tile_pipes = 2;
3268 rdev->config.evergreen.max_simds = 2;
3269 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3270 rdev->config.evergreen.max_gprs = 256;
3271 rdev->config.evergreen.max_threads = 192;
3272 rdev->config.evergreen.max_gs_threads = 16;
3273 rdev->config.evergreen.max_stack_entries = 256;
3274 rdev->config.evergreen.sx_num_of_sets = 4;
3275 rdev->config.evergreen.sx_max_export_size = 128;
3276 rdev->config.evergreen.sx_max_export_pos_size = 32;
3277 rdev->config.evergreen.sx_max_export_smx_size = 96;
3278 rdev->config.evergreen.max_hw_contexts = 4;
3279 rdev->config.evergreen.sq_num_cf_insts = 1;
3280
3281 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3282 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3283 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3284 gb_addr_config = CEDAR_GB_ADDR_CONFIG_GOLDEN;
3285 break;
3286 case CHIP_PALM:
3287 rdev->config.evergreen.num_ses = 1;
3288 rdev->config.evergreen.max_pipes = 2;
3289 rdev->config.evergreen.max_tile_pipes = 2;
3290 rdev->config.evergreen.max_simds = 2;
3291 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3292 rdev->config.evergreen.max_gprs = 256;
3293 rdev->config.evergreen.max_threads = 192;
3294 rdev->config.evergreen.max_gs_threads = 16;
3295 rdev->config.evergreen.max_stack_entries = 256;
3296 rdev->config.evergreen.sx_num_of_sets = 4;
3297 rdev->config.evergreen.sx_max_export_size = 128;
3298 rdev->config.evergreen.sx_max_export_pos_size = 32;
3299 rdev->config.evergreen.sx_max_export_smx_size = 96;
3300 rdev->config.evergreen.max_hw_contexts = 4;
3301 rdev->config.evergreen.sq_num_cf_insts = 1;
3302
3303 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3304 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3305 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3306 gb_addr_config = CEDAR_GB_ADDR_CONFIG_GOLDEN;
3307 break;
3308 case CHIP_SUMO:
3309 rdev->config.evergreen.num_ses = 1;
3310 rdev->config.evergreen.max_pipes = 4;
3311 rdev->config.evergreen.max_tile_pipes = 4;
3312 if (rdev->pdev->device == 0x9648)
3313 rdev->config.evergreen.max_simds = 3;
3314 else if ((rdev->pdev->device == 0x9647) ||
3315 (rdev->pdev->device == 0x964a))
3316 rdev->config.evergreen.max_simds = 4;
3317 else
3318 rdev->config.evergreen.max_simds = 5;
3319 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3320 rdev->config.evergreen.max_gprs = 256;
3321 rdev->config.evergreen.max_threads = 248;
3322 rdev->config.evergreen.max_gs_threads = 32;
3323 rdev->config.evergreen.max_stack_entries = 256;
3324 rdev->config.evergreen.sx_num_of_sets = 4;
3325 rdev->config.evergreen.sx_max_export_size = 256;
3326 rdev->config.evergreen.sx_max_export_pos_size = 64;
3327 rdev->config.evergreen.sx_max_export_smx_size = 192;
3328 rdev->config.evergreen.max_hw_contexts = 8;
3329 rdev->config.evergreen.sq_num_cf_insts = 2;
3330
3331 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3332 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3333 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3334 gb_addr_config = SUMO_GB_ADDR_CONFIG_GOLDEN;
3335 break;
3336 case CHIP_SUMO2:
3337 rdev->config.evergreen.num_ses = 1;
3338 rdev->config.evergreen.max_pipes = 4;
3339 rdev->config.evergreen.max_tile_pipes = 4;
3340 rdev->config.evergreen.max_simds = 2;
3341 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3342 rdev->config.evergreen.max_gprs = 256;
3343 rdev->config.evergreen.max_threads = 248;
3344 rdev->config.evergreen.max_gs_threads = 32;
3345 rdev->config.evergreen.max_stack_entries = 512;
3346 rdev->config.evergreen.sx_num_of_sets = 4;
3347 rdev->config.evergreen.sx_max_export_size = 256;
3348 rdev->config.evergreen.sx_max_export_pos_size = 64;
3349 rdev->config.evergreen.sx_max_export_smx_size = 192;
3350 rdev->config.evergreen.max_hw_contexts = 4;
3351 rdev->config.evergreen.sq_num_cf_insts = 2;
3352
3353 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3354 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3355 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3356 gb_addr_config = SUMO2_GB_ADDR_CONFIG_GOLDEN;
3357 break;
3358 case CHIP_BARTS:
3359 rdev->config.evergreen.num_ses = 2;
3360 rdev->config.evergreen.max_pipes = 4;
3361 rdev->config.evergreen.max_tile_pipes = 8;
3362 rdev->config.evergreen.max_simds = 7;
3363 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
3364 rdev->config.evergreen.max_gprs = 256;
3365 rdev->config.evergreen.max_threads = 248;
3366 rdev->config.evergreen.max_gs_threads = 32;
3367 rdev->config.evergreen.max_stack_entries = 512;
3368 rdev->config.evergreen.sx_num_of_sets = 4;
3369 rdev->config.evergreen.sx_max_export_size = 256;
3370 rdev->config.evergreen.sx_max_export_pos_size = 64;
3371 rdev->config.evergreen.sx_max_export_smx_size = 192;
3372 rdev->config.evergreen.max_hw_contexts = 8;
3373 rdev->config.evergreen.sq_num_cf_insts = 2;
3374
3375 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3376 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3377 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3378 gb_addr_config = BARTS_GB_ADDR_CONFIG_GOLDEN;
3379 break;
3380 case CHIP_TURKS:
3381 rdev->config.evergreen.num_ses = 1;
3382 rdev->config.evergreen.max_pipes = 4;
3383 rdev->config.evergreen.max_tile_pipes = 4;
3384 rdev->config.evergreen.max_simds = 6;
3385 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3386 rdev->config.evergreen.max_gprs = 256;
3387 rdev->config.evergreen.max_threads = 248;
3388 rdev->config.evergreen.max_gs_threads = 32;
3389 rdev->config.evergreen.max_stack_entries = 256;
3390 rdev->config.evergreen.sx_num_of_sets = 4;
3391 rdev->config.evergreen.sx_max_export_size = 256;
3392 rdev->config.evergreen.sx_max_export_pos_size = 64;
3393 rdev->config.evergreen.sx_max_export_smx_size = 192;
3394 rdev->config.evergreen.max_hw_contexts = 8;
3395 rdev->config.evergreen.sq_num_cf_insts = 2;
3396
3397 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3398 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3399 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3400 gb_addr_config = TURKS_GB_ADDR_CONFIG_GOLDEN;
3401 break;
3402 case CHIP_CAICOS:
3403 rdev->config.evergreen.num_ses = 1;
3404 rdev->config.evergreen.max_pipes = 2;
3405 rdev->config.evergreen.max_tile_pipes = 2;
3406 rdev->config.evergreen.max_simds = 2;
3407 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3408 rdev->config.evergreen.max_gprs = 256;
3409 rdev->config.evergreen.max_threads = 192;
3410 rdev->config.evergreen.max_gs_threads = 16;
3411 rdev->config.evergreen.max_stack_entries = 256;
3412 rdev->config.evergreen.sx_num_of_sets = 4;
3413 rdev->config.evergreen.sx_max_export_size = 128;
3414 rdev->config.evergreen.sx_max_export_pos_size = 32;
3415 rdev->config.evergreen.sx_max_export_smx_size = 96;
3416 rdev->config.evergreen.max_hw_contexts = 4;
3417 rdev->config.evergreen.sq_num_cf_insts = 1;
3418
3419 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3420 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3421 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3422 gb_addr_config = CAICOS_GB_ADDR_CONFIG_GOLDEN;
3423 break;
3424 }
3425
3426 /* Initialize HDP */
3427 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
3428 WREG32((0x2c14 + j), 0x00000000);
3429 WREG32((0x2c18 + j), 0x00000000);
3430 WREG32((0x2c1c + j), 0x00000000);
3431 WREG32((0x2c20 + j), 0x00000000);
3432 WREG32((0x2c24 + j), 0x00000000);
3433 }
3434
3435 WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
3436 WREG32(SRBM_INT_CNTL, 0x1);
3437 WREG32(SRBM_INT_ACK, 0x1);
3438
3439 evergreen_fix_pci_max_read_req_size(rdev);
3440
3441 mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
3442 if ((rdev->family == CHIP_PALM) ||
3443 (rdev->family == CHIP_SUMO) ||
3444 (rdev->family == CHIP_SUMO2))
3445 mc_arb_ramcfg = RREG32(FUS_MC_ARB_RAMCFG);
3446 else
3447 mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
3448
3449 /* setup tiling info dword. gb_addr_config is not adequate since it does
3450 * not have bank info, so create a custom tiling dword.
3451 * bits 3:0 num_pipes
3452 * bits 7:4 num_banks
3453 * bits 11:8 group_size
3454 * bits 15:12 row_size
3455 */
3456 rdev->config.evergreen.tile_config = 0;
3457 switch (rdev->config.evergreen.max_tile_pipes) {
3458 case 1:
3459 default:
3460 rdev->config.evergreen.tile_config |= (0 << 0);
3461 break;
3462 case 2:
3463 rdev->config.evergreen.tile_config |= (1 << 0);
3464 break;
3465 case 4:
3466 rdev->config.evergreen.tile_config |= (2 << 0);
3467 break;
3468 case 8:
3469 rdev->config.evergreen.tile_config |= (3 << 0);
3470 break;
3471 }
3472 /* num banks is 8 on all fusion asics. 0 = 4, 1 = 8, 2 = 16 */
3473 if (rdev->flags & RADEON_IS_IGP)
3474 rdev->config.evergreen.tile_config |= 1 << 4;
3475 else {
3476 switch ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) {
3477 case 0: /* four banks */
3478 rdev->config.evergreen.tile_config |= 0 << 4;
3479 break;
3480 case 1: /* eight banks */
3481 rdev->config.evergreen.tile_config |= 1 << 4;
3482 break;
3483 case 2: /* sixteen banks */
3484 default:
3485 rdev->config.evergreen.tile_config |= 2 << 4;
3486 break;
3487 }
3488 }
3489 rdev->config.evergreen.tile_config |= 0 << 8;
3490 rdev->config.evergreen.tile_config |=
3491 ((gb_addr_config & 0x30000000) >> 28) << 12;
3492
3493 if ((rdev->family >= CHIP_CEDAR) && (rdev->family <= CHIP_HEMLOCK)) {
3494 u32 efuse_straps_4;
3495 u32 efuse_straps_3;
3496
3497 efuse_straps_4 = RREG32_RCU(0x204);
3498 efuse_straps_3 = RREG32_RCU(0x203);
3499 tmp = (((efuse_straps_4 & 0xf) << 4) |
3500 ((efuse_straps_3 & 0xf0000000) >> 28));
3501 } else {
3502 tmp = 0;
3503 for (i = (rdev->config.evergreen.num_ses - 1); i >= 0; i--) {
3504 u32 rb_disable_bitmap;
3505
3506 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3507 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3508 rb_disable_bitmap = (RREG32(CC_RB_BACKEND_DISABLE) & 0x00ff0000) >> 16;
3509 tmp <<= 4;
3510 tmp |= rb_disable_bitmap;
3511 }
3512 }
3513 /* enabled rb are just the one not disabled :) */
3514 disabled_rb_mask = tmp;
3515 tmp = 0;
3516 for (i = 0; i < rdev->config.evergreen.max_backends; i++)
3517 tmp |= (1 << i);
3518 /* if all the backends are disabled, fix it up here */
3519 if ((disabled_rb_mask & tmp) == tmp) {
3520 for (i = 0; i < rdev->config.evergreen.max_backends; i++)
3521 disabled_rb_mask &= ~(1 << i);
3522 }
3523
3524 for (i = 0; i < rdev->config.evergreen.num_ses; i++) {
3525 u32 simd_disable_bitmap;
3526
3527 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3528 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3529 simd_disable_bitmap = (RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0xffff0000) >> 16;
3530 simd_disable_bitmap |= 0xffffffff << rdev->config.evergreen.max_simds;
3531 tmp <<= 16;
3532 tmp |= simd_disable_bitmap;
3533 }
3534 rdev->config.evergreen.active_simds = hweight32(~tmp);
3535
3536 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
3537 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
3538
3539 WREG32(GB_ADDR_CONFIG, gb_addr_config);
3540 WREG32(DMIF_ADDR_CONFIG, gb_addr_config);
3541 WREG32(HDP_ADDR_CONFIG, gb_addr_config);
3542 WREG32(DMA_TILING_CONFIG, gb_addr_config);
3543 WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
3544 WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
3545 WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
3546
3547 if ((rdev->config.evergreen.max_backends == 1) &&
3548 (rdev->flags & RADEON_IS_IGP)) {
3549 if ((disabled_rb_mask & 3) == 1) {
3550 /* RB0 disabled, RB1 enabled */
3551 tmp = 0x11111111;
3552 } else {
3553 /* RB1 disabled, RB0 enabled */
3554 tmp = 0x00000000;
3555 }
3556 } else {
3557 tmp = gb_addr_config & NUM_PIPES_MASK;
3558 tmp = r6xx_remap_render_backend(rdev, tmp, rdev->config.evergreen.max_backends,
3559 EVERGREEN_MAX_BACKENDS, disabled_rb_mask);
3560 }
3561 WREG32(GB_BACKEND_MAP, tmp);
3562
3563 WREG32(CGTS_SYS_TCC_DISABLE, 0);
3564 WREG32(CGTS_TCC_DISABLE, 0);
3565 WREG32(CGTS_USER_SYS_TCC_DISABLE, 0);
3566 WREG32(CGTS_USER_TCC_DISABLE, 0);
3567
3568 /* set HW defaults for 3D engine */
3569 WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) |
3570 ROQ_IB2_START(0x2b)));
3571
3572 WREG32(CP_MEQ_THRESHOLDS, STQ_SPLIT(0x30));
3573
3574 WREG32(TA_CNTL_AUX, (DISABLE_CUBE_ANISO |
3575 SYNC_GRADIENT |
3576 SYNC_WALKER |
3577 SYNC_ALIGNER));
3578
3579 sx_debug_1 = RREG32(SX_DEBUG_1);
3580 sx_debug_1 |= ENABLE_NEW_SMX_ADDRESS;
3581 WREG32(SX_DEBUG_1, sx_debug_1);
3582
3583
3584 smx_dc_ctl0 = RREG32(SMX_DC_CTL0);
3585 smx_dc_ctl0 &= ~NUMBER_OF_SETS(0x1ff);
3586 smx_dc_ctl0 |= NUMBER_OF_SETS(rdev->config.evergreen.sx_num_of_sets);
3587 WREG32(SMX_DC_CTL0, smx_dc_ctl0);
3588
3589 if (rdev->family <= CHIP_SUMO2)
3590 WREG32(SMX_SAR_CTL0, 0x00010000);
3591
3592 WREG32(SX_EXPORT_BUFFER_SIZES, (COLOR_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_size / 4) - 1) |
3593 POSITION_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_pos_size / 4) - 1) |
3594 SMX_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_smx_size / 4) - 1)));
3595
3596 WREG32(PA_SC_FIFO_SIZE, (SC_PRIM_FIFO_SIZE(rdev->config.evergreen.sc_prim_fifo_size) |
3597 SC_HIZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_hiz_tile_fifo_size) |
3598 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_earlyz_tile_fifo_size)));
3599
3600 WREG32(VGT_NUM_INSTANCES, 1);
3601 WREG32(SPI_CONFIG_CNTL, 0);
3602 WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
3603 WREG32(CP_PERFMON_CNTL, 0);
3604
3605 WREG32(SQ_MS_FIFO_SIZES, (CACHE_FIFO_SIZE(16 * rdev->config.evergreen.sq_num_cf_insts) |
3606 FETCH_FIFO_HIWATER(0x4) |
3607 DONE_FIFO_HIWATER(0xe0) |
3608 ALU_UPDATE_FIFO_HIWATER(0x8)));
3609
3610 sq_config = RREG32(SQ_CONFIG);
3611 sq_config &= ~(PS_PRIO(3) |
3612 VS_PRIO(3) |
3613 GS_PRIO(3) |
3614 ES_PRIO(3));
3615 sq_config |= (VC_ENABLE |
3616 EXPORT_SRC_C |
3617 PS_PRIO(0) |
3618 VS_PRIO(1) |
3619 GS_PRIO(2) |
3620 ES_PRIO(3));
3621
3622 switch (rdev->family) {
3623 case CHIP_CEDAR:
3624 case CHIP_PALM:
3625 case CHIP_SUMO:
3626 case CHIP_SUMO2:
3627 case CHIP_CAICOS:
3628 /* no vertex cache */
3629 sq_config &= ~VC_ENABLE;
3630 break;
3631 default:
3632 break;
3633 }
3634
3635 sq_lds_resource_mgmt = RREG32(SQ_LDS_RESOURCE_MGMT);
3636
3637 sq_gpr_resource_mgmt_1 = NUM_PS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2))* 12 / 32);
3638 sq_gpr_resource_mgmt_1 |= NUM_VS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 6 / 32);
3639 sq_gpr_resource_mgmt_1 |= NUM_CLAUSE_TEMP_GPRS(4);
3640 sq_gpr_resource_mgmt_2 = NUM_GS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32);
3641 sq_gpr_resource_mgmt_2 |= NUM_ES_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32);
3642 sq_gpr_resource_mgmt_3 = NUM_HS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32);
3643 sq_gpr_resource_mgmt_3 |= NUM_LS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32);
3644
3645 switch (rdev->family) {
3646 case CHIP_CEDAR:
3647 case CHIP_PALM:
3648 case CHIP_SUMO:
3649 case CHIP_SUMO2:
3650 ps_thread_count = 96;
3651 break;
3652 default:
3653 ps_thread_count = 128;
3654 break;
3655 }
3656
3657 sq_thread_resource_mgmt = NUM_PS_THREADS(ps_thread_count);
3658 sq_thread_resource_mgmt |= NUM_VS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3659 sq_thread_resource_mgmt |= NUM_GS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3660 sq_thread_resource_mgmt |= NUM_ES_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3661 sq_thread_resource_mgmt_2 = NUM_HS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3662 sq_thread_resource_mgmt_2 |= NUM_LS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3663
3664 sq_stack_resource_mgmt_1 = NUM_PS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3665 sq_stack_resource_mgmt_1 |= NUM_VS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3666 sq_stack_resource_mgmt_2 = NUM_GS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3667 sq_stack_resource_mgmt_2 |= NUM_ES_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3668 sq_stack_resource_mgmt_3 = NUM_HS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3669 sq_stack_resource_mgmt_3 |= NUM_LS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3670
3671 WREG32(SQ_CONFIG, sq_config);
3672 WREG32(SQ_GPR_RESOURCE_MGMT_1, sq_gpr_resource_mgmt_1);
3673 WREG32(SQ_GPR_RESOURCE_MGMT_2, sq_gpr_resource_mgmt_2);
3674 WREG32(SQ_GPR_RESOURCE_MGMT_3, sq_gpr_resource_mgmt_3);
3675 WREG32(SQ_THREAD_RESOURCE_MGMT, sq_thread_resource_mgmt);
3676 WREG32(SQ_THREAD_RESOURCE_MGMT_2, sq_thread_resource_mgmt_2);
3677 WREG32(SQ_STACK_RESOURCE_MGMT_1, sq_stack_resource_mgmt_1);
3678 WREG32(SQ_STACK_RESOURCE_MGMT_2, sq_stack_resource_mgmt_2);
3679 WREG32(SQ_STACK_RESOURCE_MGMT_3, sq_stack_resource_mgmt_3);
3680 WREG32(SQ_DYN_GPR_CNTL_PS_FLUSH_REQ, 0);
3681 WREG32(SQ_LDS_RESOURCE_MGMT, sq_lds_resource_mgmt);
3682
3683 WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
3684 FORCE_EOV_MAX_REZ_CNT(255)));
3685
3686 switch (rdev->family) {
3687 case CHIP_CEDAR:
3688 case CHIP_PALM:
3689 case CHIP_SUMO:
3690 case CHIP_SUMO2:
3691 case CHIP_CAICOS:
3692 vgt_cache_invalidation = CACHE_INVALIDATION(TC_ONLY);
3693 break;
3694 default:
3695 vgt_cache_invalidation = CACHE_INVALIDATION(VC_AND_TC);
3696 break;
3697 }
3698 vgt_cache_invalidation |= AUTO_INVLD_EN(ES_AND_GS_AUTO);
3699 WREG32(VGT_CACHE_INVALIDATION, vgt_cache_invalidation);
3700
3701 WREG32(VGT_GS_VERTEX_REUSE, 16);
3702 WREG32(PA_SU_LINE_STIPPLE_VALUE, 0);
3703 WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
3704
3705 WREG32(VGT_VERTEX_REUSE_BLOCK_CNTL, 14);
3706 WREG32(VGT_OUT_DEALLOC_CNTL, 16);
3707
3708 WREG32(CB_PERF_CTR0_SEL_0, 0);
3709 WREG32(CB_PERF_CTR0_SEL_1, 0);
3710 WREG32(CB_PERF_CTR1_SEL_0, 0);
3711 WREG32(CB_PERF_CTR1_SEL_1, 0);
3712 WREG32(CB_PERF_CTR2_SEL_0, 0);
3713 WREG32(CB_PERF_CTR2_SEL_1, 0);
3714 WREG32(CB_PERF_CTR3_SEL_0, 0);
3715 WREG32(CB_PERF_CTR3_SEL_1, 0);
3716
3717 /* clear render buffer base addresses */
3718 WREG32(CB_COLOR0_BASE, 0);
3719 WREG32(CB_COLOR1_BASE, 0);
3720 WREG32(CB_COLOR2_BASE, 0);
3721 WREG32(CB_COLOR3_BASE, 0);
3722 WREG32(CB_COLOR4_BASE, 0);
3723 WREG32(CB_COLOR5_BASE, 0);
3724 WREG32(CB_COLOR6_BASE, 0);
3725 WREG32(CB_COLOR7_BASE, 0);
3726 WREG32(CB_COLOR8_BASE, 0);
3727 WREG32(CB_COLOR9_BASE, 0);
3728 WREG32(CB_COLOR10_BASE, 0);
3729 WREG32(CB_COLOR11_BASE, 0);
3730
3731 /* set the shader const cache sizes to 0 */
3732 for (i = SQ_ALU_CONST_BUFFER_SIZE_PS_0; i < 0x28200; i += 4)
3733 WREG32(i, 0);
3734 for (i = SQ_ALU_CONST_BUFFER_SIZE_HS_0; i < 0x29000; i += 4)
3735 WREG32(i, 0);
3736
3737 tmp = RREG32(HDP_MISC_CNTL);
3738 tmp |= HDP_FLUSH_INVALIDATE_CACHE;
3739 WREG32(HDP_MISC_CNTL, tmp);
3740
3741 hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
3742 WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
3743
3744 WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
3745
3746 udelay(50);
3747
3748 }
3749
evergreen_mc_init(struct radeon_device * rdev)3750 int evergreen_mc_init(struct radeon_device *rdev)
3751 {
3752 u32 tmp;
3753 int chansize, numchan;
3754
3755 /* Get VRAM informations */
3756 rdev->mc.vram_is_ddr = true;
3757 if ((rdev->family == CHIP_PALM) ||
3758 (rdev->family == CHIP_SUMO) ||
3759 (rdev->family == CHIP_SUMO2))
3760 tmp = RREG32(FUS_MC_ARB_RAMCFG);
3761 else
3762 tmp = RREG32(MC_ARB_RAMCFG);
3763 if (tmp & CHANSIZE_OVERRIDE) {
3764 chansize = 16;
3765 } else if (tmp & CHANSIZE_MASK) {
3766 chansize = 64;
3767 } else {
3768 chansize = 32;
3769 }
3770 tmp = RREG32(MC_SHARED_CHMAP);
3771 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
3772 case 0:
3773 default:
3774 numchan = 1;
3775 break;
3776 case 1:
3777 numchan = 2;
3778 break;
3779 case 2:
3780 numchan = 4;
3781 break;
3782 case 3:
3783 numchan = 8;
3784 break;
3785 }
3786 rdev->mc.vram_width = numchan * chansize;
3787 /* Could aper size report 0 ? */
3788 rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
3789 rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
3790 /* Setup GPU memory space */
3791 if ((rdev->family == CHIP_PALM) ||
3792 (rdev->family == CHIP_SUMO) ||
3793 (rdev->family == CHIP_SUMO2)) {
3794 /* size in bytes on fusion */
3795 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE);
3796 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE);
3797 } else {
3798 /* size in MB on evergreen/cayman/tn */
3799 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
3800 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
3801 }
3802 rdev->mc.visible_vram_size = rdev->mc.aper_size;
3803 r700_vram_gtt_location(rdev, &rdev->mc);
3804 radeon_update_bandwidth_info(rdev);
3805
3806 return 0;
3807 }
3808
evergreen_print_gpu_status_regs(struct radeon_device * rdev)3809 void evergreen_print_gpu_status_regs(struct radeon_device *rdev)
3810 {
3811 dev_info(rdev->dev, " GRBM_STATUS = 0x%08X\n",
3812 RREG32(GRBM_STATUS));
3813 dev_info(rdev->dev, " GRBM_STATUS_SE0 = 0x%08X\n",
3814 RREG32(GRBM_STATUS_SE0));
3815 dev_info(rdev->dev, " GRBM_STATUS_SE1 = 0x%08X\n",
3816 RREG32(GRBM_STATUS_SE1));
3817 dev_info(rdev->dev, " SRBM_STATUS = 0x%08X\n",
3818 RREG32(SRBM_STATUS));
3819 dev_info(rdev->dev, " SRBM_STATUS2 = 0x%08X\n",
3820 RREG32(SRBM_STATUS2));
3821 dev_info(rdev->dev, " R_008674_CP_STALLED_STAT1 = 0x%08X\n",
3822 RREG32(CP_STALLED_STAT1));
3823 dev_info(rdev->dev, " R_008678_CP_STALLED_STAT2 = 0x%08X\n",
3824 RREG32(CP_STALLED_STAT2));
3825 dev_info(rdev->dev, " R_00867C_CP_BUSY_STAT = 0x%08X\n",
3826 RREG32(CP_BUSY_STAT));
3827 dev_info(rdev->dev, " R_008680_CP_STAT = 0x%08X\n",
3828 RREG32(CP_STAT));
3829 dev_info(rdev->dev, " R_00D034_DMA_STATUS_REG = 0x%08X\n",
3830 RREG32(DMA_STATUS_REG));
3831 if (rdev->family >= CHIP_CAYMAN) {
3832 dev_info(rdev->dev, " R_00D834_DMA_STATUS_REG = 0x%08X\n",
3833 RREG32(DMA_STATUS_REG + 0x800));
3834 }
3835 }
3836
evergreen_is_display_hung(struct radeon_device * rdev)3837 bool evergreen_is_display_hung(struct radeon_device *rdev)
3838 {
3839 u32 crtc_hung = 0;
3840 u32 crtc_status[6];
3841 u32 i, j, tmp;
3842
3843 for (i = 0; i < rdev->num_crtc; i++) {
3844 if (RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]) & EVERGREEN_CRTC_MASTER_EN) {
3845 crtc_status[i] = RREG32(EVERGREEN_CRTC_STATUS_HV_COUNT + crtc_offsets[i]);
3846 crtc_hung |= (1 << i);
3847 }
3848 }
3849
3850 for (j = 0; j < 10; j++) {
3851 for (i = 0; i < rdev->num_crtc; i++) {
3852 if (crtc_hung & (1 << i)) {
3853 tmp = RREG32(EVERGREEN_CRTC_STATUS_HV_COUNT + crtc_offsets[i]);
3854 if (tmp != crtc_status[i])
3855 crtc_hung &= ~(1 << i);
3856 }
3857 }
3858 if (crtc_hung == 0)
3859 return false;
3860 udelay(100);
3861 }
3862
3863 return true;
3864 }
3865
evergreen_gpu_check_soft_reset(struct radeon_device * rdev)3866 u32 evergreen_gpu_check_soft_reset(struct radeon_device *rdev)
3867 {
3868 u32 reset_mask = 0;
3869 u32 tmp;
3870
3871 /* GRBM_STATUS */
3872 tmp = RREG32(GRBM_STATUS);
3873 if (tmp & (PA_BUSY | SC_BUSY |
3874 SH_BUSY | SX_BUSY |
3875 TA_BUSY | VGT_BUSY |
3876 DB_BUSY | CB_BUSY |
3877 SPI_BUSY | VGT_BUSY_NO_DMA))
3878 reset_mask |= RADEON_RESET_GFX;
3879
3880 if (tmp & (CF_RQ_PENDING | PF_RQ_PENDING |
3881 CP_BUSY | CP_COHERENCY_BUSY))
3882 reset_mask |= RADEON_RESET_CP;
3883
3884 if (tmp & GRBM_EE_BUSY)
3885 reset_mask |= RADEON_RESET_GRBM | RADEON_RESET_GFX | RADEON_RESET_CP;
3886
3887 /* DMA_STATUS_REG */
3888 tmp = RREG32(DMA_STATUS_REG);
3889 if (!(tmp & DMA_IDLE))
3890 reset_mask |= RADEON_RESET_DMA;
3891
3892 /* SRBM_STATUS2 */
3893 tmp = RREG32(SRBM_STATUS2);
3894 if (tmp & DMA_BUSY)
3895 reset_mask |= RADEON_RESET_DMA;
3896
3897 /* SRBM_STATUS */
3898 tmp = RREG32(SRBM_STATUS);
3899 if (tmp & (RLC_RQ_PENDING | RLC_BUSY))
3900 reset_mask |= RADEON_RESET_RLC;
3901
3902 if (tmp & IH_BUSY)
3903 reset_mask |= RADEON_RESET_IH;
3904
3905 if (tmp & SEM_BUSY)
3906 reset_mask |= RADEON_RESET_SEM;
3907
3908 if (tmp & GRBM_RQ_PENDING)
3909 reset_mask |= RADEON_RESET_GRBM;
3910
3911 if (tmp & VMC_BUSY)
3912 reset_mask |= RADEON_RESET_VMC;
3913
3914 if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
3915 MCC_BUSY | MCD_BUSY))
3916 reset_mask |= RADEON_RESET_MC;
3917
3918 if (evergreen_is_display_hung(rdev))
3919 reset_mask |= RADEON_RESET_DISPLAY;
3920
3921 /* VM_L2_STATUS */
3922 tmp = RREG32(VM_L2_STATUS);
3923 if (tmp & L2_BUSY)
3924 reset_mask |= RADEON_RESET_VMC;
3925
3926 /* Skip MC reset as it's mostly likely not hung, just busy */
3927 if (reset_mask & RADEON_RESET_MC) {
3928 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
3929 reset_mask &= ~RADEON_RESET_MC;
3930 }
3931
3932 return reset_mask;
3933 }
3934
evergreen_gpu_soft_reset(struct radeon_device * rdev,u32 reset_mask)3935 static void evergreen_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
3936 {
3937 struct evergreen_mc_save save;
3938 u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
3939 u32 tmp;
3940
3941 if (reset_mask == 0)
3942 return;
3943
3944 dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
3945
3946 evergreen_print_gpu_status_regs(rdev);
3947
3948 /* Disable CP parsing/prefetching */
3949 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT);
3950
3951 if (reset_mask & RADEON_RESET_DMA) {
3952 /* Disable DMA */
3953 tmp = RREG32(DMA_RB_CNTL);
3954 tmp &= ~DMA_RB_ENABLE;
3955 WREG32(DMA_RB_CNTL, tmp);
3956 }
3957
3958 udelay(50);
3959
3960 evergreen_mc_stop(rdev, &save);
3961 if (evergreen_mc_wait_for_idle(rdev)) {
3962 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3963 }
3964
3965 if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE)) {
3966 grbm_soft_reset |= SOFT_RESET_DB |
3967 SOFT_RESET_CB |
3968 SOFT_RESET_PA |
3969 SOFT_RESET_SC |
3970 SOFT_RESET_SPI |
3971 SOFT_RESET_SX |
3972 SOFT_RESET_SH |
3973 SOFT_RESET_TC |
3974 SOFT_RESET_TA |
3975 SOFT_RESET_VC |
3976 SOFT_RESET_VGT;
3977 }
3978
3979 if (reset_mask & RADEON_RESET_CP) {
3980 grbm_soft_reset |= SOFT_RESET_CP |
3981 SOFT_RESET_VGT;
3982
3983 srbm_soft_reset |= SOFT_RESET_GRBM;
3984 }
3985
3986 if (reset_mask & RADEON_RESET_DMA)
3987 srbm_soft_reset |= SOFT_RESET_DMA;
3988
3989 if (reset_mask & RADEON_RESET_DISPLAY)
3990 srbm_soft_reset |= SOFT_RESET_DC;
3991
3992 if (reset_mask & RADEON_RESET_RLC)
3993 srbm_soft_reset |= SOFT_RESET_RLC;
3994
3995 if (reset_mask & RADEON_RESET_SEM)
3996 srbm_soft_reset |= SOFT_RESET_SEM;
3997
3998 if (reset_mask & RADEON_RESET_IH)
3999 srbm_soft_reset |= SOFT_RESET_IH;
4000
4001 if (reset_mask & RADEON_RESET_GRBM)
4002 srbm_soft_reset |= SOFT_RESET_GRBM;
4003
4004 if (reset_mask & RADEON_RESET_VMC)
4005 srbm_soft_reset |= SOFT_RESET_VMC;
4006
4007 if (!(rdev->flags & RADEON_IS_IGP)) {
4008 if (reset_mask & RADEON_RESET_MC)
4009 srbm_soft_reset |= SOFT_RESET_MC;
4010 }
4011
4012 if (grbm_soft_reset) {
4013 tmp = RREG32(GRBM_SOFT_RESET);
4014 tmp |= grbm_soft_reset;
4015 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
4016 WREG32(GRBM_SOFT_RESET, tmp);
4017 tmp = RREG32(GRBM_SOFT_RESET);
4018
4019 udelay(50);
4020
4021 tmp &= ~grbm_soft_reset;
4022 WREG32(GRBM_SOFT_RESET, tmp);
4023 tmp = RREG32(GRBM_SOFT_RESET);
4024 }
4025
4026 if (srbm_soft_reset) {
4027 tmp = RREG32(SRBM_SOFT_RESET);
4028 tmp |= srbm_soft_reset;
4029 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
4030 WREG32(SRBM_SOFT_RESET, tmp);
4031 tmp = RREG32(SRBM_SOFT_RESET);
4032
4033 udelay(50);
4034
4035 tmp &= ~srbm_soft_reset;
4036 WREG32(SRBM_SOFT_RESET, tmp);
4037 tmp = RREG32(SRBM_SOFT_RESET);
4038 }
4039
4040 /* Wait a little for things to settle down */
4041 udelay(50);
4042
4043 evergreen_mc_resume(rdev, &save);
4044 udelay(50);
4045
4046 evergreen_print_gpu_status_regs(rdev);
4047 }
4048
evergreen_gpu_pci_config_reset(struct radeon_device * rdev)4049 void evergreen_gpu_pci_config_reset(struct radeon_device *rdev)
4050 {
4051 struct evergreen_mc_save save;
4052 u32 tmp, i;
4053
4054 dev_info(rdev->dev, "GPU pci config reset\n");
4055
4056 /* disable dpm? */
4057
4058 /* Disable CP parsing/prefetching */
4059 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT);
4060 udelay(50);
4061 /* Disable DMA */
4062 tmp = RREG32(DMA_RB_CNTL);
4063 tmp &= ~DMA_RB_ENABLE;
4064 WREG32(DMA_RB_CNTL, tmp);
4065 /* XXX other engines? */
4066
4067 /* halt the rlc */
4068 r600_rlc_stop(rdev);
4069
4070 udelay(50);
4071
4072 /* set mclk/sclk to bypass */
4073 rv770_set_clk_bypass_mode(rdev);
4074 /* disable BM */
4075 pci_clear_master(rdev->pdev);
4076 /* disable mem access */
4077 evergreen_mc_stop(rdev, &save);
4078 if (evergreen_mc_wait_for_idle(rdev)) {
4079 dev_warn(rdev->dev, "Wait for MC idle timed out !\n");
4080 }
4081 /* reset */
4082 radeon_pci_config_reset(rdev);
4083 /* wait for asic to come out of reset */
4084 for (i = 0; i < rdev->usec_timeout; i++) {
4085 if (RREG32(CONFIG_MEMSIZE) != 0xffffffff)
4086 break;
4087 udelay(1);
4088 }
4089 }
4090
evergreen_asic_reset(struct radeon_device * rdev)4091 int evergreen_asic_reset(struct radeon_device *rdev)
4092 {
4093 u32 reset_mask;
4094
4095 reset_mask = evergreen_gpu_check_soft_reset(rdev);
4096
4097 if (reset_mask)
4098 r600_set_bios_scratch_engine_hung(rdev, true);
4099
4100 /* try soft reset */
4101 evergreen_gpu_soft_reset(rdev, reset_mask);
4102
4103 reset_mask = evergreen_gpu_check_soft_reset(rdev);
4104
4105 /* try pci config reset */
4106 if (reset_mask && radeon_hard_reset)
4107 evergreen_gpu_pci_config_reset(rdev);
4108
4109 reset_mask = evergreen_gpu_check_soft_reset(rdev);
4110
4111 if (!reset_mask)
4112 r600_set_bios_scratch_engine_hung(rdev, false);
4113
4114 return 0;
4115 }
4116
4117 /**
4118 * evergreen_gfx_is_lockup - Check if the GFX engine is locked up
4119 *
4120 * @rdev: radeon_device pointer
4121 * @ring: radeon_ring structure holding ring information
4122 *
4123 * Check if the GFX engine is locked up.
4124 * Returns true if the engine appears to be locked up, false if not.
4125 */
evergreen_gfx_is_lockup(struct radeon_device * rdev,struct radeon_ring * ring)4126 bool evergreen_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
4127 {
4128 u32 reset_mask = evergreen_gpu_check_soft_reset(rdev);
4129
4130 if (!(reset_mask & (RADEON_RESET_GFX |
4131 RADEON_RESET_COMPUTE |
4132 RADEON_RESET_CP))) {
4133 radeon_ring_lockup_update(rdev, ring);
4134 return false;
4135 }
4136 return radeon_ring_test_lockup(rdev, ring);
4137 }
4138
4139 /*
4140 * RLC
4141 */
4142 #define RLC_SAVE_RESTORE_LIST_END_MARKER 0x00000000
4143 #define RLC_CLEAR_STATE_END_MARKER 0x00000001
4144
sumo_rlc_fini(struct radeon_device * rdev)4145 void sumo_rlc_fini(struct radeon_device *rdev)
4146 {
4147 int r;
4148
4149 /* save restore block */
4150 if (rdev->rlc.save_restore_obj) {
4151 r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false);
4152 if (unlikely(r != 0))
4153 dev_warn(rdev->dev, "(%d) reserve RLC sr bo failed\n", r);
4154 radeon_bo_unpin(rdev->rlc.save_restore_obj);
4155 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
4156
4157 radeon_bo_unref(&rdev->rlc.save_restore_obj);
4158 rdev->rlc.save_restore_obj = NULL;
4159 }
4160
4161 /* clear state block */
4162 if (rdev->rlc.clear_state_obj) {
4163 r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false);
4164 if (unlikely(r != 0))
4165 dev_warn(rdev->dev, "(%d) reserve RLC c bo failed\n", r);
4166 radeon_bo_unpin(rdev->rlc.clear_state_obj);
4167 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
4168
4169 radeon_bo_unref(&rdev->rlc.clear_state_obj);
4170 rdev->rlc.clear_state_obj = NULL;
4171 }
4172
4173 /* clear state block */
4174 if (rdev->rlc.cp_table_obj) {
4175 r = radeon_bo_reserve(rdev->rlc.cp_table_obj, false);
4176 if (unlikely(r != 0))
4177 dev_warn(rdev->dev, "(%d) reserve RLC cp table bo failed\n", r);
4178 radeon_bo_unpin(rdev->rlc.cp_table_obj);
4179 radeon_bo_unreserve(rdev->rlc.cp_table_obj);
4180
4181 radeon_bo_unref(&rdev->rlc.cp_table_obj);
4182 rdev->rlc.cp_table_obj = NULL;
4183 }
4184 }
4185
4186 #define CP_ME_TABLE_SIZE 96
4187
sumo_rlc_init(struct radeon_device * rdev)4188 int sumo_rlc_init(struct radeon_device *rdev)
4189 {
4190 const u32 *src_ptr;
4191 volatile u32 *dst_ptr;
4192 u32 dws, data, i, j, k, reg_num;
4193 u32 reg_list_num, reg_list_hdr_blk_index, reg_list_blk_index = 0;
4194 u64 reg_list_mc_addr;
4195 const struct cs_section_def *cs_data;
4196 int r;
4197
4198 src_ptr = rdev->rlc.reg_list;
4199 dws = rdev->rlc.reg_list_size;
4200 if (rdev->family >= CHIP_BONAIRE) {
4201 dws += (5 * 16) + 48 + 48 + 64;
4202 }
4203 cs_data = rdev->rlc.cs_data;
4204
4205 if (src_ptr) {
4206 /* save restore block */
4207 if (rdev->rlc.save_restore_obj == NULL) {
4208 r = radeon_bo_create(rdev, dws * 4, PAGE_SIZE, true,
4209 RADEON_GEM_DOMAIN_VRAM, 0, NULL,
4210 NULL, &rdev->rlc.save_restore_obj);
4211 if (r) {
4212 dev_warn(rdev->dev, "(%d) create RLC sr bo failed\n", r);
4213 return r;
4214 }
4215 }
4216
4217 r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false);
4218 if (unlikely(r != 0)) {
4219 sumo_rlc_fini(rdev);
4220 return r;
4221 }
4222 r = radeon_bo_pin(rdev->rlc.save_restore_obj, RADEON_GEM_DOMAIN_VRAM,
4223 &rdev->rlc.save_restore_gpu_addr);
4224 if (r) {
4225 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
4226 dev_warn(rdev->dev, "(%d) pin RLC sr bo failed\n", r);
4227 sumo_rlc_fini(rdev);
4228 return r;
4229 }
4230
4231 r = radeon_bo_kmap(rdev->rlc.save_restore_obj, (void **)&rdev->rlc.sr_ptr);
4232 if (r) {
4233 dev_warn(rdev->dev, "(%d) map RLC sr bo failed\n", r);
4234 sumo_rlc_fini(rdev);
4235 return r;
4236 }
4237 /* write the sr buffer */
4238 dst_ptr = rdev->rlc.sr_ptr;
4239 if (rdev->family >= CHIP_TAHITI) {
4240 /* SI */
4241 for (i = 0; i < rdev->rlc.reg_list_size; i++)
4242 dst_ptr[i] = cpu_to_le32(src_ptr[i]);
4243 } else {
4244 /* ON/LN/TN */
4245 /* format:
4246 * dw0: (reg2 << 16) | reg1
4247 * dw1: reg1 save space
4248 * dw2: reg2 save space
4249 */
4250 for (i = 0; i < dws; i++) {
4251 data = src_ptr[i] >> 2;
4252 i++;
4253 if (i < dws)
4254 data |= (src_ptr[i] >> 2) << 16;
4255 j = (((i - 1) * 3) / 2);
4256 dst_ptr[j] = cpu_to_le32(data);
4257 }
4258 j = ((i * 3) / 2);
4259 dst_ptr[j] = cpu_to_le32(RLC_SAVE_RESTORE_LIST_END_MARKER);
4260 }
4261 radeon_bo_kunmap(rdev->rlc.save_restore_obj);
4262 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
4263 }
4264
4265 if (cs_data) {
4266 /* clear state block */
4267 if (rdev->family >= CHIP_BONAIRE) {
4268 rdev->rlc.clear_state_size = dws = cik_get_csb_size(rdev);
4269 } else if (rdev->family >= CHIP_TAHITI) {
4270 rdev->rlc.clear_state_size = si_get_csb_size(rdev);
4271 dws = rdev->rlc.clear_state_size + (256 / 4);
4272 } else {
4273 reg_list_num = 0;
4274 dws = 0;
4275 for (i = 0; cs_data[i].section != NULL; i++) {
4276 for (j = 0; cs_data[i].section[j].extent != NULL; j++) {
4277 reg_list_num++;
4278 dws += cs_data[i].section[j].reg_count;
4279 }
4280 }
4281 reg_list_blk_index = (3 * reg_list_num + 2);
4282 dws += reg_list_blk_index;
4283 rdev->rlc.clear_state_size = dws;
4284 }
4285
4286 if (rdev->rlc.clear_state_obj == NULL) {
4287 r = radeon_bo_create(rdev, dws * 4, PAGE_SIZE, true,
4288 RADEON_GEM_DOMAIN_VRAM, 0, NULL,
4289 NULL, &rdev->rlc.clear_state_obj);
4290 if (r) {
4291 dev_warn(rdev->dev, "(%d) create RLC c bo failed\n", r);
4292 sumo_rlc_fini(rdev);
4293 return r;
4294 }
4295 }
4296 r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false);
4297 if (unlikely(r != 0)) {
4298 sumo_rlc_fini(rdev);
4299 return r;
4300 }
4301 r = radeon_bo_pin(rdev->rlc.clear_state_obj, RADEON_GEM_DOMAIN_VRAM,
4302 &rdev->rlc.clear_state_gpu_addr);
4303 if (r) {
4304 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
4305 dev_warn(rdev->dev, "(%d) pin RLC c bo failed\n", r);
4306 sumo_rlc_fini(rdev);
4307 return r;
4308 }
4309
4310 r = radeon_bo_kmap(rdev->rlc.clear_state_obj, (void **)&rdev->rlc.cs_ptr);
4311 if (r) {
4312 dev_warn(rdev->dev, "(%d) map RLC c bo failed\n", r);
4313 sumo_rlc_fini(rdev);
4314 return r;
4315 }
4316 /* set up the cs buffer */
4317 dst_ptr = rdev->rlc.cs_ptr;
4318 if (rdev->family >= CHIP_BONAIRE) {
4319 cik_get_csb_buffer(rdev, dst_ptr);
4320 } else if (rdev->family >= CHIP_TAHITI) {
4321 reg_list_mc_addr = rdev->rlc.clear_state_gpu_addr + 256;
4322 dst_ptr[0] = cpu_to_le32(upper_32_bits(reg_list_mc_addr));
4323 dst_ptr[1] = cpu_to_le32(lower_32_bits(reg_list_mc_addr));
4324 dst_ptr[2] = cpu_to_le32(rdev->rlc.clear_state_size);
4325 si_get_csb_buffer(rdev, &dst_ptr[(256/4)]);
4326 } else {
4327 reg_list_hdr_blk_index = 0;
4328 reg_list_mc_addr = rdev->rlc.clear_state_gpu_addr + (reg_list_blk_index * 4);
4329 data = upper_32_bits(reg_list_mc_addr);
4330 dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
4331 reg_list_hdr_blk_index++;
4332 for (i = 0; cs_data[i].section != NULL; i++) {
4333 for (j = 0; cs_data[i].section[j].extent != NULL; j++) {
4334 reg_num = cs_data[i].section[j].reg_count;
4335 data = reg_list_mc_addr & 0xffffffff;
4336 dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
4337 reg_list_hdr_blk_index++;
4338
4339 data = (cs_data[i].section[j].reg_index * 4) & 0xffffffff;
4340 dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
4341 reg_list_hdr_blk_index++;
4342
4343 data = 0x08000000 | (reg_num * 4);
4344 dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
4345 reg_list_hdr_blk_index++;
4346
4347 for (k = 0; k < reg_num; k++) {
4348 data = cs_data[i].section[j].extent[k];
4349 dst_ptr[reg_list_blk_index + k] = cpu_to_le32(data);
4350 }
4351 reg_list_mc_addr += reg_num * 4;
4352 reg_list_blk_index += reg_num;
4353 }
4354 }
4355 dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(RLC_CLEAR_STATE_END_MARKER);
4356 }
4357 radeon_bo_kunmap(rdev->rlc.clear_state_obj);
4358 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
4359 }
4360
4361 if (rdev->rlc.cp_table_size) {
4362 if (rdev->rlc.cp_table_obj == NULL) {
4363 r = radeon_bo_create(rdev, rdev->rlc.cp_table_size,
4364 PAGE_SIZE, true,
4365 RADEON_GEM_DOMAIN_VRAM, 0, NULL,
4366 NULL, &rdev->rlc.cp_table_obj);
4367 if (r) {
4368 dev_warn(rdev->dev, "(%d) create RLC cp table bo failed\n", r);
4369 sumo_rlc_fini(rdev);
4370 return r;
4371 }
4372 }
4373
4374 r = radeon_bo_reserve(rdev->rlc.cp_table_obj, false);
4375 if (unlikely(r != 0)) {
4376 dev_warn(rdev->dev, "(%d) reserve RLC cp table bo failed\n", r);
4377 sumo_rlc_fini(rdev);
4378 return r;
4379 }
4380 r = radeon_bo_pin(rdev->rlc.cp_table_obj, RADEON_GEM_DOMAIN_VRAM,
4381 &rdev->rlc.cp_table_gpu_addr);
4382 if (r) {
4383 radeon_bo_unreserve(rdev->rlc.cp_table_obj);
4384 dev_warn(rdev->dev, "(%d) pin RLC cp_table bo failed\n", r);
4385 sumo_rlc_fini(rdev);
4386 return r;
4387 }
4388 r = radeon_bo_kmap(rdev->rlc.cp_table_obj, (void **)&rdev->rlc.cp_table_ptr);
4389 if (r) {
4390 dev_warn(rdev->dev, "(%d) map RLC cp table bo failed\n", r);
4391 sumo_rlc_fini(rdev);
4392 return r;
4393 }
4394
4395 cik_init_cp_pg_table(rdev);
4396
4397 radeon_bo_kunmap(rdev->rlc.cp_table_obj);
4398 radeon_bo_unreserve(rdev->rlc.cp_table_obj);
4399
4400 }
4401
4402 return 0;
4403 }
4404
evergreen_rlc_start(struct radeon_device * rdev)4405 static void evergreen_rlc_start(struct radeon_device *rdev)
4406 {
4407 u32 mask = RLC_ENABLE;
4408
4409 if (rdev->flags & RADEON_IS_IGP) {
4410 mask |= GFX_POWER_GATING_ENABLE | GFX_POWER_GATING_SRC;
4411 }
4412
4413 WREG32(RLC_CNTL, mask);
4414 }
4415
evergreen_rlc_resume(struct radeon_device * rdev)4416 int evergreen_rlc_resume(struct radeon_device *rdev)
4417 {
4418 u32 i;
4419 const __be32 *fw_data;
4420
4421 if (!rdev->rlc_fw)
4422 return -EINVAL;
4423
4424 r600_rlc_stop(rdev);
4425
4426 WREG32(RLC_HB_CNTL, 0);
4427
4428 if (rdev->flags & RADEON_IS_IGP) {
4429 if (rdev->family == CHIP_ARUBA) {
4430 u32 always_on_bitmap =
4431 3 | (3 << (16 * rdev->config.cayman.max_shader_engines));
4432 /* find out the number of active simds */
4433 u32 tmp = (RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0xffff0000) >> 16;
4434 tmp |= 0xffffffff << rdev->config.cayman.max_simds_per_se;
4435 tmp = hweight32(~tmp);
4436 if (tmp == rdev->config.cayman.max_simds_per_se) {
4437 WREG32(TN_RLC_LB_ALWAYS_ACTIVE_SIMD_MASK, always_on_bitmap);
4438 WREG32(TN_RLC_LB_PARAMS, 0x00601004);
4439 WREG32(TN_RLC_LB_INIT_SIMD_MASK, 0xffffffff);
4440 WREG32(TN_RLC_LB_CNTR_INIT, 0x00000000);
4441 WREG32(TN_RLC_LB_CNTR_MAX, 0x00002000);
4442 }
4443 } else {
4444 WREG32(RLC_HB_WPTR_LSB_ADDR, 0);
4445 WREG32(RLC_HB_WPTR_MSB_ADDR, 0);
4446 }
4447 WREG32(TN_RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
4448 WREG32(TN_RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
4449 } else {
4450 WREG32(RLC_HB_BASE, 0);
4451 WREG32(RLC_HB_RPTR, 0);
4452 WREG32(RLC_HB_WPTR, 0);
4453 WREG32(RLC_HB_WPTR_LSB_ADDR, 0);
4454 WREG32(RLC_HB_WPTR_MSB_ADDR, 0);
4455 }
4456 WREG32(RLC_MC_CNTL, 0);
4457 WREG32(RLC_UCODE_CNTL, 0);
4458
4459 fw_data = (const __be32 *)rdev->rlc_fw->data;
4460 if (rdev->family >= CHIP_ARUBA) {
4461 for (i = 0; i < ARUBA_RLC_UCODE_SIZE; i++) {
4462 WREG32(RLC_UCODE_ADDR, i);
4463 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4464 }
4465 } else if (rdev->family >= CHIP_CAYMAN) {
4466 for (i = 0; i < CAYMAN_RLC_UCODE_SIZE; i++) {
4467 WREG32(RLC_UCODE_ADDR, i);
4468 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4469 }
4470 } else {
4471 for (i = 0; i < EVERGREEN_RLC_UCODE_SIZE; i++) {
4472 WREG32(RLC_UCODE_ADDR, i);
4473 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4474 }
4475 }
4476 WREG32(RLC_UCODE_ADDR, 0);
4477
4478 evergreen_rlc_start(rdev);
4479
4480 return 0;
4481 }
4482
4483 /* Interrupts */
4484
evergreen_get_vblank_counter(struct radeon_device * rdev,int crtc)4485 u32 evergreen_get_vblank_counter(struct radeon_device *rdev, int crtc)
4486 {
4487 if (crtc >= rdev->num_crtc)
4488 return 0;
4489 else
4490 return RREG32(CRTC_STATUS_FRAME_COUNT + crtc_offsets[crtc]);
4491 }
4492
evergreen_disable_interrupt_state(struct radeon_device * rdev)4493 void evergreen_disable_interrupt_state(struct radeon_device *rdev)
4494 {
4495 u32 tmp;
4496
4497 if (rdev->family >= CHIP_CAYMAN) {
4498 cayman_cp_int_cntl_setup(rdev, 0,
4499 CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4500 cayman_cp_int_cntl_setup(rdev, 1, 0);
4501 cayman_cp_int_cntl_setup(rdev, 2, 0);
4502 tmp = RREG32(CAYMAN_DMA1_CNTL) & ~TRAP_ENABLE;
4503 WREG32(CAYMAN_DMA1_CNTL, tmp);
4504 } else
4505 WREG32(CP_INT_CNTL, CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4506 tmp = RREG32(DMA_CNTL) & ~TRAP_ENABLE;
4507 WREG32(DMA_CNTL, tmp);
4508 WREG32(GRBM_INT_CNTL, 0);
4509 WREG32(SRBM_INT_CNTL, 0);
4510 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
4511 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
4512 if (rdev->num_crtc >= 4) {
4513 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
4514 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
4515 }
4516 if (rdev->num_crtc >= 6) {
4517 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
4518 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
4519 }
4520
4521 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
4522 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
4523 if (rdev->num_crtc >= 4) {
4524 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
4525 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
4526 }
4527 if (rdev->num_crtc >= 6) {
4528 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
4529 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
4530 }
4531
4532 /* only one DAC on DCE5 */
4533 if (!ASIC_IS_DCE5(rdev))
4534 WREG32(DACA_AUTODETECT_INT_CONTROL, 0);
4535 WREG32(DACB_AUTODETECT_INT_CONTROL, 0);
4536
4537 tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4538 WREG32(DC_HPD1_INT_CONTROL, tmp);
4539 tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4540 WREG32(DC_HPD2_INT_CONTROL, tmp);
4541 tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4542 WREG32(DC_HPD3_INT_CONTROL, tmp);
4543 tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4544 WREG32(DC_HPD4_INT_CONTROL, tmp);
4545 tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4546 WREG32(DC_HPD5_INT_CONTROL, tmp);
4547 tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4548 WREG32(DC_HPD6_INT_CONTROL, tmp);
4549
4550 }
4551
evergreen_irq_set(struct radeon_device * rdev)4552 int evergreen_irq_set(struct radeon_device *rdev)
4553 {
4554 u32 cp_int_cntl = CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE;
4555 u32 cp_int_cntl1 = 0, cp_int_cntl2 = 0;
4556 u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;
4557 u32 hpd1, hpd2, hpd3, hpd4, hpd5, hpd6;
4558 u32 grbm_int_cntl = 0;
4559 u32 afmt1 = 0, afmt2 = 0, afmt3 = 0, afmt4 = 0, afmt5 = 0, afmt6 = 0;
4560 u32 dma_cntl, dma_cntl1 = 0;
4561 u32 thermal_int = 0;
4562
4563 if (!rdev->irq.installed) {
4564 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
4565 return -EINVAL;
4566 }
4567 /* don't enable anything if the ih is disabled */
4568 if (!rdev->ih.enabled) {
4569 r600_disable_interrupts(rdev);
4570 /* force the active interrupt state to all disabled */
4571 evergreen_disable_interrupt_state(rdev);
4572 return 0;
4573 }
4574
4575 hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
4576 hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
4577 hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
4578 hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
4579 hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
4580 hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
4581 if (rdev->family == CHIP_ARUBA)
4582 thermal_int = RREG32(TN_CG_THERMAL_INT_CTRL) &
4583 ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
4584 else
4585 thermal_int = RREG32(CG_THERMAL_INT) &
4586 ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
4587
4588 afmt1 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4589 afmt2 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4590 afmt3 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4591 afmt4 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4592 afmt5 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4593 afmt6 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4594
4595 dma_cntl = RREG32(DMA_CNTL) & ~TRAP_ENABLE;
4596
4597 if (rdev->family >= CHIP_CAYMAN) {
4598 /* enable CP interrupts on all rings */
4599 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
4600 DRM_DEBUG("evergreen_irq_set: sw int gfx\n");
4601 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
4602 }
4603 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
4604 DRM_DEBUG("evergreen_irq_set: sw int cp1\n");
4605 cp_int_cntl1 |= TIME_STAMP_INT_ENABLE;
4606 }
4607 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
4608 DRM_DEBUG("evergreen_irq_set: sw int cp2\n");
4609 cp_int_cntl2 |= TIME_STAMP_INT_ENABLE;
4610 }
4611 } else {
4612 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
4613 DRM_DEBUG("evergreen_irq_set: sw int gfx\n");
4614 cp_int_cntl |= RB_INT_ENABLE;
4615 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
4616 }
4617 }
4618
4619 if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
4620 DRM_DEBUG("r600_irq_set: sw int dma\n");
4621 dma_cntl |= TRAP_ENABLE;
4622 }
4623
4624 if (rdev->family >= CHIP_CAYMAN) {
4625 dma_cntl1 = RREG32(CAYMAN_DMA1_CNTL) & ~TRAP_ENABLE;
4626 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
4627 DRM_DEBUG("r600_irq_set: sw int dma1\n");
4628 dma_cntl1 |= TRAP_ENABLE;
4629 }
4630 }
4631
4632 if (rdev->irq.dpm_thermal) {
4633 DRM_DEBUG("dpm thermal\n");
4634 thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW;
4635 }
4636
4637 if (rdev->irq.crtc_vblank_int[0] ||
4638 atomic_read(&rdev->irq.pflip[0])) {
4639 DRM_DEBUG("evergreen_irq_set: vblank 0\n");
4640 crtc1 |= VBLANK_INT_MASK;
4641 }
4642 if (rdev->irq.crtc_vblank_int[1] ||
4643 atomic_read(&rdev->irq.pflip[1])) {
4644 DRM_DEBUG("evergreen_irq_set: vblank 1\n");
4645 crtc2 |= VBLANK_INT_MASK;
4646 }
4647 if (rdev->irq.crtc_vblank_int[2] ||
4648 atomic_read(&rdev->irq.pflip[2])) {
4649 DRM_DEBUG("evergreen_irq_set: vblank 2\n");
4650 crtc3 |= VBLANK_INT_MASK;
4651 }
4652 if (rdev->irq.crtc_vblank_int[3] ||
4653 atomic_read(&rdev->irq.pflip[3])) {
4654 DRM_DEBUG("evergreen_irq_set: vblank 3\n");
4655 crtc4 |= VBLANK_INT_MASK;
4656 }
4657 if (rdev->irq.crtc_vblank_int[4] ||
4658 atomic_read(&rdev->irq.pflip[4])) {
4659 DRM_DEBUG("evergreen_irq_set: vblank 4\n");
4660 crtc5 |= VBLANK_INT_MASK;
4661 }
4662 if (rdev->irq.crtc_vblank_int[5] ||
4663 atomic_read(&rdev->irq.pflip[5])) {
4664 DRM_DEBUG("evergreen_irq_set: vblank 5\n");
4665 crtc6 |= VBLANK_INT_MASK;
4666 }
4667 if (rdev->irq.hpd[0]) {
4668 DRM_DEBUG("evergreen_irq_set: hpd 1\n");
4669 hpd1 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
4670 }
4671 if (rdev->irq.hpd[1]) {
4672 DRM_DEBUG("evergreen_irq_set: hpd 2\n");
4673 hpd2 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
4674 }
4675 if (rdev->irq.hpd[2]) {
4676 DRM_DEBUG("evergreen_irq_set: hpd 3\n");
4677 hpd3 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
4678 }
4679 if (rdev->irq.hpd[3]) {
4680 DRM_DEBUG("evergreen_irq_set: hpd 4\n");
4681 hpd4 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
4682 }
4683 if (rdev->irq.hpd[4]) {
4684 DRM_DEBUG("evergreen_irq_set: hpd 5\n");
4685 hpd5 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
4686 }
4687 if (rdev->irq.hpd[5]) {
4688 DRM_DEBUG("evergreen_irq_set: hpd 6\n");
4689 hpd6 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
4690 }
4691 if (rdev->irq.afmt[0]) {
4692 DRM_DEBUG("evergreen_irq_set: hdmi 0\n");
4693 afmt1 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4694 }
4695 if (rdev->irq.afmt[1]) {
4696 DRM_DEBUG("evergreen_irq_set: hdmi 1\n");
4697 afmt2 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4698 }
4699 if (rdev->irq.afmt[2]) {
4700 DRM_DEBUG("evergreen_irq_set: hdmi 2\n");
4701 afmt3 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4702 }
4703 if (rdev->irq.afmt[3]) {
4704 DRM_DEBUG("evergreen_irq_set: hdmi 3\n");
4705 afmt4 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4706 }
4707 if (rdev->irq.afmt[4]) {
4708 DRM_DEBUG("evergreen_irq_set: hdmi 4\n");
4709 afmt5 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4710 }
4711 if (rdev->irq.afmt[5]) {
4712 DRM_DEBUG("evergreen_irq_set: hdmi 5\n");
4713 afmt6 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4714 }
4715
4716 if (rdev->family >= CHIP_CAYMAN) {
4717 cayman_cp_int_cntl_setup(rdev, 0, cp_int_cntl);
4718 cayman_cp_int_cntl_setup(rdev, 1, cp_int_cntl1);
4719 cayman_cp_int_cntl_setup(rdev, 2, cp_int_cntl2);
4720 } else
4721 WREG32(CP_INT_CNTL, cp_int_cntl);
4722
4723 WREG32(DMA_CNTL, dma_cntl);
4724
4725 if (rdev->family >= CHIP_CAYMAN)
4726 WREG32(CAYMAN_DMA1_CNTL, dma_cntl1);
4727
4728 WREG32(GRBM_INT_CNTL, grbm_int_cntl);
4729
4730 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1);
4731 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2);
4732 if (rdev->num_crtc >= 4) {
4733 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3);
4734 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4);
4735 }
4736 if (rdev->num_crtc >= 6) {
4737 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5);
4738 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6);
4739 }
4740
4741 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET,
4742 GRPH_PFLIP_INT_MASK);
4743 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET,
4744 GRPH_PFLIP_INT_MASK);
4745 if (rdev->num_crtc >= 4) {
4746 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET,
4747 GRPH_PFLIP_INT_MASK);
4748 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET,
4749 GRPH_PFLIP_INT_MASK);
4750 }
4751 if (rdev->num_crtc >= 6) {
4752 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET,
4753 GRPH_PFLIP_INT_MASK);
4754 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET,
4755 GRPH_PFLIP_INT_MASK);
4756 }
4757
4758 WREG32(DC_HPD1_INT_CONTROL, hpd1);
4759 WREG32(DC_HPD2_INT_CONTROL, hpd2);
4760 WREG32(DC_HPD3_INT_CONTROL, hpd3);
4761 WREG32(DC_HPD4_INT_CONTROL, hpd4);
4762 WREG32(DC_HPD5_INT_CONTROL, hpd5);
4763 WREG32(DC_HPD6_INT_CONTROL, hpd6);
4764 if (rdev->family == CHIP_ARUBA)
4765 WREG32(TN_CG_THERMAL_INT_CTRL, thermal_int);
4766 else
4767 WREG32(CG_THERMAL_INT, thermal_int);
4768
4769 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, afmt1);
4770 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, afmt2);
4771 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, afmt3);
4772 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, afmt4);
4773 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, afmt5);
4774 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, afmt6);
4775
4776 /* posting read */
4777 RREG32(SRBM_STATUS);
4778
4779 return 0;
4780 }
4781
evergreen_irq_ack(struct radeon_device * rdev)4782 static void evergreen_irq_ack(struct radeon_device *rdev)
4783 {
4784 u32 tmp;
4785
4786 rdev->irq.stat_regs.evergreen.disp_int = RREG32(DISP_INTERRUPT_STATUS);
4787 rdev->irq.stat_regs.evergreen.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
4788 rdev->irq.stat_regs.evergreen.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2);
4789 rdev->irq.stat_regs.evergreen.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3);
4790 rdev->irq.stat_regs.evergreen.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4);
4791 rdev->irq.stat_regs.evergreen.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5);
4792 rdev->irq.stat_regs.evergreen.d1grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET);
4793 rdev->irq.stat_regs.evergreen.d2grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET);
4794 if (rdev->num_crtc >= 4) {
4795 rdev->irq.stat_regs.evergreen.d3grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET);
4796 rdev->irq.stat_regs.evergreen.d4grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET);
4797 }
4798 if (rdev->num_crtc >= 6) {
4799 rdev->irq.stat_regs.evergreen.d5grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET);
4800 rdev->irq.stat_regs.evergreen.d6grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET);
4801 }
4802
4803 rdev->irq.stat_regs.evergreen.afmt_status1 = RREG32(AFMT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET);
4804 rdev->irq.stat_regs.evergreen.afmt_status2 = RREG32(AFMT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET);
4805 rdev->irq.stat_regs.evergreen.afmt_status3 = RREG32(AFMT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET);
4806 rdev->irq.stat_regs.evergreen.afmt_status4 = RREG32(AFMT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET);
4807 rdev->irq.stat_regs.evergreen.afmt_status5 = RREG32(AFMT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET);
4808 rdev->irq.stat_regs.evergreen.afmt_status6 = RREG32(AFMT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET);
4809
4810 if (rdev->irq.stat_regs.evergreen.d1grph_int & GRPH_PFLIP_INT_OCCURRED)
4811 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4812 if (rdev->irq.stat_regs.evergreen.d2grph_int & GRPH_PFLIP_INT_OCCURRED)
4813 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4814 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT)
4815 WREG32(VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK);
4816 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT)
4817 WREG32(VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK);
4818 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT)
4819 WREG32(VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK);
4820 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT)
4821 WREG32(VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK);
4822
4823 if (rdev->num_crtc >= 4) {
4824 if (rdev->irq.stat_regs.evergreen.d3grph_int & GRPH_PFLIP_INT_OCCURRED)
4825 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4826 if (rdev->irq.stat_regs.evergreen.d4grph_int & GRPH_PFLIP_INT_OCCURRED)
4827 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4828 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)
4829 WREG32(VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK);
4830 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)
4831 WREG32(VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK);
4832 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)
4833 WREG32(VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK);
4834 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)
4835 WREG32(VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK);
4836 }
4837
4838 if (rdev->num_crtc >= 6) {
4839 if (rdev->irq.stat_regs.evergreen.d5grph_int & GRPH_PFLIP_INT_OCCURRED)
4840 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4841 if (rdev->irq.stat_regs.evergreen.d6grph_int & GRPH_PFLIP_INT_OCCURRED)
4842 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4843 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)
4844 WREG32(VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK);
4845 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)
4846 WREG32(VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK);
4847 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)
4848 WREG32(VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK);
4849 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)
4850 WREG32(VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK);
4851 }
4852
4853 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
4854 tmp = RREG32(DC_HPD1_INT_CONTROL);
4855 tmp |= DC_HPDx_INT_ACK;
4856 WREG32(DC_HPD1_INT_CONTROL, tmp);
4857 }
4858 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
4859 tmp = RREG32(DC_HPD2_INT_CONTROL);
4860 tmp |= DC_HPDx_INT_ACK;
4861 WREG32(DC_HPD2_INT_CONTROL, tmp);
4862 }
4863 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
4864 tmp = RREG32(DC_HPD3_INT_CONTROL);
4865 tmp |= DC_HPDx_INT_ACK;
4866 WREG32(DC_HPD3_INT_CONTROL, tmp);
4867 }
4868 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
4869 tmp = RREG32(DC_HPD4_INT_CONTROL);
4870 tmp |= DC_HPDx_INT_ACK;
4871 WREG32(DC_HPD4_INT_CONTROL, tmp);
4872 }
4873 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
4874 tmp = RREG32(DC_HPD5_INT_CONTROL);
4875 tmp |= DC_HPDx_INT_ACK;
4876 WREG32(DC_HPD5_INT_CONTROL, tmp);
4877 }
4878 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
4879 tmp = RREG32(DC_HPD5_INT_CONTROL);
4880 tmp |= DC_HPDx_INT_ACK;
4881 WREG32(DC_HPD6_INT_CONTROL, tmp);
4882 }
4883
4884 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_RX_INTERRUPT) {
4885 tmp = RREG32(DC_HPD1_INT_CONTROL);
4886 tmp |= DC_HPDx_RX_INT_ACK;
4887 WREG32(DC_HPD1_INT_CONTROL, tmp);
4888 }
4889 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_RX_INTERRUPT) {
4890 tmp = RREG32(DC_HPD2_INT_CONTROL);
4891 tmp |= DC_HPDx_RX_INT_ACK;
4892 WREG32(DC_HPD2_INT_CONTROL, tmp);
4893 }
4894 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_RX_INTERRUPT) {
4895 tmp = RREG32(DC_HPD3_INT_CONTROL);
4896 tmp |= DC_HPDx_RX_INT_ACK;
4897 WREG32(DC_HPD3_INT_CONTROL, tmp);
4898 }
4899 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_RX_INTERRUPT) {
4900 tmp = RREG32(DC_HPD4_INT_CONTROL);
4901 tmp |= DC_HPDx_RX_INT_ACK;
4902 WREG32(DC_HPD4_INT_CONTROL, tmp);
4903 }
4904 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_RX_INTERRUPT) {
4905 tmp = RREG32(DC_HPD5_INT_CONTROL);
4906 tmp |= DC_HPDx_RX_INT_ACK;
4907 WREG32(DC_HPD5_INT_CONTROL, tmp);
4908 }
4909 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_RX_INTERRUPT) {
4910 tmp = RREG32(DC_HPD5_INT_CONTROL);
4911 tmp |= DC_HPDx_RX_INT_ACK;
4912 WREG32(DC_HPD6_INT_CONTROL, tmp);
4913 }
4914
4915 if (rdev->irq.stat_regs.evergreen.afmt_status1 & AFMT_AZ_FORMAT_WTRIG) {
4916 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET);
4917 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4918 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, tmp);
4919 }
4920 if (rdev->irq.stat_regs.evergreen.afmt_status2 & AFMT_AZ_FORMAT_WTRIG) {
4921 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET);
4922 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4923 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, tmp);
4924 }
4925 if (rdev->irq.stat_regs.evergreen.afmt_status3 & AFMT_AZ_FORMAT_WTRIG) {
4926 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET);
4927 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4928 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, tmp);
4929 }
4930 if (rdev->irq.stat_regs.evergreen.afmt_status4 & AFMT_AZ_FORMAT_WTRIG) {
4931 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET);
4932 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4933 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, tmp);
4934 }
4935 if (rdev->irq.stat_regs.evergreen.afmt_status5 & AFMT_AZ_FORMAT_WTRIG) {
4936 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET);
4937 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4938 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, tmp);
4939 }
4940 if (rdev->irq.stat_regs.evergreen.afmt_status6 & AFMT_AZ_FORMAT_WTRIG) {
4941 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET);
4942 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4943 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, tmp);
4944 }
4945 }
4946
evergreen_irq_disable(struct radeon_device * rdev)4947 static void evergreen_irq_disable(struct radeon_device *rdev)
4948 {
4949 r600_disable_interrupts(rdev);
4950 /* Wait and acknowledge irq */
4951 mdelay(1);
4952 evergreen_irq_ack(rdev);
4953 evergreen_disable_interrupt_state(rdev);
4954 }
4955
evergreen_irq_suspend(struct radeon_device * rdev)4956 void evergreen_irq_suspend(struct radeon_device *rdev)
4957 {
4958 evergreen_irq_disable(rdev);
4959 r600_rlc_stop(rdev);
4960 }
4961
evergreen_get_ih_wptr(struct radeon_device * rdev)4962 static u32 evergreen_get_ih_wptr(struct radeon_device *rdev)
4963 {
4964 u32 wptr, tmp;
4965
4966 if (rdev->wb.enabled)
4967 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
4968 else
4969 wptr = RREG32(IH_RB_WPTR);
4970
4971 if (wptr & RB_OVERFLOW) {
4972 wptr &= ~RB_OVERFLOW;
4973 /* When a ring buffer overflow happen start parsing interrupt
4974 * from the last not overwritten vector (wptr + 16). Hopefully
4975 * this should allow us to catchup.
4976 */
4977 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, 0x%08X, 0x%08X)\n",
4978 wptr, rdev->ih.rptr, (wptr + 16) & rdev->ih.ptr_mask);
4979 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
4980 tmp = RREG32(IH_RB_CNTL);
4981 tmp |= IH_WPTR_OVERFLOW_CLEAR;
4982 WREG32(IH_RB_CNTL, tmp);
4983 }
4984 return (wptr & rdev->ih.ptr_mask);
4985 }
4986
evergreen_irq_process(struct radeon_device * rdev)4987 int evergreen_irq_process(struct radeon_device *rdev)
4988 {
4989 u32 wptr;
4990 u32 rptr;
4991 u32 src_id, src_data;
4992 u32 ring_index;
4993 bool queue_hotplug = false;
4994 bool queue_hdmi = false;
4995 bool queue_dp = false;
4996 bool queue_thermal = false;
4997 u32 status, addr;
4998
4999 if (!rdev->ih.enabled || rdev->shutdown)
5000 return IRQ_NONE;
5001
5002 wptr = evergreen_get_ih_wptr(rdev);
5003
5004 restart_ih:
5005 /* is somebody else already processing irqs? */
5006 if (atomic_xchg(&rdev->ih.lock, 1))
5007 return IRQ_NONE;
5008
5009 rptr = rdev->ih.rptr;
5010 DRM_DEBUG("evergreen_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
5011
5012 /* Order reading of wptr vs. reading of IH ring data */
5013 rmb();
5014
5015 /* display interrupts */
5016 evergreen_irq_ack(rdev);
5017
5018 while (rptr != wptr) {
5019 /* wptr/rptr are in bytes! */
5020 ring_index = rptr / 4;
5021 src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
5022 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
5023
5024 switch (src_id) {
5025 case 1: /* D1 vblank/vline */
5026 switch (src_data) {
5027 case 0: /* D1 vblank */
5028 if (!(rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT))
5029 DRM_DEBUG("IH: D1 vblank - IH event w/o asserted irq bit?\n");
5030
5031 if (rdev->irq.crtc_vblank_int[0]) {
5032 drm_handle_vblank(rdev->ddev, 0);
5033 rdev->pm.vblank_sync = true;
5034 wake_up(&rdev->irq.vblank_queue);
5035 }
5036 if (atomic_read(&rdev->irq.pflip[0]))
5037 radeon_crtc_handle_vblank(rdev, 0);
5038 rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
5039 DRM_DEBUG("IH: D1 vblank\n");
5040
5041 break;
5042 case 1: /* D1 vline */
5043 if (!(rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT))
5044 DRM_DEBUG("IH: D1 vline - IH event w/o asserted irq bit?\n");
5045
5046 rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VLINE_INTERRUPT;
5047 DRM_DEBUG("IH: D1 vline\n");
5048
5049 break;
5050 default:
5051 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5052 break;
5053 }
5054 break;
5055 case 2: /* D2 vblank/vline */
5056 switch (src_data) {
5057 case 0: /* D2 vblank */
5058 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT))
5059 DRM_DEBUG("IH: D2 vblank - IH event w/o asserted irq bit?\n");
5060
5061 if (rdev->irq.crtc_vblank_int[1]) {
5062 drm_handle_vblank(rdev->ddev, 1);
5063 rdev->pm.vblank_sync = true;
5064 wake_up(&rdev->irq.vblank_queue);
5065 }
5066 if (atomic_read(&rdev->irq.pflip[1]))
5067 radeon_crtc_handle_vblank(rdev, 1);
5068 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT;
5069 DRM_DEBUG("IH: D2 vblank\n");
5070
5071 break;
5072 case 1: /* D2 vline */
5073 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT))
5074 DRM_DEBUG("IH: D2 vline - IH event w/o asserted irq bit?\n");
5075
5076 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT;
5077 DRM_DEBUG("IH: D2 vline\n");
5078
5079 break;
5080 default:
5081 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5082 break;
5083 }
5084 break;
5085 case 3: /* D3 vblank/vline */
5086 switch (src_data) {
5087 case 0: /* D3 vblank */
5088 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT))
5089 DRM_DEBUG("IH: D3 vblank - IH event w/o asserted irq bit?\n");
5090
5091 if (rdev->irq.crtc_vblank_int[2]) {
5092 drm_handle_vblank(rdev->ddev, 2);
5093 rdev->pm.vblank_sync = true;
5094 wake_up(&rdev->irq.vblank_queue);
5095 }
5096 if (atomic_read(&rdev->irq.pflip[2]))
5097 radeon_crtc_handle_vblank(rdev, 2);
5098 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT;
5099 DRM_DEBUG("IH: D3 vblank\n");
5100
5101 break;
5102 case 1: /* D3 vline */
5103 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT))
5104 DRM_DEBUG("IH: D3 vline - IH event w/o asserted irq bit?\n");
5105
5106 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT;
5107 DRM_DEBUG("IH: D3 vline\n");
5108
5109 break;
5110 default:
5111 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5112 break;
5113 }
5114 break;
5115 case 4: /* D4 vblank/vline */
5116 switch (src_data) {
5117 case 0: /* D4 vblank */
5118 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT))
5119 DRM_DEBUG("IH: D4 vblank - IH event w/o asserted irq bit?\n");
5120
5121 if (rdev->irq.crtc_vblank_int[3]) {
5122 drm_handle_vblank(rdev->ddev, 3);
5123 rdev->pm.vblank_sync = true;
5124 wake_up(&rdev->irq.vblank_queue);
5125 }
5126 if (atomic_read(&rdev->irq.pflip[3]))
5127 radeon_crtc_handle_vblank(rdev, 3);
5128 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT;
5129 DRM_DEBUG("IH: D4 vblank\n");
5130
5131 break;
5132 case 1: /* D4 vline */
5133 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT))
5134 DRM_DEBUG("IH: D4 vline - IH event w/o asserted irq bit?\n");
5135
5136 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT;
5137 DRM_DEBUG("IH: D4 vline\n");
5138
5139 break;
5140 default:
5141 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5142 break;
5143 }
5144 break;
5145 case 5: /* D5 vblank/vline */
5146 switch (src_data) {
5147 case 0: /* D5 vblank */
5148 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT))
5149 DRM_DEBUG("IH: D5 vblank - IH event w/o asserted irq bit?\n");
5150
5151 if (rdev->irq.crtc_vblank_int[4]) {
5152 drm_handle_vblank(rdev->ddev, 4);
5153 rdev->pm.vblank_sync = true;
5154 wake_up(&rdev->irq.vblank_queue);
5155 }
5156 if (atomic_read(&rdev->irq.pflip[4]))
5157 radeon_crtc_handle_vblank(rdev, 4);
5158 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT;
5159 DRM_DEBUG("IH: D5 vblank\n");
5160
5161 break;
5162 case 1: /* D5 vline */
5163 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT))
5164 DRM_DEBUG("IH: D5 vline - IH event w/o asserted irq bit?\n");
5165
5166 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT;
5167 DRM_DEBUG("IH: D5 vline\n");
5168
5169 break;
5170 default:
5171 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5172 break;
5173 }
5174 break;
5175 case 6: /* D6 vblank/vline */
5176 switch (src_data) {
5177 case 0: /* D6 vblank */
5178 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT))
5179 DRM_DEBUG("IH: D6 vblank - IH event w/o asserted irq bit?\n");
5180
5181 if (rdev->irq.crtc_vblank_int[5]) {
5182 drm_handle_vblank(rdev->ddev, 5);
5183 rdev->pm.vblank_sync = true;
5184 wake_up(&rdev->irq.vblank_queue);
5185 }
5186 if (atomic_read(&rdev->irq.pflip[5]))
5187 radeon_crtc_handle_vblank(rdev, 5);
5188 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT;
5189 DRM_DEBUG("IH: D6 vblank\n");
5190
5191 break;
5192 case 1: /* D6 vline */
5193 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT))
5194 DRM_DEBUG("IH: D6 vline - IH event w/o asserted irq bit?\n");
5195
5196 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT;
5197 DRM_DEBUG("IH: D6 vline\n");
5198
5199 break;
5200 default:
5201 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5202 break;
5203 }
5204 break;
5205 case 8: /* D1 page flip */
5206 case 10: /* D2 page flip */
5207 case 12: /* D3 page flip */
5208 case 14: /* D4 page flip */
5209 case 16: /* D5 page flip */
5210 case 18: /* D6 page flip */
5211 DRM_DEBUG("IH: D%d flip\n", ((src_id - 8) >> 1) + 1);
5212 if (radeon_use_pflipirq > 0)
5213 radeon_crtc_handle_flip(rdev, (src_id - 8) >> 1);
5214 break;
5215 case 42: /* HPD hotplug */
5216 switch (src_data) {
5217 case 0:
5218 if (!(rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT))
5219 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5220
5221 rdev->irq.stat_regs.evergreen.disp_int &= ~DC_HPD1_INTERRUPT;
5222 queue_hotplug = true;
5223 DRM_DEBUG("IH: HPD1\n");
5224 break;
5225 case 1:
5226 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT))
5227 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5228
5229 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~DC_HPD2_INTERRUPT;
5230 queue_hotplug = true;
5231 DRM_DEBUG("IH: HPD2\n");
5232 break;
5233 case 2:
5234 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT))
5235 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5236
5237 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~DC_HPD3_INTERRUPT;
5238 queue_hotplug = true;
5239 DRM_DEBUG("IH: HPD3\n");
5240 break;
5241 case 3:
5242 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT))
5243 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5244
5245 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~DC_HPD4_INTERRUPT;
5246 queue_hotplug = true;
5247 DRM_DEBUG("IH: HPD4\n");
5248 break;
5249 case 4:
5250 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT))
5251 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5252
5253 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~DC_HPD5_INTERRUPT;
5254 queue_hotplug = true;
5255 DRM_DEBUG("IH: HPD5\n");
5256 break;
5257 case 5:
5258 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT))
5259 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5260
5261 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~DC_HPD6_INTERRUPT;
5262 queue_hotplug = true;
5263 DRM_DEBUG("IH: HPD6\n");
5264 break;
5265 case 6:
5266 if (!(rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_RX_INTERRUPT))
5267 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5268
5269 rdev->irq.stat_regs.evergreen.disp_int &= ~DC_HPD1_RX_INTERRUPT;
5270 queue_dp = true;
5271 DRM_DEBUG("IH: HPD_RX 1\n");
5272 break;
5273 case 7:
5274 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_RX_INTERRUPT))
5275 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5276
5277 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~DC_HPD2_RX_INTERRUPT;
5278 queue_dp = true;
5279 DRM_DEBUG("IH: HPD_RX 2\n");
5280 break;
5281 case 8:
5282 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_RX_INTERRUPT))
5283 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5284
5285 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~DC_HPD3_RX_INTERRUPT;
5286 queue_dp = true;
5287 DRM_DEBUG("IH: HPD_RX 3\n");
5288 break;
5289 case 9:
5290 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_RX_INTERRUPT))
5291 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5292
5293 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~DC_HPD4_RX_INTERRUPT;
5294 queue_dp = true;
5295 DRM_DEBUG("IH: HPD_RX 4\n");
5296 break;
5297 case 10:
5298 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_RX_INTERRUPT))
5299 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5300
5301 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~DC_HPD5_RX_INTERRUPT;
5302 queue_dp = true;
5303 DRM_DEBUG("IH: HPD_RX 5\n");
5304 break;
5305 case 11:
5306 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_RX_INTERRUPT))
5307 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5308
5309 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~DC_HPD6_RX_INTERRUPT;
5310 queue_dp = true;
5311 DRM_DEBUG("IH: HPD_RX 6\n");
5312 break;
5313 default:
5314 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5315 break;
5316 }
5317 break;
5318 case 44: /* hdmi */
5319 switch (src_data) {
5320 case 0:
5321 if (!(rdev->irq.stat_regs.evergreen.afmt_status1 & AFMT_AZ_FORMAT_WTRIG))
5322 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5323
5324 rdev->irq.stat_regs.evergreen.afmt_status1 &= ~AFMT_AZ_FORMAT_WTRIG;
5325 queue_hdmi = true;
5326 DRM_DEBUG("IH: HDMI0\n");
5327 break;
5328 case 1:
5329 if (!(rdev->irq.stat_regs.evergreen.afmt_status2 & AFMT_AZ_FORMAT_WTRIG))
5330 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5331
5332 rdev->irq.stat_regs.evergreen.afmt_status2 &= ~AFMT_AZ_FORMAT_WTRIG;
5333 queue_hdmi = true;
5334 DRM_DEBUG("IH: HDMI1\n");
5335 break;
5336 case 2:
5337 if (!(rdev->irq.stat_regs.evergreen.afmt_status3 & AFMT_AZ_FORMAT_WTRIG))
5338 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5339
5340 rdev->irq.stat_regs.evergreen.afmt_status3 &= ~AFMT_AZ_FORMAT_WTRIG;
5341 queue_hdmi = true;
5342 DRM_DEBUG("IH: HDMI2\n");
5343 break;
5344 case 3:
5345 if (!(rdev->irq.stat_regs.evergreen.afmt_status4 & AFMT_AZ_FORMAT_WTRIG))
5346 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5347
5348 rdev->irq.stat_regs.evergreen.afmt_status4 &= ~AFMT_AZ_FORMAT_WTRIG;
5349 queue_hdmi = true;
5350 DRM_DEBUG("IH: HDMI3\n");
5351 break;
5352 case 4:
5353 if (!(rdev->irq.stat_regs.evergreen.afmt_status5 & AFMT_AZ_FORMAT_WTRIG))
5354 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5355
5356 rdev->irq.stat_regs.evergreen.afmt_status5 &= ~AFMT_AZ_FORMAT_WTRIG;
5357 queue_hdmi = true;
5358 DRM_DEBUG("IH: HDMI4\n");
5359 break;
5360 case 5:
5361 if (!(rdev->irq.stat_regs.evergreen.afmt_status6 & AFMT_AZ_FORMAT_WTRIG))
5362 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
5363
5364 rdev->irq.stat_regs.evergreen.afmt_status6 &= ~AFMT_AZ_FORMAT_WTRIG;
5365 queue_hdmi = true;
5366 DRM_DEBUG("IH: HDMI5\n");
5367 break;
5368 default:
5369 DRM_ERROR("Unhandled interrupt: %d %d\n", src_id, src_data);
5370 break;
5371 }
5372 case 96:
5373 DRM_ERROR("SRBM_READ_ERROR: 0x%x\n", RREG32(SRBM_READ_ERROR));
5374 WREG32(SRBM_INT_ACK, 0x1);
5375 break;
5376 case 124: /* UVD */
5377 DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data);
5378 radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX);
5379 break;
5380 case 146:
5381 case 147:
5382 addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR);
5383 status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS);
5384 /* reset addr and status */
5385 WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
5386 if (addr == 0x0 && status == 0x0)
5387 break;
5388 dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
5389 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
5390 addr);
5391 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
5392 status);
5393 cayman_vm_decode_fault(rdev, status, addr);
5394 break;
5395 case 176: /* CP_INT in ring buffer */
5396 case 177: /* CP_INT in IB1 */
5397 case 178: /* CP_INT in IB2 */
5398 DRM_DEBUG("IH: CP int: 0x%08x\n", src_data);
5399 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
5400 break;
5401 case 181: /* CP EOP event */
5402 DRM_DEBUG("IH: CP EOP\n");
5403 if (rdev->family >= CHIP_CAYMAN) {
5404 switch (src_data) {
5405 case 0:
5406 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
5407 break;
5408 case 1:
5409 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
5410 break;
5411 case 2:
5412 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
5413 break;
5414 }
5415 } else
5416 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
5417 break;
5418 case 224: /* DMA trap event */
5419 DRM_DEBUG("IH: DMA trap\n");
5420 radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
5421 break;
5422 case 230: /* thermal low to high */
5423 DRM_DEBUG("IH: thermal low to high\n");
5424 rdev->pm.dpm.thermal.high_to_low = false;
5425 queue_thermal = true;
5426 break;
5427 case 231: /* thermal high to low */
5428 DRM_DEBUG("IH: thermal high to low\n");
5429 rdev->pm.dpm.thermal.high_to_low = true;
5430 queue_thermal = true;
5431 break;
5432 case 233: /* GUI IDLE */
5433 DRM_DEBUG("IH: GUI idle\n");
5434 break;
5435 case 244: /* DMA trap event */
5436 if (rdev->family >= CHIP_CAYMAN) {
5437 DRM_DEBUG("IH: DMA1 trap\n");
5438 radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
5439 }
5440 break;
5441 default:
5442 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5443 break;
5444 }
5445
5446 /* wptr/rptr are in bytes! */
5447 rptr += 16;
5448 rptr &= rdev->ih.ptr_mask;
5449 WREG32(IH_RB_RPTR, rptr);
5450 }
5451 if (queue_dp)
5452 schedule_work(&rdev->dp_work);
5453 if (queue_hotplug)
5454 schedule_work(&rdev->hotplug_work);
5455 if (queue_hdmi)
5456 schedule_work(&rdev->audio_work);
5457 if (queue_thermal && rdev->pm.dpm_enabled)
5458 schedule_work(&rdev->pm.dpm.thermal.work);
5459 rdev->ih.rptr = rptr;
5460 atomic_set(&rdev->ih.lock, 0);
5461
5462 /* make sure wptr hasn't changed while processing */
5463 wptr = evergreen_get_ih_wptr(rdev);
5464 if (wptr != rptr)
5465 goto restart_ih;
5466
5467 return IRQ_HANDLED;
5468 }
5469
evergreen_startup(struct radeon_device * rdev)5470 static int evergreen_startup(struct radeon_device *rdev)
5471 {
5472 struct radeon_ring *ring;
5473 int r;
5474
5475 /* enable pcie gen2 link */
5476 evergreen_pcie_gen2_enable(rdev);
5477 /* enable aspm */
5478 evergreen_program_aspm(rdev);
5479
5480 /* scratch needs to be initialized before MC */
5481 r = r600_vram_scratch_init(rdev);
5482 if (r)
5483 return r;
5484
5485 evergreen_mc_program(rdev);
5486
5487 if (ASIC_IS_DCE5(rdev) && !rdev->pm.dpm_enabled) {
5488 r = ni_mc_load_microcode(rdev);
5489 if (r) {
5490 DRM_ERROR("Failed to load MC firmware!\n");
5491 return r;
5492 }
5493 }
5494
5495 if (rdev->flags & RADEON_IS_AGP) {
5496 evergreen_agp_enable(rdev);
5497 } else {
5498 r = evergreen_pcie_gart_enable(rdev);
5499 if (r)
5500 return r;
5501 }
5502 evergreen_gpu_init(rdev);
5503
5504 /* allocate rlc buffers */
5505 if (rdev->flags & RADEON_IS_IGP) {
5506 rdev->rlc.reg_list = sumo_rlc_save_restore_register_list;
5507 rdev->rlc.reg_list_size =
5508 (u32)ARRAY_SIZE(sumo_rlc_save_restore_register_list);
5509 rdev->rlc.cs_data = evergreen_cs_data;
5510 r = sumo_rlc_init(rdev);
5511 if (r) {
5512 DRM_ERROR("Failed to init rlc BOs!\n");
5513 return r;
5514 }
5515 }
5516
5517 /* allocate wb buffer */
5518 r = radeon_wb_init(rdev);
5519 if (r)
5520 return r;
5521
5522 r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
5523 if (r) {
5524 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
5525 return r;
5526 }
5527
5528 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
5529 if (r) {
5530 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
5531 return r;
5532 }
5533
5534 r = uvd_v2_2_resume(rdev);
5535 if (!r) {
5536 r = radeon_fence_driver_start_ring(rdev,
5537 R600_RING_TYPE_UVD_INDEX);
5538 if (r)
5539 dev_err(rdev->dev, "UVD fences init error (%d).\n", r);
5540 }
5541
5542 if (r)
5543 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
5544
5545 /* Enable IRQ */
5546 if (!rdev->irq.installed) {
5547 r = radeon_irq_kms_init(rdev);
5548 if (r)
5549 return r;
5550 }
5551
5552 r = r600_irq_init(rdev);
5553 if (r) {
5554 DRM_ERROR("radeon: IH init failed (%d).\n", r);
5555 radeon_irq_kms_fini(rdev);
5556 return r;
5557 }
5558 evergreen_irq_set(rdev);
5559
5560 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
5561 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
5562 RADEON_CP_PACKET2);
5563 if (r)
5564 return r;
5565
5566 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
5567 r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
5568 DMA_PACKET(DMA_PACKET_NOP, 0, 0));
5569 if (r)
5570 return r;
5571
5572 r = evergreen_cp_load_microcode(rdev);
5573 if (r)
5574 return r;
5575 r = evergreen_cp_resume(rdev);
5576 if (r)
5577 return r;
5578 r = r600_dma_resume(rdev);
5579 if (r)
5580 return r;
5581
5582 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
5583 if (ring->ring_size) {
5584 r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
5585 RADEON_CP_PACKET2);
5586 if (!r)
5587 r = uvd_v1_0_init(rdev);
5588
5589 if (r)
5590 DRM_ERROR("radeon: error initializing UVD (%d).\n", r);
5591 }
5592
5593 r = radeon_ib_pool_init(rdev);
5594 if (r) {
5595 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
5596 return r;
5597 }
5598
5599 r = radeon_audio_init(rdev);
5600 if (r) {
5601 DRM_ERROR("radeon: audio init failed\n");
5602 return r;
5603 }
5604
5605 return 0;
5606 }
5607
evergreen_resume(struct radeon_device * rdev)5608 int evergreen_resume(struct radeon_device *rdev)
5609 {
5610 int r;
5611
5612 /* reset the asic, the gfx blocks are often in a bad state
5613 * after the driver is unloaded or after a resume
5614 */
5615 if (radeon_asic_reset(rdev))
5616 dev_warn(rdev->dev, "GPU reset failed !\n");
5617 /* Do not reset GPU before posting, on rv770 hw unlike on r500 hw,
5618 * posting will perform necessary task to bring back GPU into good
5619 * shape.
5620 */
5621 /* post card */
5622 atom_asic_init(rdev->mode_info.atom_context);
5623
5624 /* init golden registers */
5625 evergreen_init_golden_registers(rdev);
5626
5627 if (rdev->pm.pm_method == PM_METHOD_DPM)
5628 radeon_pm_resume(rdev);
5629
5630 rdev->accel_working = true;
5631 r = evergreen_startup(rdev);
5632 if (r) {
5633 DRM_ERROR("evergreen startup failed on resume\n");
5634 rdev->accel_working = false;
5635 return r;
5636 }
5637
5638 return r;
5639
5640 }
5641
evergreen_suspend(struct radeon_device * rdev)5642 int evergreen_suspend(struct radeon_device *rdev)
5643 {
5644 radeon_pm_suspend(rdev);
5645 radeon_audio_fini(rdev);
5646 uvd_v1_0_fini(rdev);
5647 radeon_uvd_suspend(rdev);
5648 r700_cp_stop(rdev);
5649 r600_dma_stop(rdev);
5650 evergreen_irq_suspend(rdev);
5651 radeon_wb_disable(rdev);
5652 evergreen_pcie_gart_disable(rdev);
5653
5654 return 0;
5655 }
5656
5657 /* Plan is to move initialization in that function and use
5658 * helper function so that radeon_device_init pretty much
5659 * do nothing more than calling asic specific function. This
5660 * should also allow to remove a bunch of callback function
5661 * like vram_info.
5662 */
evergreen_init(struct radeon_device * rdev)5663 int evergreen_init(struct radeon_device *rdev)
5664 {
5665 int r;
5666
5667 /* Read BIOS */
5668 if (!radeon_get_bios(rdev)) {
5669 if (ASIC_IS_AVIVO(rdev))
5670 return -EINVAL;
5671 }
5672 /* Must be an ATOMBIOS */
5673 if (!rdev->is_atom_bios) {
5674 dev_err(rdev->dev, "Expecting atombios for evergreen GPU\n");
5675 return -EINVAL;
5676 }
5677 r = radeon_atombios_init(rdev);
5678 if (r)
5679 return r;
5680 /* reset the asic, the gfx blocks are often in a bad state
5681 * after the driver is unloaded or after a resume
5682 */
5683 if (radeon_asic_reset(rdev))
5684 dev_warn(rdev->dev, "GPU reset failed !\n");
5685 /* Post card if necessary */
5686 if (!radeon_card_posted(rdev)) {
5687 if (!rdev->bios) {
5688 dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
5689 return -EINVAL;
5690 }
5691 DRM_INFO("GPU not posted. posting now...\n");
5692 atom_asic_init(rdev->mode_info.atom_context);
5693 }
5694 /* init golden registers */
5695 evergreen_init_golden_registers(rdev);
5696 /* Initialize scratch registers */
5697 r600_scratch_init(rdev);
5698 /* Initialize surface registers */
5699 radeon_surface_init(rdev);
5700 /* Initialize clocks */
5701 radeon_get_clock_info(rdev->ddev);
5702 /* Fence driver */
5703 r = radeon_fence_driver_init(rdev);
5704 if (r)
5705 return r;
5706 /* initialize AGP */
5707 if (rdev->flags & RADEON_IS_AGP) {
5708 r = radeon_agp_init(rdev);
5709 if (r)
5710 radeon_agp_disable(rdev);
5711 }
5712 /* initialize memory controller */
5713 r = evergreen_mc_init(rdev);
5714 if (r)
5715 return r;
5716 /* Memory manager */
5717 r = radeon_bo_init(rdev);
5718 if (r)
5719 return r;
5720
5721 if (ASIC_IS_DCE5(rdev)) {
5722 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw || !rdev->mc_fw) {
5723 r = ni_init_microcode(rdev);
5724 if (r) {
5725 DRM_ERROR("Failed to load firmware!\n");
5726 return r;
5727 }
5728 }
5729 } else {
5730 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) {
5731 r = r600_init_microcode(rdev);
5732 if (r) {
5733 DRM_ERROR("Failed to load firmware!\n");
5734 return r;
5735 }
5736 }
5737 }
5738
5739 /* Initialize power management */
5740 radeon_pm_init(rdev);
5741
5742 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ring_obj = NULL;
5743 r600_ring_init(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX], 1024 * 1024);
5744
5745 rdev->ring[R600_RING_TYPE_DMA_INDEX].ring_obj = NULL;
5746 r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_DMA_INDEX], 64 * 1024);
5747
5748 r = radeon_uvd_init(rdev);
5749 if (!r) {
5750 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_obj = NULL;
5751 r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_UVD_INDEX],
5752 4096);
5753 }
5754
5755 rdev->ih.ring_obj = NULL;
5756 r600_ih_ring_init(rdev, 64 * 1024);
5757
5758 r = r600_pcie_gart_init(rdev);
5759 if (r)
5760 return r;
5761
5762 rdev->accel_working = true;
5763 r = evergreen_startup(rdev);
5764 if (r) {
5765 dev_err(rdev->dev, "disabling GPU acceleration\n");
5766 r700_cp_fini(rdev);
5767 r600_dma_fini(rdev);
5768 r600_irq_fini(rdev);
5769 if (rdev->flags & RADEON_IS_IGP)
5770 sumo_rlc_fini(rdev);
5771 radeon_wb_fini(rdev);
5772 radeon_ib_pool_fini(rdev);
5773 radeon_irq_kms_fini(rdev);
5774 evergreen_pcie_gart_fini(rdev);
5775 rdev->accel_working = false;
5776 }
5777
5778 /* Don't start up if the MC ucode is missing on BTC parts.
5779 * The default clocks and voltages before the MC ucode
5780 * is loaded are not suffient for advanced operations.
5781 */
5782 if (ASIC_IS_DCE5(rdev)) {
5783 if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) {
5784 DRM_ERROR("radeon: MC ucode required for NI+.\n");
5785 return -EINVAL;
5786 }
5787 }
5788
5789 return 0;
5790 }
5791
evergreen_fini(struct radeon_device * rdev)5792 void evergreen_fini(struct radeon_device *rdev)
5793 {
5794 radeon_pm_fini(rdev);
5795 radeon_audio_fini(rdev);
5796 r700_cp_fini(rdev);
5797 r600_dma_fini(rdev);
5798 r600_irq_fini(rdev);
5799 if (rdev->flags & RADEON_IS_IGP)
5800 sumo_rlc_fini(rdev);
5801 radeon_wb_fini(rdev);
5802 radeon_ib_pool_fini(rdev);
5803 radeon_irq_kms_fini(rdev);
5804 uvd_v1_0_fini(rdev);
5805 radeon_uvd_fini(rdev);
5806 evergreen_pcie_gart_fini(rdev);
5807 r600_vram_scratch_fini(rdev);
5808 radeon_gem_fini(rdev);
5809 radeon_fence_driver_fini(rdev);
5810 radeon_agp_fini(rdev);
5811 radeon_bo_fini(rdev);
5812 radeon_atombios_fini(rdev);
5813 kfree(rdev->bios);
5814 rdev->bios = NULL;
5815 }
5816
evergreen_pcie_gen2_enable(struct radeon_device * rdev)5817 void evergreen_pcie_gen2_enable(struct radeon_device *rdev)
5818 {
5819 u32 link_width_cntl, speed_cntl;
5820
5821 if (radeon_pcie_gen2 == 0)
5822 return;
5823
5824 if (rdev->flags & RADEON_IS_IGP)
5825 return;
5826
5827 if (!(rdev->flags & RADEON_IS_PCIE))
5828 return;
5829
5830 /* x2 cards have a special sequence */
5831 if (ASIC_IS_X2(rdev))
5832 return;
5833
5834 if ((rdev->pdev->bus->max_bus_speed != PCIE_SPEED_5_0GT) &&
5835 (rdev->pdev->bus->max_bus_speed != PCIE_SPEED_8_0GT))
5836 return;
5837
5838 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5839 if (speed_cntl & LC_CURRENT_DATA_RATE) {
5840 DRM_INFO("PCIE gen 2 link speeds already enabled\n");
5841 return;
5842 }
5843
5844 DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
5845
5846 if ((speed_cntl & LC_OTHER_SIDE_EVER_SENT_GEN2) ||
5847 (speed_cntl & LC_OTHER_SIDE_SUPPORTS_GEN2)) {
5848
5849 link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
5850 link_width_cntl &= ~LC_UPCONFIGURE_DIS;
5851 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
5852
5853 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5854 speed_cntl &= ~LC_TARGET_LINK_SPEED_OVERRIDE_EN;
5855 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5856
5857 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5858 speed_cntl |= LC_CLR_FAILED_SPD_CHANGE_CNT;
5859 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5860
5861 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5862 speed_cntl &= ~LC_CLR_FAILED_SPD_CHANGE_CNT;
5863 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5864
5865 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5866 speed_cntl |= LC_GEN2_EN_STRAP;
5867 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5868
5869 } else {
5870 link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
5871 /* XXX: only disable it if gen1 bridge vendor == 0x111d or 0x1106 */
5872 if (1)
5873 link_width_cntl |= LC_UPCONFIGURE_DIS;
5874 else
5875 link_width_cntl &= ~LC_UPCONFIGURE_DIS;
5876 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
5877 }
5878 }
5879
evergreen_program_aspm(struct radeon_device * rdev)5880 void evergreen_program_aspm(struct radeon_device *rdev)
5881 {
5882 u32 data, orig;
5883 u32 pcie_lc_cntl, pcie_lc_cntl_old;
5884 bool disable_l0s, disable_l1 = false, disable_plloff_in_l1 = false;
5885 /* fusion_platform = true
5886 * if the system is a fusion system
5887 * (APU or DGPU in a fusion system).
5888 * todo: check if the system is a fusion platform.
5889 */
5890 bool fusion_platform = false;
5891
5892 if (radeon_aspm == 0)
5893 return;
5894
5895 if (!(rdev->flags & RADEON_IS_PCIE))
5896 return;
5897
5898 switch (rdev->family) {
5899 case CHIP_CYPRESS:
5900 case CHIP_HEMLOCK:
5901 case CHIP_JUNIPER:
5902 case CHIP_REDWOOD:
5903 case CHIP_CEDAR:
5904 case CHIP_SUMO:
5905 case CHIP_SUMO2:
5906 case CHIP_PALM:
5907 case CHIP_ARUBA:
5908 disable_l0s = true;
5909 break;
5910 default:
5911 disable_l0s = false;
5912 break;
5913 }
5914
5915 if (rdev->flags & RADEON_IS_IGP)
5916 fusion_platform = true; /* XXX also dGPUs in a fusion system */
5917
5918 data = orig = RREG32_PIF_PHY0(PB0_PIF_PAIRING);
5919 if (fusion_platform)
5920 data &= ~MULTI_PIF;
5921 else
5922 data |= MULTI_PIF;
5923 if (data != orig)
5924 WREG32_PIF_PHY0(PB0_PIF_PAIRING, data);
5925
5926 data = orig = RREG32_PIF_PHY1(PB1_PIF_PAIRING);
5927 if (fusion_platform)
5928 data &= ~MULTI_PIF;
5929 else
5930 data |= MULTI_PIF;
5931 if (data != orig)
5932 WREG32_PIF_PHY1(PB1_PIF_PAIRING, data);
5933
5934 pcie_lc_cntl = pcie_lc_cntl_old = RREG32_PCIE_PORT(PCIE_LC_CNTL);
5935 pcie_lc_cntl &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK);
5936 if (!disable_l0s) {
5937 if (rdev->family >= CHIP_BARTS)
5938 pcie_lc_cntl |= LC_L0S_INACTIVITY(7);
5939 else
5940 pcie_lc_cntl |= LC_L0S_INACTIVITY(3);
5941 }
5942
5943 if (!disable_l1) {
5944 if (rdev->family >= CHIP_BARTS)
5945 pcie_lc_cntl |= LC_L1_INACTIVITY(7);
5946 else
5947 pcie_lc_cntl |= LC_L1_INACTIVITY(8);
5948
5949 if (!disable_plloff_in_l1) {
5950 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
5951 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
5952 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
5953 if (data != orig)
5954 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
5955
5956 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
5957 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
5958 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
5959 if (data != orig)
5960 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
5961
5962 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
5963 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
5964 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
5965 if (data != orig)
5966 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
5967
5968 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
5969 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
5970 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
5971 if (data != orig)
5972 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
5973
5974 if (rdev->family >= CHIP_BARTS) {
5975 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
5976 data &= ~PLL_RAMP_UP_TIME_0_MASK;
5977 data |= PLL_RAMP_UP_TIME_0(4);
5978 if (data != orig)
5979 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
5980
5981 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
5982 data &= ~PLL_RAMP_UP_TIME_1_MASK;
5983 data |= PLL_RAMP_UP_TIME_1(4);
5984 if (data != orig)
5985 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
5986
5987 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
5988 data &= ~PLL_RAMP_UP_TIME_0_MASK;
5989 data |= PLL_RAMP_UP_TIME_0(4);
5990 if (data != orig)
5991 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
5992
5993 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
5994 data &= ~PLL_RAMP_UP_TIME_1_MASK;
5995 data |= PLL_RAMP_UP_TIME_1(4);
5996 if (data != orig)
5997 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
5998 }
5999
6000 data = orig = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
6001 data &= ~LC_DYN_LANES_PWR_STATE_MASK;
6002 data |= LC_DYN_LANES_PWR_STATE(3);
6003 if (data != orig)
6004 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
6005
6006 if (rdev->family >= CHIP_BARTS) {
6007 data = orig = RREG32_PIF_PHY0(PB0_PIF_CNTL);
6008 data &= ~LS2_EXIT_TIME_MASK;
6009 data |= LS2_EXIT_TIME(1);
6010 if (data != orig)
6011 WREG32_PIF_PHY0(PB0_PIF_CNTL, data);
6012
6013 data = orig = RREG32_PIF_PHY1(PB1_PIF_CNTL);
6014 data &= ~LS2_EXIT_TIME_MASK;
6015 data |= LS2_EXIT_TIME(1);
6016 if (data != orig)
6017 WREG32_PIF_PHY1(PB1_PIF_CNTL, data);
6018 }
6019 }
6020 }
6021
6022 /* evergreen parts only */
6023 if (rdev->family < CHIP_BARTS)
6024 pcie_lc_cntl |= LC_PMI_TO_L1_DIS;
6025
6026 if (pcie_lc_cntl != pcie_lc_cntl_old)
6027 WREG32_PCIE_PORT(PCIE_LC_CNTL, pcie_lc_cntl);
6028 }
6029