This source file includes following definitions.
- __enable_icache_msr
- __disable_icache_msr
- __enable_dcache_msr
- __disable_dcache_msr
- __enable_icache_nomsr
- __disable_icache_nomsr
- __enable_dcache_nomsr
- __disable_dcache_nomsr
- __flush_icache_range_msr_irq
- __flush_icache_range_nomsr_irq
- __flush_icache_range_noirq
- __flush_icache_all_msr_irq
- __flush_icache_all_nomsr_irq
- __flush_icache_all_noirq
- __invalidate_dcache_all_msr_irq
- __invalidate_dcache_all_nomsr_irq
- __invalidate_dcache_all_noirq_wt
- __invalidate_dcache_all_wb
- __invalidate_dcache_range_wb
- __invalidate_dcache_range_nomsr_wt
- __invalidate_dcache_range_msr_irq_wt
- __invalidate_dcache_range_nomsr_irq
- __flush_dcache_all_wb
- __flush_dcache_range_wb
- microblaze_cache_init
1
2
3
4
5
6
7
8
9
10
11
12
13 #include <asm/cacheflush.h>
14 #include <linux/cache.h>
15 #include <asm/cpuinfo.h>
16 #include <asm/pvr.h>
17
18 static inline void __enable_icache_msr(void)
19 {
20 __asm__ __volatile__ (" msrset r0, %0;" \
21 "nop;" \
22 : : "i" (MSR_ICE) : "memory");
23 }
24
25 static inline void __disable_icache_msr(void)
26 {
27 __asm__ __volatile__ (" msrclr r0, %0;" \
28 "nop;" \
29 : : "i" (MSR_ICE) : "memory");
30 }
31
32 static inline void __enable_dcache_msr(void)
33 {
34 __asm__ __volatile__ (" msrset r0, %0;" \
35 "nop;" \
36 : : "i" (MSR_DCE) : "memory");
37 }
38
39 static inline void __disable_dcache_msr(void)
40 {
41 __asm__ __volatile__ (" msrclr r0, %0;" \
42 "nop; " \
43 : : "i" (MSR_DCE) : "memory");
44 }
45
46 static inline void __enable_icache_nomsr(void)
47 {
48 __asm__ __volatile__ (" mfs r12, rmsr;" \
49 "nop;" \
50 "ori r12, r12, %0;" \
51 "mts rmsr, r12;" \
52 "nop;" \
53 : : "i" (MSR_ICE) : "memory", "r12");
54 }
55
56 static inline void __disable_icache_nomsr(void)
57 {
58 __asm__ __volatile__ (" mfs r12, rmsr;" \
59 "nop;" \
60 "andi r12, r12, ~%0;" \
61 "mts rmsr, r12;" \
62 "nop;" \
63 : : "i" (MSR_ICE) : "memory", "r12");
64 }
65
66 static inline void __enable_dcache_nomsr(void)
67 {
68 __asm__ __volatile__ (" mfs r12, rmsr;" \
69 "nop;" \
70 "ori r12, r12, %0;" \
71 "mts rmsr, r12;" \
72 "nop;" \
73 : : "i" (MSR_DCE) : "memory", "r12");
74 }
75
76 static inline void __disable_dcache_nomsr(void)
77 {
78 __asm__ __volatile__ (" mfs r12, rmsr;" \
79 "nop;" \
80 "andi r12, r12, ~%0;" \
81 "mts rmsr, r12;" \
82 "nop;" \
83 : : "i" (MSR_DCE) : "memory", "r12");
84 }
85
86
87
88
89
90
91
92 #define CACHE_LOOP_LIMITS(start, end, cache_line_length, cache_size) \
93 do { \
94 int align = ~(cache_line_length - 1); \
95 if (start < UINT_MAX - cache_size) \
96 end = min(start + cache_size, end); \
97 start &= align; \
98 } while (0)
99
100
101
102
103
104 #define CACHE_ALL_LOOP(cache_size, line_length, op) \
105 do { \
106 unsigned int len = cache_size - line_length; \
107 int step = -line_length; \
108 WARN_ON(step >= 0); \
109 \
110 __asm__ __volatile__ (" 1: " #op " %0, r0;" \
111 "bgtid %0, 1b;" \
112 "addk %0, %0, %1;" \
113 : : "r" (len), "r" (step) \
114 : "memory"); \
115 } while (0)
116
117
118
119
120
121
122
123
124
125 #define CACHE_RANGE_LOOP_2(start, end, line_length, op) \
126 do { \
127 int step = -line_length; \
128 int align = ~(line_length - 1); \
129 int count; \
130 end = ((end & align) == end) ? end - line_length : end & align; \
131 count = end - start; \
132 WARN_ON(count < 0); \
133 \
134 __asm__ __volatile__ (" 1: " #op " %0, %1;" \
135 "bgtid %1, 1b;" \
136 "addk %1, %1, %2;" \
137 : : "r" (start), "r" (count), \
138 "r" (step) : "memory"); \
139 } while (0)
140
141
142 #define CACHE_RANGE_LOOP_1(start, end, line_length, op) \
143 do { \
144 unsigned int volatile temp = 0; \
145 unsigned int align = ~(line_length - 1); \
146 end = ((end & align) == end) ? end - line_length : end & align; \
147 WARN_ON(end < start); \
148 \
149 __asm__ __volatile__ (" 1: " #op " %1, r0;" \
150 "cmpu %0, %1, %2;" \
151 "bgtid %0, 1b;" \
152 "addk %1, %1, %3;" \
153 : : "r" (temp), "r" (start), "r" (end), \
154 "r" (line_length) : "memory"); \
155 } while (0)
156
157 #define ASM_LOOP
158
159 static void __flush_icache_range_msr_irq(unsigned long start, unsigned long end)
160 {
161 unsigned long flags;
162 #ifndef ASM_LOOP
163 int i;
164 #endif
165 pr_debug("%s: start 0x%x, end 0x%x\n", __func__,
166 (unsigned int)start, (unsigned int) end);
167
168 CACHE_LOOP_LIMITS(start, end,
169 cpuinfo.icache_line_length, cpuinfo.icache_size);
170
171 local_irq_save(flags);
172 __disable_icache_msr();
173
174 #ifdef ASM_LOOP
175 CACHE_RANGE_LOOP_1(start, end, cpuinfo.icache_line_length, wic);
176 #else
177 for (i = start; i < end; i += cpuinfo.icache_line_length)
178 __asm__ __volatile__ ("wic %0, r0;" \
179 : : "r" (i));
180 #endif
181 __enable_icache_msr();
182 local_irq_restore(flags);
183 }
184
185 static void __flush_icache_range_nomsr_irq(unsigned long start,
186 unsigned long end)
187 {
188 unsigned long flags;
189 #ifndef ASM_LOOP
190 int i;
191 #endif
192 pr_debug("%s: start 0x%x, end 0x%x\n", __func__,
193 (unsigned int)start, (unsigned int) end);
194
195 CACHE_LOOP_LIMITS(start, end,
196 cpuinfo.icache_line_length, cpuinfo.icache_size);
197
198 local_irq_save(flags);
199 __disable_icache_nomsr();
200
201 #ifdef ASM_LOOP
202 CACHE_RANGE_LOOP_1(start, end, cpuinfo.icache_line_length, wic);
203 #else
204 for (i = start; i < end; i += cpuinfo.icache_line_length)
205 __asm__ __volatile__ ("wic %0, r0;" \
206 : : "r" (i));
207 #endif
208
209 __enable_icache_nomsr();
210 local_irq_restore(flags);
211 }
212
213 static void __flush_icache_range_noirq(unsigned long start,
214 unsigned long end)
215 {
216 #ifndef ASM_LOOP
217 int i;
218 #endif
219 pr_debug("%s: start 0x%x, end 0x%x\n", __func__,
220 (unsigned int)start, (unsigned int) end);
221
222 CACHE_LOOP_LIMITS(start, end,
223 cpuinfo.icache_line_length, cpuinfo.icache_size);
224 #ifdef ASM_LOOP
225 CACHE_RANGE_LOOP_1(start, end, cpuinfo.icache_line_length, wic);
226 #else
227 for (i = start; i < end; i += cpuinfo.icache_line_length)
228 __asm__ __volatile__ ("wic %0, r0;" \
229 : : "r" (i));
230 #endif
231 }
232
233 static void __flush_icache_all_msr_irq(void)
234 {
235 unsigned long flags;
236 #ifndef ASM_LOOP
237 int i;
238 #endif
239 pr_debug("%s\n", __func__);
240
241 local_irq_save(flags);
242 __disable_icache_msr();
243 #ifdef ASM_LOOP
244 CACHE_ALL_LOOP(cpuinfo.icache_size, cpuinfo.icache_line_length, wic);
245 #else
246 for (i = 0; i < cpuinfo.icache_size;
247 i += cpuinfo.icache_line_length)
248 __asm__ __volatile__ ("wic %0, r0;" \
249 : : "r" (i));
250 #endif
251 __enable_icache_msr();
252 local_irq_restore(flags);
253 }
254
255 static void __flush_icache_all_nomsr_irq(void)
256 {
257 unsigned long flags;
258 #ifndef ASM_LOOP
259 int i;
260 #endif
261 pr_debug("%s\n", __func__);
262
263 local_irq_save(flags);
264 __disable_icache_nomsr();
265 #ifdef ASM_LOOP
266 CACHE_ALL_LOOP(cpuinfo.icache_size, cpuinfo.icache_line_length, wic);
267 #else
268 for (i = 0; i < cpuinfo.icache_size;
269 i += cpuinfo.icache_line_length)
270 __asm__ __volatile__ ("wic %0, r0;" \
271 : : "r" (i));
272 #endif
273 __enable_icache_nomsr();
274 local_irq_restore(flags);
275 }
276
277 static void __flush_icache_all_noirq(void)
278 {
279 #ifndef ASM_LOOP
280 int i;
281 #endif
282 pr_debug("%s\n", __func__);
283 #ifdef ASM_LOOP
284 CACHE_ALL_LOOP(cpuinfo.icache_size, cpuinfo.icache_line_length, wic);
285 #else
286 for (i = 0; i < cpuinfo.icache_size;
287 i += cpuinfo.icache_line_length)
288 __asm__ __volatile__ ("wic %0, r0;" \
289 : : "r" (i));
290 #endif
291 }
292
293 static void __invalidate_dcache_all_msr_irq(void)
294 {
295 unsigned long flags;
296 #ifndef ASM_LOOP
297 int i;
298 #endif
299 pr_debug("%s\n", __func__);
300
301 local_irq_save(flags);
302 __disable_dcache_msr();
303 #ifdef ASM_LOOP
304 CACHE_ALL_LOOP(cpuinfo.dcache_size, cpuinfo.dcache_line_length, wdc);
305 #else
306 for (i = 0; i < cpuinfo.dcache_size;
307 i += cpuinfo.dcache_line_length)
308 __asm__ __volatile__ ("wdc %0, r0;" \
309 : : "r" (i));
310 #endif
311 __enable_dcache_msr();
312 local_irq_restore(flags);
313 }
314
315 static void __invalidate_dcache_all_nomsr_irq(void)
316 {
317 unsigned long flags;
318 #ifndef ASM_LOOP
319 int i;
320 #endif
321 pr_debug("%s\n", __func__);
322
323 local_irq_save(flags);
324 __disable_dcache_nomsr();
325 #ifdef ASM_LOOP
326 CACHE_ALL_LOOP(cpuinfo.dcache_size, cpuinfo.dcache_line_length, wdc);
327 #else
328 for (i = 0; i < cpuinfo.dcache_size;
329 i += cpuinfo.dcache_line_length)
330 __asm__ __volatile__ ("wdc %0, r0;" \
331 : : "r" (i));
332 #endif
333 __enable_dcache_nomsr();
334 local_irq_restore(flags);
335 }
336
337 static void __invalidate_dcache_all_noirq_wt(void)
338 {
339 #ifndef ASM_LOOP
340 int i;
341 #endif
342 pr_debug("%s\n", __func__);
343 #ifdef ASM_LOOP
344 CACHE_ALL_LOOP(cpuinfo.dcache_size, cpuinfo.dcache_line_length, wdc);
345 #else
346 for (i = 0; i < cpuinfo.dcache_size;
347 i += cpuinfo.dcache_line_length)
348 __asm__ __volatile__ ("wdc %0, r0;" \
349 : : "r" (i));
350 #endif
351 }
352
353
354
355
356
357
358
359
360 static void __invalidate_dcache_all_wb(void)
361 {
362 #ifndef ASM_LOOP
363 int i;
364 #endif
365 pr_debug("%s\n", __func__);
366 #ifdef ASM_LOOP
367 CACHE_ALL_LOOP(cpuinfo.dcache_size, cpuinfo.dcache_line_length,
368 wdc);
369 #else
370 for (i = 0; i < cpuinfo.dcache_size;
371 i += cpuinfo.dcache_line_length)
372 __asm__ __volatile__ ("wdc %0, r0;" \
373 : : "r" (i));
374 #endif
375 }
376
377 static void __invalidate_dcache_range_wb(unsigned long start,
378 unsigned long end)
379 {
380 #ifndef ASM_LOOP
381 int i;
382 #endif
383 pr_debug("%s: start 0x%x, end 0x%x\n", __func__,
384 (unsigned int)start, (unsigned int) end);
385
386 CACHE_LOOP_LIMITS(start, end,
387 cpuinfo.dcache_line_length, cpuinfo.dcache_size);
388 #ifdef ASM_LOOP
389 CACHE_RANGE_LOOP_2(start, end, cpuinfo.dcache_line_length, wdc.clear);
390 #else
391 for (i = start; i < end; i += cpuinfo.dcache_line_length)
392 __asm__ __volatile__ ("wdc.clear %0, r0;" \
393 : : "r" (i));
394 #endif
395 }
396
397 static void __invalidate_dcache_range_nomsr_wt(unsigned long start,
398 unsigned long end)
399 {
400 #ifndef ASM_LOOP
401 int i;
402 #endif
403 pr_debug("%s: start 0x%x, end 0x%x\n", __func__,
404 (unsigned int)start, (unsigned int) end);
405 CACHE_LOOP_LIMITS(start, end,
406 cpuinfo.dcache_line_length, cpuinfo.dcache_size);
407
408 #ifdef ASM_LOOP
409 CACHE_RANGE_LOOP_1(start, end, cpuinfo.dcache_line_length, wdc);
410 #else
411 for (i = start; i < end; i += cpuinfo.dcache_line_length)
412 __asm__ __volatile__ ("wdc %0, r0;" \
413 : : "r" (i));
414 #endif
415 }
416
417 static void __invalidate_dcache_range_msr_irq_wt(unsigned long start,
418 unsigned long end)
419 {
420 unsigned long flags;
421 #ifndef ASM_LOOP
422 int i;
423 #endif
424 pr_debug("%s: start 0x%x, end 0x%x\n", __func__,
425 (unsigned int)start, (unsigned int) end);
426 CACHE_LOOP_LIMITS(start, end,
427 cpuinfo.dcache_line_length, cpuinfo.dcache_size);
428
429 local_irq_save(flags);
430 __disable_dcache_msr();
431
432 #ifdef ASM_LOOP
433 CACHE_RANGE_LOOP_1(start, end, cpuinfo.dcache_line_length, wdc);
434 #else
435 for (i = start; i < end; i += cpuinfo.dcache_line_length)
436 __asm__ __volatile__ ("wdc %0, r0;" \
437 : : "r" (i));
438 #endif
439
440 __enable_dcache_msr();
441 local_irq_restore(flags);
442 }
443
444 static void __invalidate_dcache_range_nomsr_irq(unsigned long start,
445 unsigned long end)
446 {
447 unsigned long flags;
448 #ifndef ASM_LOOP
449 int i;
450 #endif
451 pr_debug("%s: start 0x%x, end 0x%x\n", __func__,
452 (unsigned int)start, (unsigned int) end);
453
454 CACHE_LOOP_LIMITS(start, end,
455 cpuinfo.dcache_line_length, cpuinfo.dcache_size);
456
457 local_irq_save(flags);
458 __disable_dcache_nomsr();
459
460 #ifdef ASM_LOOP
461 CACHE_RANGE_LOOP_1(start, end, cpuinfo.dcache_line_length, wdc);
462 #else
463 for (i = start; i < end; i += cpuinfo.dcache_line_length)
464 __asm__ __volatile__ ("wdc %0, r0;" \
465 : : "r" (i));
466 #endif
467
468 __enable_dcache_nomsr();
469 local_irq_restore(flags);
470 }
471
472 static void __flush_dcache_all_wb(void)
473 {
474 #ifndef ASM_LOOP
475 int i;
476 #endif
477 pr_debug("%s\n", __func__);
478 #ifdef ASM_LOOP
479 CACHE_ALL_LOOP(cpuinfo.dcache_size, cpuinfo.dcache_line_length,
480 wdc.flush);
481 #else
482 for (i = 0; i < cpuinfo.dcache_size;
483 i += cpuinfo.dcache_line_length)
484 __asm__ __volatile__ ("wdc.flush %0, r0;" \
485 : : "r" (i));
486 #endif
487 }
488
489 static void __flush_dcache_range_wb(unsigned long start, unsigned long end)
490 {
491 #ifndef ASM_LOOP
492 int i;
493 #endif
494 pr_debug("%s: start 0x%x, end 0x%x\n", __func__,
495 (unsigned int)start, (unsigned int) end);
496
497 CACHE_LOOP_LIMITS(start, end,
498 cpuinfo.dcache_line_length, cpuinfo.dcache_size);
499 #ifdef ASM_LOOP
500 CACHE_RANGE_LOOP_2(start, end, cpuinfo.dcache_line_length, wdc.flush);
501 #else
502 for (i = start; i < end; i += cpuinfo.dcache_line_length)
503 __asm__ __volatile__ ("wdc.flush %0, r0;" \
504 : : "r" (i));
505 #endif
506 }
507
508
509 struct scache *mbc;
510
511
512 static const struct scache wb_msr = {
513 .ie = __enable_icache_msr,
514 .id = __disable_icache_msr,
515 .ifl = __flush_icache_all_noirq,
516 .iflr = __flush_icache_range_noirq,
517 .iin = __flush_icache_all_noirq,
518 .iinr = __flush_icache_range_noirq,
519 .de = __enable_dcache_msr,
520 .dd = __disable_dcache_msr,
521 .dfl = __flush_dcache_all_wb,
522 .dflr = __flush_dcache_range_wb,
523 .din = __invalidate_dcache_all_wb,
524 .dinr = __invalidate_dcache_range_wb,
525 };
526
527
528 static const struct scache wb_nomsr = {
529 .ie = __enable_icache_nomsr,
530 .id = __disable_icache_nomsr,
531 .ifl = __flush_icache_all_noirq,
532 .iflr = __flush_icache_range_noirq,
533 .iin = __flush_icache_all_noirq,
534 .iinr = __flush_icache_range_noirq,
535 .de = __enable_dcache_nomsr,
536 .dd = __disable_dcache_nomsr,
537 .dfl = __flush_dcache_all_wb,
538 .dflr = __flush_dcache_range_wb,
539 .din = __invalidate_dcache_all_wb,
540 .dinr = __invalidate_dcache_range_wb,
541 };
542
543
544 static const struct scache wt_msr = {
545 .ie = __enable_icache_msr,
546 .id = __disable_icache_msr,
547 .ifl = __flush_icache_all_msr_irq,
548 .iflr = __flush_icache_range_msr_irq,
549 .iin = __flush_icache_all_msr_irq,
550 .iinr = __flush_icache_range_msr_irq,
551 .de = __enable_dcache_msr,
552 .dd = __disable_dcache_msr,
553 .dfl = __invalidate_dcache_all_msr_irq,
554 .dflr = __invalidate_dcache_range_msr_irq_wt,
555 .din = __invalidate_dcache_all_msr_irq,
556 .dinr = __invalidate_dcache_range_msr_irq_wt,
557 };
558
559 static const struct scache wt_nomsr = {
560 .ie = __enable_icache_nomsr,
561 .id = __disable_icache_nomsr,
562 .ifl = __flush_icache_all_nomsr_irq,
563 .iflr = __flush_icache_range_nomsr_irq,
564 .iin = __flush_icache_all_nomsr_irq,
565 .iinr = __flush_icache_range_nomsr_irq,
566 .de = __enable_dcache_nomsr,
567 .dd = __disable_dcache_nomsr,
568 .dfl = __invalidate_dcache_all_nomsr_irq,
569 .dflr = __invalidate_dcache_range_nomsr_irq,
570 .din = __invalidate_dcache_all_nomsr_irq,
571 .dinr = __invalidate_dcache_range_nomsr_irq,
572 };
573
574
575 static const struct scache wt_msr_noirq = {
576 .ie = __enable_icache_msr,
577 .id = __disable_icache_msr,
578 .ifl = __flush_icache_all_noirq,
579 .iflr = __flush_icache_range_noirq,
580 .iin = __flush_icache_all_noirq,
581 .iinr = __flush_icache_range_noirq,
582 .de = __enable_dcache_msr,
583 .dd = __disable_dcache_msr,
584 .dfl = __invalidate_dcache_all_noirq_wt,
585 .dflr = __invalidate_dcache_range_nomsr_wt,
586 .din = __invalidate_dcache_all_noirq_wt,
587 .dinr = __invalidate_dcache_range_nomsr_wt,
588 };
589
590 static const struct scache wt_nomsr_noirq = {
591 .ie = __enable_icache_nomsr,
592 .id = __disable_icache_nomsr,
593 .ifl = __flush_icache_all_noirq,
594 .iflr = __flush_icache_range_noirq,
595 .iin = __flush_icache_all_noirq,
596 .iinr = __flush_icache_range_noirq,
597 .de = __enable_dcache_nomsr,
598 .dd = __disable_dcache_nomsr,
599 .dfl = __invalidate_dcache_all_noirq_wt,
600 .dflr = __invalidate_dcache_range_nomsr_wt,
601 .din = __invalidate_dcache_all_noirq_wt,
602 .dinr = __invalidate_dcache_range_nomsr_wt,
603 };
604
605
606 #define CPUVER_7_20_A 0x0c
607 #define CPUVER_7_20_D 0x0f
608
609 void microblaze_cache_init(void)
610 {
611 if (cpuinfo.use_instr & PVR2_USE_MSR_INSTR) {
612 if (cpuinfo.dcache_wb) {
613 pr_info("wb_msr\n");
614 mbc = (struct scache *)&wb_msr;
615 if (cpuinfo.ver_code <= CPUVER_7_20_D) {
616
617 pr_info("WB won't work properly\n");
618 }
619 } else {
620 if (cpuinfo.ver_code >= CPUVER_7_20_A) {
621 pr_info("wt_msr_noirq\n");
622 mbc = (struct scache *)&wt_msr_noirq;
623 } else {
624 pr_info("wt_msr\n");
625 mbc = (struct scache *)&wt_msr;
626 }
627 }
628 } else {
629 if (cpuinfo.dcache_wb) {
630 pr_info("wb_nomsr\n");
631 mbc = (struct scache *)&wb_nomsr;
632 if (cpuinfo.ver_code <= CPUVER_7_20_D) {
633
634 pr_info("WB won't work properly\n");
635 }
636 } else {
637 if (cpuinfo.ver_code >= CPUVER_7_20_A) {
638 pr_info("wt_nomsr_noirq\n");
639 mbc = (struct scache *)&wt_nomsr_noirq;
640 } else {
641 pr_info("wt_nomsr\n");
642 mbc = (struct scache *)&wt_nomsr;
643 }
644 }
645 }
646
647
648
649
650
651
652 enable_dcache();
653
654 invalidate_icache();
655 enable_icache();
656 }