1/* 2 * This file is subject to the terms and conditions of the GNU General Public 3 * License. See the file "COPYING" in the main directory of this archive 4 * for more details. 5 * 6 * Copyright (C) 2003, 04, 05 Ralf Baechle (ralf@linux-mips.org) 7 * Copyright (C) 2007 Maciej W. Rozycki 8 * Copyright (C) 2008 Thiemo Seufer 9 * Copyright (C) 2012 MIPS Technologies, Inc. 10 */ 11#include <linux/kernel.h> 12#include <linux/sched.h> 13#include <linux/smp.h> 14#include <linux/mm.h> 15#include <linux/module.h> 16#include <linux/proc_fs.h> 17 18#include <asm/bugs.h> 19#include <asm/cacheops.h> 20#include <asm/cpu-type.h> 21#include <asm/inst.h> 22#include <asm/io.h> 23#include <asm/page.h> 24#include <asm/pgtable.h> 25#include <asm/prefetch.h> 26#include <asm/bootinfo.h> 27#include <asm/mipsregs.h> 28#include <asm/mmu_context.h> 29#include <asm/cpu.h> 30#include <asm/war.h> 31 32#ifdef CONFIG_SIBYTE_DMA_PAGEOPS 33#include <asm/sibyte/sb1250.h> 34#include <asm/sibyte/sb1250_regs.h> 35#include <asm/sibyte/sb1250_dma.h> 36#endif 37 38#include <asm/uasm.h> 39 40/* Registers used in the assembled routines. */ 41#define ZERO 0 42#define AT 2 43#define A0 4 44#define A1 5 45#define A2 6 46#define T0 8 47#define T1 9 48#define T2 10 49#define T3 11 50#define T9 25 51#define RA 31 52 53/* Handle labels (which must be positive integers). */ 54enum label_id { 55 label_clear_nopref = 1, 56 label_clear_pref, 57 label_copy_nopref, 58 label_copy_pref_both, 59 label_copy_pref_store, 60}; 61 62UASM_L_LA(_clear_nopref) 63UASM_L_LA(_clear_pref) 64UASM_L_LA(_copy_nopref) 65UASM_L_LA(_copy_pref_both) 66UASM_L_LA(_copy_pref_store) 67 68/* We need one branch and therefore one relocation per target label. */ 69static struct uasm_label labels[5]; 70static struct uasm_reloc relocs[5]; 71 72#define cpu_is_r4600_v1_x() ((read_c0_prid() & 0xfffffff0) == 0x00002010) 73#define cpu_is_r4600_v2_x() ((read_c0_prid() & 0xfffffff0) == 0x00002020) 74 75/* 76 * R6 has a limited offset of the pref instruction. 77 * Skip it if the offset is more than 9 bits. 78 */ 79#define _uasm_i_pref(a, b, c, d) \ 80do { \ 81 if (cpu_has_mips_r6) { \ 82 if (c <= 0xff && c >= -0x100) \ 83 uasm_i_pref(a, b, c, d);\ 84 } else { \ 85 uasm_i_pref(a, b, c, d); \ 86 } \ 87} while(0) 88 89static int pref_bias_clear_store; 90static int pref_bias_copy_load; 91static int pref_bias_copy_store; 92 93static u32 pref_src_mode; 94static u32 pref_dst_mode; 95 96static int clear_word_size; 97static int copy_word_size; 98 99static int half_clear_loop_size; 100static int half_copy_loop_size; 101 102static int cache_line_size; 103#define cache_line_mask() (cache_line_size - 1) 104 105static inline void 106pg_addiu(u32 **buf, unsigned int reg1, unsigned int reg2, unsigned int off) 107{ 108 if (cpu_has_64bit_gp_regs && DADDI_WAR && r4k_daddiu_bug()) { 109 if (off > 0x7fff) { 110 uasm_i_lui(buf, T9, uasm_rel_hi(off)); 111 uasm_i_addiu(buf, T9, T9, uasm_rel_lo(off)); 112 } else 113 uasm_i_addiu(buf, T9, ZERO, off); 114 uasm_i_daddu(buf, reg1, reg2, T9); 115 } else { 116 if (off > 0x7fff) { 117 uasm_i_lui(buf, T9, uasm_rel_hi(off)); 118 uasm_i_addiu(buf, T9, T9, uasm_rel_lo(off)); 119 UASM_i_ADDU(buf, reg1, reg2, T9); 120 } else 121 UASM_i_ADDIU(buf, reg1, reg2, off); 122 } 123} 124 125static void set_prefetch_parameters(void) 126{ 127 if (cpu_has_64bit_gp_regs || cpu_has_64bit_zero_reg) 128 clear_word_size = 8; 129 else 130 clear_word_size = 4; 131 132 if (cpu_has_64bit_gp_regs) 133 copy_word_size = 8; 134 else 135 copy_word_size = 4; 136 137 /* 138 * The pref's used here are using "streaming" hints, which cause the 139 * copied data to be kicked out of the cache sooner. A page copy often 140 * ends up copying a lot more data than is commonly used, so this seems 141 * to make sense in terms of reducing cache pollution, but I've no real 142 * performance data to back this up. 143 */ 144 if (cpu_has_prefetch) { 145 /* 146 * XXX: Most prefetch bias values in here are based on 147 * guesswork. 148 */ 149 cache_line_size = cpu_dcache_line_size(); 150 switch (current_cpu_type()) { 151 case CPU_R5500: 152 case CPU_TX49XX: 153 /* These processors only support the Pref_Load. */ 154 pref_bias_copy_load = 256; 155 break; 156 157 case CPU_R10000: 158 case CPU_R12000: 159 case CPU_R14000: 160 case CPU_R16000: 161 /* 162 * Those values have been experimentally tuned for an 163 * Origin 200. 164 */ 165 pref_bias_clear_store = 512; 166 pref_bias_copy_load = 256; 167 pref_bias_copy_store = 256; 168 pref_src_mode = Pref_LoadStreamed; 169 pref_dst_mode = Pref_StoreStreamed; 170 break; 171 172 case CPU_SB1: 173 case CPU_SB1A: 174 pref_bias_clear_store = 128; 175 pref_bias_copy_load = 128; 176 pref_bias_copy_store = 128; 177 /* 178 * SB1 pass1 Pref_LoadStreamed/Pref_StoreStreamed 179 * hints are broken. 180 */ 181 if (current_cpu_type() == CPU_SB1 && 182 (current_cpu_data.processor_id & 0xff) < 0x02) { 183 pref_src_mode = Pref_Load; 184 pref_dst_mode = Pref_Store; 185 } else { 186 pref_src_mode = Pref_LoadStreamed; 187 pref_dst_mode = Pref_StoreStreamed; 188 } 189 break; 190 191 default: 192 pref_bias_clear_store = 128; 193 pref_bias_copy_load = 256; 194 pref_bias_copy_store = 128; 195 pref_src_mode = Pref_LoadStreamed; 196 if (cpu_has_mips_r6) 197 /* 198 * Bit 30 (Pref_PrepareForStore) has been 199 * removed from MIPS R6. Use bit 5 200 * (Pref_StoreStreamed). 201 */ 202 pref_dst_mode = Pref_StoreStreamed; 203 else 204 pref_dst_mode = Pref_PrepareForStore; 205 break; 206 } 207 } else { 208 if (cpu_has_cache_cdex_s) 209 cache_line_size = cpu_scache_line_size(); 210 else if (cpu_has_cache_cdex_p) 211 cache_line_size = cpu_dcache_line_size(); 212 } 213 /* 214 * Too much unrolling will overflow the available space in 215 * clear_space_array / copy_page_array. 216 */ 217 half_clear_loop_size = min(16 * clear_word_size, 218 max(cache_line_size >> 1, 219 4 * clear_word_size)); 220 half_copy_loop_size = min(16 * copy_word_size, 221 max(cache_line_size >> 1, 222 4 * copy_word_size)); 223} 224 225static void build_clear_store(u32 **buf, int off) 226{ 227 if (cpu_has_64bit_gp_regs || cpu_has_64bit_zero_reg) { 228 uasm_i_sd(buf, ZERO, off, A0); 229 } else { 230 uasm_i_sw(buf, ZERO, off, A0); 231 } 232} 233 234static inline void build_clear_pref(u32 **buf, int off) 235{ 236 if (off & cache_line_mask()) 237 return; 238 239 if (pref_bias_clear_store) { 240 _uasm_i_pref(buf, pref_dst_mode, pref_bias_clear_store + off, 241 A0); 242 } else if (cache_line_size == (half_clear_loop_size << 1)) { 243 if (cpu_has_cache_cdex_s) { 244 uasm_i_cache(buf, Create_Dirty_Excl_SD, off, A0); 245 } else if (cpu_has_cache_cdex_p) { 246 if (R4600_V1_HIT_CACHEOP_WAR && cpu_is_r4600_v1_x()) { 247 uasm_i_nop(buf); 248 uasm_i_nop(buf); 249 uasm_i_nop(buf); 250 uasm_i_nop(buf); 251 } 252 253 if (R4600_V2_HIT_CACHEOP_WAR && cpu_is_r4600_v2_x()) 254 uasm_i_lw(buf, ZERO, ZERO, AT); 255 256 uasm_i_cache(buf, Create_Dirty_Excl_D, off, A0); 257 } 258 } 259} 260 261extern u32 __clear_page_start; 262extern u32 __clear_page_end; 263extern u32 __copy_page_start; 264extern u32 __copy_page_end; 265 266void build_clear_page(void) 267{ 268 int off; 269 u32 *buf = &__clear_page_start; 270 struct uasm_label *l = labels; 271 struct uasm_reloc *r = relocs; 272 int i; 273 static atomic_t run_once = ATOMIC_INIT(0); 274 275 if (atomic_xchg(&run_once, 1)) { 276 return; 277 } 278 279 memset(labels, 0, sizeof(labels)); 280 memset(relocs, 0, sizeof(relocs)); 281 282 set_prefetch_parameters(); 283 284 /* 285 * This algorithm makes the following assumptions: 286 * - The prefetch bias is a multiple of 2 words. 287 * - The prefetch bias is less than one page. 288 */ 289 BUG_ON(pref_bias_clear_store % (2 * clear_word_size)); 290 BUG_ON(PAGE_SIZE < pref_bias_clear_store); 291 292 off = PAGE_SIZE - pref_bias_clear_store; 293 if (off > 0xffff || !pref_bias_clear_store) 294 pg_addiu(&buf, A2, A0, off); 295 else 296 uasm_i_ori(&buf, A2, A0, off); 297 298 if (R4600_V2_HIT_CACHEOP_WAR && cpu_is_r4600_v2_x()) 299 uasm_i_lui(&buf, AT, uasm_rel_hi(0xa0000000)); 300 301 off = cache_line_size ? min(8, pref_bias_clear_store / cache_line_size) 302 * cache_line_size : 0; 303 while (off) { 304 build_clear_pref(&buf, -off); 305 off -= cache_line_size; 306 } 307 uasm_l_clear_pref(&l, buf); 308 do { 309 build_clear_pref(&buf, off); 310 build_clear_store(&buf, off); 311 off += clear_word_size; 312 } while (off < half_clear_loop_size); 313 pg_addiu(&buf, A0, A0, 2 * off); 314 off = -off; 315 do { 316 build_clear_pref(&buf, off); 317 if (off == -clear_word_size) 318 uasm_il_bne(&buf, &r, A0, A2, label_clear_pref); 319 build_clear_store(&buf, off); 320 off += clear_word_size; 321 } while (off < 0); 322 323 if (pref_bias_clear_store) { 324 pg_addiu(&buf, A2, A0, pref_bias_clear_store); 325 uasm_l_clear_nopref(&l, buf); 326 off = 0; 327 do { 328 build_clear_store(&buf, off); 329 off += clear_word_size; 330 } while (off < half_clear_loop_size); 331 pg_addiu(&buf, A0, A0, 2 * off); 332 off = -off; 333 do { 334 if (off == -clear_word_size) 335 uasm_il_bne(&buf, &r, A0, A2, 336 label_clear_nopref); 337 build_clear_store(&buf, off); 338 off += clear_word_size; 339 } while (off < 0); 340 } 341 342 uasm_i_jr(&buf, RA); 343 uasm_i_nop(&buf); 344 345 BUG_ON(buf > &__clear_page_end); 346 347 uasm_resolve_relocs(relocs, labels); 348 349 pr_debug("Synthesized clear page handler (%u instructions).\n", 350 (u32)(buf - &__clear_page_start)); 351 352 pr_debug("\t.set push\n"); 353 pr_debug("\t.set noreorder\n"); 354 for (i = 0; i < (buf - &__clear_page_start); i++) 355 pr_debug("\t.word 0x%08x\n", (&__clear_page_start)[i]); 356 pr_debug("\t.set pop\n"); 357} 358 359static void build_copy_load(u32 **buf, int reg, int off) 360{ 361 if (cpu_has_64bit_gp_regs) { 362 uasm_i_ld(buf, reg, off, A1); 363 } else { 364 uasm_i_lw(buf, reg, off, A1); 365 } 366} 367 368static void build_copy_store(u32 **buf, int reg, int off) 369{ 370 if (cpu_has_64bit_gp_regs) { 371 uasm_i_sd(buf, reg, off, A0); 372 } else { 373 uasm_i_sw(buf, reg, off, A0); 374 } 375} 376 377static inline void build_copy_load_pref(u32 **buf, int off) 378{ 379 if (off & cache_line_mask()) 380 return; 381 382 if (pref_bias_copy_load) 383 _uasm_i_pref(buf, pref_src_mode, pref_bias_copy_load + off, A1); 384} 385 386static inline void build_copy_store_pref(u32 **buf, int off) 387{ 388 if (off & cache_line_mask()) 389 return; 390 391 if (pref_bias_copy_store) { 392 _uasm_i_pref(buf, pref_dst_mode, pref_bias_copy_store + off, 393 A0); 394 } else if (cache_line_size == (half_copy_loop_size << 1)) { 395 if (cpu_has_cache_cdex_s) { 396 uasm_i_cache(buf, Create_Dirty_Excl_SD, off, A0); 397 } else if (cpu_has_cache_cdex_p) { 398 if (R4600_V1_HIT_CACHEOP_WAR && cpu_is_r4600_v1_x()) { 399 uasm_i_nop(buf); 400 uasm_i_nop(buf); 401 uasm_i_nop(buf); 402 uasm_i_nop(buf); 403 } 404 405 if (R4600_V2_HIT_CACHEOP_WAR && cpu_is_r4600_v2_x()) 406 uasm_i_lw(buf, ZERO, ZERO, AT); 407 408 uasm_i_cache(buf, Create_Dirty_Excl_D, off, A0); 409 } 410 } 411} 412 413void build_copy_page(void) 414{ 415 int off; 416 u32 *buf = &__copy_page_start; 417 struct uasm_label *l = labels; 418 struct uasm_reloc *r = relocs; 419 int i; 420 static atomic_t run_once = ATOMIC_INIT(0); 421 422 if (atomic_xchg(&run_once, 1)) { 423 return; 424 } 425 426 memset(labels, 0, sizeof(labels)); 427 memset(relocs, 0, sizeof(relocs)); 428 429 set_prefetch_parameters(); 430 431 /* 432 * This algorithm makes the following assumptions: 433 * - All prefetch biases are multiples of 8 words. 434 * - The prefetch biases are less than one page. 435 * - The store prefetch bias isn't greater than the load 436 * prefetch bias. 437 */ 438 BUG_ON(pref_bias_copy_load % (8 * copy_word_size)); 439 BUG_ON(pref_bias_copy_store % (8 * copy_word_size)); 440 BUG_ON(PAGE_SIZE < pref_bias_copy_load); 441 BUG_ON(pref_bias_copy_store > pref_bias_copy_load); 442 443 off = PAGE_SIZE - pref_bias_copy_load; 444 if (off > 0xffff || !pref_bias_copy_load) 445 pg_addiu(&buf, A2, A0, off); 446 else 447 uasm_i_ori(&buf, A2, A0, off); 448 449 if (R4600_V2_HIT_CACHEOP_WAR && cpu_is_r4600_v2_x()) 450 uasm_i_lui(&buf, AT, uasm_rel_hi(0xa0000000)); 451 452 off = cache_line_size ? min(8, pref_bias_copy_load / cache_line_size) * 453 cache_line_size : 0; 454 while (off) { 455 build_copy_load_pref(&buf, -off); 456 off -= cache_line_size; 457 } 458 off = cache_line_size ? min(8, pref_bias_copy_store / cache_line_size) * 459 cache_line_size : 0; 460 while (off) { 461 build_copy_store_pref(&buf, -off); 462 off -= cache_line_size; 463 } 464 uasm_l_copy_pref_both(&l, buf); 465 do { 466 build_copy_load_pref(&buf, off); 467 build_copy_load(&buf, T0, off); 468 build_copy_load_pref(&buf, off + copy_word_size); 469 build_copy_load(&buf, T1, off + copy_word_size); 470 build_copy_load_pref(&buf, off + 2 * copy_word_size); 471 build_copy_load(&buf, T2, off + 2 * copy_word_size); 472 build_copy_load_pref(&buf, off + 3 * copy_word_size); 473 build_copy_load(&buf, T3, off + 3 * copy_word_size); 474 build_copy_store_pref(&buf, off); 475 build_copy_store(&buf, T0, off); 476 build_copy_store_pref(&buf, off + copy_word_size); 477 build_copy_store(&buf, T1, off + copy_word_size); 478 build_copy_store_pref(&buf, off + 2 * copy_word_size); 479 build_copy_store(&buf, T2, off + 2 * copy_word_size); 480 build_copy_store_pref(&buf, off + 3 * copy_word_size); 481 build_copy_store(&buf, T3, off + 3 * copy_word_size); 482 off += 4 * copy_word_size; 483 } while (off < half_copy_loop_size); 484 pg_addiu(&buf, A1, A1, 2 * off); 485 pg_addiu(&buf, A0, A0, 2 * off); 486 off = -off; 487 do { 488 build_copy_load_pref(&buf, off); 489 build_copy_load(&buf, T0, off); 490 build_copy_load_pref(&buf, off + copy_word_size); 491 build_copy_load(&buf, T1, off + copy_word_size); 492 build_copy_load_pref(&buf, off + 2 * copy_word_size); 493 build_copy_load(&buf, T2, off + 2 * copy_word_size); 494 build_copy_load_pref(&buf, off + 3 * copy_word_size); 495 build_copy_load(&buf, T3, off + 3 * copy_word_size); 496 build_copy_store_pref(&buf, off); 497 build_copy_store(&buf, T0, off); 498 build_copy_store_pref(&buf, off + copy_word_size); 499 build_copy_store(&buf, T1, off + copy_word_size); 500 build_copy_store_pref(&buf, off + 2 * copy_word_size); 501 build_copy_store(&buf, T2, off + 2 * copy_word_size); 502 build_copy_store_pref(&buf, off + 3 * copy_word_size); 503 if (off == -(4 * copy_word_size)) 504 uasm_il_bne(&buf, &r, A2, A0, label_copy_pref_both); 505 build_copy_store(&buf, T3, off + 3 * copy_word_size); 506 off += 4 * copy_word_size; 507 } while (off < 0); 508 509 if (pref_bias_copy_load - pref_bias_copy_store) { 510 pg_addiu(&buf, A2, A0, 511 pref_bias_copy_load - pref_bias_copy_store); 512 uasm_l_copy_pref_store(&l, buf); 513 off = 0; 514 do { 515 build_copy_load(&buf, T0, off); 516 build_copy_load(&buf, T1, off + copy_word_size); 517 build_copy_load(&buf, T2, off + 2 * copy_word_size); 518 build_copy_load(&buf, T3, off + 3 * copy_word_size); 519 build_copy_store_pref(&buf, off); 520 build_copy_store(&buf, T0, off); 521 build_copy_store_pref(&buf, off + copy_word_size); 522 build_copy_store(&buf, T1, off + copy_word_size); 523 build_copy_store_pref(&buf, off + 2 * copy_word_size); 524 build_copy_store(&buf, T2, off + 2 * copy_word_size); 525 build_copy_store_pref(&buf, off + 3 * copy_word_size); 526 build_copy_store(&buf, T3, off + 3 * copy_word_size); 527 off += 4 * copy_word_size; 528 } while (off < half_copy_loop_size); 529 pg_addiu(&buf, A1, A1, 2 * off); 530 pg_addiu(&buf, A0, A0, 2 * off); 531 off = -off; 532 do { 533 build_copy_load(&buf, T0, off); 534 build_copy_load(&buf, T1, off + copy_word_size); 535 build_copy_load(&buf, T2, off + 2 * copy_word_size); 536 build_copy_load(&buf, T3, off + 3 * copy_word_size); 537 build_copy_store_pref(&buf, off); 538 build_copy_store(&buf, T0, off); 539 build_copy_store_pref(&buf, off + copy_word_size); 540 build_copy_store(&buf, T1, off + copy_word_size); 541 build_copy_store_pref(&buf, off + 2 * copy_word_size); 542 build_copy_store(&buf, T2, off + 2 * copy_word_size); 543 build_copy_store_pref(&buf, off + 3 * copy_word_size); 544 if (off == -(4 * copy_word_size)) 545 uasm_il_bne(&buf, &r, A2, A0, 546 label_copy_pref_store); 547 build_copy_store(&buf, T3, off + 3 * copy_word_size); 548 off += 4 * copy_word_size; 549 } while (off < 0); 550 } 551 552 if (pref_bias_copy_store) { 553 pg_addiu(&buf, A2, A0, pref_bias_copy_store); 554 uasm_l_copy_nopref(&l, buf); 555 off = 0; 556 do { 557 build_copy_load(&buf, T0, off); 558 build_copy_load(&buf, T1, off + copy_word_size); 559 build_copy_load(&buf, T2, off + 2 * copy_word_size); 560 build_copy_load(&buf, T3, off + 3 * copy_word_size); 561 build_copy_store(&buf, T0, off); 562 build_copy_store(&buf, T1, off + copy_word_size); 563 build_copy_store(&buf, T2, off + 2 * copy_word_size); 564 build_copy_store(&buf, T3, off + 3 * copy_word_size); 565 off += 4 * copy_word_size; 566 } while (off < half_copy_loop_size); 567 pg_addiu(&buf, A1, A1, 2 * off); 568 pg_addiu(&buf, A0, A0, 2 * off); 569 off = -off; 570 do { 571 build_copy_load(&buf, T0, off); 572 build_copy_load(&buf, T1, off + copy_word_size); 573 build_copy_load(&buf, T2, off + 2 * copy_word_size); 574 build_copy_load(&buf, T3, off + 3 * copy_word_size); 575 build_copy_store(&buf, T0, off); 576 build_copy_store(&buf, T1, off + copy_word_size); 577 build_copy_store(&buf, T2, off + 2 * copy_word_size); 578 if (off == -(4 * copy_word_size)) 579 uasm_il_bne(&buf, &r, A2, A0, 580 label_copy_nopref); 581 build_copy_store(&buf, T3, off + 3 * copy_word_size); 582 off += 4 * copy_word_size; 583 } while (off < 0); 584 } 585 586 uasm_i_jr(&buf, RA); 587 uasm_i_nop(&buf); 588 589 BUG_ON(buf > &__copy_page_end); 590 591 uasm_resolve_relocs(relocs, labels); 592 593 pr_debug("Synthesized copy page handler (%u instructions).\n", 594 (u32)(buf - &__copy_page_start)); 595 596 pr_debug("\t.set push\n"); 597 pr_debug("\t.set noreorder\n"); 598 for (i = 0; i < (buf - &__copy_page_start); i++) 599 pr_debug("\t.word 0x%08x\n", (&__copy_page_start)[i]); 600 pr_debug("\t.set pop\n"); 601} 602 603#ifdef CONFIG_SIBYTE_DMA_PAGEOPS 604extern void clear_page_cpu(void *page); 605extern void copy_page_cpu(void *to, void *from); 606 607/* 608 * Pad descriptors to cacheline, since each is exclusively owned by a 609 * particular CPU. 610 */ 611struct dmadscr { 612 u64 dscr_a; 613 u64 dscr_b; 614 u64 pad_a; 615 u64 pad_b; 616} ____cacheline_aligned_in_smp page_descr[DM_NUM_CHANNELS]; 617 618void sb1_dma_init(void) 619{ 620 int i; 621 622 for (i = 0; i < DM_NUM_CHANNELS; i++) { 623 const u64 base_val = CPHYSADDR((unsigned long)&page_descr[i]) | 624 V_DM_DSCR_BASE_RINGSZ(1); 625 void *base_reg = IOADDR(A_DM_REGISTER(i, R_DM_DSCR_BASE)); 626 627 __raw_writeq(base_val, base_reg); 628 __raw_writeq(base_val | M_DM_DSCR_BASE_RESET, base_reg); 629 __raw_writeq(base_val | M_DM_DSCR_BASE_ENABL, base_reg); 630 } 631} 632 633void clear_page(void *page) 634{ 635 u64 to_phys = CPHYSADDR((unsigned long)page); 636 unsigned int cpu = smp_processor_id(); 637 638 /* if the page is not in KSEG0, use old way */ 639 if ((long)KSEGX((unsigned long)page) != (long)CKSEG0) 640 return clear_page_cpu(page); 641 642 page_descr[cpu].dscr_a = to_phys | M_DM_DSCRA_ZERO_MEM | 643 M_DM_DSCRA_L2C_DEST | M_DM_DSCRA_INTERRUPT; 644 page_descr[cpu].dscr_b = V_DM_DSCRB_SRC_LENGTH(PAGE_SIZE); 645 __raw_writeq(1, IOADDR(A_DM_REGISTER(cpu, R_DM_DSCR_COUNT))); 646 647 /* 648 * Don't really want to do it this way, but there's no 649 * reliable way to delay completion detection. 650 */ 651 while (!(__raw_readq(IOADDR(A_DM_REGISTER(cpu, R_DM_DSCR_BASE_DEBUG))) 652 & M_DM_DSCR_BASE_INTERRUPT)) 653 ; 654 __raw_readq(IOADDR(A_DM_REGISTER(cpu, R_DM_DSCR_BASE))); 655} 656 657void copy_page(void *to, void *from) 658{ 659 u64 from_phys = CPHYSADDR((unsigned long)from); 660 u64 to_phys = CPHYSADDR((unsigned long)to); 661 unsigned int cpu = smp_processor_id(); 662 663 /* if any page is not in KSEG0, use old way */ 664 if ((long)KSEGX((unsigned long)to) != (long)CKSEG0 665 || (long)KSEGX((unsigned long)from) != (long)CKSEG0) 666 return copy_page_cpu(to, from); 667 668 page_descr[cpu].dscr_a = to_phys | M_DM_DSCRA_L2C_DEST | 669 M_DM_DSCRA_INTERRUPT; 670 page_descr[cpu].dscr_b = from_phys | V_DM_DSCRB_SRC_LENGTH(PAGE_SIZE); 671 __raw_writeq(1, IOADDR(A_DM_REGISTER(cpu, R_DM_DSCR_COUNT))); 672 673 /* 674 * Don't really want to do it this way, but there's no 675 * reliable way to delay completion detection. 676 */ 677 while (!(__raw_readq(IOADDR(A_DM_REGISTER(cpu, R_DM_DSCR_BASE_DEBUG))) 678 & M_DM_DSCR_BASE_INTERRUPT)) 679 ; 680 __raw_readq(IOADDR(A_DM_REGISTER(cpu, R_DM_DSCR_BASE))); 681} 682 683#endif /* CONFIG_SIBYTE_DMA_PAGEOPS */ 684