1/* 2 * ppc64 MMU hashtable management routines 3 * 4 * (c) Copyright IBM Corp. 2003, 2005 5 * 6 * Maintained by: Benjamin Herrenschmidt 7 * <benh@kernel.crashing.org> 8 * 9 * This file is covered by the GNU Public Licence v2 as 10 * described in the kernel's COPYING file. 11 */ 12 13#include <asm/reg.h> 14#include <asm/pgtable.h> 15#include <asm/mmu.h> 16#include <asm/page.h> 17#include <asm/types.h> 18#include <asm/ppc_asm.h> 19#include <asm/asm-offsets.h> 20#include <asm/cputable.h> 21 22 .text 23 24/* 25 * Stackframe: 26 * 27 * +-> Back chain (SP + 256) 28 * | General register save area (SP + 112) 29 * | Parameter save area (SP + 48) 30 * | TOC save area (SP + 40) 31 * | link editor doubleword (SP + 32) 32 * | compiler doubleword (SP + 24) 33 * | LR save area (SP + 16) 34 * | CR save area (SP + 8) 35 * SP ---> +-- Back chain (SP + 0) 36 */ 37 38#ifndef CONFIG_PPC_64K_PAGES 39 40/***************************************************************************** 41 * * 42 * 4K SW & 4K HW pages implementation * 43 * * 44 *****************************************************************************/ 45 46 47/* 48 * _hash_page_4K(unsigned long ea, unsigned long access, unsigned long vsid, 49 * pte_t *ptep, unsigned long trap, unsigned long flags, 50 * int ssize) 51 * 52 * Adds a 4K page to the hash table in a segment of 4K pages only 53 */ 54 55_GLOBAL(__hash_page_4K) 56 mflr r0 57 std r0,16(r1) 58 stdu r1,-STACKFRAMESIZE(r1) 59 /* Save all params that we need after a function call */ 60 std r6,STK_PARAM(R6)(r1) 61 std r8,STK_PARAM(R8)(r1) 62 std r9,STK_PARAM(R9)(r1) 63 64 /* Save non-volatile registers. 65 * r31 will hold "old PTE" 66 * r30 is "new PTE" 67 * r29 is vpn 68 * r28 is a hash value 69 * r27 is hashtab mask (maybe dynamic patched instead ?) 70 */ 71 std r27,STK_REG(R27)(r1) 72 std r28,STK_REG(R28)(r1) 73 std r29,STK_REG(R29)(r1) 74 std r30,STK_REG(R30)(r1) 75 std r31,STK_REG(R31)(r1) 76 77 /* Step 1: 78 * 79 * Check permissions, atomically mark the linux PTE busy 80 * and hashed. 81 */ 821: 83 ldarx r31,0,r6 84 /* Check access rights (access & ~(pte_val(*ptep))) */ 85 andc. r0,r4,r31 86 bne- htab_wrong_access 87 /* Check if PTE is busy */ 88 andi. r0,r31,_PAGE_BUSY 89 /* If so, just bail out and refault if needed. Someone else 90 * is changing this PTE anyway and might hash it. 91 */ 92 bne- htab_bail_ok 93 94 /* Prepare new PTE value (turn access RW into DIRTY, then 95 * add BUSY,HASHPTE and ACCESSED) 96 */ 97 rlwinm r30,r4,32-9+7,31-7,31-7 /* _PAGE_RW -> _PAGE_DIRTY */ 98 or r30,r30,r31 99 ori r30,r30,_PAGE_BUSY | _PAGE_ACCESSED | _PAGE_HASHPTE 100 /* Write the linux PTE atomically (setting busy) */ 101 stdcx. r30,0,r6 102 bne- 1b 103 isync 104 105 /* Step 2: 106 * 107 * Insert/Update the HPTE in the hash table. At this point, 108 * r4 (access) is re-useable, we use it for the new HPTE flags 109 */ 110 111BEGIN_FTR_SECTION 112 cmpdi r9,0 /* check segment size */ 113 bne 3f 114END_MMU_FTR_SECTION_IFSET(MMU_FTR_1T_SEGMENT) 115 /* Calc vpn and put it in r29 */ 116 sldi r29,r5,SID_SHIFT - VPN_SHIFT 117 rldicl r28,r3,64 - VPN_SHIFT,64 - (SID_SHIFT - VPN_SHIFT) 118 or r29,r28,r29 119 /* 120 * Calculate hash value for primary slot and store it in r28 121 * r3 = va, r5 = vsid 122 * r0 = (va >> 12) & ((1ul << (28 - 12)) -1) 123 */ 124 rldicl r0,r3,64-12,48 125 xor r28,r5,r0 /* hash */ 126 b 4f 127 1283: /* Calc vpn and put it in r29 */ 129 sldi r29,r5,SID_SHIFT_1T - VPN_SHIFT 130 rldicl r28,r3,64 - VPN_SHIFT,64 - (SID_SHIFT_1T - VPN_SHIFT) 131 or r29,r28,r29 132 133 /* 134 * calculate hash value for primary slot and 135 * store it in r28 for 1T segment 136 * r3 = va, r5 = vsid 137 */ 138 sldi r28,r5,25 /* vsid << 25 */ 139 /* r0 = (va >> 12) & ((1ul << (40 - 12)) -1) */ 140 rldicl r0,r3,64-12,36 141 xor r28,r28,r5 /* vsid ^ ( vsid << 25) */ 142 xor r28,r28,r0 /* hash */ 143 144 /* Convert linux PTE bits into HW equivalents */ 1454: andi. r3,r30,0x1fe /* Get basic set of flags */ 146 xori r3,r3,HPTE_R_N /* _PAGE_EXEC -> NOEXEC */ 147 rlwinm r0,r30,32-9+1,30,30 /* _PAGE_RW -> _PAGE_USER (r0) */ 148 rlwinm r4,r30,32-7+1,30,30 /* _PAGE_DIRTY -> _PAGE_USER (r4) */ 149 and r0,r0,r4 /* _PAGE_RW & _PAGE_DIRTY ->r0 bit 30*/ 150 andc r0,r30,r0 /* r0 = pte & ~r0 */ 151 rlwimi r3,r0,32-1,31,31 /* Insert result into PP lsb */ 152 /* 153 * Always add "C" bit for perf. Memory coherence is always enabled 154 */ 155 ori r3,r3,HPTE_R_C | HPTE_R_M 156 157 /* We eventually do the icache sync here (maybe inline that 158 * code rather than call a C function...) 159 */ 160BEGIN_FTR_SECTION 161 mr r4,r30 162 mr r5,r7 163 bl hash_page_do_lazy_icache 164END_FTR_SECTION(CPU_FTR_NOEXECUTE|CPU_FTR_COHERENT_ICACHE, CPU_FTR_NOEXECUTE) 165 166 /* At this point, r3 contains new PP bits, save them in 167 * place of "access" in the param area (sic) 168 */ 169 std r3,STK_PARAM(R4)(r1) 170 171 /* Get htab_hash_mask */ 172 ld r4,htab_hash_mask@got(2) 173 ld r27,0(r4) /* htab_hash_mask -> r27 */ 174 175 /* Check if we may already be in the hashtable, in this case, we 176 * go to out-of-line code to try to modify the HPTE 177 */ 178 andi. r0,r31,_PAGE_HASHPTE 179 bne htab_modify_pte 180 181htab_insert_pte: 182 /* Clear hpte bits in new pte (we also clear BUSY btw) and 183 * add _PAGE_HASHPTE 184 */ 185 lis r0,_PAGE_HPTEFLAGS@h 186 ori r0,r0,_PAGE_HPTEFLAGS@l 187 andc r30,r30,r0 188 ori r30,r30,_PAGE_HASHPTE 189 190 /* physical address r5 */ 191 rldicl r5,r31,64-PTE_RPN_SHIFT,PTE_RPN_SHIFT 192 sldi r5,r5,PAGE_SHIFT 193 194 /* Calculate primary group hash */ 195 and r0,r28,r27 196 rldicr r3,r0,3,63-3 /* r3 = (hash & mask) << 3 */ 197 198 /* Call ppc_md.hpte_insert */ 199 ld r6,STK_PARAM(R4)(r1) /* Retrieve new pp bits */ 200 mr r4,r29 /* Retrieve vpn */ 201 li r7,0 /* !bolted, !secondary */ 202 li r8,MMU_PAGE_4K /* page size */ 203 li r9,MMU_PAGE_4K /* actual page size */ 204 ld r10,STK_PARAM(R9)(r1) /* segment size */ 205.globl htab_call_hpte_insert1 206htab_call_hpte_insert1: 207 bl . /* Patched by htab_finish_init() */ 208 cmpdi 0,r3,0 209 bge htab_pte_insert_ok /* Insertion successful */ 210 cmpdi 0,r3,-2 /* Critical failure */ 211 beq- htab_pte_insert_failure 212 213 /* Now try secondary slot */ 214 215 /* physical address r5 */ 216 rldicl r5,r31,64-PTE_RPN_SHIFT,PTE_RPN_SHIFT 217 sldi r5,r5,PAGE_SHIFT 218 219 /* Calculate secondary group hash */ 220 andc r0,r27,r28 221 rldicr r3,r0,3,63-3 /* r0 = (~hash & mask) << 3 */ 222 223 /* Call ppc_md.hpte_insert */ 224 ld r6,STK_PARAM(R4)(r1) /* Retrieve new pp bits */ 225 mr r4,r29 /* Retrieve vpn */ 226 li r7,HPTE_V_SECONDARY /* !bolted, secondary */ 227 li r8,MMU_PAGE_4K /* page size */ 228 li r9,MMU_PAGE_4K /* actual page size */ 229 ld r10,STK_PARAM(R9)(r1) /* segment size */ 230.globl htab_call_hpte_insert2 231htab_call_hpte_insert2: 232 bl . /* Patched by htab_finish_init() */ 233 cmpdi 0,r3,0 234 bge+ htab_pte_insert_ok /* Insertion successful */ 235 cmpdi 0,r3,-2 /* Critical failure */ 236 beq- htab_pte_insert_failure 237 238 /* Both are full, we need to evict something */ 239 mftb r0 240 /* Pick a random group based on TB */ 241 andi. r0,r0,1 242 mr r5,r28 243 bne 2f 244 not r5,r5 2452: and r0,r5,r27 246 rldicr r3,r0,3,63-3 /* r0 = (hash & mask) << 3 */ 247 /* Call ppc_md.hpte_remove */ 248.globl htab_call_hpte_remove 249htab_call_hpte_remove: 250 bl . /* Patched by htab_finish_init() */ 251 252 /* Try all again */ 253 b htab_insert_pte 254 255htab_bail_ok: 256 li r3,0 257 b htab_bail 258 259htab_pte_insert_ok: 260 /* Insert slot number & secondary bit in PTE */ 261 rldimi r30,r3,12,63-15 262 263 /* Write out the PTE with a normal write 264 * (maybe add eieio may be good still ?) 265 */ 266htab_write_out_pte: 267 ld r6,STK_PARAM(R6)(r1) 268 std r30,0(r6) 269 li r3, 0 270htab_bail: 271 ld r27,STK_REG(R27)(r1) 272 ld r28,STK_REG(R28)(r1) 273 ld r29,STK_REG(R29)(r1) 274 ld r30,STK_REG(R30)(r1) 275 ld r31,STK_REG(R31)(r1) 276 addi r1,r1,STACKFRAMESIZE 277 ld r0,16(r1) 278 mtlr r0 279 blr 280 281htab_modify_pte: 282 /* Keep PP bits in r4 and slot idx from the PTE around in r3 */ 283 mr r4,r3 284 rlwinm r3,r31,32-12,29,31 285 286 /* Secondary group ? if yes, get a inverted hash value */ 287 mr r5,r28 288 andi. r0,r31,_PAGE_SECONDARY 289 beq 1f 290 not r5,r5 2911: 292 /* Calculate proper slot value for ppc_md.hpte_updatepp */ 293 and r0,r5,r27 294 rldicr r0,r0,3,63-3 /* r0 = (hash & mask) << 3 */ 295 add r3,r0,r3 /* add slot idx */ 296 297 /* Call ppc_md.hpte_updatepp */ 298 mr r5,r29 /* vpn */ 299 li r6,MMU_PAGE_4K /* base page size */ 300 li r7,MMU_PAGE_4K /* actual page size */ 301 ld r8,STK_PARAM(R9)(r1) /* segment size */ 302 ld r9,STK_PARAM(R8)(r1) /* get "flags" param */ 303.globl htab_call_hpte_updatepp 304htab_call_hpte_updatepp: 305 bl . /* Patched by htab_finish_init() */ 306 307 /* if we failed because typically the HPTE wasn't really here 308 * we try an insertion. 309 */ 310 cmpdi 0,r3,-1 311 beq- htab_insert_pte 312 313 /* Clear the BUSY bit and Write out the PTE */ 314 li r0,_PAGE_BUSY 315 andc r30,r30,r0 316 b htab_write_out_pte 317 318htab_wrong_access: 319 /* Bail out clearing reservation */ 320 stdcx. r31,0,r6 321 li r3,1 322 b htab_bail 323 324htab_pte_insert_failure: 325 /* Bail out restoring old PTE */ 326 ld r6,STK_PARAM(R6)(r1) 327 std r31,0(r6) 328 li r3,-1 329 b htab_bail 330 331 332#else /* CONFIG_PPC_64K_PAGES */ 333 334 335/***************************************************************************** 336 * * 337 * 64K SW & 4K or 64K HW in a 4K segment pages implementation * 338 * * 339 *****************************************************************************/ 340 341/* _hash_page_4K(unsigned long ea, unsigned long access, unsigned long vsid, 342 * pte_t *ptep, unsigned long trap, unsigned local flags, 343 * int ssize, int subpg_prot) 344 */ 345 346/* 347 * For now, we do NOT implement Admixed pages 348 */ 349_GLOBAL(__hash_page_4K) 350 mflr r0 351 std r0,16(r1) 352 stdu r1,-STACKFRAMESIZE(r1) 353 /* Save all params that we need after a function call */ 354 std r6,STK_PARAM(R6)(r1) 355 std r8,STK_PARAM(R8)(r1) 356 std r9,STK_PARAM(R9)(r1) 357 358 /* Save non-volatile registers. 359 * r31 will hold "old PTE" 360 * r30 is "new PTE" 361 * r29 is vpn 362 * r28 is a hash value 363 * r27 is hashtab mask (maybe dynamic patched instead ?) 364 * r26 is the hidx mask 365 * r25 is the index in combo page 366 */ 367 std r25,STK_REG(R25)(r1) 368 std r26,STK_REG(R26)(r1) 369 std r27,STK_REG(R27)(r1) 370 std r28,STK_REG(R28)(r1) 371 std r29,STK_REG(R29)(r1) 372 std r30,STK_REG(R30)(r1) 373 std r31,STK_REG(R31)(r1) 374 375 /* Step 1: 376 * 377 * Check permissions, atomically mark the linux PTE busy 378 * and hashed. 379 */ 3801: 381 ldarx r31,0,r6 382 /* Check access rights (access & ~(pte_val(*ptep))) */ 383 andc. r0,r4,r31 384 bne- htab_wrong_access 385 /* Check if PTE is busy */ 386 andi. r0,r31,_PAGE_BUSY 387 /* If so, just bail out and refault if needed. Someone else 388 * is changing this PTE anyway and might hash it. 389 */ 390 bne- htab_bail_ok 391 /* Prepare new PTE value (turn access RW into DIRTY, then 392 * add BUSY and ACCESSED) 393 */ 394 rlwinm r30,r4,32-9+7,31-7,31-7 /* _PAGE_RW -> _PAGE_DIRTY */ 395 or r30,r30,r31 396 ori r30,r30,_PAGE_BUSY | _PAGE_ACCESSED 397 oris r30,r30,_PAGE_COMBO@h 398 /* Write the linux PTE atomically (setting busy) */ 399 stdcx. r30,0,r6 400 bne- 1b 401 isync 402 403 /* Step 2: 404 * 405 * Insert/Update the HPTE in the hash table. At this point, 406 * r4 (access) is re-useable, we use it for the new HPTE flags 407 */ 408 409 /* Load the hidx index */ 410 rldicl r25,r3,64-12,60 411 412BEGIN_FTR_SECTION 413 cmpdi r9,0 /* check segment size */ 414 bne 3f 415END_MMU_FTR_SECTION_IFSET(MMU_FTR_1T_SEGMENT) 416 /* Calc vpn and put it in r29 */ 417 sldi r29,r5,SID_SHIFT - VPN_SHIFT 418 /* 419 * clrldi r3,r3,64 - SID_SHIFT --> ea & 0xfffffff 420 * srdi r28,r3,VPN_SHIFT 421 */ 422 rldicl r28,r3,64 - VPN_SHIFT,64 - (SID_SHIFT - VPN_SHIFT) 423 or r29,r28,r29 424 /* 425 * Calculate hash value for primary slot and store it in r28 426 * r3 = va, r5 = vsid 427 * r0 = (va >> 12) & ((1ul << (28 - 12)) -1) 428 */ 429 rldicl r0,r3,64-12,48 430 xor r28,r5,r0 /* hash */ 431 b 4f 432 4333: /* Calc vpn and put it in r29 */ 434 sldi r29,r5,SID_SHIFT_1T - VPN_SHIFT 435 /* 436 * clrldi r3,r3,64 - SID_SHIFT_1T --> ea & 0xffffffffff 437 * srdi r28,r3,VPN_SHIFT 438 */ 439 rldicl r28,r3,64 - VPN_SHIFT,64 - (SID_SHIFT_1T - VPN_SHIFT) 440 or r29,r28,r29 441 442 /* 443 * Calculate hash value for primary slot and 444 * store it in r28 for 1T segment 445 * r3 = va, r5 = vsid 446 */ 447 sldi r28,r5,25 /* vsid << 25 */ 448 /* r0 = (va >> 12) & ((1ul << (40 - 12)) -1) */ 449 rldicl r0,r3,64-12,36 450 xor r28,r28,r5 /* vsid ^ ( vsid << 25) */ 451 xor r28,r28,r0 /* hash */ 452 453 /* Convert linux PTE bits into HW equivalents */ 4544: 455#ifdef CONFIG_PPC_SUBPAGE_PROT 456 andc r10,r30,r10 457 andi. r3,r10,0x1fe /* Get basic set of flags */ 458 rlwinm r0,r10,32-9+1,30,30 /* _PAGE_RW -> _PAGE_USER (r0) */ 459#else 460 andi. r3,r30,0x1fe /* Get basic set of flags */ 461 rlwinm r0,r30,32-9+1,30,30 /* _PAGE_RW -> _PAGE_USER (r0) */ 462#endif 463 xori r3,r3,HPTE_R_N /* _PAGE_EXEC -> NOEXEC */ 464 rlwinm r4,r30,32-7+1,30,30 /* _PAGE_DIRTY -> _PAGE_USER (r4) */ 465 and r0,r0,r4 /* _PAGE_RW & _PAGE_DIRTY ->r0 bit 30*/ 466 andc r0,r3,r0 /* r0 = pte & ~r0 */ 467 rlwimi r3,r0,32-1,31,31 /* Insert result into PP lsb */ 468 /* 469 * Always add "C" bit for perf. Memory coherence is always enabled 470 */ 471 ori r3,r3,HPTE_R_C | HPTE_R_M 472 473 /* We eventually do the icache sync here (maybe inline that 474 * code rather than call a C function...) 475 */ 476BEGIN_FTR_SECTION 477 mr r4,r30 478 mr r5,r7 479 bl hash_page_do_lazy_icache 480END_FTR_SECTION(CPU_FTR_NOEXECUTE|CPU_FTR_COHERENT_ICACHE, CPU_FTR_NOEXECUTE) 481 482 /* At this point, r3 contains new PP bits, save them in 483 * place of "access" in the param area (sic) 484 */ 485 std r3,STK_PARAM(R4)(r1) 486 487 /* Get htab_hash_mask */ 488 ld r4,htab_hash_mask@got(2) 489 ld r27,0(r4) /* htab_hash_mask -> r27 */ 490 491 /* Check if we may already be in the hashtable, in this case, we 492 * go to out-of-line code to try to modify the HPTE. We look for 493 * the bit at (1 >> (index + 32)) 494 */ 495 rldicl. r0,r31,64-12,48 496 li r26,0 /* Default hidx */ 497 beq htab_insert_pte 498 499 /* 500 * Check if the pte was already inserted into the hash table 501 * as a 64k HW page, and invalidate the 64k HPTE if so. 502 */ 503 andis. r0,r31,_PAGE_COMBO@h 504 beq htab_inval_old_hpte 505 506 ld r6,STK_PARAM(R6)(r1) 507 ori r26,r6,PTE_PAGE_HIDX_OFFSET /* Load the hidx mask. */ 508 ld r26,0(r26) 509 addi r5,r25,36 /* Check actual HPTE_SUB bit, this */ 510 rldcr. r0,r31,r5,0 /* must match pgtable.h definition */ 511 bne htab_modify_pte 512 513htab_insert_pte: 514 /* real page number in r5, PTE RPN value + index */ 515 andis. r0,r31,_PAGE_4K_PFN@h 516 srdi r5,r31,PTE_RPN_SHIFT 517 bne- htab_special_pfn 518 sldi r5,r5,PAGE_FACTOR 519 add r5,r5,r25 520htab_special_pfn: 521 sldi r5,r5,HW_PAGE_SHIFT 522 523 /* Calculate primary group hash */ 524 and r0,r28,r27 525 rldicr r3,r0,3,63-3 /* r0 = (hash & mask) << 3 */ 526 527 /* Call ppc_md.hpte_insert */ 528 ld r6,STK_PARAM(R4)(r1) /* Retrieve new pp bits */ 529 mr r4,r29 /* Retrieve vpn */ 530 li r7,0 /* !bolted, !secondary */ 531 li r8,MMU_PAGE_4K /* page size */ 532 li r9,MMU_PAGE_4K /* actual page size */ 533 ld r10,STK_PARAM(R9)(r1) /* segment size */ 534.globl htab_call_hpte_insert1 535htab_call_hpte_insert1: 536 bl . /* patched by htab_finish_init() */ 537 cmpdi 0,r3,0 538 bge htab_pte_insert_ok /* Insertion successful */ 539 cmpdi 0,r3,-2 /* Critical failure */ 540 beq- htab_pte_insert_failure 541 542 /* Now try secondary slot */ 543 544 /* real page number in r5, PTE RPN value + index */ 545 andis. r0,r31,_PAGE_4K_PFN@h 546 srdi r5,r31,PTE_RPN_SHIFT 547 bne- 3f 548 sldi r5,r5,PAGE_FACTOR 549 add r5,r5,r25 5503: sldi r5,r5,HW_PAGE_SHIFT 551 552 /* Calculate secondary group hash */ 553 andc r0,r27,r28 554 rldicr r3,r0,3,63-3 /* r0 = (~hash & mask) << 3 */ 555 556 /* Call ppc_md.hpte_insert */ 557 ld r6,STK_PARAM(R4)(r1) /* Retrieve new pp bits */ 558 mr r4,r29 /* Retrieve vpn */ 559 li r7,HPTE_V_SECONDARY /* !bolted, secondary */ 560 li r8,MMU_PAGE_4K /* page size */ 561 li r9,MMU_PAGE_4K /* actual page size */ 562 ld r10,STK_PARAM(R9)(r1) /* segment size */ 563.globl htab_call_hpte_insert2 564htab_call_hpte_insert2: 565 bl . /* patched by htab_finish_init() */ 566 cmpdi 0,r3,0 567 bge+ htab_pte_insert_ok /* Insertion successful */ 568 cmpdi 0,r3,-2 /* Critical failure */ 569 beq- htab_pte_insert_failure 570 571 /* Both are full, we need to evict something */ 572 mftb r0 573 /* Pick a random group based on TB */ 574 andi. r0,r0,1 575 mr r5,r28 576 bne 2f 577 not r5,r5 5782: and r0,r5,r27 579 rldicr r3,r0,3,63-3 /* r0 = (hash & mask) << 3 */ 580 /* Call ppc_md.hpte_remove */ 581.globl htab_call_hpte_remove 582htab_call_hpte_remove: 583 bl . /* patched by htab_finish_init() */ 584 585 /* Try all again */ 586 b htab_insert_pte 587 588 /* 589 * Call out to C code to invalidate an 64k HW HPTE that is 590 * useless now that the segment has been switched to 4k pages. 591 */ 592htab_inval_old_hpte: 593 mr r3,r29 /* vpn */ 594 mr r4,r31 /* PTE.pte */ 595 li r5,0 /* PTE.hidx */ 596 li r6,MMU_PAGE_64K /* psize */ 597 ld r7,STK_PARAM(R9)(r1) /* ssize */ 598 ld r8,STK_PARAM(R8)(r1) /* flags */ 599 bl flush_hash_page 600 /* Clear out _PAGE_HPTE_SUB bits in the new linux PTE */ 601 lis r0,_PAGE_HPTE_SUB@h 602 ori r0,r0,_PAGE_HPTE_SUB@l 603 andc r30,r30,r0 604 b htab_insert_pte 605 606htab_bail_ok: 607 li r3,0 608 b htab_bail 609 610htab_pte_insert_ok: 611 /* Insert slot number & secondary bit in PTE second half, 612 * clear _PAGE_BUSY and set approriate HPTE slot bit 613 */ 614 ld r6,STK_PARAM(R6)(r1) 615 li r0,_PAGE_BUSY 616 andc r30,r30,r0 617 /* HPTE SUB bit */ 618 li r0,1 619 subfic r5,r25,27 /* Must match bit position in */ 620 sld r0,r0,r5 /* pgtable.h */ 621 or r30,r30,r0 622 /* hindx */ 623 sldi r5,r25,2 624 sld r3,r3,r5 625 li r4,0xf 626 sld r4,r4,r5 627 andc r26,r26,r4 628 or r26,r26,r3 629 ori r5,r6,PTE_PAGE_HIDX_OFFSET 630 std r26,0(r5) 631 lwsync 632 std r30,0(r6) 633 li r3, 0 634htab_bail: 635 ld r25,STK_REG(R25)(r1) 636 ld r26,STK_REG(R26)(r1) 637 ld r27,STK_REG(R27)(r1) 638 ld r28,STK_REG(R28)(r1) 639 ld r29,STK_REG(R29)(r1) 640 ld r30,STK_REG(R30)(r1) 641 ld r31,STK_REG(R31)(r1) 642 addi r1,r1,STACKFRAMESIZE 643 ld r0,16(r1) 644 mtlr r0 645 blr 646 647htab_modify_pte: 648 /* Keep PP bits in r4 and slot idx from the PTE around in r3 */ 649 mr r4,r3 650 sldi r5,r25,2 651 srd r3,r26,r5 652 653 /* Secondary group ? if yes, get a inverted hash value */ 654 mr r5,r28 655 andi. r0,r3,0x8 /* page secondary ? */ 656 beq 1f 657 not r5,r5 6581: andi. r3,r3,0x7 /* extract idx alone */ 659 660 /* Calculate proper slot value for ppc_md.hpte_updatepp */ 661 and r0,r5,r27 662 rldicr r0,r0,3,63-3 /* r0 = (hash & mask) << 3 */ 663 add r3,r0,r3 /* add slot idx */ 664 665 /* Call ppc_md.hpte_updatepp */ 666 mr r5,r29 /* vpn */ 667 li r6,MMU_PAGE_4K /* base page size */ 668 li r7,MMU_PAGE_4K /* actual page size */ 669 ld r8,STK_PARAM(R9)(r1) /* segment size */ 670 ld r9,STK_PARAM(R8)(r1) /* get "flags" param */ 671.globl htab_call_hpte_updatepp 672htab_call_hpte_updatepp: 673 bl . /* patched by htab_finish_init() */ 674 675 /* if we failed because typically the HPTE wasn't really here 676 * we try an insertion. 677 */ 678 cmpdi 0,r3,-1 679 beq- htab_insert_pte 680 681 /* Clear the BUSY bit and Write out the PTE */ 682 li r0,_PAGE_BUSY 683 andc r30,r30,r0 684 ld r6,STK_PARAM(R6)(r1) 685 std r30,0(r6) 686 li r3,0 687 b htab_bail 688 689htab_wrong_access: 690 /* Bail out clearing reservation */ 691 stdcx. r31,0,r6 692 li r3,1 693 b htab_bail 694 695htab_pte_insert_failure: 696 /* Bail out restoring old PTE */ 697 ld r6,STK_PARAM(R6)(r1) 698 std r31,0(r6) 699 li r3,-1 700 b htab_bail 701 702#endif /* CONFIG_PPC_64K_PAGES */ 703 704#ifdef CONFIG_PPC_HAS_HASH_64K 705 706/***************************************************************************** 707 * * 708 * 64K SW & 64K HW in a 64K segment pages implementation * 709 * * 710 *****************************************************************************/ 711 712_GLOBAL(__hash_page_64K) 713 mflr r0 714 std r0,16(r1) 715 stdu r1,-STACKFRAMESIZE(r1) 716 /* Save all params that we need after a function call */ 717 std r6,STK_PARAM(R6)(r1) 718 std r8,STK_PARAM(R8)(r1) 719 std r9,STK_PARAM(R9)(r1) 720 721 /* Save non-volatile registers. 722 * r31 will hold "old PTE" 723 * r30 is "new PTE" 724 * r29 is vpn 725 * r28 is a hash value 726 * r27 is hashtab mask (maybe dynamic patched instead ?) 727 */ 728 std r27,STK_REG(R27)(r1) 729 std r28,STK_REG(R28)(r1) 730 std r29,STK_REG(R29)(r1) 731 std r30,STK_REG(R30)(r1) 732 std r31,STK_REG(R31)(r1) 733 734 /* Step 1: 735 * 736 * Check permissions, atomically mark the linux PTE busy 737 * and hashed. 738 */ 7391: 740 ldarx r31,0,r6 741 /* Check access rights (access & ~(pte_val(*ptep))) */ 742 andc. r0,r4,r31 743 bne- ht64_wrong_access 744 /* Check if PTE is busy */ 745 andi. r0,r31,_PAGE_BUSY 746 /* If so, just bail out and refault if needed. Someone else 747 * is changing this PTE anyway and might hash it. 748 */ 749 bne- ht64_bail_ok 750BEGIN_FTR_SECTION 751 /* Check if PTE has the cache-inhibit bit set */ 752 andi. r0,r31,_PAGE_NO_CACHE 753 /* If so, bail out and refault as a 4k page */ 754 bne- ht64_bail_ok 755END_MMU_FTR_SECTION_IFCLR(MMU_FTR_CI_LARGE_PAGE) 756 /* Prepare new PTE value (turn access RW into DIRTY, then 757 * add BUSY and ACCESSED) 758 */ 759 rlwinm r30,r4,32-9+7,31-7,31-7 /* _PAGE_RW -> _PAGE_DIRTY */ 760 or r30,r30,r31 761 ori r30,r30,_PAGE_BUSY | _PAGE_ACCESSED 762 /* Write the linux PTE atomically (setting busy) */ 763 stdcx. r30,0,r6 764 bne- 1b 765 isync 766 767 /* Step 2: 768 * 769 * Insert/Update the HPTE in the hash table. At this point, 770 * r4 (access) is re-useable, we use it for the new HPTE flags 771 */ 772 773BEGIN_FTR_SECTION 774 cmpdi r9,0 /* check segment size */ 775 bne 3f 776END_MMU_FTR_SECTION_IFSET(MMU_FTR_1T_SEGMENT) 777 /* Calc vpn and put it in r29 */ 778 sldi r29,r5,SID_SHIFT - VPN_SHIFT 779 rldicl r28,r3,64 - VPN_SHIFT,64 - (SID_SHIFT - VPN_SHIFT) 780 or r29,r28,r29 781 782 /* Calculate hash value for primary slot and store it in r28 783 * r3 = va, r5 = vsid 784 * r0 = (va >> 16) & ((1ul << (28 - 16)) -1) 785 */ 786 rldicl r0,r3,64-16,52 787 xor r28,r5,r0 /* hash */ 788 b 4f 789 7903: /* Calc vpn and put it in r29 */ 791 sldi r29,r5,SID_SHIFT_1T - VPN_SHIFT 792 rldicl r28,r3,64 - VPN_SHIFT,64 - (SID_SHIFT_1T - VPN_SHIFT) 793 or r29,r28,r29 794 /* 795 * calculate hash value for primary slot and 796 * store it in r28 for 1T segment 797 * r3 = va, r5 = vsid 798 */ 799 sldi r28,r5,25 /* vsid << 25 */ 800 /* r0 = (va >> 16) & ((1ul << (40 - 16)) -1) */ 801 rldicl r0,r3,64-16,40 802 xor r28,r28,r5 /* vsid ^ ( vsid << 25) */ 803 xor r28,r28,r0 /* hash */ 804 805 /* Convert linux PTE bits into HW equivalents */ 8064: andi. r3,r30,0x1fe /* Get basic set of flags */ 807 xori r3,r3,HPTE_R_N /* _PAGE_EXEC -> NOEXEC */ 808 rlwinm r0,r30,32-9+1,30,30 /* _PAGE_RW -> _PAGE_USER (r0) */ 809 rlwinm r4,r30,32-7+1,30,30 /* _PAGE_DIRTY -> _PAGE_USER (r4) */ 810 and r0,r0,r4 /* _PAGE_RW & _PAGE_DIRTY ->r0 bit 30*/ 811 andc r0,r30,r0 /* r0 = pte & ~r0 */ 812 rlwimi r3,r0,32-1,31,31 /* Insert result into PP lsb */ 813 /* 814 * Always add "C" bit for perf. Memory coherence is always enabled 815 */ 816 ori r3,r3,HPTE_R_C | HPTE_R_M 817 818 /* We eventually do the icache sync here (maybe inline that 819 * code rather than call a C function...) 820 */ 821BEGIN_FTR_SECTION 822 mr r4,r30 823 mr r5,r7 824 bl hash_page_do_lazy_icache 825END_FTR_SECTION(CPU_FTR_NOEXECUTE|CPU_FTR_COHERENT_ICACHE, CPU_FTR_NOEXECUTE) 826 827 /* At this point, r3 contains new PP bits, save them in 828 * place of "access" in the param area (sic) 829 */ 830 std r3,STK_PARAM(R4)(r1) 831 832 /* Get htab_hash_mask */ 833 ld r4,htab_hash_mask@got(2) 834 ld r27,0(r4) /* htab_hash_mask -> r27 */ 835 836 /* Check if we may already be in the hashtable, in this case, we 837 * go to out-of-line code to try to modify the HPTE 838 */ 839 rldicl. r0,r31,64-12,48 840 bne ht64_modify_pte 841 842ht64_insert_pte: 843 /* Clear hpte bits in new pte (we also clear BUSY btw) and 844 * add _PAGE_HPTE_SUB0 845 */ 846 lis r0,_PAGE_HPTEFLAGS@h 847 ori r0,r0,_PAGE_HPTEFLAGS@l 848 andc r30,r30,r0 849#ifdef CONFIG_PPC_64K_PAGES 850 oris r30,r30,_PAGE_HPTE_SUB0@h 851#else 852 ori r30,r30,_PAGE_HASHPTE 853#endif 854 /* Phyical address in r5 */ 855 rldicl r5,r31,64-PTE_RPN_SHIFT,PTE_RPN_SHIFT 856 sldi r5,r5,PAGE_SHIFT 857 858 /* Calculate primary group hash */ 859 and r0,r28,r27 860 rldicr r3,r0,3,63-3 /* r0 = (hash & mask) << 3 */ 861 862 /* Call ppc_md.hpte_insert */ 863 ld r6,STK_PARAM(R4)(r1) /* Retrieve new pp bits */ 864 mr r4,r29 /* Retrieve vpn */ 865 li r7,0 /* !bolted, !secondary */ 866 li r8,MMU_PAGE_64K 867 li r9,MMU_PAGE_64K /* actual page size */ 868 ld r10,STK_PARAM(R9)(r1) /* segment size */ 869.globl ht64_call_hpte_insert1 870ht64_call_hpte_insert1: 871 bl . /* patched by htab_finish_init() */ 872 cmpdi 0,r3,0 873 bge ht64_pte_insert_ok /* Insertion successful */ 874 cmpdi 0,r3,-2 /* Critical failure */ 875 beq- ht64_pte_insert_failure 876 877 /* Now try secondary slot */ 878 879 /* Phyical address in r5 */ 880 rldicl r5,r31,64-PTE_RPN_SHIFT,PTE_RPN_SHIFT 881 sldi r5,r5,PAGE_SHIFT 882 883 /* Calculate secondary group hash */ 884 andc r0,r27,r28 885 rldicr r3,r0,3,63-3 /* r0 = (~hash & mask) << 3 */ 886 887 /* Call ppc_md.hpte_insert */ 888 ld r6,STK_PARAM(R4)(r1) /* Retrieve new pp bits */ 889 mr r4,r29 /* Retrieve vpn */ 890 li r7,HPTE_V_SECONDARY /* !bolted, secondary */ 891 li r8,MMU_PAGE_64K 892 li r9,MMU_PAGE_64K /* actual page size */ 893 ld r10,STK_PARAM(R9)(r1) /* segment size */ 894.globl ht64_call_hpte_insert2 895ht64_call_hpte_insert2: 896 bl . /* patched by htab_finish_init() */ 897 cmpdi 0,r3,0 898 bge+ ht64_pte_insert_ok /* Insertion successful */ 899 cmpdi 0,r3,-2 /* Critical failure */ 900 beq- ht64_pte_insert_failure 901 902 /* Both are full, we need to evict something */ 903 mftb r0 904 /* Pick a random group based on TB */ 905 andi. r0,r0,1 906 mr r5,r28 907 bne 2f 908 not r5,r5 9092: and r0,r5,r27 910 rldicr r3,r0,3,63-3 /* r0 = (hash & mask) << 3 */ 911 /* Call ppc_md.hpte_remove */ 912.globl ht64_call_hpte_remove 913ht64_call_hpte_remove: 914 bl . /* patched by htab_finish_init() */ 915 916 /* Try all again */ 917 b ht64_insert_pte 918 919ht64_bail_ok: 920 li r3,0 921 b ht64_bail 922 923ht64_pte_insert_ok: 924 /* Insert slot number & secondary bit in PTE */ 925 rldimi r30,r3,12,63-15 926 927 /* Write out the PTE with a normal write 928 * (maybe add eieio may be good still ?) 929 */ 930ht64_write_out_pte: 931 ld r6,STK_PARAM(R6)(r1) 932 std r30,0(r6) 933 li r3, 0 934ht64_bail: 935 ld r27,STK_REG(R27)(r1) 936 ld r28,STK_REG(R28)(r1) 937 ld r29,STK_REG(R29)(r1) 938 ld r30,STK_REG(R30)(r1) 939 ld r31,STK_REG(R31)(r1) 940 addi r1,r1,STACKFRAMESIZE 941 ld r0,16(r1) 942 mtlr r0 943 blr 944 945ht64_modify_pte: 946 /* Keep PP bits in r4 and slot idx from the PTE around in r3 */ 947 mr r4,r3 948 rlwinm r3,r31,32-12,29,31 949 950 /* Secondary group ? if yes, get a inverted hash value */ 951 mr r5,r28 952 andi. r0,r31,_PAGE_F_SECOND 953 beq 1f 954 not r5,r5 9551: 956 /* Calculate proper slot value for ppc_md.hpte_updatepp */ 957 and r0,r5,r27 958 rldicr r0,r0,3,63-3 /* r0 = (hash & mask) << 3 */ 959 add r3,r0,r3 /* add slot idx */ 960 961 /* Call ppc_md.hpte_updatepp */ 962 mr r5,r29 /* vpn */ 963 li r6,MMU_PAGE_64K /* base page size */ 964 li r7,MMU_PAGE_64K /* actual page size */ 965 ld r8,STK_PARAM(R9)(r1) /* segment size */ 966 ld r9,STK_PARAM(R8)(r1) /* get "flags" param */ 967.globl ht64_call_hpte_updatepp 968ht64_call_hpte_updatepp: 969 bl . /* patched by htab_finish_init() */ 970 971 /* if we failed because typically the HPTE wasn't really here 972 * we try an insertion. 973 */ 974 cmpdi 0,r3,-1 975 beq- ht64_insert_pte 976 977 /* Clear the BUSY bit and Write out the PTE */ 978 li r0,_PAGE_BUSY 979 andc r30,r30,r0 980 b ht64_write_out_pte 981 982ht64_wrong_access: 983 /* Bail out clearing reservation */ 984 stdcx. r31,0,r6 985 li r3,1 986 b ht64_bail 987 988ht64_pte_insert_failure: 989 /* Bail out restoring old PTE */ 990 ld r6,STK_PARAM(R6)(r1) 991 std r31,0(r6) 992 li r3,-1 993 b ht64_bail 994 995 996#endif /* CONFIG_PPC_HAS_HASH_64K */ 997 998 999/***************************************************************************** 1000 * * 1001 * Huge pages implementation is in hugetlbpage.c * 1002 * * 1003 *****************************************************************************/ 1004