Lines Matching refs:p
199 static void uasm_bgezl_hazard(u32 **p, struct uasm_reloc **r, int instance) in uasm_bgezl_hazard() argument
203 uasm_il_bgezl(p, r, 0, label_tlbw_hazard_0 + instance); in uasm_bgezl_hazard()
210 static void uasm_bgezl_label(struct uasm_label **l, u32 **p, int instance) in uasm_bgezl_label() argument
214 uasm_build_label(l, *p, label_tlbw_hazard_0 + instance); in uasm_bgezl_label()
349 static struct work_registers build_get_work_registers(u32 **p) in build_get_work_registers() argument
355 UASM_i_MTC0(p, 1, c0_kscratch(), scratch_reg); in build_get_work_registers()
364 UASM_i_CPUID_MFC0(p, K0, SMP_CPUID_REG); in build_get_work_registers()
365 UASM_i_SRL_SAFE(p, K0, K0, SMP_CPUID_REGSHIFT); in build_get_work_registers()
368 UASM_i_SLL(p, K0, K0, ilog2(sizeof(struct tlb_reg_save))); in build_get_work_registers()
370 UASM_i_LA(p, K1, (long)&handler_reg_save); in build_get_work_registers()
371 UASM_i_ADDU(p, K0, K0, K1); in build_get_work_registers()
373 UASM_i_LA(p, K0, (long)&handler_reg_save); in build_get_work_registers()
376 UASM_i_SW(p, 1, offsetof(struct tlb_reg_save, a), K0); in build_get_work_registers()
377 UASM_i_SW(p, 2, offsetof(struct tlb_reg_save, b), K0); in build_get_work_registers()
385 static void build_restore_work_registers(u32 **p) in build_restore_work_registers() argument
388 UASM_i_MFC0(p, 1, c0_kscratch(), scratch_reg); in build_restore_work_registers()
392 UASM_i_LW(p, 1, offsetof(struct tlb_reg_save, a), K0); in build_restore_work_registers()
393 UASM_i_LW(p, 2, offsetof(struct tlb_reg_save, b), K0); in build_restore_work_registers()
413 u32 *p; in build_r3000_tlb_refill_handler() local
416 p = tlb_handler; in build_r3000_tlb_refill_handler()
418 uasm_i_mfc0(&p, K0, C0_BADVADDR); in build_r3000_tlb_refill_handler()
419 uasm_i_lui(&p, K1, uasm_rel_hi(pgdc)); /* cp0 delay */ in build_r3000_tlb_refill_handler()
420 uasm_i_lw(&p, K1, uasm_rel_lo(pgdc), K1); in build_r3000_tlb_refill_handler()
421 uasm_i_srl(&p, K0, K0, 22); /* load delay */ in build_r3000_tlb_refill_handler()
422 uasm_i_sll(&p, K0, K0, 2); in build_r3000_tlb_refill_handler()
423 uasm_i_addu(&p, K1, K1, K0); in build_r3000_tlb_refill_handler()
424 uasm_i_mfc0(&p, K0, C0_CONTEXT); in build_r3000_tlb_refill_handler()
425 uasm_i_lw(&p, K1, 0, K1); /* cp0 delay */ in build_r3000_tlb_refill_handler()
426 uasm_i_andi(&p, K0, K0, 0xffc); /* load delay */ in build_r3000_tlb_refill_handler()
427 uasm_i_addu(&p, K1, K1, K0); in build_r3000_tlb_refill_handler()
428 uasm_i_lw(&p, K0, 0, K1); in build_r3000_tlb_refill_handler()
429 uasm_i_nop(&p); /* load delay */ in build_r3000_tlb_refill_handler()
430 uasm_i_mtc0(&p, K0, C0_ENTRYLO0); in build_r3000_tlb_refill_handler()
431 uasm_i_mfc0(&p, K1, C0_EPC); /* cp0 delay */ in build_r3000_tlb_refill_handler()
432 uasm_i_tlbwr(&p); /* cp0 delay */ in build_r3000_tlb_refill_handler()
433 uasm_i_jr(&p, K1); in build_r3000_tlb_refill_handler()
434 uasm_i_rfe(&p); /* branch delay */ in build_r3000_tlb_refill_handler()
436 if (p > tlb_handler + 32) in build_r3000_tlb_refill_handler()
440 (unsigned int)(p - tlb_handler)); in build_r3000_tlb_refill_handler()
480 static void __maybe_unused build_tlb_probe_entry(u32 **p) in build_tlb_probe_entry() argument
488 uasm_i_nop(p); in build_tlb_probe_entry()
489 uasm_i_tlbp(p); in build_tlb_probe_entry()
493 uasm_i_tlbp(p); in build_tlb_probe_entry()
504 static void build_tlb_write_entry(u32 **p, struct uasm_label **l, in build_tlb_write_entry() argument
517 uasm_i_ehb(p); in build_tlb_write_entry()
518 tlbw(p); in build_tlb_write_entry()
533 uasm_bgezl_hazard(p, r, hazard_instance); in build_tlb_write_entry()
534 tlbw(p); in build_tlb_write_entry()
535 uasm_bgezl_label(l, p, hazard_instance); in build_tlb_write_entry()
537 uasm_i_nop(p); in build_tlb_write_entry()
542 uasm_i_nop(p); in build_tlb_write_entry()
543 tlbw(p); in build_tlb_write_entry()
544 uasm_i_nop(p); in build_tlb_write_entry()
549 uasm_i_nop(p); /* QED specifies 2 nops hazard */ in build_tlb_write_entry()
550 uasm_i_nop(p); /* QED specifies 2 nops hazard */ in build_tlb_write_entry()
551 tlbw(p); in build_tlb_write_entry()
559 uasm_i_nop(p); in build_tlb_write_entry()
560 tlbw(p); in build_tlb_write_entry()
585 uasm_i_nop(p); in build_tlb_write_entry()
587 tlbw(p); in build_tlb_write_entry()
591 uasm_i_nop(p); in build_tlb_write_entry()
592 uasm_i_nop(p); in build_tlb_write_entry()
593 uasm_i_nop(p); in build_tlb_write_entry()
594 uasm_i_nop(p); in build_tlb_write_entry()
595 tlbw(p); in build_tlb_write_entry()
603 uasm_i_nop(p); in build_tlb_write_entry()
604 uasm_i_nop(p); in build_tlb_write_entry()
605 tlbw(p); in build_tlb_write_entry()
606 uasm_i_nop(p); in build_tlb_write_entry()
607 uasm_i_nop(p); in build_tlb_write_entry()
613 uasm_i_nop(p); in build_tlb_write_entry()
614 uasm_i_nop(p); in build_tlb_write_entry()
615 tlbw(p); in build_tlb_write_entry()
619 tlbw(p); in build_tlb_write_entry()
620 uasm_i_nop(p); in build_tlb_write_entry()
630 static __maybe_unused void build_convert_pte_to_entrylo(u32 **p, in build_convert_pte_to_entrylo() argument
634 UASM_i_ROTR(p, reg, reg, ilog2(_PAGE_GLOBAL)); in build_convert_pte_to_entrylo()
637 uasm_i_dsrl_safe(p, reg, reg, ilog2(_PAGE_GLOBAL)); in build_convert_pte_to_entrylo()
639 UASM_i_SRL(p, reg, reg, ilog2(_PAGE_GLOBAL)); in build_convert_pte_to_entrylo()
646 static void build_restore_pagemask(u32 **p, struct uasm_reloc **r, in build_restore_pagemask() argument
653 uasm_i_lui(p, tmp, PM_DEFAULT_MASK >> 16); in build_restore_pagemask()
654 uasm_i_ori(p, tmp, tmp, PM_DEFAULT_MASK & 0xffff); in build_restore_pagemask()
655 uasm_i_mtc0(p, tmp, C0_PAGEMASK); in build_restore_pagemask()
656 uasm_il_b(p, r, lid); in build_restore_pagemask()
658 uasm_i_ori(p, tmp, 0, PM_DEFAULT_MASK); in build_restore_pagemask()
659 uasm_i_mtc0(p, tmp, C0_PAGEMASK); in build_restore_pagemask()
660 uasm_il_b(p, r, lid); in build_restore_pagemask()
662 uasm_i_mtc0(p, 0, C0_PAGEMASK); in build_restore_pagemask()
663 uasm_il_b(p, r, lid); in build_restore_pagemask()
666 UASM_i_MFC0(p, 1, c0_kscratch(), scratch_reg); in build_restore_pagemask()
668 UASM_i_LW(p, 1, scratchpad_offset(0), 0); in build_restore_pagemask()
672 uasm_i_lui(p, tmp, PM_DEFAULT_MASK >> 16); in build_restore_pagemask()
673 uasm_i_ori(p, tmp, tmp, PM_DEFAULT_MASK & 0xffff); in build_restore_pagemask()
674 uasm_il_b(p, r, lid); in build_restore_pagemask()
675 uasm_i_mtc0(p, tmp, C0_PAGEMASK); in build_restore_pagemask()
677 uasm_i_ori(p, tmp, 0, PM_DEFAULT_MASK); in build_restore_pagemask()
678 uasm_il_b(p, r, lid); in build_restore_pagemask()
679 uasm_i_mtc0(p, tmp, C0_PAGEMASK); in build_restore_pagemask()
681 uasm_il_b(p, r, lid); in build_restore_pagemask()
682 uasm_i_mtc0(p, 0, C0_PAGEMASK); in build_restore_pagemask()
687 static void build_huge_tlb_write_entry(u32 **p, struct uasm_label **l, in build_huge_tlb_write_entry() argument
694 uasm_i_lui(p, tmp, PM_HUGE_MASK >> 16); in build_huge_tlb_write_entry()
695 uasm_i_ori(p, tmp, tmp, PM_HUGE_MASK & 0xffff); in build_huge_tlb_write_entry()
696 uasm_i_mtc0(p, tmp, C0_PAGEMASK); in build_huge_tlb_write_entry()
698 build_tlb_write_entry(p, l, r, wmode); in build_huge_tlb_write_entry()
700 build_restore_pagemask(p, r, tmp, label_leave, restore_scratch); in build_huge_tlb_write_entry()
707 build_is_huge_pte(u32 **p, struct uasm_reloc **r, unsigned int tmp, in build_is_huge_pte() argument
710 UASM_i_LW(p, tmp, 0, pmd); in build_is_huge_pte()
712 uasm_il_bbit1(p, r, tmp, ilog2(_PAGE_HUGE), lid); in build_is_huge_pte()
714 uasm_i_andi(p, tmp, tmp, _PAGE_HUGE); in build_is_huge_pte()
715 uasm_il_bnez(p, r, tmp, lid); in build_is_huge_pte()
719 static void build_huge_update_entries(u32 **p, unsigned int pte, in build_huge_update_entries() argument
737 uasm_i_lui(p, tmp, HPAGE_SIZE >> (7 + 16)); in build_huge_update_entries()
739 build_convert_pte_to_entrylo(p, pte); in build_huge_update_entries()
740 UASM_i_MTC0(p, pte, C0_ENTRYLO0); /* load it */ in build_huge_update_entries()
743 UASM_i_ADDIU(p, pte, pte, HPAGE_SIZE >> 7); in build_huge_update_entries()
745 UASM_i_ADDU(p, pte, pte, tmp); in build_huge_update_entries()
747 UASM_i_MTC0(p, pte, C0_ENTRYLO1); /* load it */ in build_huge_update_entries()
750 static void build_huge_handler_tail(u32 **p, struct uasm_reloc **r, in build_huge_handler_tail() argument
756 UASM_i_SC(p, pte, 0, ptr); in build_huge_handler_tail()
757 uasm_il_beqz(p, r, pte, label_tlb_huge_update); in build_huge_handler_tail()
758 UASM_i_LW(p, pte, 0, ptr); /* Needed because SC killed our PTE */ in build_huge_handler_tail()
760 UASM_i_SW(p, pte, 0, ptr); in build_huge_handler_tail()
762 build_huge_update_entries(p, pte, ptr); in build_huge_handler_tail()
763 build_huge_tlb_write_entry(p, l, r, pte, tlb_indexed, 0); in build_huge_handler_tail()
773 build_get_pmde64(u32 **p, struct uasm_label **l, struct uasm_reloc **r, in build_get_pmde64() argument
782 uasm_i_dmfc0(p, tmp, C0_BADVADDR); in build_get_pmde64()
796 uasm_i_dsrl_safe(p, ptr, tmp, PGDIR_SHIFT + PGD_ORDER + PAGE_SHIFT - 3); in build_get_pmde64()
797 uasm_il_bnez(p, r, ptr, label_vmalloc); in build_get_pmde64()
799 uasm_il_bltz(p, r, tmp, label_vmalloc); in build_get_pmde64()
805 UASM_i_MFC0(p, ptr, c0_kscratch(), pgd_reg); in build_get_pmde64()
811 UASM_i_MFC0(p, ptr, C0_CONTEXT); in build_get_pmde64()
814 uasm_i_dins(p, ptr, 0, 0, 23); in build_get_pmde64()
817 uasm_i_ori(p, ptr, ptr, 0x540); in build_get_pmde64()
818 uasm_i_drotr(p, ptr, ptr, 11); in build_get_pmde64()
820 UASM_i_CPUID_MFC0(p, ptr, SMP_CPUID_REG); in build_get_pmde64()
821 uasm_i_dsrl_safe(p, ptr, ptr, SMP_CPUID_PTRSHIFT); in build_get_pmde64()
822 UASM_i_LA_mostly(p, tmp, pgdc); in build_get_pmde64()
823 uasm_i_daddu(p, ptr, ptr, tmp); in build_get_pmde64()
824 uasm_i_dmfc0(p, tmp, C0_BADVADDR); in build_get_pmde64()
825 uasm_i_ld(p, ptr, uasm_rel_lo(pgdc), ptr); in build_get_pmde64()
827 UASM_i_LA_mostly(p, ptr, pgdc); in build_get_pmde64()
828 uasm_i_ld(p, ptr, uasm_rel_lo(pgdc), ptr); in build_get_pmde64()
832 uasm_l_vmalloc_done(l, *p); in build_get_pmde64()
835 uasm_i_dsrl_safe(p, tmp, tmp, PGDIR_SHIFT - 3); in build_get_pmde64()
837 uasm_i_andi(p, tmp, tmp, (PTRS_PER_PGD - 1)<<3); in build_get_pmde64()
838 uasm_i_daddu(p, ptr, ptr, tmp); /* add in pgd offset */ in build_get_pmde64()
840 uasm_i_dmfc0(p, tmp, C0_BADVADDR); /* get faulting address */ in build_get_pmde64()
841 uasm_i_ld(p, ptr, 0, ptr); /* get pmd pointer */ in build_get_pmde64()
842 uasm_i_dsrl_safe(p, tmp, tmp, PMD_SHIFT-3); /* get pmd offset in bytes */ in build_get_pmde64()
843 uasm_i_andi(p, tmp, tmp, (PTRS_PER_PMD - 1)<<3); in build_get_pmde64()
844 uasm_i_daddu(p, ptr, ptr, tmp); /* add in pmd offset */ in build_get_pmde64()
853 build_get_pgd_vmalloc64(u32 **p, struct uasm_label **l, struct uasm_reloc **r, in build_get_pgd_vmalloc64() argument
863 uasm_l_vmalloc(l, *p); in build_get_pgd_vmalloc64()
867 uasm_il_bltz(p, r, bvaddr, label_vmalloc_done); in build_get_pgd_vmalloc64()
868 uasm_i_lui(p, ptr, uasm_rel_hi(swpd)); in build_get_pgd_vmalloc64()
872 uasm_il_bgez(p, r, bvaddr, label_large_segbits_fault); in build_get_pgd_vmalloc64()
877 uasm_il_b(p, r, label_vmalloc_done); in build_get_pgd_vmalloc64()
878 uasm_i_lui(p, ptr, uasm_rel_hi(swpd)); in build_get_pgd_vmalloc64()
880 UASM_i_LA_mostly(p, ptr, swpd); in build_get_pgd_vmalloc64()
881 uasm_il_b(p, r, label_vmalloc_done); in build_get_pgd_vmalloc64()
883 uasm_i_addiu(p, ptr, ptr, uasm_rel_lo(swpd)); in build_get_pgd_vmalloc64()
885 uasm_i_daddiu(p, ptr, ptr, uasm_rel_lo(swpd)); in build_get_pgd_vmalloc64()
889 uasm_l_large_segbits_fault(l, *p); in build_get_pgd_vmalloc64()
902 UASM_i_LA(p, ptr, (unsigned long)tlb_do_page_fault_0); in build_get_pgd_vmalloc64()
903 uasm_i_jr(p, ptr); in build_get_pgd_vmalloc64()
907 UASM_i_MFC0(p, 1, c0_kscratch(), scratch_reg); in build_get_pgd_vmalloc64()
909 UASM_i_LW(p, 1, scratchpad_offset(0), 0); in build_get_pgd_vmalloc64()
911 uasm_i_nop(p); in build_get_pgd_vmalloc64()
923 build_get_pgde32(u32 **p, unsigned int tmp, unsigned int ptr) in build_get_pgde32() argument
927 uasm_i_mfc0(p, ptr, c0_kscratch(), pgd_reg); in build_get_pgde32()
928 uasm_i_mfc0(p, tmp, C0_BADVADDR); /* get faulting address */ in build_get_pgde32()
934 uasm_i_mfc0(p, ptr, SMP_CPUID_REG); in build_get_pgde32()
935 UASM_i_LA_mostly(p, tmp, pgdc); in build_get_pgde32()
936 uasm_i_srl(p, ptr, ptr, SMP_CPUID_PTRSHIFT); in build_get_pgde32()
937 uasm_i_addu(p, ptr, tmp, ptr); in build_get_pgde32()
939 UASM_i_LA_mostly(p, ptr, pgdc); in build_get_pgde32()
941 uasm_i_mfc0(p, tmp, C0_BADVADDR); /* get faulting address */ in build_get_pgde32()
942 uasm_i_lw(p, ptr, uasm_rel_lo(pgdc), ptr); in build_get_pgde32()
944 uasm_i_srl(p, tmp, tmp, PGDIR_SHIFT); /* get pgd only bits */ in build_get_pgde32()
945 uasm_i_sll(p, tmp, tmp, PGD_T_LOG2); in build_get_pgde32()
946 uasm_i_addu(p, ptr, ptr, tmp); /* add in pgd offset */ in build_get_pgde32()
951 static void build_adjust_context(u32 **p, unsigned int ctx) in build_adjust_context() argument
973 UASM_i_SRL(p, ctx, ctx, shift); in build_adjust_context()
974 uasm_i_andi(p, ctx, ctx, mask); in build_adjust_context()
977 static void build_get_ptep(u32 **p, unsigned int tmp, unsigned int ptr) in build_get_ptep() argument
988 UASM_i_LW(p, ptr, 0, ptr); in build_get_ptep()
989 GET_CONTEXT(p, tmp); /* get context reg */ in build_get_ptep()
993 GET_CONTEXT(p, tmp); /* get context reg */ in build_get_ptep()
994 UASM_i_LW(p, ptr, 0, ptr); in build_get_ptep()
998 build_adjust_context(p, tmp); in build_get_ptep()
999 UASM_i_ADDU(p, ptr, ptr, tmp); /* add in offset */ in build_get_ptep()
1002 static void build_update_entries(u32 **p, unsigned int tmp, unsigned int ptep) in build_update_entries() argument
1010 uasm_i_ld(p, tmp, 0, ptep); /* get even pte */ in build_update_entries()
1011 uasm_i_ld(p, ptep, sizeof(pte_t), ptep); /* get odd pte */ in build_update_entries()
1013 UASM_i_ROTR(p, tmp, tmp, ilog2(_PAGE_GLOBAL)); in build_update_entries()
1014 UASM_i_MTC0(p, tmp, C0_ENTRYLO0); /* load it */ in build_update_entries()
1015 UASM_i_ROTR(p, ptep, ptep, ilog2(_PAGE_GLOBAL)); in build_update_entries()
1017 uasm_i_dsrl_safe(p, tmp, tmp, ilog2(_PAGE_GLOBAL)); /* convert to entrylo0 */ in build_update_entries()
1018 UASM_i_MTC0(p, tmp, C0_ENTRYLO0); /* load it */ in build_update_entries()
1019 uasm_i_dsrl_safe(p, ptep, ptep, ilog2(_PAGE_GLOBAL)); /* convert to entrylo1 */ in build_update_entries()
1021 UASM_i_MTC0(p, ptep, C0_ENTRYLO1); /* load it */ in build_update_entries()
1028 uasm_i_addu(p, scratch, 0, ptep); in build_update_entries()
1030 uasm_i_lw(p, tmp, pte_off_even, ptep); /* even pte */ in build_update_entries()
1031 uasm_i_lw(p, ptep, pte_off_odd, ptep); /* odd pte */ in build_update_entries()
1032 UASM_i_ROTR(p, tmp, tmp, ilog2(_PAGE_GLOBAL)); in build_update_entries()
1033 UASM_i_ROTR(p, ptep, ptep, ilog2(_PAGE_GLOBAL)); in build_update_entries()
1034 UASM_i_MTC0(p, tmp, C0_ENTRYLO0); in build_update_entries()
1035 UASM_i_MTC0(p, ptep, C0_ENTRYLO1); in build_update_entries()
1037 uasm_i_lw(p, tmp, 0, scratch); in build_update_entries()
1038 uasm_i_lw(p, ptep, sizeof(pte_t), scratch); in build_update_entries()
1039 uasm_i_lui(p, scratch, 0xff); in build_update_entries()
1040 uasm_i_ori(p, scratch, scratch, 0xffff); in build_update_entries()
1041 uasm_i_and(p, tmp, scratch, tmp); in build_update_entries()
1042 uasm_i_and(p, ptep, scratch, ptep); in build_update_entries()
1043 uasm_i_mthc0(p, tmp, C0_ENTRYLO0); in build_update_entries()
1044 uasm_i_mthc0(p, ptep, C0_ENTRYLO1); in build_update_entries()
1048 UASM_i_LW(p, tmp, 0, ptep); /* get even pte */ in build_update_entries()
1049 UASM_i_LW(p, ptep, sizeof(pte_t), ptep); /* get odd pte */ in build_update_entries()
1051 build_tlb_probe_entry(p); in build_update_entries()
1053 UASM_i_ROTR(p, tmp, tmp, ilog2(_PAGE_GLOBAL)); in build_update_entries()
1055 UASM_i_MTC0(p, 0, C0_ENTRYLO0); in build_update_entries()
1056 UASM_i_MTC0(p, tmp, C0_ENTRYLO0); /* load it */ in build_update_entries()
1057 UASM_i_ROTR(p, ptep, ptep, ilog2(_PAGE_GLOBAL)); in build_update_entries()
1059 UASM_i_SRL(p, tmp, tmp, ilog2(_PAGE_GLOBAL)); /* convert to entrylo0 */ in build_update_entries()
1061 UASM_i_MTC0(p, 0, C0_ENTRYLO0); in build_update_entries()
1062 UASM_i_MTC0(p, tmp, C0_ENTRYLO0); /* load it */ in build_update_entries()
1063 UASM_i_SRL(p, ptep, ptep, ilog2(_PAGE_GLOBAL)); /* convert to entrylo1 */ in build_update_entries()
1065 uasm_i_mfc0(p, tmp, C0_INDEX); in build_update_entries()
1068 UASM_i_MTC0(p, 0, C0_ENTRYLO1); in build_update_entries()
1069 UASM_i_MTC0(p, ptep, C0_ENTRYLO1); /* load it */ in build_update_entries()
1080 build_fast_tlb_refill_handler (u32 **p, struct uasm_label **l, in build_fast_tlb_refill_handler() argument
1094 UASM_i_MFC0(p, tmp, C0_BADVADDR); in build_fast_tlb_refill_handler()
1097 UASM_i_MFC0(p, ptr, c0_kscratch(), pgd_reg); in build_fast_tlb_refill_handler()
1099 UASM_i_MFC0(p, ptr, C0_CONTEXT); in build_fast_tlb_refill_handler()
1102 UASM_i_MTC0(p, scratch, c0_kscratch(), c0_scratch_reg); in build_fast_tlb_refill_handler()
1104 UASM_i_SW(p, scratch, scratchpad_offset(0), 0); in build_fast_tlb_refill_handler()
1106 uasm_i_dsrl_safe(p, scratch, tmp, in build_fast_tlb_refill_handler()
1108 uasm_il_bnez(p, r, scratch, label_vmalloc); in build_fast_tlb_refill_handler()
1113 uasm_i_dins(p, ptr, 0, 0, 23); in build_fast_tlb_refill_handler()
1117 UASM_i_MFC0(p, ptr, c0_kscratch(), pgd_reg); in build_fast_tlb_refill_handler()
1119 UASM_i_MFC0(p, ptr, C0_CONTEXT); in build_fast_tlb_refill_handler()
1121 UASM_i_MFC0(p, tmp, C0_BADVADDR); in build_fast_tlb_refill_handler()
1124 UASM_i_MTC0(p, scratch, c0_kscratch(), c0_scratch_reg); in build_fast_tlb_refill_handler()
1126 UASM_i_SW(p, scratch, scratchpad_offset(0), 0); in build_fast_tlb_refill_handler()
1130 uasm_i_dins(p, ptr, 0, 0, 23); in build_fast_tlb_refill_handler()
1132 uasm_il_bltz(p, r, tmp, label_vmalloc); in build_fast_tlb_refill_handler()
1138 uasm_i_ori(p, ptr, ptr, 0x540); in build_fast_tlb_refill_handler()
1139 uasm_i_drotr(p, ptr, ptr, 11); in build_fast_tlb_refill_handler()
1150 uasm_i_dsrl_safe(p, scratch, tmp, PGDIR_SHIFT - 3); in build_fast_tlb_refill_handler()
1152 uasm_l_vmalloc_done(l, *p); in build_fast_tlb_refill_handler()
1162 uasm_i_dsrl_safe(p, scratch, tmp, PGDIR_SHIFT - 3); in build_fast_tlb_refill_handler()
1165 GET_CONTEXT(p, tmp); /* get context reg */ in build_fast_tlb_refill_handler()
1167 uasm_i_andi(p, scratch, scratch, (PTRS_PER_PGD - 1) << 3); in build_fast_tlb_refill_handler()
1170 UASM_i_LWX(p, LOC_PTEP, scratch, ptr); in build_fast_tlb_refill_handler()
1172 uasm_i_daddu(p, ptr, ptr, scratch); /* add in pgd offset */ in build_fast_tlb_refill_handler()
1173 uasm_i_ld(p, LOC_PTEP, 0, ptr); /* get pmd pointer */ in build_fast_tlb_refill_handler()
1178 uasm_i_dsrl_safe(p, scratch, tmp, PMD_SHIFT - 3); in build_fast_tlb_refill_handler()
1179 uasm_i_andi(p, scratch, scratch, (PTRS_PER_PMD - 1) << 3); in build_fast_tlb_refill_handler()
1180 GET_CONTEXT(p, tmp); /* get context reg */ in build_fast_tlb_refill_handler()
1183 UASM_i_LWX(p, scratch, scratch, ptr); in build_fast_tlb_refill_handler()
1185 uasm_i_daddu(p, ptr, ptr, scratch); /* add in pmd offset */ in build_fast_tlb_refill_handler()
1186 UASM_i_LW(p, scratch, 0, ptr); in build_fast_tlb_refill_handler()
1190 build_adjust_context(p, tmp); in build_fast_tlb_refill_handler()
1193 uasm_il_bbit1(p, r, scratch, ilog2(_PAGE_HUGE), label_tlb_huge_update); in build_fast_tlb_refill_handler()
1200 uasm_i_nop(p); in build_fast_tlb_refill_handler()
1208 UASM_i_LWX(p, even, scratch, tmp); in build_fast_tlb_refill_handler()
1209 UASM_i_ADDIU(p, tmp, tmp, sizeof(pte_t)); in build_fast_tlb_refill_handler()
1210 UASM_i_LWX(p, odd, scratch, tmp); in build_fast_tlb_refill_handler()
1212 UASM_i_ADDU(p, ptr, scratch, tmp); /* add in offset */ in build_fast_tlb_refill_handler()
1215 UASM_i_LW(p, even, 0, ptr); /* get even pte */ in build_fast_tlb_refill_handler()
1216 UASM_i_LW(p, odd, sizeof(pte_t), ptr); /* get odd pte */ in build_fast_tlb_refill_handler()
1219 uasm_i_drotr(p, even, even, ilog2(_PAGE_GLOBAL)); in build_fast_tlb_refill_handler()
1220 UASM_i_MTC0(p, even, C0_ENTRYLO0); /* load it */ in build_fast_tlb_refill_handler()
1221 uasm_i_drotr(p, odd, odd, ilog2(_PAGE_GLOBAL)); in build_fast_tlb_refill_handler()
1223 uasm_i_dsrl_safe(p, even, even, ilog2(_PAGE_GLOBAL)); in build_fast_tlb_refill_handler()
1224 UASM_i_MTC0(p, even, C0_ENTRYLO0); /* load it */ in build_fast_tlb_refill_handler()
1225 uasm_i_dsrl_safe(p, odd, odd, ilog2(_PAGE_GLOBAL)); in build_fast_tlb_refill_handler()
1227 UASM_i_MTC0(p, odd, C0_ENTRYLO1); /* load it */ in build_fast_tlb_refill_handler()
1230 UASM_i_MFC0(p, scratch, c0_kscratch(), c0_scratch_reg); in build_fast_tlb_refill_handler()
1231 build_tlb_write_entry(p, l, r, tlb_random); in build_fast_tlb_refill_handler()
1232 uasm_l_leave(l, *p); in build_fast_tlb_refill_handler()
1235 build_tlb_write_entry(p, l, r, tlb_random); in build_fast_tlb_refill_handler()
1236 uasm_l_leave(l, *p); in build_fast_tlb_refill_handler()
1237 UASM_i_LW(p, scratch, scratchpad_offset(0), 0); in build_fast_tlb_refill_handler()
1239 UASM_i_LW(p, scratch, scratchpad_offset(0), 0); in build_fast_tlb_refill_handler()
1240 build_tlb_write_entry(p, l, r, tlb_random); in build_fast_tlb_refill_handler()
1241 uasm_l_leave(l, *p); in build_fast_tlb_refill_handler()
1245 uasm_i_eret(p); /* return from trap */ in build_fast_tlb_refill_handler()
1260 u32 *p = tlb_handler; in build_r4000_tlb_refill_handler() local
1274 htlb_info = build_fast_tlb_refill_handler(&p, &l, &r, K0, K1, in build_r4000_tlb_refill_handler()
1288 uasm_i_dmfc0(&p, K0, C0_BADVADDR); in build_r4000_tlb_refill_handler()
1289 uasm_i_dmfc0(&p, K1, C0_ENTRYHI); in build_r4000_tlb_refill_handler()
1290 uasm_i_xor(&p, K0, K0, K1); in build_r4000_tlb_refill_handler()
1291 uasm_i_dsrl_safe(&p, K1, K0, 62); in build_r4000_tlb_refill_handler()
1292 uasm_i_dsrl_safe(&p, K0, K0, 12 + 1); in build_r4000_tlb_refill_handler()
1293 uasm_i_dsll_safe(&p, K0, K0, 64 + 12 + 1 - segbits); in build_r4000_tlb_refill_handler()
1294 uasm_i_or(&p, K0, K0, K1); in build_r4000_tlb_refill_handler()
1295 uasm_il_bnez(&p, &r, K0, label_leave); in build_r4000_tlb_refill_handler()
1300 build_get_pmde64(&p, &l, &r, K0, K1); /* get pmd in K1 */ in build_r4000_tlb_refill_handler()
1302 build_get_pgde32(&p, K0, K1); /* get pgd in K1 */ in build_r4000_tlb_refill_handler()
1306 build_is_huge_pte(&p, &r, K0, K1, label_tlb_huge_update); in build_r4000_tlb_refill_handler()
1309 build_get_ptep(&p, K0, K1); in build_r4000_tlb_refill_handler()
1310 build_update_entries(&p, K0, K1); in build_r4000_tlb_refill_handler()
1311 build_tlb_write_entry(&p, &l, &r, tlb_random); in build_r4000_tlb_refill_handler()
1312 uasm_l_leave(&l, p); in build_r4000_tlb_refill_handler()
1313 uasm_i_eret(&p); /* return from trap */ in build_r4000_tlb_refill_handler()
1316 uasm_l_tlb_huge_update(&l, p); in build_r4000_tlb_refill_handler()
1318 UASM_i_LW(&p, htlb_info.huge_pte, 0, K1); in build_r4000_tlb_refill_handler()
1319 build_huge_update_entries(&p, htlb_info.huge_pte, K1); in build_r4000_tlb_refill_handler()
1320 build_huge_tlb_write_entry(&p, &l, &r, K0, tlb_random, in build_r4000_tlb_refill_handler()
1325 build_get_pgd_vmalloc64(&p, &l, &r, K0, K1, vmalloc_mode); in build_r4000_tlb_refill_handler()
1340 if ((p - tlb_handler) > 64) in build_r4000_tlb_refill_handler()
1347 uasm_copy_handler(relocs, labels, tlb_handler, p, f); in build_r4000_tlb_refill_handler()
1348 final_len = p - tlb_handler; in build_r4000_tlb_refill_handler()
1351 if (((p - tlb_handler) > (MIPS64_REFILL_INSNS * 2) - 1) in build_r4000_tlb_refill_handler()
1352 || (((p - tlb_handler) > (MIPS64_REFILL_INSNS * 2) - 3) in build_r4000_tlb_refill_handler()
1360 if ((p - tlb_handler) <= MIPS64_REFILL_INSNS) { in build_r4000_tlb_refill_handler()
1362 uasm_copy_handler(relocs, labels, tlb_handler, p, f); in build_r4000_tlb_refill_handler()
1363 final_len = p - tlb_handler; in build_r4000_tlb_refill_handler()
1383 split < p - MIPS64_REFILL_INSNS) in build_r4000_tlb_refill_handler()
1422 uasm_copy_handler(relocs, labels, split, p, final_handler); in build_r4000_tlb_refill_handler()
1424 (p - split); in build_r4000_tlb_refill_handler()
1451 u32 *p = tlbmiss_handler_setup_pgd_start; in build_setup_pgd() local
1475 UASM_i_SRA(&p, a1, a0, 29); in build_setup_pgd()
1476 UASM_i_ADDIU(&p, a1, a1, 4); in build_setup_pgd()
1477 uasm_il_bnez(&p, &r, a1, label_tlbl_goaround1); in build_setup_pgd()
1478 uasm_i_nop(&p); in build_setup_pgd()
1479 uasm_i_dinsm(&p, a0, 0, 29, 64 - 29); in build_setup_pgd()
1480 uasm_l_tlbl_goaround1(&l, p); in build_setup_pgd()
1481 UASM_i_SLL(&p, a0, a0, 11); in build_setup_pgd()
1482 uasm_i_jr(&p, 31); in build_setup_pgd()
1483 UASM_i_MTC0(&p, a0, C0_CONTEXT); in build_setup_pgd()
1486 uasm_i_jr(&p, 31); in build_setup_pgd()
1487 UASM_i_MTC0(&p, a0, c0_kscratch(), pgd_reg); in build_setup_pgd()
1492 UASM_i_CPUID_MFC0(&p, a1, SMP_CPUID_REG); in build_setup_pgd()
1493 UASM_i_SRL_SAFE(&p, a1, a1, SMP_CPUID_PTRSHIFT); in build_setup_pgd()
1494 UASM_i_LA_mostly(&p, a2, pgdc); in build_setup_pgd()
1495 UASM_i_ADDU(&p, a2, a2, a1); in build_setup_pgd()
1496 UASM_i_SW(&p, a0, uasm_rel_lo(pgdc), a2); in build_setup_pgd()
1498 UASM_i_LA_mostly(&p, a2, pgdc); in build_setup_pgd()
1499 UASM_i_SW(&p, a0, uasm_rel_lo(pgdc), a2); in build_setup_pgd()
1501 uasm_i_jr(&p, 31); in build_setup_pgd()
1505 UASM_i_MTC0(&p, a0, c0_kscratch(), pgd_reg); in build_setup_pgd()
1507 uasm_i_nop(&p); in build_setup_pgd()
1509 if (p >= tlbmiss_handler_setup_pgd_end) in build_setup_pgd()
1514 (unsigned int)(p - tlbmiss_handler_setup_pgd)); in build_setup_pgd()
1521 iPTE_LW(u32 **p, unsigned int pte, unsigned int ptr) in iPTE_LW() argument
1526 uasm_i_lld(p, pte, 0, ptr); in iPTE_LW()
1529 UASM_i_LL(p, pte, 0, ptr); in iPTE_LW()
1533 uasm_i_ld(p, pte, 0, ptr); in iPTE_LW()
1536 UASM_i_LW(p, pte, 0, ptr); in iPTE_LW()
1541 iPTE_SW(u32 **p, struct uasm_reloc **r, unsigned int pte, unsigned int ptr, in iPTE_SW() argument
1550 uasm_i_lui(p, scratch, (mode >> 16)); in iPTE_SW()
1551 uasm_i_or(p, pte, pte, scratch); in iPTE_SW()
1554 uasm_i_ori(p, pte, pte, mode); in iPTE_SW()
1558 uasm_i_scd(p, pte, 0, ptr); in iPTE_SW()
1561 UASM_i_SC(p, pte, 0, ptr); in iPTE_SW()
1564 uasm_il_beqzl(p, r, pte, label_smp_pgtable_change); in iPTE_SW()
1566 uasm_il_beqz(p, r, pte, label_smp_pgtable_change); in iPTE_SW()
1571 uasm_i_ll(p, pte, sizeof(pte_t) / 2, ptr); in iPTE_SW()
1572 uasm_i_ori(p, pte, pte, hwmode); in iPTE_SW()
1573 uasm_i_sc(p, pte, sizeof(pte_t) / 2, ptr); in iPTE_SW()
1574 uasm_il_beqz(p, r, pte, label_smp_pgtable_change); in iPTE_SW()
1576 uasm_i_lw(p, pte, 0, ptr); in iPTE_SW()
1578 uasm_i_nop(p); in iPTE_SW()
1580 uasm_i_nop(p); in iPTE_SW()
1585 uasm_i_sd(p, pte, 0, ptr); in iPTE_SW()
1588 UASM_i_SW(p, pte, 0, ptr); in iPTE_SW()
1592 uasm_i_lw(p, pte, sizeof(pte_t) / 2, ptr); in iPTE_SW()
1593 uasm_i_ori(p, pte, pte, hwmode); in iPTE_SW()
1594 uasm_i_sw(p, pte, sizeof(pte_t) / 2, ptr); in iPTE_SW()
1595 uasm_i_lw(p, pte, 0, ptr); in iPTE_SW()
1607 build_pte_present(u32 **p, struct uasm_reloc **r, in build_pte_present() argument
1614 uasm_il_bbit0(p, r, pte, ilog2(_PAGE_PRESENT), lid); in build_pte_present()
1615 uasm_i_nop(p); in build_pte_present()
1617 uasm_i_srl(p, t, pte, _PAGE_PRESENT_SHIFT); in build_pte_present()
1618 uasm_i_andi(p, t, t, 1); in build_pte_present()
1619 uasm_il_beqz(p, r, t, lid); in build_pte_present()
1622 iPTE_LW(p, pte, ptr); in build_pte_present()
1625 uasm_i_srl(p, t, pte, _PAGE_PRESENT_SHIFT); in build_pte_present()
1626 uasm_i_andi(p, t, t, 3); in build_pte_present()
1627 uasm_i_xori(p, t, t, 3); in build_pte_present()
1628 uasm_il_bnez(p, r, t, lid); in build_pte_present()
1631 iPTE_LW(p, pte, ptr); in build_pte_present()
1637 build_make_valid(u32 **p, struct uasm_reloc **r, unsigned int pte, in build_make_valid() argument
1642 iPTE_SW(p, r, pte, ptr, mode); in build_make_valid()
1650 build_pte_writable(u32 **p, struct uasm_reloc **r, in build_pte_writable() argument
1656 uasm_i_srl(p, t, pte, _PAGE_PRESENT_SHIFT); in build_pte_writable()
1657 uasm_i_andi(p, t, t, 5); in build_pte_writable()
1658 uasm_i_xori(p, t, t, 5); in build_pte_writable()
1659 uasm_il_bnez(p, r, t, lid); in build_pte_writable()
1662 iPTE_LW(p, pte, ptr); in build_pte_writable()
1664 uasm_i_nop(p); in build_pte_writable()
1671 build_make_write(u32 **p, struct uasm_reloc **r, unsigned int pte, in build_make_write() argument
1677 iPTE_SW(p, r, pte, ptr, mode); in build_make_write()
1685 build_pte_modifiable(u32 **p, struct uasm_reloc **r, in build_pte_modifiable() argument
1690 uasm_il_bbit0(p, r, pte, ilog2(_PAGE_WRITE), lid); in build_pte_modifiable()
1691 uasm_i_nop(p); in build_pte_modifiable()
1694 uasm_i_srl(p, t, pte, _PAGE_WRITE_SHIFT); in build_pte_modifiable()
1695 uasm_i_andi(p, t, t, 1); in build_pte_modifiable()
1696 uasm_il_beqz(p, r, t, lid); in build_pte_modifiable()
1699 iPTE_LW(p, pte, ptr); in build_pte_modifiable()
1715 build_r3000_pte_reload_tlbwi(u32 **p, unsigned int pte, unsigned int tmp) in build_r3000_pte_reload_tlbwi() argument
1717 uasm_i_mtc0(p, pte, C0_ENTRYLO0); /* cp0 delay */ in build_r3000_pte_reload_tlbwi()
1718 uasm_i_mfc0(p, tmp, C0_EPC); /* cp0 delay */ in build_r3000_pte_reload_tlbwi()
1719 uasm_i_tlbwi(p); in build_r3000_pte_reload_tlbwi()
1720 uasm_i_jr(p, tmp); in build_r3000_pte_reload_tlbwi()
1721 uasm_i_rfe(p); /* branch delay */ in build_r3000_pte_reload_tlbwi()
1731 build_r3000_tlb_reload_write(u32 **p, struct uasm_label **l, in build_r3000_tlb_reload_write() argument
1735 uasm_i_mfc0(p, tmp, C0_INDEX); in build_r3000_tlb_reload_write()
1736 uasm_i_mtc0(p, pte, C0_ENTRYLO0); /* cp0 delay */ in build_r3000_tlb_reload_write()
1737 uasm_il_bltz(p, r, tmp, label_r3000_write_probe_fail); /* cp0 delay */ in build_r3000_tlb_reload_write()
1738 uasm_i_mfc0(p, tmp, C0_EPC); /* branch delay */ in build_r3000_tlb_reload_write()
1739 uasm_i_tlbwi(p); /* cp0 delay */ in build_r3000_tlb_reload_write()
1740 uasm_i_jr(p, tmp); in build_r3000_tlb_reload_write()
1741 uasm_i_rfe(p); /* branch delay */ in build_r3000_tlb_reload_write()
1742 uasm_l_r3000_write_probe_fail(l, *p); in build_r3000_tlb_reload_write()
1743 uasm_i_tlbwr(p); /* cp0 delay */ in build_r3000_tlb_reload_write()
1744 uasm_i_jr(p, tmp); in build_r3000_tlb_reload_write()
1745 uasm_i_rfe(p); /* branch delay */ in build_r3000_tlb_reload_write()
1749 build_r3000_tlbchange_handler_head(u32 **p, unsigned int pte, in build_r3000_tlbchange_handler_head() argument
1754 uasm_i_mfc0(p, pte, C0_BADVADDR); in build_r3000_tlbchange_handler_head()
1755 uasm_i_lui(p, ptr, uasm_rel_hi(pgdc)); /* cp0 delay */ in build_r3000_tlbchange_handler_head()
1756 uasm_i_lw(p, ptr, uasm_rel_lo(pgdc), ptr); in build_r3000_tlbchange_handler_head()
1757 uasm_i_srl(p, pte, pte, 22); /* load delay */ in build_r3000_tlbchange_handler_head()
1758 uasm_i_sll(p, pte, pte, 2); in build_r3000_tlbchange_handler_head()
1759 uasm_i_addu(p, ptr, ptr, pte); in build_r3000_tlbchange_handler_head()
1760 uasm_i_mfc0(p, pte, C0_CONTEXT); in build_r3000_tlbchange_handler_head()
1761 uasm_i_lw(p, ptr, 0, ptr); /* cp0 delay */ in build_r3000_tlbchange_handler_head()
1762 uasm_i_andi(p, pte, pte, 0xffc); /* load delay */ in build_r3000_tlbchange_handler_head()
1763 uasm_i_addu(p, ptr, ptr, pte); in build_r3000_tlbchange_handler_head()
1764 uasm_i_lw(p, pte, 0, ptr); in build_r3000_tlbchange_handler_head()
1765 uasm_i_tlbp(p); /* load delay */ in build_r3000_tlbchange_handler_head()
1770 u32 *p = handle_tlbl; in build_r3000_tlb_load_handler() local
1779 build_r3000_tlbchange_handler_head(&p, K0, K1); in build_r3000_tlb_load_handler()
1780 build_pte_present(&p, &r, K0, K1, -1, label_nopage_tlbl); in build_r3000_tlb_load_handler()
1781 uasm_i_nop(&p); /* load delay */ in build_r3000_tlb_load_handler()
1782 build_make_valid(&p, &r, K0, K1); in build_r3000_tlb_load_handler()
1783 build_r3000_tlb_reload_write(&p, &l, &r, K0, K1); in build_r3000_tlb_load_handler()
1785 uasm_l_nopage_tlbl(&l, p); in build_r3000_tlb_load_handler()
1786 uasm_i_j(&p, (unsigned long)tlb_do_page_fault_0 & 0x0fffffff); in build_r3000_tlb_load_handler()
1787 uasm_i_nop(&p); in build_r3000_tlb_load_handler()
1789 if (p >= handle_tlbl_end) in build_r3000_tlb_load_handler()
1794 (unsigned int)(p - handle_tlbl)); in build_r3000_tlb_load_handler()
1801 u32 *p = handle_tlbs; in build_r3000_tlb_store_handler() local
1810 build_r3000_tlbchange_handler_head(&p, K0, K1); in build_r3000_tlb_store_handler()
1811 build_pte_writable(&p, &r, K0, K1, -1, label_nopage_tlbs); in build_r3000_tlb_store_handler()
1812 uasm_i_nop(&p); /* load delay */ in build_r3000_tlb_store_handler()
1813 build_make_write(&p, &r, K0, K1); in build_r3000_tlb_store_handler()
1814 build_r3000_tlb_reload_write(&p, &l, &r, K0, K1); in build_r3000_tlb_store_handler()
1816 uasm_l_nopage_tlbs(&l, p); in build_r3000_tlb_store_handler()
1817 uasm_i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff); in build_r3000_tlb_store_handler()
1818 uasm_i_nop(&p); in build_r3000_tlb_store_handler()
1820 if (p >= handle_tlbs_end) in build_r3000_tlb_store_handler()
1825 (unsigned int)(p - handle_tlbs)); in build_r3000_tlb_store_handler()
1832 u32 *p = handle_tlbm; in build_r3000_tlb_modify_handler() local
1841 build_r3000_tlbchange_handler_head(&p, K0, K1); in build_r3000_tlb_modify_handler()
1842 build_pte_modifiable(&p, &r, K0, K1, -1, label_nopage_tlbm); in build_r3000_tlb_modify_handler()
1843 uasm_i_nop(&p); /* load delay */ in build_r3000_tlb_modify_handler()
1844 build_make_write(&p, &r, K0, K1); in build_r3000_tlb_modify_handler()
1845 build_r3000_pte_reload_tlbwi(&p, K0, K1); in build_r3000_tlb_modify_handler()
1847 uasm_l_nopage_tlbm(&l, p); in build_r3000_tlb_modify_handler()
1848 uasm_i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff); in build_r3000_tlb_modify_handler()
1849 uasm_i_nop(&p); in build_r3000_tlb_modify_handler()
1851 if (p >= handle_tlbm_end) in build_r3000_tlb_modify_handler()
1856 (unsigned int)(p - handle_tlbm)); in build_r3000_tlb_modify_handler()
1866 build_r4000_tlbchange_handler_head(u32 **p, struct uasm_label **l, in build_r4000_tlbchange_handler_head() argument
1869 struct work_registers wr = build_get_work_registers(p); in build_r4000_tlbchange_handler_head()
1872 build_get_pmde64(p, l, r, wr.r1, wr.r2); /* get pmd in ptr */ in build_r4000_tlbchange_handler_head()
1874 build_get_pgde32(p, wr.r1, wr.r2); /* get pgd in ptr */ in build_r4000_tlbchange_handler_head()
1883 build_is_huge_pte(p, r, wr.r1, wr.r2, label_tlb_huge_update); in build_r4000_tlbchange_handler_head()
1886 UASM_i_MFC0(p, wr.r1, C0_BADVADDR); in build_r4000_tlbchange_handler_head()
1887 UASM_i_LW(p, wr.r2, 0, wr.r2); in build_r4000_tlbchange_handler_head()
1888 UASM_i_SRL(p, wr.r1, wr.r1, PAGE_SHIFT + PTE_ORDER - PTE_T_LOG2); in build_r4000_tlbchange_handler_head()
1889 uasm_i_andi(p, wr.r1, wr.r1, (PTRS_PER_PTE - 1) << PTE_T_LOG2); in build_r4000_tlbchange_handler_head()
1890 UASM_i_ADDU(p, wr.r2, wr.r2, wr.r1); in build_r4000_tlbchange_handler_head()
1893 uasm_l_smp_pgtable_change(l, *p); in build_r4000_tlbchange_handler_head()
1895 iPTE_LW(p, wr.r1, wr.r2); /* get even pte */ in build_r4000_tlbchange_handler_head()
1897 build_tlb_probe_entry(p); in build_r4000_tlbchange_handler_head()
1900 uasm_i_ehb(p); in build_r4000_tlbchange_handler_head()
1901 uasm_i_mfc0(p, wr.r3, C0_INDEX); in build_r4000_tlbchange_handler_head()
1902 uasm_il_bltz(p, r, wr.r3, label_leave); in build_r4000_tlbchange_handler_head()
1903 uasm_i_nop(p); in build_r4000_tlbchange_handler_head()
1910 build_r4000_tlbchange_handler_tail(u32 **p, struct uasm_label **l, in build_r4000_tlbchange_handler_tail() argument
1914 uasm_i_ori(p, ptr, ptr, sizeof(pte_t)); in build_r4000_tlbchange_handler_tail()
1915 uasm_i_xori(p, ptr, ptr, sizeof(pte_t)); in build_r4000_tlbchange_handler_tail()
1916 build_update_entries(p, tmp, ptr); in build_r4000_tlbchange_handler_tail()
1917 build_tlb_write_entry(p, l, r, tlb_indexed); in build_r4000_tlbchange_handler_tail()
1918 uasm_l_leave(l, *p); in build_r4000_tlbchange_handler_tail()
1919 build_restore_work_registers(p); in build_r4000_tlbchange_handler_tail()
1920 uasm_i_eret(p); /* return from trap */ in build_r4000_tlbchange_handler_tail()
1923 build_get_pgd_vmalloc64(p, l, r, tmp, ptr, not_refill); in build_r4000_tlbchange_handler_tail()
1929 u32 *p = handle_tlbl; in build_r4000_tlb_load_handler() local
1942 uasm_i_dmfc0(&p, K0, C0_BADVADDR); in build_r4000_tlb_load_handler()
1943 uasm_i_dmfc0(&p, K1, C0_ENTRYHI); in build_r4000_tlb_load_handler()
1944 uasm_i_xor(&p, K0, K0, K1); in build_r4000_tlb_load_handler()
1945 uasm_i_dsrl_safe(&p, K1, K0, 62); in build_r4000_tlb_load_handler()
1946 uasm_i_dsrl_safe(&p, K0, K0, 12 + 1); in build_r4000_tlb_load_handler()
1947 uasm_i_dsll_safe(&p, K0, K0, 64 + 12 + 1 - segbits); in build_r4000_tlb_load_handler()
1948 uasm_i_or(&p, K0, K0, K1); in build_r4000_tlb_load_handler()
1949 uasm_il_bnez(&p, &r, K0, label_leave); in build_r4000_tlb_load_handler()
1953 wr = build_r4000_tlbchange_handler_head(&p, &l, &r); in build_r4000_tlb_load_handler()
1954 build_pte_present(&p, &r, wr.r1, wr.r2, wr.r3, label_nopage_tlbl); in build_r4000_tlb_load_handler()
1956 build_tlb_probe_entry(&p); in build_r4000_tlb_load_handler()
1964 uasm_il_bbit0(&p, &r, wr.r1, ilog2(_PAGE_VALID), in build_r4000_tlb_load_handler()
1967 uasm_i_andi(&p, wr.r3, wr.r1, _PAGE_VALID); in build_r4000_tlb_load_handler()
1968 uasm_il_beqz(&p, &r, wr.r3, label_tlbl_goaround1); in build_r4000_tlb_load_handler()
1970 uasm_i_nop(&p); in build_r4000_tlb_load_handler()
1972 uasm_i_tlbr(&p); in build_r4000_tlb_load_handler()
1977 uasm_i_ehb(&p); in build_r4000_tlb_load_handler()
1988 uasm_i_bbit0(&p, wr.r2, ilog2(sizeof(pte_t)), 8); in build_r4000_tlb_load_handler()
1990 uasm_i_andi(&p, wr.r3, wr.r2, sizeof(pte_t)); in build_r4000_tlb_load_handler()
1991 uasm_i_beqz(&p, wr.r3, 8); in build_r4000_tlb_load_handler()
1994 UASM_i_MFC0(&p, wr.r3, C0_ENTRYLO0); in build_r4000_tlb_load_handler()
1996 UASM_i_MFC0(&p, wr.r3, C0_ENTRYLO1); in build_r4000_tlb_load_handler()
2002 uasm_il_bbit1(&p, &r, wr.r3, 1, label_nopage_tlbl); in build_r4000_tlb_load_handler()
2003 uasm_i_nop(&p); in build_r4000_tlb_load_handler()
2004 uasm_l_tlbl_goaround1(&l, p); in build_r4000_tlb_load_handler()
2006 uasm_i_andi(&p, wr.r3, wr.r3, 2); in build_r4000_tlb_load_handler()
2007 uasm_il_bnez(&p, &r, wr.r3, label_nopage_tlbl); in build_r4000_tlb_load_handler()
2008 uasm_i_nop(&p); in build_r4000_tlb_load_handler()
2010 uasm_l_tlbl_goaround1(&l, p); in build_r4000_tlb_load_handler()
2012 build_make_valid(&p, &r, wr.r1, wr.r2); in build_r4000_tlb_load_handler()
2013 build_r4000_tlbchange_handler_tail(&p, &l, &r, wr.r1, wr.r2); in build_r4000_tlb_load_handler()
2020 uasm_l_tlb_huge_update(&l, p); in build_r4000_tlb_load_handler()
2021 iPTE_LW(&p, wr.r1, wr.r2); in build_r4000_tlb_load_handler()
2022 build_pte_present(&p, &r, wr.r1, wr.r2, wr.r3, label_nopage_tlbl); in build_r4000_tlb_load_handler()
2023 build_tlb_probe_entry(&p); in build_r4000_tlb_load_handler()
2031 uasm_il_bbit0(&p, &r, wr.r1, ilog2(_PAGE_VALID), in build_r4000_tlb_load_handler()
2034 uasm_i_andi(&p, wr.r3, wr.r1, _PAGE_VALID); in build_r4000_tlb_load_handler()
2035 uasm_il_beqz(&p, &r, wr.r3, label_tlbl_goaround2); in build_r4000_tlb_load_handler()
2037 uasm_i_nop(&p); in build_r4000_tlb_load_handler()
2039 uasm_i_tlbr(&p); in build_r4000_tlb_load_handler()
2044 uasm_i_ehb(&p); in build_r4000_tlb_load_handler()
2055 uasm_i_bbit0(&p, wr.r2, ilog2(sizeof(pte_t)), 8); in build_r4000_tlb_load_handler()
2057 uasm_i_andi(&p, wr.r3, wr.r2, sizeof(pte_t)); in build_r4000_tlb_load_handler()
2058 uasm_i_beqz(&p, wr.r3, 8); in build_r4000_tlb_load_handler()
2061 UASM_i_MFC0(&p, wr.r3, C0_ENTRYLO0); in build_r4000_tlb_load_handler()
2063 UASM_i_MFC0(&p, wr.r3, C0_ENTRYLO1); in build_r4000_tlb_load_handler()
2069 uasm_il_bbit0(&p, &r, wr.r3, 1, label_tlbl_goaround2); in build_r4000_tlb_load_handler()
2071 uasm_i_andi(&p, wr.r3, wr.r3, 2); in build_r4000_tlb_load_handler()
2072 uasm_il_beqz(&p, &r, wr.r3, label_tlbl_goaround2); in build_r4000_tlb_load_handler()
2075 uasm_i_nop(&p); in build_r4000_tlb_load_handler()
2080 build_restore_pagemask(&p, &r, wr.r3, label_nopage_tlbl, 0); in build_r4000_tlb_load_handler()
2082 uasm_l_tlbl_goaround2(&l, p); in build_r4000_tlb_load_handler()
2084 uasm_i_ori(&p, wr.r1, wr.r1, (_PAGE_ACCESSED | _PAGE_VALID)); in build_r4000_tlb_load_handler()
2085 build_huge_handler_tail(&p, &r, &l, wr.r1, wr.r2); in build_r4000_tlb_load_handler()
2088 uasm_l_nopage_tlbl(&l, p); in build_r4000_tlb_load_handler()
2089 build_restore_work_registers(&p); in build_r4000_tlb_load_handler()
2092 uasm_i_lui(&p, K0, uasm_rel_hi((long)tlb_do_page_fault_0)); in build_r4000_tlb_load_handler()
2093 uasm_i_addiu(&p, K0, K0, uasm_rel_lo((long)tlb_do_page_fault_0)); in build_r4000_tlb_load_handler()
2094 uasm_i_jr(&p, K0); in build_r4000_tlb_load_handler()
2097 uasm_i_j(&p, (unsigned long)tlb_do_page_fault_0 & 0x0fffffff); in build_r4000_tlb_load_handler()
2098 uasm_i_nop(&p); in build_r4000_tlb_load_handler()
2100 if (p >= handle_tlbl_end) in build_r4000_tlb_load_handler()
2105 (unsigned int)(p - handle_tlbl)); in build_r4000_tlb_load_handler()
2112 u32 *p = handle_tlbs; in build_r4000_tlb_store_handler() local
2122 wr = build_r4000_tlbchange_handler_head(&p, &l, &r); in build_r4000_tlb_store_handler()
2123 build_pte_writable(&p, &r, wr.r1, wr.r2, wr.r3, label_nopage_tlbs); in build_r4000_tlb_store_handler()
2125 build_tlb_probe_entry(&p); in build_r4000_tlb_store_handler()
2126 build_make_write(&p, &r, wr.r1, wr.r2); in build_r4000_tlb_store_handler()
2127 build_r4000_tlbchange_handler_tail(&p, &l, &r, wr.r1, wr.r2); in build_r4000_tlb_store_handler()
2134 uasm_l_tlb_huge_update(&l, p); in build_r4000_tlb_store_handler()
2135 iPTE_LW(&p, wr.r1, wr.r2); in build_r4000_tlb_store_handler()
2136 build_pte_writable(&p, &r, wr.r1, wr.r2, wr.r3, label_nopage_tlbs); in build_r4000_tlb_store_handler()
2137 build_tlb_probe_entry(&p); in build_r4000_tlb_store_handler()
2138 uasm_i_ori(&p, wr.r1, wr.r1, in build_r4000_tlb_store_handler()
2140 build_huge_handler_tail(&p, &r, &l, wr.r1, wr.r2); in build_r4000_tlb_store_handler()
2143 uasm_l_nopage_tlbs(&l, p); in build_r4000_tlb_store_handler()
2144 build_restore_work_registers(&p); in build_r4000_tlb_store_handler()
2147 uasm_i_lui(&p, K0, uasm_rel_hi((long)tlb_do_page_fault_1)); in build_r4000_tlb_store_handler()
2148 uasm_i_addiu(&p, K0, K0, uasm_rel_lo((long)tlb_do_page_fault_1)); in build_r4000_tlb_store_handler()
2149 uasm_i_jr(&p, K0); in build_r4000_tlb_store_handler()
2152 uasm_i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff); in build_r4000_tlb_store_handler()
2153 uasm_i_nop(&p); in build_r4000_tlb_store_handler()
2155 if (p >= handle_tlbs_end) in build_r4000_tlb_store_handler()
2160 (unsigned int)(p - handle_tlbs)); in build_r4000_tlb_store_handler()
2167 u32 *p = handle_tlbm; in build_r4000_tlb_modify_handler() local
2177 wr = build_r4000_tlbchange_handler_head(&p, &l, &r); in build_r4000_tlb_modify_handler()
2178 build_pte_modifiable(&p, &r, wr.r1, wr.r2, wr.r3, label_nopage_tlbm); in build_r4000_tlb_modify_handler()
2180 build_tlb_probe_entry(&p); in build_r4000_tlb_modify_handler()
2182 build_make_write(&p, &r, wr.r1, wr.r2); in build_r4000_tlb_modify_handler()
2183 build_r4000_tlbchange_handler_tail(&p, &l, &r, wr.r1, wr.r2); in build_r4000_tlb_modify_handler()
2190 uasm_l_tlb_huge_update(&l, p); in build_r4000_tlb_modify_handler()
2191 iPTE_LW(&p, wr.r1, wr.r2); in build_r4000_tlb_modify_handler()
2192 build_pte_modifiable(&p, &r, wr.r1, wr.r2, wr.r3, label_nopage_tlbm); in build_r4000_tlb_modify_handler()
2193 build_tlb_probe_entry(&p); in build_r4000_tlb_modify_handler()
2194 uasm_i_ori(&p, wr.r1, wr.r1, in build_r4000_tlb_modify_handler()
2196 build_huge_handler_tail(&p, &r, &l, wr.r1, wr.r2); in build_r4000_tlb_modify_handler()
2199 uasm_l_nopage_tlbm(&l, p); in build_r4000_tlb_modify_handler()
2200 build_restore_work_registers(&p); in build_r4000_tlb_modify_handler()
2203 uasm_i_lui(&p, K0, uasm_rel_hi((long)tlb_do_page_fault_1)); in build_r4000_tlb_modify_handler()
2204 uasm_i_addiu(&p, K0, K0, uasm_rel_lo((long)tlb_do_page_fault_1)); in build_r4000_tlb_modify_handler()
2205 uasm_i_jr(&p, K0); in build_r4000_tlb_modify_handler()
2208 uasm_i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff); in build_r4000_tlb_modify_handler()
2209 uasm_i_nop(&p); in build_r4000_tlb_modify_handler()
2211 if (p >= handle_tlbm_end) in build_r4000_tlb_modify_handler()
2216 (unsigned int)(p - handle_tlbm)); in build_r4000_tlb_modify_handler()