43 void crst_table_free(struct mm_struct *mm, unsigned long *table)  in crst_table_free()  argument
45 	free_pages((unsigned long) table, ALLOC_ORDER);  in crst_table_free()
61 	unsigned long *table, *pgd;  in crst_table_upgrade()  local
68 	table = crst_table_alloc(mm);  in crst_table_upgrade()
69 	if (!table)  in crst_table_upgrade()
87 		crst_table_init(table, entry);  in crst_table_upgrade()
88 		pgd_populate(mm, (pgd_t *) table, (pud_t *) pgd);  in crst_table_upgrade()
89 		mm->pgd = (pgd_t *) table;  in crst_table_upgrade()
91 		table = NULL;  in crst_table_upgrade()
95 	if (table)  in crst_table_upgrade()
96 		crst_table_free(mm, table);  in crst_table_upgrade()
151 	unsigned long *table;  in gmap_alloc()  local
184 	table = (unsigned long *) page_to_phys(page);  in gmap_alloc()
185 	crst_table_init(table, etype);  in gmap_alloc()
186 	gmap->table = table;  in gmap_alloc()
188 		_ASCE_USER_BITS | __pa(table);  in gmap_alloc()
283 static int gmap_alloc_table(struct gmap *gmap, unsigned long *table,  in gmap_alloc_table()  argument
296 	if (*table & _REGION_ENTRY_INVALID) {  in gmap_alloc_table()
298 		*table = (unsigned long) new | _REGION_ENTRY_LENGTH |  in gmap_alloc_table()
299 			(*table & _REGION_ENTRY_TYPE_MASK);  in gmap_alloc_table()
483 static void gmap_unlink(struct mm_struct *mm, unsigned long *table,  in gmap_unlink()  argument
510 	unsigned long *table;  in __gmap_link()  local
518 	table = gmap->table;  in __gmap_link()
520 		table += (gaddr >> 53) & 0x7ff;  in __gmap_link()
521 		if ((*table & _REGION_ENTRY_INVALID) &&  in __gmap_link()
522 		    gmap_alloc_table(gmap, table, _REGION2_ENTRY_EMPTY,  in __gmap_link()
525 		table = (unsigned long *)(*table & _REGION_ENTRY_ORIGIN);  in __gmap_link()
528 		table += (gaddr >> 42) & 0x7ff;  in __gmap_link()
529 		if ((*table & _REGION_ENTRY_INVALID) &&  in __gmap_link()
530 		    gmap_alloc_table(gmap, table, _REGION3_ENTRY_EMPTY,  in __gmap_link()
533 		table = (unsigned long *)(*table & _REGION_ENTRY_ORIGIN);  in __gmap_link()
536 		table += (gaddr >> 31) & 0x7ff;  in __gmap_link()
537 		if ((*table & _REGION_ENTRY_INVALID) &&  in __gmap_link()
538 		    gmap_alloc_table(gmap, table, _SEGMENT_ENTRY_EMPTY,  in __gmap_link()
541 		table = (unsigned long *)(*table & _REGION_ENTRY_ORIGIN);  in __gmap_link()
543 	table += (gaddr >> 20) & 0x7ff;  in __gmap_link()
561 	if (*table == _SEGMENT_ENTRY_INVALID) {  in __gmap_link()
563 				       vmaddr >> PMD_SHIFT, table);  in __gmap_link()
565 			*table = pmd_val(*pmd);  in __gmap_link()
778 	unsigned long *table;  in gmap_do_ipte_notify()  local
786 		table = radix_tree_lookup(&gmap->host_to_guest,  in gmap_do_ipte_notify()
788 		if (!table)  in gmap_do_ipte_notify()
790 		gaddr = __gmap_segment_gaddr(table) + offset;  in gmap_do_ipte_notify()
806 	unsigned long *table;  in page_table_alloc_pgste()  local
816 	table = (unsigned long *) page_to_phys(page);  in page_table_alloc_pgste()
817 	clear_table(table, _PAGE_INVALID, PAGE_SIZE/2);  in page_table_alloc_pgste()
818 	clear_table(table + PTRS_PER_PTE, 0, PAGE_SIZE/2);  in page_table_alloc_pgste()
819 	return table;  in page_table_alloc_pgste()
822 static inline void page_table_free_pgste(unsigned long *table)  in page_table_free_pgste()  argument
826 	page = pfn_to_page(__pa(table) >> PAGE_SHIFT);  in page_table_free_pgste()
970 static inline void page_table_free_pgste(unsigned long *table)  in page_table_free_pgste()  argument
974 static inline void gmap_unlink(struct mm_struct *mm, unsigned long *table,  in gmap_unlink()  argument
997 	unsigned long *uninitialized_var(table);  in page_table_alloc()
1009 		table = (unsigned long *) page_to_phys(page);  in page_table_alloc()
1023 		table = (unsigned long *) page_to_phys(page);  in page_table_alloc()
1024 		clear_table(table, _PAGE_INVALID, PAGE_SIZE);  in page_table_alloc()
1029 			table += PTRS_PER_PTE;  in page_table_alloc()
1035 	return table;  in page_table_alloc()
1038 void page_table_free(struct mm_struct *mm, unsigned long *table)  in page_table_free()  argument
1043 	page = pfn_to_page(__pa(table) >> PAGE_SHIFT);  in page_table_free()
1045 		return page_table_free_pgste(table);  in page_table_free()
1047 	bit = 1 << ((__pa(table) & ~PAGE_MASK)/(PTRS_PER_PTE*sizeof(pte_t)));  in page_table_free()
1062 static void __page_table_free_rcu(void *table, unsigned bit)  in __page_table_free_rcu()  argument
1067 		return page_table_free_pgste(table);  in __page_table_free_rcu()
1069 	page = pfn_to_page(__pa(table) >> PAGE_SHIFT);  in __page_table_free_rcu()
1077 void page_table_free_rcu(struct mmu_gather *tlb, unsigned long *table,  in page_table_free_rcu()  argument
1085 	page = pfn_to_page(__pa(table) >> PAGE_SHIFT);  in page_table_free_rcu()
1087 		gmap_unlink(mm, table, vmaddr);  in page_table_free_rcu()
1088 		table = (unsigned long *) (__pa(table) | FRAG_MASK);  in page_table_free_rcu()
1089 		tlb_remove_table(tlb, table);  in page_table_free_rcu()
1092 	bit = 1 << ((__pa(table) & ~PAGE_MASK) / (PTRS_PER_PTE*sizeof(pte_t)));  in page_table_free_rcu()
1100 	table = (unsigned long *) (__pa(table) | (bit << 4));  in page_table_free_rcu()
1101 	tlb_remove_table(tlb, table);  in page_table_free_rcu()
1107 	void *table = (void *)((unsigned long) _table & ~mask);  in __tlb_remove_table()  local
1111 		__page_table_free_rcu(table, type);  in __tlb_remove_table()
1113 		free_pages((unsigned long) table, ALLOC_ORDER);  in __tlb_remove_table()
1121 static void tlb_remove_table_one(void *table)  in tlb_remove_table_one()  argument
1131 	__tlb_remove_table(table);  in tlb_remove_table_one()
1157 void tlb_remove_table(struct mmu_gather *tlb, void *table)  in tlb_remove_table()  argument
1167 			tlb_remove_table_one(table);  in tlb_remove_table()
1172 	(*batch)->tables[(*batch)->nr++] = table;  in tlb_remove_table()