36 void crst_table_free(struct mm_struct *mm, unsigned long *table)  in crst_table_free()  argument
38 	free_pages((unsigned long) table, 2);  in crst_table_free()
54 	unsigned long *table, *pgd;  in crst_table_upgrade()  local
59 	table = crst_table_alloc(mm);  in crst_table_upgrade()
60 	if (!table)  in crst_table_upgrade()
65 	crst_table_init(table, _REGION2_ENTRY_EMPTY);  in crst_table_upgrade()
66 	pgd_populate(mm, (pgd_t *) table, (pud_t *) pgd);  in crst_table_upgrade()
67 	mm->pgd = (pgd_t *) table;  in crst_table_upgrade()
115 	unsigned long *table;  in gmap_alloc()  local
148 	table = (unsigned long *) page_to_phys(page);  in gmap_alloc()
149 	crst_table_init(table, etype);  in gmap_alloc()
150 	gmap->table = table;  in gmap_alloc()
152 		_ASCE_USER_BITS | __pa(table);  in gmap_alloc()
247 static int gmap_alloc_table(struct gmap *gmap, unsigned long *table,  in gmap_alloc_table()  argument
260 	if (*table & _REGION_ENTRY_INVALID) {  in gmap_alloc_table()
262 		*table = (unsigned long) new | _REGION_ENTRY_LENGTH |  in gmap_alloc_table()
263 			(*table & _REGION_ENTRY_TYPE_MASK);  in gmap_alloc_table()
447 static void gmap_unlink(struct mm_struct *mm, unsigned long *table,  in gmap_unlink()  argument
474 	unsigned long *table;  in __gmap_link()  local
482 	table = gmap->table;  in __gmap_link()
484 		table += (gaddr >> 53) & 0x7ff;  in __gmap_link()
485 		if ((*table & _REGION_ENTRY_INVALID) &&  in __gmap_link()
486 		    gmap_alloc_table(gmap, table, _REGION2_ENTRY_EMPTY,  in __gmap_link()
489 		table = (unsigned long *)(*table & _REGION_ENTRY_ORIGIN);  in __gmap_link()
492 		table += (gaddr >> 42) & 0x7ff;  in __gmap_link()
493 		if ((*table & _REGION_ENTRY_INVALID) &&  in __gmap_link()
494 		    gmap_alloc_table(gmap, table, _REGION3_ENTRY_EMPTY,  in __gmap_link()
497 		table = (unsigned long *)(*table & _REGION_ENTRY_ORIGIN);  in __gmap_link()
500 		table += (gaddr >> 31) & 0x7ff;  in __gmap_link()
501 		if ((*table & _REGION_ENTRY_INVALID) &&  in __gmap_link()
502 		    gmap_alloc_table(gmap, table, _SEGMENT_ENTRY_EMPTY,  in __gmap_link()
505 		table = (unsigned long *)(*table & _REGION_ENTRY_ORIGIN);  in __gmap_link()
507 	table += (gaddr >> 20) & 0x7ff;  in __gmap_link()
525 	if (*table == _SEGMENT_ENTRY_INVALID) {  in __gmap_link()
527 				       vmaddr >> PMD_SHIFT, table);  in __gmap_link()
529 			*table = pmd_val(*pmd);  in __gmap_link()
742 	unsigned long *table;  in gmap_do_ipte_notify()  local
750 		table = radix_tree_lookup(&gmap->host_to_guest,  in gmap_do_ipte_notify()
752 		if (!table)  in gmap_do_ipte_notify()
754 		gaddr = __gmap_segment_gaddr(table) + offset;  in gmap_do_ipte_notify()
890 static inline void gmap_unlink(struct mm_struct *mm, unsigned long *table,  in gmap_unlink()  argument
913 	unsigned long *table;  in page_table_alloc()  local
919 		table = NULL;  in page_table_alloc()
927 				table = (unsigned long *) page_to_phys(page);  in page_table_alloc()
930 					table += PTRS_PER_PTE;  in page_table_alloc()
936 		if (table)  in page_table_alloc()
937 			return table;  in page_table_alloc()
948 	table = (unsigned long *) page_to_phys(page);  in page_table_alloc()
952 		clear_table(table, _PAGE_INVALID, PAGE_SIZE/2);  in page_table_alloc()
953 		clear_table(table + PTRS_PER_PTE, 0, PAGE_SIZE/2);  in page_table_alloc()
957 		clear_table(table, _PAGE_INVALID, PAGE_SIZE);  in page_table_alloc()
962 	return table;  in page_table_alloc()
965 void page_table_free(struct mm_struct *mm, unsigned long *table)  in page_table_free()  argument
970 	page = pfn_to_page(__pa(table) >> PAGE_SHIFT);  in page_table_free()
973 		bit = (__pa(table) & ~PAGE_MASK)/(PTRS_PER_PTE*sizeof(pte_t));  in page_table_free()
990 void page_table_free_rcu(struct mmu_gather *tlb, unsigned long *table,  in page_table_free_rcu()  argument
998 	page = pfn_to_page(__pa(table) >> PAGE_SHIFT);  in page_table_free_rcu()
1000 		gmap_unlink(mm, table, vmaddr);  in page_table_free_rcu()
1001 		table = (unsigned long *) (__pa(table) | 3);  in page_table_free_rcu()
1002 		tlb_remove_table(tlb, table);  in page_table_free_rcu()
1005 	bit = (__pa(table) & ~PAGE_MASK) / (PTRS_PER_PTE*sizeof(pte_t));  in page_table_free_rcu()
1013 	table = (unsigned long *) (__pa(table) | (1U << bit));  in page_table_free_rcu()
1014 	tlb_remove_table(tlb, table);  in page_table_free_rcu()
1020 	void *table = (void *)((unsigned long) _table ^ mask);  in __tlb_remove_table()  local
1021 	struct page *page = pfn_to_page(__pa(table) >> PAGE_SHIFT);  in __tlb_remove_table()
1025 		free_pages((unsigned long) table, 2);  in __tlb_remove_table()
1045 static void tlb_remove_table_one(void *table)  in tlb_remove_table_one()  argument
1055 	__tlb_remove_table(table);  in tlb_remove_table_one()
1081 void tlb_remove_table(struct mmu_gather *tlb, void *table)  in tlb_remove_table()  argument
1091 			tlb_remove_table_one(table);  in tlb_remove_table()
1096 	(*batch)->tables[(*batch)->nr++] = table;  in tlb_remove_table()