26 #define tlb_flush(tlb)	((void) tlb)  argument
53 extern void tlb_table_flush(struct mmu_gather *tlb);
54 extern void tlb_remove_table(struct mmu_gather *tlb, void *table);
56 #define tlb_remove_entry(tlb, entry)	tlb_remove_table(tlb, entry)  argument
58 #define tlb_remove_entry(tlb, entry)	tlb_remove_page(tlb, entry)  argument
97 static inline void tlb_flush(struct mmu_gather *tlb)  in tlb_flush()  argument
99 	if (tlb->fullmm || !tlb->vma)  in tlb_flush()
100 		flush_tlb_mm(tlb->mm);  in tlb_flush()
101 	else if (tlb->range_end > 0) {  in tlb_flush()
102 		flush_tlb_range(tlb->vma, tlb->range_start, tlb->range_end);  in tlb_flush()
103 		tlb->range_start = TASK_SIZE;  in tlb_flush()
104 		tlb->range_end = 0;  in tlb_flush()
108 static inline void tlb_add_flush(struct mmu_gather *tlb, unsigned long addr)  in tlb_add_flush()  argument
110 	if (!tlb->fullmm) {  in tlb_add_flush()
111 		if (addr < tlb->range_start)  in tlb_add_flush()
112 			tlb->range_start = addr;  in tlb_add_flush()
113 		if (addr + PAGE_SIZE > tlb->range_end)  in tlb_add_flush()
114 			tlb->range_end = addr + PAGE_SIZE;  in tlb_add_flush()
118 static inline void __tlb_alloc_page(struct mmu_gather *tlb)  in __tlb_alloc_page()  argument
123 		tlb->pages = (void *)addr;  in __tlb_alloc_page()
124 		tlb->max = PAGE_SIZE / sizeof(struct page *);  in __tlb_alloc_page()
128 static inline void tlb_flush_mmu_tlbonly(struct mmu_gather *tlb)  in tlb_flush_mmu_tlbonly()  argument
130 	tlb_flush(tlb);  in tlb_flush_mmu_tlbonly()
132 	tlb_table_flush(tlb);  in tlb_flush_mmu_tlbonly()
136 static inline void tlb_flush_mmu_free(struct mmu_gather *tlb)  in tlb_flush_mmu_free()  argument
138 	free_pages_and_swap_cache(tlb->pages, tlb->nr);  in tlb_flush_mmu_free()
139 	tlb->nr = 0;  in tlb_flush_mmu_free()
140 	if (tlb->pages == tlb->local)  in tlb_flush_mmu_free()
141 		__tlb_alloc_page(tlb);  in tlb_flush_mmu_free()
144 static inline void tlb_flush_mmu(struct mmu_gather *tlb)  in tlb_flush_mmu()  argument
146 	tlb_flush_mmu_tlbonly(tlb);  in tlb_flush_mmu()
147 	tlb_flush_mmu_free(tlb);  in tlb_flush_mmu()
151 tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm, unsigned long start, unsigned long end)  in tlb_gather_mmu()  argument
153 	tlb->mm = mm;  in tlb_gather_mmu()
154 	tlb->fullmm = !(start | (end+1));  in tlb_gather_mmu()
155 	tlb->start = start;  in tlb_gather_mmu()
156 	tlb->end = end;  in tlb_gather_mmu()
157 	tlb->vma = NULL;  in tlb_gather_mmu()
158 	tlb->max = ARRAY_SIZE(tlb->local);  in tlb_gather_mmu()
159 	tlb->pages = tlb->local;  in tlb_gather_mmu()
160 	tlb->nr = 0;  in tlb_gather_mmu()
161 	__tlb_alloc_page(tlb);  in tlb_gather_mmu()
164 	tlb->batch = NULL;  in tlb_gather_mmu()
169 tlb_finish_mmu(struct mmu_gather *tlb, unsigned long start, unsigned long end)  in tlb_finish_mmu()  argument
171 	tlb_flush_mmu(tlb);  in tlb_finish_mmu()
176 	if (tlb->pages != tlb->local)  in tlb_finish_mmu()
177 		free_pages((unsigned long)tlb->pages, 0);  in tlb_finish_mmu()
184 tlb_remove_tlb_entry(struct mmu_gather *tlb, pte_t *ptep, unsigned long addr)  in tlb_remove_tlb_entry()  argument
186 	tlb_add_flush(tlb, addr);  in tlb_remove_tlb_entry()
195 tlb_start_vma(struct mmu_gather *tlb, struct vm_area_struct *vma)  in tlb_start_vma()  argument
197 	if (!tlb->fullmm) {  in tlb_start_vma()
199 		tlb->vma = vma;  in tlb_start_vma()
200 		tlb->range_start = TASK_SIZE;  in tlb_start_vma()
201 		tlb->range_end = 0;  in tlb_start_vma()
206 tlb_end_vma(struct mmu_gather *tlb, struct vm_area_struct *vma)  in tlb_end_vma()  argument
208 	if (!tlb->fullmm)  in tlb_end_vma()
209 		tlb_flush(tlb);  in tlb_end_vma()
212 static inline int __tlb_remove_page(struct mmu_gather *tlb, struct page *page)  in __tlb_remove_page()  argument
214 	tlb->pages[tlb->nr++] = page;  in __tlb_remove_page()
215 	VM_BUG_ON(tlb->nr > tlb->max);  in __tlb_remove_page()
216 	return tlb->max - tlb->nr;  in __tlb_remove_page()
219 static inline void tlb_remove_page(struct mmu_gather *tlb, struct page *page)  in tlb_remove_page()  argument
221 	if (!__tlb_remove_page(tlb, page))  in tlb_remove_page()
222 		tlb_flush_mmu(tlb);  in tlb_remove_page()
225 static inline void __pte_free_tlb(struct mmu_gather *tlb, pgtable_t pte,  in __pte_free_tlb()  argument
231 	tlb_add_flush(tlb, addr);  in __pte_free_tlb()
238 	tlb_add_flush(tlb, addr + SZ_1M - PAGE_SIZE);  in __pte_free_tlb()
239 	tlb_add_flush(tlb, addr + SZ_1M);  in __pte_free_tlb()
242 	tlb_remove_entry(tlb, pte);  in __pte_free_tlb()
245 static inline void __pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmdp,  in __pmd_free_tlb()  argument
249 	tlb_add_flush(tlb, addr);  in __pmd_free_tlb()
250 	tlb_remove_entry(tlb, virt_to_page(pmdp));  in __pmd_free_tlb()
255 tlb_remove_pmd_tlb_entry(struct mmu_gather *tlb, pmd_t *pmdp, unsigned long addr)  in tlb_remove_pmd_tlb_entry()  argument
257 	tlb_add_flush(tlb, addr);  in tlb_remove_pmd_tlb_entry()
260 #define pte_free_tlb(tlb, ptep, addr)	__pte_free_tlb(tlb, ptep, addr)  argument
261 #define pmd_free_tlb(tlb, pmdp, addr)	__pmd_free_tlb(tlb, pmdp, addr)  argument
262 #define pud_free_tlb(tlb, pudp, addr)	pud_free((tlb)->mm, pudp)  argument