113 within(unsigned long addr, unsigned long start, unsigned long end)  in within()  argument
115 	return addr >= start && addr < end;  in within()
177 static void cpa_flush_range(unsigned long start, int numpages, int cache)  in cpa_flush_range()  argument
183 	WARN_ON(PAGE_ALIGN(start) != start);  in cpa_flush_range()
196 	for (i = 0, addr = start; i < numpages; i++, addr += PAGE_SIZE) {  in cpa_flush_range()
207 static void cpa_flush_array(unsigned long *start, int numpages, int cache,  in cpa_flush_array()  argument
233 			addr = start[i];  in cpa_flush_array()
762 static bool unmap_pte_range(pmd_t *pmd, unsigned long start, unsigned long end)  in unmap_pte_range()  argument
764 	pte_t *pte = pte_offset_kernel(pmd, start);  in unmap_pte_range()
766 	while (start < end) {  in unmap_pte_range()
769 		start += PAGE_SIZE;  in unmap_pte_range()
781 			      unsigned long start, unsigned long end)  in __unmap_pmd_range()  argument
783 	if (unmap_pte_range(pmd, start, end))  in __unmap_pmd_range()
788 static void unmap_pmd_range(pud_t *pud, unsigned long start, unsigned long end)  in unmap_pmd_range()  argument
790 	pmd_t *pmd = pmd_offset(pud, start);  in unmap_pmd_range()
795 	if (start & (PMD_SIZE - 1)) {  in unmap_pmd_range()
796 		unsigned long next_page = (start + PMD_SIZE) & PMD_MASK;  in unmap_pmd_range()
799 		__unmap_pmd_range(pud, pmd, start, pre_end);  in unmap_pmd_range()
801 		start = pre_end;  in unmap_pmd_range()
808 	while (end - start >= PMD_SIZE) {  in unmap_pmd_range()
812 			__unmap_pmd_range(pud, pmd, start, start + PMD_SIZE);  in unmap_pmd_range()
814 		start += PMD_SIZE;  in unmap_pmd_range()
821 	if (start < end)  in unmap_pmd_range()
822 		return __unmap_pmd_range(pud, pmd, start, end);  in unmap_pmd_range()
832 static void unmap_pud_range(pgd_t *pgd, unsigned long start, unsigned long end)  in unmap_pud_range()  argument
834 	pud_t *pud = pud_offset(pgd, start);  in unmap_pud_range()
839 	if (start & (PUD_SIZE - 1)) {  in unmap_pud_range()
840 		unsigned long next_page = (start + PUD_SIZE) & PUD_MASK;  in unmap_pud_range()
843 		unmap_pmd_range(pud, start, pre_end);  in unmap_pud_range()
845 		start = pre_end;  in unmap_pud_range()
852 	while (end - start >= PUD_SIZE) {  in unmap_pud_range()
857 			unmap_pmd_range(pud, start, start + PUD_SIZE);  in unmap_pud_range()
859 		start += PUD_SIZE;  in unmap_pud_range()
866 	if (start < end)  in unmap_pud_range()
867 		unmap_pmd_range(pud, start, end);  in unmap_pud_range()
906 			 unsigned long start, unsigned long end,  in populate_pte()  argument
911 	pte = pte_offset_kernel(pmd, start);  in populate_pte()
913 	while (num_pages-- && start < end) {  in populate_pte()
921 		start	 += PAGE_SIZE;  in populate_pte()
928 			unsigned long start, unsigned long end,  in populate_pmd()  argument
938 	if (start & (PMD_SIZE - 1)) {  in populate_pmd()
939 		unsigned long pre_end = start + (num_pages << PAGE_SHIFT);  in populate_pmd()
940 		unsigned long next_page = (start + PMD_SIZE) & PMD_MASK;  in populate_pmd()
943 		cur_pages = (pre_end - start) >> PAGE_SHIFT;  in populate_pmd()
949 		pmd = pmd_offset(pud, start);  in populate_pmd()
954 		populate_pte(cpa, start, pre_end, cur_pages, pmd, pgprot);  in populate_pmd()
956 		start = pre_end;  in populate_pmd()
967 	while (end - start >= PMD_SIZE) {  in populate_pmd()
976 		pmd = pmd_offset(pud, start);  in populate_pmd()
981 		start	  += PMD_SIZE;  in populate_pmd()
989 	if (start < end) {  in populate_pmd()
990 		pmd = pmd_offset(pud, start);  in populate_pmd()
995 		populate_pte(cpa, start, end, num_pages - cur_pages,  in populate_pmd()
1001 static int populate_pud(struct cpa_data *cpa, unsigned long start, pgd_t *pgd,  in populate_pud()  argument
1009 	end = start + (cpa->numpages << PAGE_SHIFT);  in populate_pud()
1015 	if (start & (PUD_SIZE - 1)) {  in populate_pud()
1017 		unsigned long next_page = (start + PUD_SIZE) & PUD_MASK;  in populate_pud()
1020 		cur_pages = (pre_end - start) >> PAGE_SHIFT;  in populate_pud()
1023 		pud = pud_offset(pgd, start);  in populate_pud()
1032 		cur_pages = populate_pmd(cpa, start, pre_end, cur_pages,  in populate_pud()
1037 		start = pre_end;  in populate_pud()
1044 	pud = pud_offset(pgd, start);  in populate_pud()
1050 	while (end - start >= PUD_SIZE) {  in populate_pud()
1054 		start	  += PUD_SIZE;  in populate_pud()
1061 	if (start < end) {  in populate_pud()
1064 		pud = pud_offset(pgd, start);  in populate_pud()
1069 		tmp = populate_pmd(cpa, start, end, cpa->numpages - cur_pages,  in populate_pud()
1735 	unsigned long start;  in _set_pages_array()  local
1745 		start = page_to_pfn(pages[i]) << PAGE_SHIFT;  in _set_pages_array()
1746 		end = start + PAGE_SIZE;  in _set_pages_array()
1747 		if (reserve_memtype(start, end, new_type, NULL))  in _set_pages_array()
1771 		start = page_to_pfn(pages[i]) << PAGE_SHIFT;  in _set_pages_array()
1772 		end = start + PAGE_SIZE;  in _set_pages_array()
1773 		free_memtype(start, end);  in _set_pages_array()
1807 	unsigned long start;  in set_pages_array_wb()  local
1820 		start = page_to_pfn(pages[i]) << PAGE_SHIFT;  in set_pages_array_wb()
1821 		end = start + PAGE_SIZE;  in set_pages_array_wb()
1822 		free_memtype(start, end);  in set_pages_array_wb()