259 static u32 rk_iova_dte_index(dma_addr_t iova)  in rk_iova_dte_index()  argument
261 	return (u32)(iova & RK_IOVA_DTE_MASK) >> RK_IOVA_DTE_SHIFT;  in rk_iova_dte_index()
264 static u32 rk_iova_pte_index(dma_addr_t iova)  in rk_iova_pte_index()  argument
266 	return (u32)(iova & RK_IOVA_PTE_MASK) >> RK_IOVA_PTE_SHIFT;  in rk_iova_pte_index()
269 static u32 rk_iova_page_offset(dma_addr_t iova)  in rk_iova_page_offset()  argument
271 	return (u32)(iova & RK_IOVA_PAGE_MASK) >> RK_IOVA_PAGE_SHIFT;  in rk_iova_page_offset()
289 static void rk_iommu_zap_lines(struct rk_iommu *iommu, dma_addr_t iova,  in rk_iommu_zap_lines()  argument
292 	dma_addr_t iova_end = iova + size;  in rk_iommu_zap_lines()
297 	for (; iova < iova_end; iova += SPAGE_SIZE)  in rk_iommu_zap_lines()
298 		rk_iommu_write(iommu, RK_MMU_ZAP_ONE_LINE, iova);  in rk_iommu_zap_lines()
411 static void log_iova(struct rk_iommu *iommu, dma_addr_t iova)  in log_iova()  argument
424 	dte_index = rk_iova_dte_index(iova);  in log_iova()
425 	pte_index = rk_iova_pte_index(iova);  in log_iova()
426 	page_offset = rk_iova_page_offset(iova);  in log_iova()
450 		&iova, dte_index, pte_index, page_offset);  in log_iova()
462 	dma_addr_t iova;  in rk_iommu_irq()  local
468 	iova = rk_iommu_read(iommu, RK_MMU_PAGE_FAULT_ADDR);  in rk_iommu_irq()
478 			&iova,  in rk_iommu_irq()
481 		log_iova(iommu, iova);  in rk_iommu_irq()
489 			report_iommu_fault(iommu->domain, iommu->dev, iova,  in rk_iommu_irq()
499 		dev_err(iommu->dev, "BUS_ERROR occurred at %pad\n", &iova);  in rk_iommu_irq()
511 					 dma_addr_t iova)  in rk_iommu_iova_to_phys()  argument
521 	dte = rk_domain->dt[rk_iova_dte_index(iova)];  in rk_iommu_iova_to_phys()
527 	pte = page_table[rk_iova_pte_index(iova)];  in rk_iommu_iova_to_phys()
531 	phys = rk_pte_page_address(pte) + rk_iova_page_offset(iova);  in rk_iommu_iova_to_phys()
539 			      dma_addr_t iova, size_t size)  in rk_iommu_zap_iova()  argument
549 		rk_iommu_zap_lines(iommu, iova, size);  in rk_iommu_zap_iova()
555 					 dma_addr_t iova, size_t size)  in rk_iommu_zap_iova_first_last()  argument
557 	rk_iommu_zap_iova(rk_domain, iova, SPAGE_SIZE);  in rk_iommu_zap_iova_first_last()
559 		rk_iommu_zap_iova(rk_domain, iova + size - SPAGE_SIZE,  in rk_iommu_zap_iova_first_last()
564 				  dma_addr_t iova)  in rk_dte_get_page_table()  argument
572 	dte_addr = &rk_domain->dt[rk_iova_dte_index(iova)];  in rk_dte_get_page_table()
593 				  u32 *pte_addr, dma_addr_t iova, size_t size)  in rk_iommu_unmap_iova()  argument
614 			     dma_addr_t iova, phys_addr_t paddr, size_t size,  in rk_iommu_map_iova()  argument
642 	rk_iommu_zap_iova_first_last(rk_domain, iova, size);  in rk_iommu_map_iova()
647 	rk_iommu_unmap_iova(rk_domain, pte_addr, iova, pte_count * SPAGE_SIZE);  in rk_iommu_map_iova()
649 	iova += pte_count * SPAGE_SIZE;  in rk_iommu_map_iova()
652 	       &iova, &page_phys, &paddr, prot);  in rk_iommu_map_iova()
662 	dma_addr_t iova = (dma_addr_t)_iova;  in rk_iommu_map()  local
675 	page_table = rk_dte_get_page_table(rk_domain, iova);  in rk_iommu_map()
681 	pte_addr = &page_table[rk_iova_pte_index(iova)];  in rk_iommu_map()
682 	ret = rk_iommu_map_iova(rk_domain, pte_addr, iova, paddr, size, prot);  in rk_iommu_map()
693 	dma_addr_t iova = (dma_addr_t)_iova;  in rk_iommu_unmap()  local
708 	dte = rk_domain->dt[rk_iova_dte_index(iova)];  in rk_iommu_unmap()
716 	pte_addr = (u32 *)phys_to_virt(pt_phys) + rk_iova_pte_index(iova);  in rk_iommu_unmap()
717 	unmap_size = rk_iommu_unmap_iova(rk_domain, pte_addr, iova, size);  in rk_iommu_unmap()
722 	rk_iommu_zap_iova(rk_domain, iova, unmap_size);  in rk_iommu_unmap()