135 static dma_addr_t swiotlb_virt_to_bus(struct device *hwdev,  in swiotlb_virt_to_bus()  argument
138 	return phys_to_dma(hwdev, virt_to_phys(address));  in swiotlb_virt_to_bus()
425 phys_addr_t swiotlb_tbl_map_single(struct device *hwdev,  in swiotlb_tbl_map_single()  argument
441 	mask = dma_get_seg_boundary(hwdev);  in swiotlb_tbl_map_single()
517 		dev_warn(hwdev, "swiotlb buffer is full (sz: %zd bytes)\n", size);  in swiotlb_tbl_map_single()
541 map_single(struct device *hwdev, phys_addr_t phys, size_t size,  in map_single()  argument
544 	dma_addr_t start_dma_addr = phys_to_dma(hwdev, io_tlb_start);  in map_single()
546 	return swiotlb_tbl_map_single(hwdev, start_dma_addr, phys, size, dir);  in map_single()
552 void swiotlb_tbl_unmap_single(struct device *hwdev, phys_addr_t tlb_addr,  in swiotlb_tbl_unmap_single()  argument
596 void swiotlb_tbl_sync_single(struct device *hwdev, phys_addr_t tlb_addr,  in swiotlb_tbl_sync_single()  argument
629 swiotlb_alloc_coherent(struct device *hwdev, size_t size,  in swiotlb_alloc_coherent()  argument
637 	if (hwdev && hwdev->coherent_dma_mask)  in swiotlb_alloc_coherent()
638 		dma_mask = hwdev->coherent_dma_mask;  in swiotlb_alloc_coherent()
642 		dev_addr = swiotlb_virt_to_bus(hwdev, ret);  in swiotlb_alloc_coherent()
657 		phys_addr_t paddr = map_single(hwdev, 0, size, DMA_FROM_DEVICE);  in swiotlb_alloc_coherent()
662 		dev_addr = phys_to_dma(hwdev, paddr);  in swiotlb_alloc_coherent()
671 			swiotlb_tbl_unmap_single(hwdev, paddr,  in swiotlb_alloc_coherent()
684 		dev_name(hwdev), size);  in swiotlb_alloc_coherent()
692 swiotlb_free_coherent(struct device *hwdev, size_t size, void *vaddr,  in swiotlb_free_coherent()  argument
695 	phys_addr_t paddr = dma_to_phys(hwdev, dev_addr);  in swiotlb_free_coherent()
702 		swiotlb_tbl_unmap_single(hwdev, paddr, size, DMA_TO_DEVICE);  in swiotlb_free_coherent()
784 static void unmap_single(struct device *hwdev, dma_addr_t dev_addr,  in unmap_single()  argument
787 	phys_addr_t paddr = dma_to_phys(hwdev, dev_addr);  in unmap_single()
792 		swiotlb_tbl_unmap_single(hwdev, paddr, size, dir);  in unmap_single()
808 void swiotlb_unmap_page(struct device *hwdev, dma_addr_t dev_addr,  in swiotlb_unmap_page()  argument
812 	unmap_single(hwdev, dev_addr, size, dir);  in swiotlb_unmap_page()
827 swiotlb_sync_single(struct device *hwdev, dma_addr_t dev_addr,  in swiotlb_sync_single()  argument
831 	phys_addr_t paddr = dma_to_phys(hwdev, dev_addr);  in swiotlb_sync_single()
836 		swiotlb_tbl_sync_single(hwdev, paddr, size, dir, target);  in swiotlb_sync_single()
847 swiotlb_sync_single_for_cpu(struct device *hwdev, dma_addr_t dev_addr,  in swiotlb_sync_single_for_cpu()  argument
850 	swiotlb_sync_single(hwdev, dev_addr, size, dir, SYNC_FOR_CPU);  in swiotlb_sync_single_for_cpu()
855 swiotlb_sync_single_for_device(struct device *hwdev, dma_addr_t dev_addr,  in swiotlb_sync_single_for_device()  argument
858 	swiotlb_sync_single(hwdev, dev_addr, size, dir, SYNC_FOR_DEVICE);  in swiotlb_sync_single_for_device()
879 swiotlb_map_sg_attrs(struct device *hwdev, struct scatterlist *sgl, int nelems,  in swiotlb_map_sg_attrs()  argument
889 		dma_addr_t dev_addr = phys_to_dma(hwdev, paddr);  in swiotlb_map_sg_attrs()
892 		    !dma_capable(hwdev, dev_addr, sg->length)) {  in swiotlb_map_sg_attrs()
893 			phys_addr_t map = map_single(hwdev, sg_phys(sg),  in swiotlb_map_sg_attrs()
898 				swiotlb_full(hwdev, sg->length, dir, 0);  in swiotlb_map_sg_attrs()
899 				swiotlb_unmap_sg_attrs(hwdev, sgl, i, dir,  in swiotlb_map_sg_attrs()
904 			sg->dma_address = phys_to_dma(hwdev, map);  in swiotlb_map_sg_attrs()
914 swiotlb_map_sg(struct device *hwdev, struct scatterlist *sgl, int nelems,  in swiotlb_map_sg()  argument
917 	return swiotlb_map_sg_attrs(hwdev, sgl, nelems, dir, NULL);  in swiotlb_map_sg()
926 swiotlb_unmap_sg_attrs(struct device *hwdev, struct scatterlist *sgl,  in swiotlb_unmap_sg_attrs()  argument
935 		unmap_single(hwdev, sg->dma_address, sg_dma_len(sg), dir);  in swiotlb_unmap_sg_attrs()
941 swiotlb_unmap_sg(struct device *hwdev, struct scatterlist *sgl, int nelems,  in swiotlb_unmap_sg()  argument
944 	return swiotlb_unmap_sg_attrs(hwdev, sgl, nelems, dir, NULL);  in swiotlb_unmap_sg()
956 swiotlb_sync_sg(struct device *hwdev, struct scatterlist *sgl,  in swiotlb_sync_sg()  argument
964 		swiotlb_sync_single(hwdev, sg->dma_address,  in swiotlb_sync_sg()
969 swiotlb_sync_sg_for_cpu(struct device *hwdev, struct scatterlist *sg,  in swiotlb_sync_sg_for_cpu()  argument
972 	swiotlb_sync_sg(hwdev, sg, nelems, dir, SYNC_FOR_CPU);  in swiotlb_sync_sg_for_cpu()
977 swiotlb_sync_sg_for_device(struct device *hwdev, struct scatterlist *sg,  in swiotlb_sync_sg_for_device()  argument
980 	swiotlb_sync_sg(hwdev, sg, nelems, dir, SYNC_FOR_DEVICE);  in swiotlb_sync_sg_for_device()
985 swiotlb_dma_mapping_error(struct device *hwdev, dma_addr_t dma_addr)  in swiotlb_dma_mapping_error()  argument
987 	return (dma_addr == phys_to_dma(hwdev, io_tlb_overflow_buffer));  in swiotlb_dma_mapping_error()
998 swiotlb_dma_supported(struct device *hwdev, u64 mask)  in swiotlb_dma_supported()  argument
1000 	return phys_to_dma(hwdev, io_tlb_end - 1) <= mask;  in swiotlb_dma_supported()