27 static inline u64 mic_system_page_mask(struct mic_device *mdev)  in mic_system_page_mask()  argument
29 	return (1ULL << mdev->smpt->info.page_shift) - 1ULL;  in mic_system_page_mask()
32 static inline u8 mic_sys_addr_to_smpt(struct mic_device *mdev, dma_addr_t pa)  in mic_sys_addr_to_smpt()  argument
34 	return (pa - mdev->smpt->info.base) >> mdev->smpt->info.page_shift;  in mic_sys_addr_to_smpt()
37 static inline u64 mic_smpt_to_pa(struct mic_device *mdev, u8 index)  in mic_smpt_to_pa()  argument
39 	return mdev->smpt->info.base + (index * mdev->smpt->info.page_size);  in mic_smpt_to_pa()
42 static inline u64 mic_smpt_offset(struct mic_device *mdev, dma_addr_t pa)  in mic_smpt_offset()  argument
44 	return pa & mic_system_page_mask(mdev);  in mic_smpt_offset()
47 static inline u64 mic_smpt_align_low(struct mic_device *mdev, dma_addr_t pa)  in mic_smpt_align_low()  argument
49 	return ALIGN(pa - mic_system_page_mask(mdev),  in mic_smpt_align_low()
50 		mdev->smpt->info.page_size);  in mic_smpt_align_low()
53 static inline u64 mic_smpt_align_high(struct mic_device *mdev, dma_addr_t pa)  in mic_smpt_align_high()  argument
55 	return ALIGN(pa, mdev->smpt->info.page_size);  in mic_smpt_align_high()
59 static inline u64 mic_max_system_memory(struct mic_device *mdev)  in mic_max_system_memory()  argument
61 	return mdev->smpt->info.num_reg * mdev->smpt->info.page_size;  in mic_max_system_memory()
65 static inline u64 mic_max_system_addr(struct mic_device *mdev)  in mic_max_system_addr()  argument
67 	return mdev->smpt->info.base + mic_max_system_memory(mdev) - 1ULL;  in mic_max_system_addr()
72 mic_is_system_addr(struct mic_device *mdev, dma_addr_t pa)  in mic_is_system_addr()  argument
74 	return pa >= mdev->smpt->info.base && pa <= mic_max_system_addr(mdev);  in mic_is_system_addr()
79 			       int entries, struct mic_device *mdev)  in mic_add_smpt_entry()  argument
81 	struct mic_smpt_info *smpt_info = mdev->smpt;  in mic_add_smpt_entry()
88 			mdev->smpt_ops->set(mdev, addr, i);  in mic_add_smpt_entry()
99 static dma_addr_t mic_smpt_op(struct mic_device *mdev, u64 dma_addr,  in mic_smpt_op()  argument
108 	struct mic_smpt_info *smpt_info = mdev->smpt;  in mic_smpt_op()
137 	mic_addr = mic_smpt_to_pa(mdev, spt);  in mic_smpt_op()
138 	mic_add_smpt_entry(spt, ref, dma_addr, entries, mdev);  in mic_smpt_op()
150 static int mic_get_smpt_ref_count(struct mic_device *mdev, dma_addr_t dma_addr,  in mic_get_smpt_ref_count()  argument
158 		ref[i++] = min(mic_smpt_align_high(mdev, start + 1),  in mic_get_smpt_ref_count()
160 		start = mic_smpt_align_high(mdev, start + 1);  in mic_get_smpt_ref_count()
164 		*smpt_start = mic_smpt_align_low(mdev, dma_addr);  in mic_get_smpt_ref_count()
177 dma_addr_t mic_to_dma_addr(struct mic_device *mdev, dma_addr_t mic_addr)  in mic_to_dma_addr()  argument
179 	struct mic_smpt_info *smpt_info = mdev->smpt;  in mic_to_dma_addr()
183 	if (!mic_is_system_addr(mdev, mic_addr)) {  in mic_to_dma_addr()
184 		dev_err(&mdev->pdev->dev,  in mic_to_dma_addr()
188 	spt = mic_sys_addr_to_smpt(mdev, mic_addr);  in mic_to_dma_addr()
190 		mic_smpt_offset(mdev, mic_addr);  in mic_to_dma_addr()
206 dma_addr_t mic_map(struct mic_device *mdev, dma_addr_t dma_addr, size_t size)  in mic_map()  argument
213 	if (!size || size > mic_max_system_memory(mdev))  in mic_map()
216 	ref = kmalloc_array(mdev->smpt->info.num_reg, sizeof(s64), GFP_ATOMIC);  in mic_map()
220 	num_entries = mic_get_smpt_ref_count(mdev, dma_addr, size,  in mic_map()
224 	mic_addr = mic_smpt_op(mdev, smpt_start, num_entries, ref, size);  in mic_map()
233 	if (!mic_addr && MIC_FAMILY_X100 == mdev->family) {  in mic_map()
234 		dev_err(&mdev->pdev->dev,  in mic_map()
239 		return mic_addr + mic_smpt_offset(mdev, dma_addr);  in mic_map()
254 void mic_unmap(struct mic_device *mdev, dma_addr_t mic_addr, size_t size)  in mic_unmap()  argument
256 	struct mic_smpt_info *smpt_info = mdev->smpt;  in mic_unmap()
266 	if (!mic_is_system_addr(mdev, mic_addr)) {  in mic_unmap()
267 		dev_err(&mdev->pdev->dev,  in mic_unmap()
272 	spt = mic_sys_addr_to_smpt(mdev, mic_addr);  in mic_unmap()
273 	ref = kmalloc_array(mdev->smpt->info.num_reg, sizeof(s64), GFP_ATOMIC);  in mic_unmap()
278 	num_smpt = mic_get_smpt_ref_count(mdev, mic_addr, size, ref, NULL);  in mic_unmap()
287 			dev_warn(&mdev->pdev->dev,  in mic_unmap()
307 dma_addr_t mic_map_single(struct mic_device *mdev, void *va, size_t size)  in mic_map_single()  argument
310 	struct pci_dev *pdev = mdev->pdev;  in mic_map_single()
315 		mic_addr = mic_map(mdev, dma_addr, size);  in mic_map_single()
317 			dev_err(&mdev->pdev->dev,  in mic_map_single()
339 mic_unmap_single(struct mic_device *mdev, dma_addr_t mic_addr, size_t size)  in mic_unmap_single()  argument
341 	struct pci_dev *pdev = mdev->pdev;  in mic_unmap_single()
342 	dma_addr_t dma_addr = mic_to_dma_addr(mdev, mic_addr);  in mic_unmap_single()
343 	mic_unmap(mdev, mic_addr, size);  in mic_unmap_single()
354 int mic_smpt_init(struct mic_device *mdev)  in mic_smpt_init()  argument
360 	mdev->smpt = kmalloc(sizeof(*mdev->smpt), GFP_KERNEL);  in mic_smpt_init()
361 	if (!mdev->smpt)  in mic_smpt_init()
364 	smpt_info = mdev->smpt;  in mic_smpt_init()
365 	mdev->smpt_ops->init(mdev);  in mic_smpt_init()
377 		mdev->smpt_ops->set(mdev, dma_addr, i);  in mic_smpt_init()
395 void mic_smpt_uninit(struct mic_device *mdev)  in mic_smpt_uninit()  argument
397 	struct mic_smpt_info *smpt_info = mdev->smpt;  in mic_smpt_uninit()
400 	dev_dbg(&mdev->pdev->dev,  in mic_smpt_uninit()
402 		mdev->id, smpt_info->ref_count,  in mic_smpt_uninit()
406 		dev_dbg(&mdev->pdev->dev,  in mic_smpt_uninit()
411 			dev_warn(&mdev->pdev->dev,  in mic_smpt_uninit()
430 void mic_smpt_restore(struct mic_device *mdev)  in mic_smpt_restore()  argument
435 	for (i = 0; i < mdev->smpt->info.num_reg; i++) {  in mic_smpt_restore()
436 		dma_addr = mdev->smpt->entry[i].dma_addr;  in mic_smpt_restore()
437 		mdev->smpt_ops->set(mdev, dma_addr, i);  in mic_smpt_restore()