27 static inline u64 mic_system_page_mask(struct mic_device *mdev)  in mic_system_page_mask()  argument
29 	return (1ULL << mdev->smpt->info.page_shift) - 1ULL;  in mic_system_page_mask()
32 static inline u8 mic_sys_addr_to_smpt(struct mic_device *mdev, dma_addr_t pa)  in mic_sys_addr_to_smpt()  argument
34 	return (pa - mdev->smpt->info.base) >> mdev->smpt->info.page_shift;  in mic_sys_addr_to_smpt()
37 static inline u64 mic_smpt_to_pa(struct mic_device *mdev, u8 index)  in mic_smpt_to_pa()  argument
39 	return mdev->smpt->info.base + (index * mdev->smpt->info.page_size);  in mic_smpt_to_pa()
42 static inline u64 mic_smpt_offset(struct mic_device *mdev, dma_addr_t pa)  in mic_smpt_offset()  argument
44 	return pa & mic_system_page_mask(mdev);  in mic_smpt_offset()
47 static inline u64 mic_smpt_align_low(struct mic_device *mdev, dma_addr_t pa)  in mic_smpt_align_low()  argument
49 	return ALIGN(pa - mic_system_page_mask(mdev),  in mic_smpt_align_low()
50 		mdev->smpt->info.page_size);  in mic_smpt_align_low()
53 static inline u64 mic_smpt_align_high(struct mic_device *mdev, dma_addr_t pa)  in mic_smpt_align_high()  argument
55 	return ALIGN(pa, mdev->smpt->info.page_size);  in mic_smpt_align_high()
59 static inline u64 mic_max_system_memory(struct mic_device *mdev)  in mic_max_system_memory()  argument
61 	return mdev->smpt->info.num_reg * mdev->smpt->info.page_size;  in mic_max_system_memory()
65 static inline u64 mic_max_system_addr(struct mic_device *mdev)  in mic_max_system_addr()  argument
67 	return mdev->smpt->info.base + mic_max_system_memory(mdev) - 1ULL;  in mic_max_system_addr()
72 mic_is_system_addr(struct mic_device *mdev, dma_addr_t pa)  in mic_is_system_addr()  argument
74 	return pa >= mdev->smpt->info.base && pa <= mic_max_system_addr(mdev);  in mic_is_system_addr()
79 		int entries, struct mic_device *mdev)  in mic_add_smpt_entry()  argument
81 	struct mic_smpt_info *smpt_info = mdev->smpt;  in mic_add_smpt_entry()
88 			mdev->smpt_ops->set(mdev, addr, i);  in mic_add_smpt_entry()
99 static dma_addr_t mic_smpt_op(struct mic_device *mdev, u64 dma_addr,  in mic_smpt_op()  argument
108 	struct mic_smpt_info *smpt_info = mdev->smpt;  in mic_smpt_op()
137 	mic_addr = mic_smpt_to_pa(mdev, spt);  in mic_smpt_op()
138 	mic_add_smpt_entry(spt, ref, dma_addr, entries, mdev);  in mic_smpt_op()
150 static int mic_get_smpt_ref_count(struct mic_device *mdev, dma_addr_t dma_addr,  in mic_get_smpt_ref_count()  argument
158 		ref[i++] = min(mic_smpt_align_high(mdev, start + 1),  in mic_get_smpt_ref_count()
160 		start = mic_smpt_align_high(mdev, start + 1);  in mic_get_smpt_ref_count()
164 		*smpt_start = mic_smpt_align_low(mdev, dma_addr);  in mic_get_smpt_ref_count()
178 mic_to_dma_addr(struct mic_device *mdev, dma_addr_t mic_addr)  in mic_to_dma_addr()  argument
180 	struct mic_smpt_info *smpt_info = mdev->smpt;  in mic_to_dma_addr()
184 	if (!mic_is_system_addr(mdev, mic_addr)) {  in mic_to_dma_addr()
185 		dev_err(mdev->sdev->parent,  in mic_to_dma_addr()
189 	spt = mic_sys_addr_to_smpt(mdev, mic_addr);  in mic_to_dma_addr()
191 		mic_smpt_offset(mdev, mic_addr);  in mic_to_dma_addr()
207 dma_addr_t mic_map(struct mic_device *mdev, dma_addr_t dma_addr, size_t size)  in mic_map()  argument
214 	if (!size || size > mic_max_system_memory(mdev))  in mic_map()
217 	ref = kmalloc(mdev->smpt->info.num_reg * sizeof(s64), GFP_KERNEL);  in mic_map()
221 	num_entries = mic_get_smpt_ref_count(mdev, dma_addr, size,  in mic_map()
225 	mic_addr = mic_smpt_op(mdev, smpt_start, num_entries, ref, size);  in mic_map()
234 	if (!mic_addr && MIC_FAMILY_X100 == mdev->family) {  in mic_map()
235 		dev_err(mdev->sdev->parent,  in mic_map()
240 		return mic_addr + mic_smpt_offset(mdev, dma_addr);  in mic_map()
255 void mic_unmap(struct mic_device *mdev, dma_addr_t mic_addr, size_t size)  in mic_unmap()  argument
257 	struct mic_smpt_info *smpt_info = mdev->smpt;  in mic_unmap()
267 	if (!mic_is_system_addr(mdev, mic_addr)) {  in mic_unmap()
268 		dev_err(mdev->sdev->parent,  in mic_unmap()
273 	spt = mic_sys_addr_to_smpt(mdev, mic_addr);  in mic_unmap()
274 	ref = kmalloc(mdev->smpt->info.num_reg * sizeof(s64), GFP_KERNEL);  in mic_unmap()
279 	num_smpt = mic_get_smpt_ref_count(mdev, mic_addr, size, ref, NULL);  in mic_unmap()
288 			dev_warn(mdev->sdev->parent,  in mic_unmap()
308 dma_addr_t mic_map_single(struct mic_device *mdev, void *va, size_t size)  in mic_map_single()  argument
311 	struct pci_dev *pdev = container_of(mdev->sdev->parent,  in mic_map_single()
317 		mic_addr = mic_map(mdev, dma_addr, size);  in mic_map_single()
319 			dev_err(mdev->sdev->parent,  in mic_map_single()
341 mic_unmap_single(struct mic_device *mdev, dma_addr_t mic_addr, size_t size)  in mic_unmap_single()  argument
343 	struct pci_dev *pdev = container_of(mdev->sdev->parent,  in mic_unmap_single()
345 	dma_addr_t dma_addr = mic_to_dma_addr(mdev, mic_addr);  in mic_unmap_single()
346 	mic_unmap(mdev, mic_addr, size);  in mic_unmap_single()
357 int mic_smpt_init(struct mic_device *mdev)  in mic_smpt_init()  argument
363 	mdev->smpt = kmalloc(sizeof(*mdev->smpt), GFP_KERNEL);  in mic_smpt_init()
364 	if (!mdev->smpt)  in mic_smpt_init()
367 	smpt_info = mdev->smpt;  in mic_smpt_init()
368 	mdev->smpt_ops->init(mdev);  in mic_smpt_init()
380 		mdev->smpt_ops->set(mdev, dma_addr, i);  in mic_smpt_init()
398 void mic_smpt_uninit(struct mic_device *mdev)  in mic_smpt_uninit()  argument
400 	struct mic_smpt_info *smpt_info = mdev->smpt;  in mic_smpt_uninit()
403 	dev_dbg(mdev->sdev->parent,  in mic_smpt_uninit()
405 		mdev->id, smpt_info->ref_count,  in mic_smpt_uninit()
409 		dev_dbg(mdev->sdev->parent,  in mic_smpt_uninit()
414 			dev_warn(mdev->sdev->parent,  in mic_smpt_uninit()
433 void mic_smpt_restore(struct mic_device *mdev)  in mic_smpt_restore()  argument
438 	for (i = 0; i < mdev->smpt->info.num_reg; i++) {  in mic_smpt_restore()
439 		dma_addr = mdev->smpt->entry[i].dma_addr;  in mic_smpt_restore()
440 		mdev->smpt_ops->set(mdev, dma_addr, i);  in mic_smpt_restore()