79 static int kfd_gtt_sa_init(struct kfd_dev *kfd, unsigned int buf_size,
81 static void kfd_gtt_sa_fini(struct kfd_dev *kfd);
100 	struct kfd_dev *kfd;  in kgd2kfd_probe()  local
108 	kfd = kzalloc(sizeof(*kfd), GFP_KERNEL);  in kgd2kfd_probe()
109 	if (!kfd)  in kgd2kfd_probe()
112 	kfd->kgd = kgd;  in kgd2kfd_probe()
113 	kfd->device_info = device_info;  in kgd2kfd_probe()
114 	kfd->pdev = pdev;  in kgd2kfd_probe()
115 	kfd->init_complete = false;  in kgd2kfd_probe()
116 	kfd->kfd2kgd = f2g;  in kgd2kfd_probe()
118 	mutex_init(&kfd->doorbell_mutex);  in kgd2kfd_probe()
119 	memset(&kfd->doorbell_available_index, 0,  in kgd2kfd_probe()
120 		sizeof(kfd->doorbell_available_index));  in kgd2kfd_probe()
122 	return kfd;  in kgd2kfd_probe()
125 static bool device_iommu_pasid_init(struct kfd_dev *kfd)  in device_iommu_pasid_init()  argument
135 	err = amd_iommu_device_info(kfd->pdev, &iommu_info);  in device_iommu_pasid_init()
151 			(unsigned int)1 << kfd->device_info->max_pasid_bits,  in device_iommu_pasid_init()
159 				kfd->doorbell_process_limit - 1);  in device_iommu_pasid_init()
161 	err = amd_iommu_init_device(kfd->pdev, pasid_limit);  in device_iommu_pasid_init()
169 		amd_iommu_free_device(kfd->pdev);  in device_iommu_pasid_init()
184 bool kgd2kfd_device_init(struct kfd_dev *kfd,  in kgd2kfd_device_init()  argument
189 	kfd->shared_resources = *gpu_resources;  in kgd2kfd_device_init()
193 			kfd->device_info->mqd_size_aligned;  in kgd2kfd_device_init()
209 	if (kfd->kfd2kgd->init_gtt_mem_allocation(  in kgd2kfd_device_init()
210 			kfd->kgd, size, &kfd->gtt_mem,  in kgd2kfd_device_init()
211 			&kfd->gtt_start_gpu_addr, &kfd->gtt_start_cpu_ptr)){  in kgd2kfd_device_init()
214 			size, kfd->pdev->vendor, kfd->pdev->device);  in kgd2kfd_device_init()
220 		size, kfd->pdev->vendor, kfd->pdev->device);  in kgd2kfd_device_init()
223 	if (kfd_gtt_sa_init(kfd, size, 512) != 0) {  in kgd2kfd_device_init()
229 	kfd_doorbell_init(kfd);  in kgd2kfd_device_init()
231 	if (kfd_topology_add_device(kfd) != 0) {  in kgd2kfd_device_init()
234 			kfd->pdev->vendor, kfd->pdev->device);  in kgd2kfd_device_init()
238 	if (!device_iommu_pasid_init(kfd)) {  in kgd2kfd_device_init()
241 			kfd->pdev->vendor, kfd->pdev->device);  in kgd2kfd_device_init()
244 	amd_iommu_set_invalidate_ctx_cb(kfd->pdev,  in kgd2kfd_device_init()
247 	kfd->dqm = device_queue_manager_init(kfd);  in kgd2kfd_device_init()
248 	if (!kfd->dqm) {  in kgd2kfd_device_init()
251 			kfd->pdev->vendor, kfd->pdev->device);  in kgd2kfd_device_init()
255 	if (kfd->dqm->ops.start(kfd->dqm) != 0) {  in kgd2kfd_device_init()
258 			kfd->pdev->vendor, kfd->pdev->device);  in kgd2kfd_device_init()
262 	kfd->init_complete = true;  in kgd2kfd_device_init()
263 	dev_info(kfd_device, "added device (%x:%x)\n", kfd->pdev->vendor,  in kgd2kfd_device_init()
264 		 kfd->pdev->device);  in kgd2kfd_device_init()
272 	device_queue_manager_uninit(kfd->dqm);  in kgd2kfd_device_init()
274 	amd_iommu_free_device(kfd->pdev);  in kgd2kfd_device_init()
276 	kfd_topology_remove_device(kfd);  in kgd2kfd_device_init()
278 	kfd_gtt_sa_fini(kfd);  in kgd2kfd_device_init()
280 	kfd->kfd2kgd->free_gtt_mem(kfd->kgd, kfd->gtt_mem);  in kgd2kfd_device_init()
283 		kfd->pdev->vendor, kfd->pdev->device);  in kgd2kfd_device_init()
285 	return kfd->init_complete;  in kgd2kfd_device_init()
288 void kgd2kfd_device_exit(struct kfd_dev *kfd)  in kgd2kfd_device_exit()  argument
290 	if (kfd->init_complete) {  in kgd2kfd_device_exit()
291 		device_queue_manager_uninit(kfd->dqm);  in kgd2kfd_device_exit()
292 		amd_iommu_free_device(kfd->pdev);  in kgd2kfd_device_exit()
293 		kfd_topology_remove_device(kfd);  in kgd2kfd_device_exit()
294 		kfd_gtt_sa_fini(kfd);  in kgd2kfd_device_exit()
295 		kfd->kfd2kgd->free_gtt_mem(kfd->kgd, kfd->gtt_mem);  in kgd2kfd_device_exit()
298 	kfree(kfd);  in kgd2kfd_device_exit()
301 void kgd2kfd_suspend(struct kfd_dev *kfd)  in kgd2kfd_suspend()  argument
303 	BUG_ON(kfd == NULL);  in kgd2kfd_suspend()
305 	if (kfd->init_complete) {  in kgd2kfd_suspend()
306 		kfd->dqm->ops.stop(kfd->dqm);  in kgd2kfd_suspend()
307 		amd_iommu_set_invalidate_ctx_cb(kfd->pdev, NULL);  in kgd2kfd_suspend()
308 		amd_iommu_free_device(kfd->pdev);  in kgd2kfd_suspend()
312 int kgd2kfd_resume(struct kfd_dev *kfd)  in kgd2kfd_resume()  argument
317 	BUG_ON(kfd == NULL);  in kgd2kfd_resume()
321 	if (kfd->init_complete) {  in kgd2kfd_resume()
322 		err = amd_iommu_init_device(kfd->pdev, pasid_limit);  in kgd2kfd_resume()
325 		amd_iommu_set_invalidate_ctx_cb(kfd->pdev,  in kgd2kfd_resume()
327 		kfd->dqm->ops.start(kfd->dqm);  in kgd2kfd_resume()
334 void kgd2kfd_interrupt(struct kfd_dev *kfd, const void *ih_ring_entry)  in kgd2kfd_interrupt()  argument
339 static int kfd_gtt_sa_init(struct kfd_dev *kfd, unsigned int buf_size,  in kfd_gtt_sa_init()  argument
344 	BUG_ON(!kfd);  in kfd_gtt_sa_init()
345 	BUG_ON(!kfd->gtt_mem);  in kfd_gtt_sa_init()
350 	kfd->gtt_sa_chunk_size = chunk_size;  in kfd_gtt_sa_init()
351 	kfd->gtt_sa_num_of_chunks = buf_size / chunk_size;  in kfd_gtt_sa_init()
353 	num_of_bits = kfd->gtt_sa_num_of_chunks / BITS_PER_BYTE;  in kfd_gtt_sa_init()
356 	kfd->gtt_sa_bitmap = kzalloc(num_of_bits, GFP_KERNEL);  in kfd_gtt_sa_init()
358 	if (!kfd->gtt_sa_bitmap)  in kfd_gtt_sa_init()
362 			kfd->gtt_sa_num_of_chunks, kfd->gtt_sa_bitmap);  in kfd_gtt_sa_init()
364 	mutex_init(&kfd->gtt_sa_lock);  in kfd_gtt_sa_init()
370 static void kfd_gtt_sa_fini(struct kfd_dev *kfd)  in kfd_gtt_sa_fini()  argument
372 	mutex_destroy(&kfd->gtt_sa_lock);  in kfd_gtt_sa_fini()
373 	kfree(kfd->gtt_sa_bitmap);  in kfd_gtt_sa_fini()
390 int kfd_gtt_sa_allocate(struct kfd_dev *kfd, unsigned int size,  in kfd_gtt_sa_allocate()  argument
395 	BUG_ON(!kfd);  in kfd_gtt_sa_allocate()
400 	if (size > kfd->gtt_sa_num_of_chunks * kfd->gtt_sa_chunk_size)  in kfd_gtt_sa_allocate()
411 	mutex_lock(&kfd->gtt_sa_lock);  in kfd_gtt_sa_allocate()
415 	found = find_next_zero_bit(kfd->gtt_sa_bitmap,  in kfd_gtt_sa_allocate()
416 					kfd->gtt_sa_num_of_chunks,  in kfd_gtt_sa_allocate()
422 	if (found == kfd->gtt_sa_num_of_chunks)  in kfd_gtt_sa_allocate()
429 					kfd->gtt_start_gpu_addr,  in kfd_gtt_sa_allocate()
431 					kfd->gtt_sa_chunk_size);  in kfd_gtt_sa_allocate()
433 					kfd->gtt_start_cpu_ptr,  in kfd_gtt_sa_allocate()
435 					kfd->gtt_sa_chunk_size);  in kfd_gtt_sa_allocate()
441 	if (size <= kfd->gtt_sa_chunk_size) {  in kfd_gtt_sa_allocate()
443 		set_bit(found, kfd->gtt_sa_bitmap);  in kfd_gtt_sa_allocate()
448 	cur_size = size - kfd->gtt_sa_chunk_size;  in kfd_gtt_sa_allocate()
451 			find_next_zero_bit(kfd->gtt_sa_bitmap,  in kfd_gtt_sa_allocate()
452 					kfd->gtt_sa_num_of_chunks, ++found);  in kfd_gtt_sa_allocate()
466 		if (found == kfd->gtt_sa_num_of_chunks)  in kfd_gtt_sa_allocate()
470 		if (cur_size <= kfd->gtt_sa_chunk_size)  in kfd_gtt_sa_allocate()
473 			cur_size -= kfd->gtt_sa_chunk_size;  in kfd_gtt_sa_allocate()
484 		set_bit(found, kfd->gtt_sa_bitmap);  in kfd_gtt_sa_allocate()
487 	mutex_unlock(&kfd->gtt_sa_lock);  in kfd_gtt_sa_allocate()
492 	mutex_unlock(&kfd->gtt_sa_lock);  in kfd_gtt_sa_allocate()
497 int kfd_gtt_sa_free(struct kfd_dev *kfd, struct kfd_mem_obj *mem_obj)  in kfd_gtt_sa_free()  argument
501 	BUG_ON(!kfd);  in kfd_gtt_sa_free()
510 	mutex_lock(&kfd->gtt_sa_lock);  in kfd_gtt_sa_free()
516 		clear_bit(bit, kfd->gtt_sa_bitmap);  in kfd_gtt_sa_free()
518 	mutex_unlock(&kfd->gtt_sa_lock);  in kfd_gtt_sa_free()