54 static void pm_calc_rlib_size(struct packet_manager *pm,  in pm_calc_rlib_size()  argument
61 	BUG_ON(!pm || !rlib_size || !over_subscription);  in pm_calc_rlib_size()
63 	process_count = pm->dqm->processes_count;  in pm_calc_rlib_size()
64 	queue_count = pm->dqm->queue_count;  in pm_calc_rlib_size()
75 		(pm->dqm->dev->device_info->asic_family == CHIP_CARRIZO) ?  in pm_calc_rlib_size()
92 static int pm_allocate_runlist_ib(struct packet_manager *pm,  in pm_allocate_runlist_ib()  argument
100 	BUG_ON(!pm);  in pm_allocate_runlist_ib()
101 	BUG_ON(pm->allocated == true);  in pm_allocate_runlist_ib()
104 	pm_calc_rlib_size(pm, rl_buffer_size, is_over_subscription);  in pm_allocate_runlist_ib()
106 	retval = kfd_gtt_sa_allocate(pm->dqm->dev, *rl_buffer_size,  in pm_allocate_runlist_ib()
107 					&pm->ib_buffer_obj);  in pm_allocate_runlist_ib()
114 	*(void **)rl_buffer = pm->ib_buffer_obj->cpu_ptr;  in pm_allocate_runlist_ib()
115 	*rl_gpu_buffer = pm->ib_buffer_obj->gpu_addr;  in pm_allocate_runlist_ib()
118 	pm->allocated = true;  in pm_allocate_runlist_ib()
122 static int pm_create_runlist(struct packet_manager *pm, uint32_t *buffer,  in pm_create_runlist()  argument
127 	BUG_ON(!pm || !buffer || !ib);  in pm_create_runlist()
145 static int pm_create_map_process(struct packet_manager *pm, uint32_t *buffer,  in pm_create_map_process()  argument
152 	BUG_ON(!pm || !buffer || !qpd);  in pm_create_map_process()
185 static int pm_create_map_queue_vi(struct packet_manager *pm, uint32_t *buffer,  in pm_create_map_queue_vi()  argument
191 	BUG_ON(!pm || !buffer || !q);  in pm_create_map_queue_vi()
250 static int pm_create_map_queue(struct packet_manager *pm, uint32_t *buffer,  in pm_create_map_queue()  argument
256 	BUG_ON(!pm || !buffer || !q);  in pm_create_map_queue()
312 static int pm_create_runlist_ib(struct packet_manager *pm,  in pm_create_runlist_ib()  argument
326 	BUG_ON(!pm || !queues || !rl_size_bytes || !rl_gpu_addr);  in pm_create_runlist_ib()
330 	retval = pm_allocate_runlist_ib(pm, &rl_buffer, rl_gpu_addr,  in pm_create_runlist_ib()
339 		pm->dqm->processes_count, pm->dqm->queue_count);  in pm_create_runlist_ib()
345 		if (proccesses_mapped >= pm->dqm->processes_count) {  in pm_create_runlist_ib()
347 			pm_release_ib(pm);  in pm_create_runlist_ib()
351 		retval = pm_create_map_process(pm, &rl_buffer[rl_wptr], qpd);  in pm_create_runlist_ib()
366 			if (pm->dqm->dev->device_info->asic_family ==  in pm_create_runlist_ib()
368 				retval = pm_create_map_queue_vi(pm,  in pm_create_runlist_ib()
373 				retval = pm_create_map_queue(pm,  in pm_create_runlist_ib()
392 			if (pm->dqm->dev->device_info->asic_family ==  in pm_create_runlist_ib()
394 				retval = pm_create_map_queue_vi(pm,  in pm_create_runlist_ib()
399 				retval = pm_create_map_queue(pm,  in pm_create_runlist_ib()
416 		pm_create_runlist(pm, &rl_buffer[rl_wptr], *rl_gpu_addr,  in pm_create_runlist_ib()
426 int pm_init(struct packet_manager *pm, struct device_queue_manager *dqm)  in pm_init()  argument
430 	pm->dqm = dqm;  in pm_init()
431 	mutex_init(&pm->lock);  in pm_init()
432 	pm->priv_queue = kernel_queue_init(dqm->dev, KFD_QUEUE_TYPE_HIQ);  in pm_init()
433 	if (pm->priv_queue == NULL) {  in pm_init()
434 		mutex_destroy(&pm->lock);  in pm_init()
437 	pm->allocated = false;  in pm_init()
442 void pm_uninit(struct packet_manager *pm)  in pm_uninit()  argument
444 	BUG_ON(!pm);  in pm_uninit()
446 	mutex_destroy(&pm->lock);  in pm_uninit()
447 	kernel_queue_uninit(pm->priv_queue);  in pm_uninit()
450 int pm_send_set_resources(struct packet_manager *pm,  in pm_send_set_resources()  argument
455 	BUG_ON(!pm || !res);  in pm_send_set_resources()
459 	mutex_lock(&pm->lock);  in pm_send_set_resources()
460 	pm->priv_queue->ops.acquire_packet_buffer(pm->priv_queue,  in pm_send_set_resources()
464 		mutex_unlock(&pm->lock);  in pm_send_set_resources()
487 	pm->priv_queue->ops.submit_packet(pm->priv_queue);  in pm_send_set_resources()
489 	mutex_unlock(&pm->lock);  in pm_send_set_resources()
494 int pm_send_runlist(struct packet_manager *pm, struct list_head *dqm_queues)  in pm_send_runlist()  argument
501 	BUG_ON(!pm || !dqm_queues);  in pm_send_runlist()
503 	retval = pm_create_runlist_ib(pm, dqm_queues, &rl_gpu_ib_addr,  in pm_send_runlist()
511 	mutex_lock(&pm->lock);  in pm_send_runlist()
513 	retval = pm->priv_queue->ops.acquire_packet_buffer(pm->priv_queue,  in pm_send_runlist()
518 	retval = pm_create_runlist(pm, rl_buffer, rl_gpu_ib_addr,  in pm_send_runlist()
523 	pm->priv_queue->ops.submit_packet(pm->priv_queue);  in pm_send_runlist()
525 	mutex_unlock(&pm->lock);  in pm_send_runlist()
530 	pm->priv_queue->ops.rollback_packet(pm->priv_queue);  in pm_send_runlist()
532 	mutex_unlock(&pm->lock);  in pm_send_runlist()
534 	if (pm->allocated == true)  in pm_send_runlist()
535 		pm_release_ib(pm);  in pm_send_runlist()
539 int pm_send_query_status(struct packet_manager *pm, uint64_t fence_address,  in pm_send_query_status()  argument
545 	BUG_ON(!pm || !fence_address);  in pm_send_query_status()
547 	mutex_lock(&pm->lock);  in pm_send_query_status()
548 	retval = pm->priv_queue->ops.acquire_packet_buffer(  in pm_send_query_status()
549 			pm->priv_queue,  in pm_send_query_status()
569 	pm->priv_queue->ops.submit_packet(pm->priv_queue);  in pm_send_query_status()
570 	mutex_unlock(&pm->lock);  in pm_send_query_status()
575 	mutex_unlock(&pm->lock);  in pm_send_query_status()
579 int pm_send_unmap_queue(struct packet_manager *pm, enum kfd_queue_type type,  in pm_send_unmap_queue()  argument
588 	BUG_ON(!pm);  in pm_send_unmap_queue()
590 	mutex_lock(&pm->lock);  in pm_send_unmap_queue()
591 	retval = pm->priv_queue->ops.acquire_packet_buffer(  in pm_send_unmap_queue()
592 			pm->priv_queue,  in pm_send_unmap_queue()
652 	pm->priv_queue->ops.submit_packet(pm->priv_queue);  in pm_send_unmap_queue()
654 	mutex_unlock(&pm->lock);  in pm_send_unmap_queue()
658 	mutex_unlock(&pm->lock);  in pm_send_unmap_queue()
662 void pm_release_ib(struct packet_manager *pm)  in pm_release_ib()  argument
664 	BUG_ON(!pm);  in pm_release_ib()
666 	mutex_lock(&pm->lock);  in pm_release_ib()
667 	if (pm->allocated) {  in pm_release_ib()
668 		kfd_gtt_sa_free(pm->dqm->dev, pm->ib_buffer_obj);  in pm_release_ib()
669 		pm->allocated = false;  in pm_release_ib()
671 	mutex_unlock(&pm->lock);  in pm_release_ib()