33 static bool amd_sched_entity_is_ready(struct amd_sched_entity *entity);
48 				    struct amd_sched_entity *entity)  in amd_sched_rq_add_entity()  argument
51 	list_add_tail(&entity->list, &rq->entities);  in amd_sched_rq_add_entity()
56 				       struct amd_sched_entity *entity)  in amd_sched_rq_remove_entity()  argument
59 	list_del_init(&entity->list);  in amd_sched_rq_remove_entity()
60 	if (rq->current_entity == entity)  in amd_sched_rq_remove_entity()
75 	struct amd_sched_entity *entity;  in amd_sched_rq_select_entity()  local
79 	entity = rq->current_entity;  in amd_sched_rq_select_entity()
80 	if (entity) {  in amd_sched_rq_select_entity()
81 		list_for_each_entry_continue(entity, &rq->entities, list) {  in amd_sched_rq_select_entity()
82 			if (amd_sched_entity_is_ready(entity)) {  in amd_sched_rq_select_entity()
83 				rq->current_entity = entity;  in amd_sched_rq_select_entity()
85 				return entity;  in amd_sched_rq_select_entity()
90 	list_for_each_entry(entity, &rq->entities, list) {  in amd_sched_rq_select_entity()
92 		if (amd_sched_entity_is_ready(entity)) {  in amd_sched_rq_select_entity()
93 			rq->current_entity = entity;  in amd_sched_rq_select_entity()
95 			return entity;  in amd_sched_rq_select_entity()
98 		if (entity == rq->current_entity)  in amd_sched_rq_select_entity()
119 			  struct amd_sched_entity *entity,  in amd_sched_entity_init()  argument
125 	if (!(sched && entity && rq))  in amd_sched_entity_init()
128 	memset(entity, 0, sizeof(struct amd_sched_entity));  in amd_sched_entity_init()
129 	INIT_LIST_HEAD(&entity->list);  in amd_sched_entity_init()
130 	entity->rq = rq;  in amd_sched_entity_init()
131 	entity->sched = sched;  in amd_sched_entity_init()
133 	spin_lock_init(&entity->queue_lock);  in amd_sched_entity_init()
134 	r = kfifo_alloc(&entity->job_queue, jobs * sizeof(void *), GFP_KERNEL);  in amd_sched_entity_init()
138 	atomic_set(&entity->fence_seq, 0);  in amd_sched_entity_init()
139 	entity->fence_context = fence_context_alloc(1);  in amd_sched_entity_init()
142 	amd_sched_rq_add_entity(rq, entity);  in amd_sched_entity_init()
156 					    struct amd_sched_entity *entity)  in amd_sched_entity_is_initialized()  argument
158 	return entity->sched == sched &&  in amd_sched_entity_is_initialized()
159 		entity->rq != NULL;  in amd_sched_entity_is_initialized()
169 static bool amd_sched_entity_is_idle(struct amd_sched_entity *entity)  in amd_sched_entity_is_idle()  argument
172 	if (kfifo_is_empty(&entity->job_queue))  in amd_sched_entity_is_idle()
185 static bool amd_sched_entity_is_ready(struct amd_sched_entity *entity)  in amd_sched_entity_is_ready()  argument
187 	if (kfifo_is_empty(&entity->job_queue))  in amd_sched_entity_is_ready()
190 	if (ACCESS_ONCE(entity->dependency))  in amd_sched_entity_is_ready()
205 			   struct amd_sched_entity *entity)  in amd_sched_entity_fini()  argument
207 	struct amd_sched_rq *rq = entity->rq;  in amd_sched_entity_fini()
209 	if (!amd_sched_entity_is_initialized(sched, entity))  in amd_sched_entity_fini()
216 	wait_event(sched->job_scheduled, amd_sched_entity_is_idle(entity));  in amd_sched_entity_fini()
218 	amd_sched_rq_remove_entity(rq, entity);  in amd_sched_entity_fini()
219 	kfifo_free(&entity->job_queue);  in amd_sched_entity_fini()
224 	struct amd_sched_entity *entity =  in amd_sched_entity_wakeup()  local
226 	entity->dependency = NULL;  in amd_sched_entity_wakeup()
228 	amd_sched_wakeup(entity->sched);  in amd_sched_entity_wakeup()
231 static bool amd_sched_entity_add_dependency_cb(struct amd_sched_entity *entity)  in amd_sched_entity_add_dependency_cb()  argument
233 	struct amd_gpu_scheduler *sched = entity->sched;  in amd_sched_entity_add_dependency_cb()
234 	struct fence * fence = entity->dependency;  in amd_sched_entity_add_dependency_cb()
237 	if (fence->context == entity->fence_context) {  in amd_sched_entity_add_dependency_cb()
239 		fence_put(entity->dependency);  in amd_sched_entity_add_dependency_cb()
248 			fence_put(entity->dependency);  in amd_sched_entity_add_dependency_cb()
253 		entity->cb.func = amd_sched_entity_wakeup;  in amd_sched_entity_add_dependency_cb()
254 		list_add_tail(&entity->cb.node, &s_fence->scheduled_cb);  in amd_sched_entity_add_dependency_cb()
258 	if (!fence_add_callback(entity->dependency, &entity->cb,  in amd_sched_entity_add_dependency_cb()
262 	fence_put(entity->dependency);  in amd_sched_entity_add_dependency_cb()
267 amd_sched_entity_pop_job(struct amd_sched_entity *entity)  in amd_sched_entity_pop_job()  argument
269 	struct amd_gpu_scheduler *sched = entity->sched;  in amd_sched_entity_pop_job()
272 	if (!kfifo_out_peek(&entity->job_queue, &sched_job, sizeof(sched_job)))  in amd_sched_entity_pop_job()
275 	while ((entity->dependency = sched->ops->dependency(sched_job)))  in amd_sched_entity_pop_job()
276 		if (amd_sched_entity_add_dependency_cb(entity))  in amd_sched_entity_pop_job()
292 	struct amd_sched_entity *entity = sched_job->s_entity;  in amd_sched_entity_in()  local
295 	spin_lock(&entity->queue_lock);  in amd_sched_entity_in()
296 	added = kfifo_in(&entity->job_queue, &sched_job,  in amd_sched_entity_in()
299 	if (added && kfifo_len(&entity->job_queue) == sizeof(sched_job))  in amd_sched_entity_in()
302 	spin_unlock(&entity->queue_lock);  in amd_sched_entity_in()
320 	struct amd_sched_entity *entity = sched_job->s_entity;  in amd_sched_entity_push_job()  local
323 	wait_event(entity->sched->job_scheduled,  in amd_sched_entity_push_job()
351 	struct amd_sched_entity *entity;  in amd_sched_select_entity()  local
357 	entity = amd_sched_rq_select_entity(&sched->kernel_rq);  in amd_sched_select_entity()
358 	if (entity == NULL)  in amd_sched_select_entity()
359 		entity = amd_sched_rq_select_entity(&sched->sched_rq);  in amd_sched_select_entity()
361 	return entity;  in amd_sched_select_entity()
389 	struct amd_sched_fence *entity, *tmp;  in amd_sched_fence_work_func()  local
396 	list_for_each_entry_safe(entity, tmp, &sched->fence_list, list) {  in amd_sched_fence_work_func()
397 		DRM_ERROR("  fence no %d\n", entity->base.seqno);  in amd_sched_fence_work_func()
398 		cancel_delayed_work(&entity->dwork);  in amd_sched_fence_work_func()
399 		list_del_init(&entity->list);  in amd_sched_fence_work_func()
400 		fence_put(&entity->base);  in amd_sched_fence_work_func()
416 		struct amd_sched_entity *entity;  in amd_sched_main()  local
423 			(entity = amd_sched_select_entity(sched)) ||  in amd_sched_main()
426 		if (!entity)  in amd_sched_main()
429 		sched_job = amd_sched_entity_pop_job(entity);  in amd_sched_main()
459 		count = kfifo_out(&entity->job_queue, &sched_job,  in amd_sched_main()