117 	struct vmw_fence_manager *fman = fman_from_fence(fence);  in vmw_fence_obj_destroy()  local
120 	spin_lock_irqsave(&fman->lock, irq_flags);  in vmw_fence_obj_destroy()
122 	--fman->num_fence_objects;  in vmw_fence_obj_destroy()
123 	spin_unlock_irqrestore(&fman->lock, irq_flags);  in vmw_fence_obj_destroy()
142 	struct vmw_fence_manager *fman = fman_from_fence(fence);  in vmw_fence_enable_signaling()  local
143 	struct vmw_private *dev_priv = fman->dev_priv;  in vmw_fence_enable_signaling()
169 static void __vmw_fences_update(struct vmw_fence_manager *fman);
176 	struct vmw_fence_manager *fman = fman_from_fence(fence);  in vmw_fence_wait()  local
177 	struct vmw_private *dev_priv = fman->dev_priv;  in vmw_fence_wait()
200 		__vmw_fences_update(fman);  in vmw_fence_wait()
246 	struct vmw_fence_manager *fman =  in vmw_fence_work_func()  local
254 		mutex_lock(&fman->goal_irq_mutex);  in vmw_fence_work_func()
256 		spin_lock_irq(&fman->lock);  in vmw_fence_work_func()
257 		list_splice_init(&fman->cleanup_list, &list);  in vmw_fence_work_func()
258 		seqno_valid = fman->seqno_valid;  in vmw_fence_work_func()
259 		spin_unlock_irq(&fman->lock);  in vmw_fence_work_func()
261 		if (!seqno_valid && fman->goal_irq_on) {  in vmw_fence_work_func()
262 			fman->goal_irq_on = false;  in vmw_fence_work_func()
263 			vmw_goal_waiter_remove(fman->dev_priv);  in vmw_fence_work_func()
265 		mutex_unlock(&fman->goal_irq_mutex);  in vmw_fence_work_func()
286 	struct vmw_fence_manager *fman = kzalloc(sizeof(*fman), GFP_KERNEL);  in vmw_fence_manager_init()  local
288 	if (unlikely(fman == NULL))  in vmw_fence_manager_init()
291 	fman->dev_priv = dev_priv;  in vmw_fence_manager_init()
292 	spin_lock_init(&fman->lock);  in vmw_fence_manager_init()
293 	INIT_LIST_HEAD(&fman->fence_list);  in vmw_fence_manager_init()
294 	INIT_LIST_HEAD(&fman->cleanup_list);  in vmw_fence_manager_init()
295 	INIT_WORK(&fman->work, &vmw_fence_work_func);  in vmw_fence_manager_init()
296 	fman->fifo_down = true;  in vmw_fence_manager_init()
297 	fman->user_fence_size = ttm_round_pot(sizeof(struct vmw_user_fence));  in vmw_fence_manager_init()
298 	fman->fence_size = ttm_round_pot(sizeof(struct vmw_fence_obj));  in vmw_fence_manager_init()
299 	fman->event_fence_action_size =  in vmw_fence_manager_init()
301 	mutex_init(&fman->goal_irq_mutex);  in vmw_fence_manager_init()
302 	fman->ctx = fence_context_alloc(1);  in vmw_fence_manager_init()
304 	return fman;  in vmw_fence_manager_init()
307 void vmw_fence_manager_takedown(struct vmw_fence_manager *fman)  in vmw_fence_manager_takedown()  argument
312 	(void) cancel_work_sync(&fman->work);  in vmw_fence_manager_takedown()
314 	spin_lock_irqsave(&fman->lock, irq_flags);  in vmw_fence_manager_takedown()
315 	lists_empty = list_empty(&fman->fence_list) &&  in vmw_fence_manager_takedown()
316 		list_empty(&fman->cleanup_list);  in vmw_fence_manager_takedown()
317 	spin_unlock_irqrestore(&fman->lock, irq_flags);  in vmw_fence_manager_takedown()
320 	kfree(fman);  in vmw_fence_manager_takedown()
323 static int vmw_fence_obj_init(struct vmw_fence_manager *fman,  in vmw_fence_obj_init()  argument
330 	fence_init(&fence->base, &vmw_fence_ops, &fman->lock,  in vmw_fence_obj_init()
331 		   fman->ctx, seqno);  in vmw_fence_obj_init()
335 	spin_lock_irqsave(&fman->lock, irq_flags);  in vmw_fence_obj_init()
336 	if (unlikely(fman->fifo_down)) {  in vmw_fence_obj_init()
340 	list_add_tail(&fence->head, &fman->fence_list);  in vmw_fence_obj_init()
341 	++fman->num_fence_objects;  in vmw_fence_obj_init()
344 	spin_unlock_irqrestore(&fman->lock, irq_flags);  in vmw_fence_obj_init()
349 static void vmw_fences_perform_actions(struct vmw_fence_manager *fman,  in vmw_fences_perform_actions()  argument
356 		fman->pending_actions[action->type]--;  in vmw_fences_perform_actions()
365 		list_add_tail(&action->head, &fman->cleanup_list);  in vmw_fences_perform_actions()
385 static bool vmw_fence_goal_new_locked(struct vmw_fence_manager *fman,  in vmw_fence_goal_new_locked()  argument
392 	if (likely(!fman->seqno_valid))  in vmw_fence_goal_new_locked()
395 	fifo_mem = fman->dev_priv->mmio_virt;  in vmw_fence_goal_new_locked()
400 	fman->seqno_valid = false;  in vmw_fence_goal_new_locked()
401 	list_for_each_entry(fence, &fman->fence_list, head) {  in vmw_fence_goal_new_locked()
403 			fman->seqno_valid = true;  in vmw_fence_goal_new_locked()
431 	struct vmw_fence_manager *fman = fman_from_fence(fence);  in vmw_fence_goal_check_locked()  local
438 	fifo_mem = fman->dev_priv->mmio_virt;  in vmw_fence_goal_check_locked()
440 	if (likely(fman->seqno_valid &&  in vmw_fence_goal_check_locked()
445 	fman->seqno_valid = true;  in vmw_fence_goal_check_locked()
450 static void __vmw_fences_update(struct vmw_fence_manager *fman)  in __vmw_fences_update()  argument
456 	__le32 __iomem *fifo_mem = fman->dev_priv->mmio_virt;  in __vmw_fences_update()
460 	list_for_each_entry_safe(fence, next_fence, &fman->fence_list, head) {  in __vmw_fences_update()
467 			vmw_fences_perform_actions(fman, &action_list);  in __vmw_fences_update()
478 	needs_rerun = vmw_fence_goal_new_locked(fman, seqno);  in __vmw_fences_update()
487 	if (!list_empty(&fman->cleanup_list))  in __vmw_fences_update()
488 		(void) schedule_work(&fman->work);  in __vmw_fences_update()
491 void vmw_fences_update(struct vmw_fence_manager *fman)  in vmw_fences_update()  argument
495 	spin_lock_irqsave(&fman->lock, irq_flags);  in vmw_fences_update()
496 	__vmw_fences_update(fman);  in vmw_fences_update()
497 	spin_unlock_irqrestore(&fman->lock, irq_flags);  in vmw_fences_update()
502 	struct vmw_fence_manager *fman = fman_from_fence(fence);  in vmw_fence_obj_signaled()  local
507 	vmw_fences_update(fman);  in vmw_fence_obj_signaled()
537 int vmw_fence_create(struct vmw_fence_manager *fman,  in vmw_fence_create()  argument
548 	ret = vmw_fence_obj_init(fman, fence, seqno,  in vmw_fence_create()
566 	struct vmw_fence_manager *fman = fman_from_fence(fence);  in vmw_user_fence_destroy()  local
572 	ttm_mem_global_free(vmw_mem_glob(fman->dev_priv),  in vmw_user_fence_destroy()
573 			    fman->user_fence_size);  in vmw_user_fence_destroy()
588 			  struct vmw_fence_manager *fman,  in vmw_user_fence_create()  argument
596 	struct ttm_mem_global *mem_glob = vmw_mem_glob(fman->dev_priv);  in vmw_user_fence_create()
604 	ret = ttm_mem_global_alloc(mem_glob, fman->user_fence_size,  in vmw_user_fence_create()
615 	ret = vmw_fence_obj_init(fman, &ufence->fence, seqno,  in vmw_user_fence_create()
648 	ttm_mem_global_free(mem_glob, fman->user_fence_size);  in vmw_user_fence_create()
657 void vmw_fence_fifo_down(struct vmw_fence_manager *fman)  in vmw_fence_fifo_down()  argument
667 	spin_lock_irq(&fman->lock);  in vmw_fence_fifo_down()
668 	fman->fifo_down = true;  in vmw_fence_fifo_down()
669 	while (!list_empty(&fman->fence_list)) {  in vmw_fence_fifo_down()
671 			list_entry(fman->fence_list.prev, struct vmw_fence_obj,  in vmw_fence_fifo_down()
674 		spin_unlock_irq(&fman->lock);  in vmw_fence_fifo_down()
685 			vmw_fences_perform_actions(fman, &action_list);  in vmw_fence_fifo_down()
690 		spin_lock_irq(&fman->lock);  in vmw_fence_fifo_down()
692 	spin_unlock_irq(&fman->lock);  in vmw_fence_fifo_down()
695 void vmw_fence_fifo_up(struct vmw_fence_manager *fman)  in vmw_fence_fifo_up()  argument
699 	spin_lock_irqsave(&fman->lock, irq_flags);  in vmw_fence_fifo_up()
700 	fman->fifo_down = false;  in vmw_fence_fifo_up()
701 	spin_unlock_irqrestore(&fman->lock, irq_flags);  in vmw_fence_fifo_up()
771 	struct vmw_fence_manager *fman;  in vmw_fence_obj_signaled_ioctl()  local
784 	fman = fman_from_fence(fence);  in vmw_fence_obj_signaled_ioctl()
789 	spin_lock_irq(&fman->lock);  in vmw_fence_obj_signaled_ioctl()
791 	spin_unlock_irq(&fman->lock);  in vmw_fence_obj_signaled_ioctl()
823 void vmw_event_fence_fpriv_gone(struct vmw_fence_manager *fman,  in vmw_event_fence_fpriv_gone()  argument
831 		spin_lock_irqsave(&fman->lock, irq_flags);  in vmw_event_fence_fpriv_gone()
840 		spin_unlock_irqrestore(&fman->lock, irq_flags);  in vmw_event_fence_fpriv_gone()
844 	spin_unlock_irqrestore(&fman->lock, irq_flags);  in vmw_event_fence_fpriv_gone()
902 	struct vmw_fence_manager *fman = fman_from_fence(eaction->fence);  in vmw_event_fence_action_cleanup()  local
905 	spin_lock_irqsave(&fman->lock, irq_flags);  in vmw_event_fence_action_cleanup()
907 	spin_unlock_irqrestore(&fman->lock, irq_flags);  in vmw_event_fence_action_cleanup()
926 	struct vmw_fence_manager *fman = fman_from_fence(fence);  in vmw_fence_obj_add_action()  local
930 	mutex_lock(&fman->goal_irq_mutex);  in vmw_fence_obj_add_action()
931 	spin_lock_irqsave(&fman->lock, irq_flags);  in vmw_fence_obj_add_action()
933 	fman->pending_actions[action->type]++;  in vmw_fence_obj_add_action()
939 		vmw_fences_perform_actions(fman, &action_list);  in vmw_fence_obj_add_action()
950 	spin_unlock_irqrestore(&fman->lock, irq_flags);  in vmw_fence_obj_add_action()
953 		if (!fman->goal_irq_on) {  in vmw_fence_obj_add_action()
954 			fman->goal_irq_on = true;  in vmw_fence_obj_add_action()
955 			vmw_goal_waiter_add(fman->dev_priv);  in vmw_fence_obj_add_action()
957 		vmw_fences_update(fman);  in vmw_fence_obj_add_action()
959 	mutex_unlock(&fman->goal_irq_mutex);  in vmw_fence_obj_add_action()
986 	struct vmw_fence_manager *fman = fman_from_fence(fence);  in vmw_event_fence_action_queue()  local
1001 	eaction->dev = fman->dev_priv->dev;  in vmw_event_fence_action_queue()
1005 	spin_lock_irqsave(&fman->lock, irq_flags);  in vmw_event_fence_action_queue()
1007 	spin_unlock_irqrestore(&fman->lock, irq_flags);  in vmw_event_fence_action_queue()
1026 	struct vmw_fence_manager *fman = fman_from_fence(fence);  in vmw_event_fence_action_create()  local
1027 	struct drm_device *dev = fman->dev_priv->dev;  in vmw_event_fence_action_create()