160 	struct tcmu_dev *udev = TCMU_DEV(se_dev);  in tcmu_alloc_cmd()  local
169 	tcmu_cmd->tcmu_dev = udev;  in tcmu_alloc_cmd()
180 	spin_lock_irq(&udev->commands_lock);  in tcmu_alloc_cmd()
181 	cmd_id = idr_alloc(&udev->commands, tcmu_cmd, 0,  in tcmu_alloc_cmd()
183 	spin_unlock_irq(&udev->commands_lock);  in tcmu_alloc_cmd()
235 static void alloc_and_scatter_data_area(struct tcmu_dev *udev,  in alloc_and_scatter_data_area()  argument
246 				 head_to_end(udev->data_head, udev->data_size));  in alloc_and_scatter_data_area()
248 		to = (void *) udev->mb_addr + udev->data_off + udev->data_head;  in alloc_and_scatter_data_area()
257 		(*iov)->iov_base = (void __user *) udev->data_off +  in alloc_and_scatter_data_area()
258 						udev->data_head;  in alloc_and_scatter_data_area()
262 		UPDATE_HEAD(udev->data_head, copy_bytes, udev->data_size);  in alloc_and_scatter_data_area()
271 			(*iov)->iov_base = (void __user *) udev->data_off +  in alloc_and_scatter_data_area()
272 							udev->data_head;  in alloc_and_scatter_data_area()
275 				to = (void *) udev->mb_addr +  in alloc_and_scatter_data_area()
276 					udev->data_off + udev->data_head;  in alloc_and_scatter_data_area()
284 			UPDATE_HEAD(udev->data_head,  in alloc_and_scatter_data_area()
285 				copy_bytes, udev->data_size);  in alloc_and_scatter_data_area()
292 static void gather_and_free_data_area(struct tcmu_dev *udev,  in gather_and_free_data_area()  argument
303 				 head_to_end(udev->data_tail, udev->data_size));  in gather_and_free_data_area()
307 		from = (void *) udev->mb_addr +  in gather_and_free_data_area()
308 			udev->data_off + udev->data_tail;  in gather_and_free_data_area()
312 		UPDATE_HEAD(udev->data_tail, copy_bytes, udev->data_size);  in gather_and_free_data_area()
318 			from = (void *) udev->mb_addr +  in gather_and_free_data_area()
319 				udev->data_off + udev->data_tail;  in gather_and_free_data_area()
320 			WARN_ON(udev->data_tail);  in gather_and_free_data_area()
325 			UPDATE_HEAD(udev->data_tail,  in gather_and_free_data_area()
326 				copy_bytes, udev->data_size);  in gather_and_free_data_area()
338 static bool is_ring_space_avail(struct tcmu_dev *udev, size_t cmd_size, size_t data_needed)  in is_ring_space_avail()  argument
340 	struct tcmu_mailbox *mb = udev->mb_addr;  in is_ring_space_avail()
347 	cmd_head = mb->cmd_head % udev->cmdr_size; /* UAM */  in is_ring_space_avail()
353 	if (head_to_end(cmd_head, udev->cmdr_size) >= cmd_size)  in is_ring_space_avail()
356 		cmd_needed = cmd_size + head_to_end(cmd_head, udev->cmdr_size);  in is_ring_space_avail()
358 	space = spc_free(cmd_head, udev->cmdr_last_cleaned, udev->cmdr_size);  in is_ring_space_avail()
361 		       udev->cmdr_last_cleaned, udev->cmdr_size);  in is_ring_space_avail()
365 	space = spc_free(udev->data_head, udev->data_tail, udev->data_size);  in is_ring_space_avail()
367 		pr_debug("no data space: %zu %zu %zu\n", udev->data_head,  in is_ring_space_avail()
368 		       udev->data_tail, udev->data_size);  in is_ring_space_avail()
377 	struct tcmu_dev *udev = tcmu_cmd->tcmu_dev;  in tcmu_queue_cmd_ring()  local
388 	if (test_bit(TCMU_DEV_BIT_BROKEN, &udev->flags))  in tcmu_queue_cmd_ring()
407 	spin_lock_irq(&udev->cmdr_lock);  in tcmu_queue_cmd_ring()
409 	mb = udev->mb_addr;  in tcmu_queue_cmd_ring()
410 	cmd_head = mb->cmd_head % udev->cmdr_size; /* UAM */  in tcmu_queue_cmd_ring()
411 	if ((command_size > (udev->cmdr_size / 2))  in tcmu_queue_cmd_ring()
412 	    || tcmu_cmd->data_length > (udev->data_size - 1))  in tcmu_queue_cmd_ring()
415 			udev->cmdr_size, udev->data_size);  in tcmu_queue_cmd_ring()
417 	while (!is_ring_space_avail(udev, command_size, tcmu_cmd->data_length)) {  in tcmu_queue_cmd_ring()
421 		prepare_to_wait(&udev->wait_cmdr, &__wait, TASK_INTERRUPTIBLE);  in tcmu_queue_cmd_ring()
424 		spin_unlock_irq(&udev->cmdr_lock);  in tcmu_queue_cmd_ring()
426 		finish_wait(&udev->wait_cmdr, &__wait);  in tcmu_queue_cmd_ring()
432 		spin_lock_irq(&udev->cmdr_lock);  in tcmu_queue_cmd_ring()
435 		cmd_head = mb->cmd_head % udev->cmdr_size; /* UAM */  in tcmu_queue_cmd_ring()
439 	if (head_to_end(cmd_head, udev->cmdr_size) < command_size) {  in tcmu_queue_cmd_ring()
440 		size_t pad_size = head_to_end(cmd_head, udev->cmdr_size);  in tcmu_queue_cmd_ring()
450 		UPDATE_HEAD(mb->cmd_head, pad_size, udev->cmdr_size);  in tcmu_queue_cmd_ring()
452 		cmd_head = mb->cmd_head % udev->cmdr_size; /* UAM */  in tcmu_queue_cmd_ring()
471 	alloc_and_scatter_data_area(udev, se_cmd->t_data_sg,  in tcmu_queue_cmd_ring()
478 	alloc_and_scatter_data_area(udev, se_cmd->t_bidi_data_sg,  in tcmu_queue_cmd_ring()
488 	UPDATE_HEAD(mb->cmd_head, command_size, udev->cmdr_size);  in tcmu_queue_cmd_ring()
491 	spin_unlock_irq(&udev->cmdr_lock);  in tcmu_queue_cmd_ring()
494 	uio_event_notify(&udev->uio_info);  in tcmu_queue_cmd_ring()
496 	mod_timer(&udev->timeout,  in tcmu_queue_cmd_ring()
505 	struct tcmu_dev *udev = TCMU_DEV(se_dev);  in tcmu_queue_cmd()  local
516 		spin_lock_irq(&udev->commands_lock);  in tcmu_queue_cmd()
517 		idr_remove(&udev->commands, tcmu_cmd->cmd_id);  in tcmu_queue_cmd()
518 		spin_unlock_irq(&udev->commands_lock);  in tcmu_queue_cmd()
529 	struct tcmu_dev *udev = cmd->tcmu_dev;  in tcmu_handle_completion()  local
534 		UPDATE_HEAD(udev->data_tail, cmd->data_length, udev->data_size);  in tcmu_handle_completion()
539 		UPDATE_HEAD(udev->data_tail, cmd->data_length, udev->data_size);  in tcmu_handle_completion()
547 		UPDATE_HEAD(udev->data_tail, cmd->data_length, udev->data_size);  in tcmu_handle_completion()
550 		UPDATE_HEAD(udev->data_tail,  in tcmu_handle_completion()
551 			(size_t)se_cmd->t_data_sg->length, udev->data_size);  in tcmu_handle_completion()
554 		gather_and_free_data_area(udev,  in tcmu_handle_completion()
557 		gather_and_free_data_area(udev,  in tcmu_handle_completion()
560 		UPDATE_HEAD(udev->data_tail, cmd->data_length, udev->data_size);  in tcmu_handle_completion()
572 static unsigned int tcmu_handle_completions(struct tcmu_dev *udev)  in tcmu_handle_completions()  argument
578 	if (test_bit(TCMU_DEV_BIT_BROKEN, &udev->flags)) {  in tcmu_handle_completions()
583 	spin_lock_irqsave(&udev->cmdr_lock, flags);  in tcmu_handle_completions()
585 	mb = udev->mb_addr;  in tcmu_handle_completions()
588 	while (udev->cmdr_last_cleaned != ACCESS_ONCE(mb->cmd_tail)) {  in tcmu_handle_completions()
590 		struct tcmu_cmd_entry *entry = (void *) mb + CMDR_OFF + udev->cmdr_last_cleaned;  in tcmu_handle_completions()
596 			UPDATE_HEAD(udev->cmdr_last_cleaned,  in tcmu_handle_completions()
598 				    udev->cmdr_size);  in tcmu_handle_completions()
603 		spin_lock(&udev->commands_lock);  in tcmu_handle_completions()
604 		cmd = idr_find(&udev->commands, entry->hdr.cmd_id);  in tcmu_handle_completions()
606 			idr_remove(&udev->commands, cmd->cmd_id);  in tcmu_handle_completions()
607 		spin_unlock(&udev->commands_lock);  in tcmu_handle_completions()
611 			set_bit(TCMU_DEV_BIT_BROKEN, &udev->flags);  in tcmu_handle_completions()
617 		UPDATE_HEAD(udev->cmdr_last_cleaned,  in tcmu_handle_completions()
619 			    udev->cmdr_size);  in tcmu_handle_completions()
625 		del_timer(&udev->timeout); /* no more pending cmds */  in tcmu_handle_completions()
627 	spin_unlock_irqrestore(&udev->cmdr_lock, flags);  in tcmu_handle_completions()
629 	wake_up(&udev->wait_cmdr);  in tcmu_handle_completions()
655 	struct tcmu_dev *udev = (struct tcmu_dev *)data;  in tcmu_device_timedout()  local
659 	handled = tcmu_handle_completions(udev);  in tcmu_device_timedout()
663 	spin_lock_irqsave(&udev->commands_lock, flags);  in tcmu_device_timedout()
664 	idr_for_each(&udev->commands, tcmu_check_expired_cmd, NULL);  in tcmu_device_timedout()
665 	spin_unlock_irqrestore(&udev->commands_lock, flags);  in tcmu_device_timedout()
695 	struct tcmu_dev *udev;  in tcmu_alloc_device()  local
697 	udev = kzalloc(sizeof(struct tcmu_dev), GFP_KERNEL);  in tcmu_alloc_device()
698 	if (!udev)  in tcmu_alloc_device()
701 	udev->name = kstrdup(name, GFP_KERNEL);  in tcmu_alloc_device()
702 	if (!udev->name) {  in tcmu_alloc_device()
703 		kfree(udev);  in tcmu_alloc_device()
707 	udev->hba = hba;  in tcmu_alloc_device()
709 	init_waitqueue_head(&udev->wait_cmdr);  in tcmu_alloc_device()
710 	spin_lock_init(&udev->cmdr_lock);  in tcmu_alloc_device()
712 	idr_init(&udev->commands);  in tcmu_alloc_device()
713 	spin_lock_init(&udev->commands_lock);  in tcmu_alloc_device()
715 	setup_timer(&udev->timeout, tcmu_device_timedout,  in tcmu_alloc_device()
716 		(unsigned long)udev);  in tcmu_alloc_device()
718 	return &udev->se_dev;  in tcmu_alloc_device()
736 	struct tcmu_dev *udev = vma->vm_private_data;  in tcmu_find_mem_index()  local
737 	struct uio_info *info = &udev->uio_info;  in tcmu_find_mem_index()
749 	struct tcmu_dev *udev = vma->vm_private_data;  in tcmu_vma_fault()  local
750 	struct uio_info *info = &udev->uio_info;  in tcmu_vma_fault()
781 	struct tcmu_dev *udev = container_of(info, struct tcmu_dev, uio_info);  in tcmu_mmap()  local
786 	vma->vm_private_data = udev;  in tcmu_mmap()
797 	struct tcmu_dev *udev = container_of(info, struct tcmu_dev, uio_info);  in tcmu_open()  local
800 	if (test_and_set_bit(TCMU_DEV_BIT_OPEN, &udev->flags))  in tcmu_open()
810 	struct tcmu_dev *udev = container_of(info, struct tcmu_dev, uio_info);  in tcmu_release()  local
812 	clear_bit(TCMU_DEV_BIT_OPEN, &udev->flags);  in tcmu_release()
858 	struct tcmu_dev *udev = TCMU_DEV(dev);  in tcmu_configure_device()  local
859 	struct tcmu_hba *hba = udev->hba->hba_ptr;  in tcmu_configure_device()
867 	info = &udev->uio_info;  in tcmu_configure_device()
869 	size = snprintf(NULL, 0, "tcm-user/%u/%s/%s", hba->host_id, udev->name,  in tcmu_configure_device()
870 			udev->dev_config);  in tcmu_configure_device()
876 	used = snprintf(str, size, "tcm-user/%u/%s", hba->host_id, udev->name);  in tcmu_configure_device()
878 	if (udev->dev_config[0])  in tcmu_configure_device()
879 		snprintf(str + used, size - used, "/%s", udev->dev_config);  in tcmu_configure_device()
883 	udev->mb_addr = vzalloc(TCMU_RING_SIZE);  in tcmu_configure_device()
884 	if (!udev->mb_addr) {  in tcmu_configure_device()
890 	udev->cmdr_size = CMDR_SIZE - CMDR_OFF;  in tcmu_configure_device()
891 	udev->data_off = CMDR_SIZE;  in tcmu_configure_device()
892 	udev->data_size = TCMU_RING_SIZE - CMDR_SIZE;  in tcmu_configure_device()
894 	mb = udev->mb_addr;  in tcmu_configure_device()
897 	mb->cmdr_size = udev->cmdr_size;  in tcmu_configure_device()
899 	WARN_ON(!PAGE_ALIGNED(udev->data_off));  in tcmu_configure_device()
900 	WARN_ON(udev->data_size % PAGE_SIZE);  in tcmu_configure_device()
905 	info->mem[0].addr = (phys_addr_t) udev->mb_addr;  in tcmu_configure_device()
925 	ret = tcmu_netlink_event(TCMU_CMD_ADDED_DEVICE, udev->uio_info.name,  in tcmu_configure_device()
926 				 udev->uio_info.uio_dev->minor);  in tcmu_configure_device()
933 	uio_unregister_device(&udev->uio_info);  in tcmu_configure_device()
935 	vfree(udev->mb_addr);  in tcmu_configure_device()
954 	struct tcmu_dev *udev = TCMU_DEV(dev);  in tcmu_dev_call_rcu()  local
956 	kfree(udev);  in tcmu_dev_call_rcu()
961 	struct tcmu_dev *udev = TCMU_DEV(dev);  in tcmu_free_device()  local
964 	del_timer_sync(&udev->timeout);  in tcmu_free_device()
966 	vfree(udev->mb_addr);  in tcmu_free_device()
969 	spin_lock_irq(&udev->commands_lock);  in tcmu_free_device()
970 	i = idr_for_each(&udev->commands, tcmu_check_pending_cmd, NULL);  in tcmu_free_device()
971 	idr_destroy(&udev->commands);  in tcmu_free_device()
972 	spin_unlock_irq(&udev->commands_lock);  in tcmu_free_device()
976 	if (udev->uio_info.uio_dev) {  in tcmu_free_device()
977 		tcmu_netlink_event(TCMU_CMD_REMOVED_DEVICE, udev->uio_info.name,  in tcmu_free_device()
978 				   udev->uio_info.uio_dev->minor);  in tcmu_free_device()
980 		uio_unregister_device(&udev->uio_info);  in tcmu_free_device()
981 		kfree(udev->uio_info.name);  in tcmu_free_device()
982 		kfree(udev->name);  in tcmu_free_device()
1001 	struct tcmu_dev *udev = TCMU_DEV(dev);  in tcmu_set_configfs_dev_params()  local
1020 			if (match_strlcpy(udev->dev_config, &args[0],  in tcmu_set_configfs_dev_params()
1025 			pr_debug("TCMU: Referencing Path: %s\n", udev->dev_config);  in tcmu_set_configfs_dev_params()
1033 			ret = kstrtoul(arg_p, 0, (unsigned long *) &udev->dev_size);  in tcmu_set_configfs_dev_params()
1067 	struct tcmu_dev *udev = TCMU_DEV(dev);  in tcmu_show_configfs_dev_params()  local
1071 		     udev->dev_config[0] ? udev->dev_config : "NULL");  in tcmu_show_configfs_dev_params()
1072 	bl += sprintf(b + bl, "Size: %zu\n", udev->dev_size);  in tcmu_show_configfs_dev_params()
1079 	struct tcmu_dev *udev = TCMU_DEV(dev);  in tcmu_get_blocks()  local
1081 	return div_u64(udev->dev_size - dev->dev_attrib.block_size,  in tcmu_get_blocks()