157 	struct tcmu_dev *udev = TCMU_DEV(se_dev);  in tcmu_alloc_cmd()  local
166 	tcmu_cmd->tcmu_dev = udev;  in tcmu_alloc_cmd()
172 	spin_lock_irq(&udev->commands_lock);  in tcmu_alloc_cmd()
173 	cmd_id = idr_alloc(&udev->commands, tcmu_cmd, 0,  in tcmu_alloc_cmd()
175 	spin_unlock_irq(&udev->commands_lock);  in tcmu_alloc_cmd()
233 static bool is_ring_space_avail(struct tcmu_dev *udev, size_t cmd_size, size_t data_needed)  in is_ring_space_avail()  argument
235 	struct tcmu_mailbox *mb = udev->mb_addr;  in is_ring_space_avail()
242 	cmd_head = mb->cmd_head % udev->cmdr_size; /* UAM */  in is_ring_space_avail()
248 	if (head_to_end(cmd_head, udev->cmdr_size) >= cmd_size)  in is_ring_space_avail()
251 		cmd_needed = cmd_size + head_to_end(cmd_head, udev->cmdr_size);  in is_ring_space_avail()
253 	space = spc_free(cmd_head, udev->cmdr_last_cleaned, udev->cmdr_size);  in is_ring_space_avail()
256 		       udev->cmdr_last_cleaned, udev->cmdr_size);  in is_ring_space_avail()
260 	space = spc_free(udev->data_head, udev->data_tail, udev->data_size);  in is_ring_space_avail()
262 		pr_debug("no data space: %zu %zu %zu\n", udev->data_head,  in is_ring_space_avail()
263 		       udev->data_tail, udev->data_size);  in is_ring_space_avail()
272 	struct tcmu_dev *udev = tcmu_cmd->tcmu_dev;  in tcmu_queue_cmd_ring()  local
284 	if (test_bit(TCMU_DEV_BIT_BROKEN, &udev->flags))  in tcmu_queue_cmd_ring()
302 	spin_lock_irq(&udev->cmdr_lock);  in tcmu_queue_cmd_ring()
304 	mb = udev->mb_addr;  in tcmu_queue_cmd_ring()
305 	cmd_head = mb->cmd_head % udev->cmdr_size; /* UAM */  in tcmu_queue_cmd_ring()
306 	if ((command_size > (udev->cmdr_size / 2))  in tcmu_queue_cmd_ring()
307 	    || tcmu_cmd->data_length > (udev->data_size - 1))  in tcmu_queue_cmd_ring()
310 			udev->cmdr_size, udev->data_size);  in tcmu_queue_cmd_ring()
312 	while (!is_ring_space_avail(udev, command_size, tcmu_cmd->data_length)) {  in tcmu_queue_cmd_ring()
316 		prepare_to_wait(&udev->wait_cmdr, &__wait, TASK_INTERRUPTIBLE);  in tcmu_queue_cmd_ring()
319 		spin_unlock_irq(&udev->cmdr_lock);  in tcmu_queue_cmd_ring()
321 		finish_wait(&udev->wait_cmdr, &__wait);  in tcmu_queue_cmd_ring()
327 		spin_lock_irq(&udev->cmdr_lock);  in tcmu_queue_cmd_ring()
330 		cmd_head = mb->cmd_head % udev->cmdr_size; /* UAM */  in tcmu_queue_cmd_ring()
334 	if (head_to_end(cmd_head, udev->cmdr_size) < command_size) {  in tcmu_queue_cmd_ring()
335 		size_t pad_size = head_to_end(cmd_head, udev->cmdr_size);  in tcmu_queue_cmd_ring()
345 		UPDATE_HEAD(mb->cmd_head, pad_size, udev->cmdr_size);  in tcmu_queue_cmd_ring()
347 		cmd_head = mb->cmd_head % udev->cmdr_size; /* UAM */  in tcmu_queue_cmd_ring()
365 				     head_to_end(udev->data_head, udev->data_size));  in tcmu_queue_cmd_ring()
367 		void *to = (void *) mb + udev->data_off + udev->data_head;  in tcmu_queue_cmd_ring()
376 		iov->iov_base = (void __user *) udev->data_off +  in tcmu_queue_cmd_ring()
377 						udev->data_head;  in tcmu_queue_cmd_ring()
381 		UPDATE_HEAD(udev->data_head, copy_bytes, udev->data_size);  in tcmu_queue_cmd_ring()
389 			iov->iov_base = (void __user *) udev->data_off +  in tcmu_queue_cmd_ring()
390 							udev->data_head;  in tcmu_queue_cmd_ring()
393 				to = (void *) mb + udev->data_off + udev->data_head;  in tcmu_queue_cmd_ring()
401 			UPDATE_HEAD(udev->data_head, copy_bytes, udev->data_size);  in tcmu_queue_cmd_ring()
416 	UPDATE_HEAD(mb->cmd_head, command_size, udev->cmdr_size);  in tcmu_queue_cmd_ring()
419 	spin_unlock_irq(&udev->cmdr_lock);  in tcmu_queue_cmd_ring()
422 	uio_event_notify(&udev->uio_info);  in tcmu_queue_cmd_ring()
424 	mod_timer(&udev->timeout,  in tcmu_queue_cmd_ring()
433 	struct tcmu_dev *udev = TCMU_DEV(se_dev);  in tcmu_queue_cmd()  local
444 		spin_lock_irq(&udev->commands_lock);  in tcmu_queue_cmd()
445 		idr_remove(&udev->commands, tcmu_cmd->cmd_id);  in tcmu_queue_cmd()
446 		spin_unlock_irq(&udev->commands_lock);  in tcmu_queue_cmd()
457 	struct tcmu_dev *udev = cmd->tcmu_dev;  in tcmu_handle_completion()  local
462 		UPDATE_HEAD(udev->data_tail, cmd->data_length, udev->data_size);  in tcmu_handle_completion()
467 		UPDATE_HEAD(udev->data_tail, cmd->data_length, udev->data_size);  in tcmu_handle_completion()
481 		UPDATE_HEAD(udev->data_tail, cmd->data_length, udev->data_size);  in tcmu_handle_completion()
494 					 head_to_end(udev->data_tail, udev->data_size));  in tcmu_handle_completion()
498 			from = (void *) udev->mb_addr + udev->data_off + udev->data_tail;  in tcmu_handle_completion()
502 			UPDATE_HEAD(udev->data_tail, copy_bytes, udev->data_size);  in tcmu_handle_completion()
506 				from = (void *) udev->mb_addr + udev->data_off + udev->data_tail;  in tcmu_handle_completion()
507 				WARN_ON(udev->data_tail);  in tcmu_handle_completion()
513 				UPDATE_HEAD(udev->data_tail, copy_bytes, udev->data_size);  in tcmu_handle_completion()
520 		UPDATE_HEAD(udev->data_tail, cmd->data_length, udev->data_size);  in tcmu_handle_completion()
531 static unsigned int tcmu_handle_completions(struct tcmu_dev *udev)  in tcmu_handle_completions()  argument
538 	if (test_bit(TCMU_DEV_BIT_BROKEN, &udev->flags)) {  in tcmu_handle_completions()
543 	spin_lock_irqsave(&udev->cmdr_lock, flags);  in tcmu_handle_completions()
545 	mb = udev->mb_addr;  in tcmu_handle_completions()
548 	while (udev->cmdr_last_cleaned != ACCESS_ONCE(mb->cmd_tail)) {  in tcmu_handle_completions()
550 		struct tcmu_cmd_entry *entry = (void *) mb + CMDR_OFF + udev->cmdr_last_cleaned;  in tcmu_handle_completions()
556 			UPDATE_HEAD(udev->cmdr_last_cleaned,  in tcmu_handle_completions()
558 				    udev->cmdr_size);  in tcmu_handle_completions()
563 		spin_lock(&udev->commands_lock);  in tcmu_handle_completions()
564 		cmd = idr_find(&udev->commands, entry->hdr.cmd_id);  in tcmu_handle_completions()
566 			idr_remove(&udev->commands, cmd->cmd_id);  in tcmu_handle_completions()
567 		spin_unlock(&udev->commands_lock);  in tcmu_handle_completions()
571 			set_bit(TCMU_DEV_BIT_BROKEN, &udev->flags);  in tcmu_handle_completions()
577 		UPDATE_HEAD(udev->cmdr_last_cleaned,  in tcmu_handle_completions()
579 			    udev->cmdr_size);  in tcmu_handle_completions()
585 		del_timer(&udev->timeout); /* no more pending cmds */  in tcmu_handle_completions()
587 	spin_unlock_irqrestore(&udev->cmdr_lock, flags);  in tcmu_handle_completions()
589 	wake_up(&udev->wait_cmdr);  in tcmu_handle_completions()
615 	struct tcmu_dev *udev = (struct tcmu_dev *)data;  in tcmu_device_timedout()  local
619 	handled = tcmu_handle_completions(udev);  in tcmu_device_timedout()
623 	spin_lock_irqsave(&udev->commands_lock, flags);  in tcmu_device_timedout()
624 	idr_for_each(&udev->commands, tcmu_check_expired_cmd, NULL);  in tcmu_device_timedout()
625 	spin_unlock_irqrestore(&udev->commands_lock, flags);  in tcmu_device_timedout()
655 	struct tcmu_dev *udev;  in tcmu_alloc_device()  local
657 	udev = kzalloc(sizeof(struct tcmu_dev), GFP_KERNEL);  in tcmu_alloc_device()
658 	if (!udev)  in tcmu_alloc_device()
661 	udev->name = kstrdup(name, GFP_KERNEL);  in tcmu_alloc_device()
662 	if (!udev->name) {  in tcmu_alloc_device()
663 		kfree(udev);  in tcmu_alloc_device()
667 	udev->hba = hba;  in tcmu_alloc_device()
669 	init_waitqueue_head(&udev->wait_cmdr);  in tcmu_alloc_device()
670 	spin_lock_init(&udev->cmdr_lock);  in tcmu_alloc_device()
672 	idr_init(&udev->commands);  in tcmu_alloc_device()
673 	spin_lock_init(&udev->commands_lock);  in tcmu_alloc_device()
675 	setup_timer(&udev->timeout, tcmu_device_timedout,  in tcmu_alloc_device()
676 		(unsigned long)udev);  in tcmu_alloc_device()
678 	return &udev->se_dev;  in tcmu_alloc_device()
696 	struct tcmu_dev *udev = vma->vm_private_data;  in tcmu_find_mem_index()  local
697 	struct uio_info *info = &udev->uio_info;  in tcmu_find_mem_index()
709 	struct tcmu_dev *udev = vma->vm_private_data;  in tcmu_vma_fault()  local
710 	struct uio_info *info = &udev->uio_info;  in tcmu_vma_fault()
741 	struct tcmu_dev *udev = container_of(info, struct tcmu_dev, uio_info);  in tcmu_mmap()  local
746 	vma->vm_private_data = udev;  in tcmu_mmap()
757 	struct tcmu_dev *udev = container_of(info, struct tcmu_dev, uio_info);  in tcmu_open()  local
760 	if (test_and_set_bit(TCMU_DEV_BIT_OPEN, &udev->flags))  in tcmu_open()
770 	struct tcmu_dev *udev = container_of(info, struct tcmu_dev, uio_info);  in tcmu_release()  local
772 	clear_bit(TCMU_DEV_BIT_OPEN, &udev->flags);  in tcmu_release()
818 	struct tcmu_dev *udev = TCMU_DEV(dev);  in tcmu_configure_device()  local
819 	struct tcmu_hba *hba = udev->hba->hba_ptr;  in tcmu_configure_device()
827 	info = &udev->uio_info;  in tcmu_configure_device()
829 	size = snprintf(NULL, 0, "tcm-user/%u/%s/%s", hba->host_id, udev->name,  in tcmu_configure_device()
830 			udev->dev_config);  in tcmu_configure_device()
836 	used = snprintf(str, size, "tcm-user/%u/%s", hba->host_id, udev->name);  in tcmu_configure_device()
838 	if (udev->dev_config[0])  in tcmu_configure_device()
839 		snprintf(str + used, size - used, "/%s", udev->dev_config);  in tcmu_configure_device()
843 	udev->mb_addr = vzalloc(TCMU_RING_SIZE);  in tcmu_configure_device()
844 	if (!udev->mb_addr) {  in tcmu_configure_device()
850 	udev->cmdr_size = CMDR_SIZE - CMDR_OFF;  in tcmu_configure_device()
851 	udev->data_off = CMDR_SIZE;  in tcmu_configure_device()
852 	udev->data_size = TCMU_RING_SIZE - CMDR_SIZE;  in tcmu_configure_device()
854 	mb = udev->mb_addr;  in tcmu_configure_device()
857 	mb->cmdr_size = udev->cmdr_size;  in tcmu_configure_device()
859 	WARN_ON(!PAGE_ALIGNED(udev->data_off));  in tcmu_configure_device()
860 	WARN_ON(udev->data_size % PAGE_SIZE);  in tcmu_configure_device()
865 	info->mem[0].addr = (phys_addr_t) udev->mb_addr;  in tcmu_configure_device()
885 	ret = tcmu_netlink_event(TCMU_CMD_ADDED_DEVICE, udev->uio_info.name,  in tcmu_configure_device()
886 				 udev->uio_info.uio_dev->minor);  in tcmu_configure_device()
893 	uio_unregister_device(&udev->uio_info);  in tcmu_configure_device()
895 	vfree(udev->mb_addr);  in tcmu_configure_device()
913 	struct tcmu_dev *udev = TCMU_DEV(dev);  in tcmu_free_device()  local
916 	del_timer_sync(&udev->timeout);  in tcmu_free_device()
918 	vfree(udev->mb_addr);  in tcmu_free_device()
921 	spin_lock_irq(&udev->commands_lock);  in tcmu_free_device()
922 	i = idr_for_each(&udev->commands, tcmu_check_pending_cmd, NULL);  in tcmu_free_device()
923 	idr_destroy(&udev->commands);  in tcmu_free_device()
924 	spin_unlock_irq(&udev->commands_lock);  in tcmu_free_device()
928 	if (udev->uio_info.uio_dev) {  in tcmu_free_device()
929 		tcmu_netlink_event(TCMU_CMD_REMOVED_DEVICE, udev->uio_info.name,  in tcmu_free_device()
930 				   udev->uio_info.uio_dev->minor);  in tcmu_free_device()
932 		uio_unregister_device(&udev->uio_info);  in tcmu_free_device()
933 		kfree(udev->uio_info.name);  in tcmu_free_device()
934 		kfree(udev->name);  in tcmu_free_device()
937 	kfree(udev);  in tcmu_free_device()
954 	struct tcmu_dev *udev = TCMU_DEV(dev);  in tcmu_set_configfs_dev_params()  local
973 			if (match_strlcpy(udev->dev_config, &args[0],  in tcmu_set_configfs_dev_params()
978 			pr_debug("TCMU: Referencing Path: %s\n", udev->dev_config);  in tcmu_set_configfs_dev_params()
986 			ret = kstrtoul(arg_p, 0, (unsigned long *) &udev->dev_size);  in tcmu_set_configfs_dev_params()
1020 	struct tcmu_dev *udev = TCMU_DEV(dev);  in tcmu_show_configfs_dev_params()  local
1024 		     udev->dev_config[0] ? udev->dev_config : "NULL");  in tcmu_show_configfs_dev_params()
1025 	bl += sprintf(b + bl, "Size: %zu\n", udev->dev_size);  in tcmu_show_configfs_dev_params()
1032 	struct tcmu_dev *udev = TCMU_DEV(dev);  in tcmu_get_blocks()  local
1034 	return div_u64(udev->dev_size - dev->dev_attrib.block_size,  in tcmu_get_blocks()