56 	struct virtio_scsi *vscsi;  member
148 static void virtscsi_complete_cmd(struct virtio_scsi *vscsi, void *buf)  in virtscsi_complete_cmd()  argument
161 	virtscsi_compute_resid(sc, virtio32_to_cpu(vscsi->vdev, resp->resid));  in virtscsi_complete_cmd()
199 	WARN_ON(virtio32_to_cpu(vscsi->vdev, resp->sense_len) >  in virtscsi_complete_cmd()
204 			     virtio32_to_cpu(vscsi->vdev, resp->sense_len),  in virtscsi_complete_cmd()
215 static void virtscsi_vq_done(struct virtio_scsi *vscsi,  in virtscsi_vq_done()  argument
217 			     void (*fn)(struct virtio_scsi *vscsi, void *buf))  in virtscsi_vq_done()  argument
228 			fn(vscsi, buf);  in virtscsi_vq_done()
239 	struct virtio_scsi *vscsi = shost_priv(sh);  in virtscsi_req_done()  local
241 	struct virtio_scsi_vq *req_vq = &vscsi->req_vqs[index];  in virtscsi_req_done()
243 	virtscsi_vq_done(vscsi, req_vq, virtscsi_complete_cmd);  in virtscsi_req_done()
246 static void virtscsi_poll_requests(struct virtio_scsi *vscsi)  in virtscsi_poll_requests()  argument
250 	num_vqs = vscsi->num_queues;  in virtscsi_poll_requests()
252 		virtscsi_vq_done(vscsi, &vscsi->req_vqs[i],  in virtscsi_poll_requests()
256 static void virtscsi_complete_free(struct virtio_scsi *vscsi, void *buf)  in virtscsi_complete_free()  argument
267 	struct virtio_scsi *vscsi = shost_priv(sh);  in virtscsi_ctrl_done()  local
269 	virtscsi_vq_done(vscsi, &vscsi->ctrl_vq, virtscsi_complete_free);  in virtscsi_ctrl_done()
274 static int virtscsi_kick_event(struct virtio_scsi *vscsi,  in virtscsi_kick_event()  argument
284 	spin_lock_irqsave(&vscsi->event_vq.vq_lock, flags);  in virtscsi_kick_event()
286 	err = virtqueue_add_inbuf(vscsi->event_vq.vq, &sg, 1, event_node,  in virtscsi_kick_event()
289 		virtqueue_kick(vscsi->event_vq.vq);  in virtscsi_kick_event()
291 	spin_unlock_irqrestore(&vscsi->event_vq.vq_lock, flags);  in virtscsi_kick_event()
296 static int virtscsi_kick_event_all(struct virtio_scsi *vscsi)  in virtscsi_kick_event_all()  argument
301 		vscsi->event_list[i].vscsi = vscsi;  in virtscsi_kick_event_all()
302 		virtscsi_kick_event(vscsi, &vscsi->event_list[i]);  in virtscsi_kick_event_all()
308 static void virtscsi_cancel_event_work(struct virtio_scsi *vscsi)  in virtscsi_cancel_event_work()  argument
313 	spin_lock_irq(&vscsi->event_vq.vq_lock);  in virtscsi_cancel_event_work()
314 	vscsi->stop_events = true;  in virtscsi_cancel_event_work()
315 	spin_unlock_irq(&vscsi->event_vq.vq_lock);  in virtscsi_cancel_event_work()
318 		cancel_work_sync(&vscsi->event_list[i].work);  in virtscsi_cancel_event_work()
321 static void virtscsi_handle_transport_reset(struct virtio_scsi *vscsi,  in virtscsi_handle_transport_reset()  argument
325 	struct Scsi_Host *shost = virtio_scsi_host(vscsi->vdev);  in virtscsi_handle_transport_reset()
329 	switch (virtio32_to_cpu(vscsi->vdev, event->reason)) {  in virtscsi_handle_transport_reset()
348 static void virtscsi_handle_param_change(struct virtio_scsi *vscsi,  in virtscsi_handle_param_change()  argument
352 	struct Scsi_Host *shost = virtio_scsi_host(vscsi->vdev);  in virtscsi_handle_param_change()
355 	u8 asc = virtio32_to_cpu(vscsi->vdev, event->reason) & 255;  in virtscsi_handle_param_change()
356 	u8 ascq = virtio32_to_cpu(vscsi->vdev, event->reason) >> 8;  in virtscsi_handle_param_change()
377 	struct virtio_scsi *vscsi = event_node->vscsi;  in virtscsi_handle_event()  local
381 	    cpu_to_virtio32(vscsi->vdev, VIRTIO_SCSI_T_EVENTS_MISSED)) {  in virtscsi_handle_event()
382 		event->event &= ~cpu_to_virtio32(vscsi->vdev,  in virtscsi_handle_event()
384 		scsi_scan_host(virtio_scsi_host(vscsi->vdev));  in virtscsi_handle_event()
387 	switch (virtio32_to_cpu(vscsi->vdev, event->event)) {  in virtscsi_handle_event()
391 		virtscsi_handle_transport_reset(vscsi, event);  in virtscsi_handle_event()
394 		virtscsi_handle_param_change(vscsi, event);  in virtscsi_handle_event()
399 	virtscsi_kick_event(vscsi, event_node);  in virtscsi_handle_event()
402 static void virtscsi_complete_event(struct virtio_scsi *vscsi, void *buf)  in virtscsi_complete_event()  argument
406 	if (!vscsi->stop_events)  in virtscsi_complete_event()
413 	struct virtio_scsi *vscsi = shost_priv(sh);  in virtscsi_event_done()  local
415 	virtscsi_vq_done(vscsi, &vscsi->event_vq, virtscsi_complete_event);  in virtscsi_event_done()
530 static int virtscsi_queuecommand(struct virtio_scsi *vscsi,  in virtscsi_queuecommand()  argument
534 	struct Scsi_Host *shost = virtio_scsi_host(vscsi->vdev);  in virtscsi_queuecommand()
552 	if (virtio_has_feature(vscsi->vdev, VIRTIO_SCSI_F_T10_PI)) {  in virtscsi_queuecommand()
553 		virtio_scsi_init_hdr_pi(vscsi->vdev, &cmd->req.cmd_pi, sc);  in virtscsi_queuecommand()
559 		virtio_scsi_init_hdr(vscsi->vdev, &cmd->req.cmd, sc);  in virtscsi_queuecommand()
572 	struct virtio_scsi *vscsi = shost_priv(sh);  in virtscsi_queuecommand_single()  local
577 	return virtscsi_queuecommand(vscsi, &vscsi->req_vqs[0], sc);  in virtscsi_queuecommand_single()
580 static struct virtio_scsi_vq *virtscsi_pick_vq_mq(struct virtio_scsi *vscsi,  in virtscsi_pick_vq_mq()  argument
586 	return &vscsi->req_vqs[hwq];  in virtscsi_pick_vq_mq()
589 static struct virtio_scsi_vq *virtscsi_pick_vq(struct virtio_scsi *vscsi,  in virtscsi_pick_vq()  argument
615 		while (unlikely(queue_num >= vscsi->num_queues))  in virtscsi_pick_vq()
616 			queue_num -= vscsi->num_queues;  in virtscsi_pick_vq()
617 		tgt->req_vq = vq = &vscsi->req_vqs[queue_num];  in virtscsi_pick_vq()
629 	struct virtio_scsi *vscsi = shost_priv(sh);  in virtscsi_queuecommand_multi()  local
635 		req_vq = virtscsi_pick_vq_mq(vscsi, sc);  in virtscsi_queuecommand_multi()
637 		req_vq = virtscsi_pick_vq(vscsi, tgt);  in virtscsi_queuecommand_multi()
639 	return virtscsi_queuecommand(vscsi, req_vq, sc);  in virtscsi_queuecommand_multi()
642 static int virtscsi_tmf(struct virtio_scsi *vscsi, struct virtio_scsi_cmd *cmd)  in virtscsi_tmf()  argument
648 	if (virtscsi_kick_cmd(&vscsi->ctrl_vq, cmd,  in virtscsi_tmf()
667 	virtscsi_poll_requests(vscsi);  in virtscsi_tmf()
676 	struct virtio_scsi *vscsi = shost_priv(sc->device->host);  in virtscsi_device_reset()  local
688 		.subtype = cpu_to_virtio32(vscsi->vdev,  in virtscsi_device_reset()
695 	return virtscsi_tmf(vscsi, cmd);  in virtscsi_device_reset()
713 	struct virtio_scsi *vscsi = shost_priv(sc->device->host);  in virtscsi_abort()  local
730 		.tag = cpu_to_virtio64(vscsi->vdev, (unsigned long)sc),  in virtscsi_abort()
732 	return virtscsi_tmf(vscsi, cmd);  in virtscsi_abort()
738 	struct virtio_scsi *vscsi = shost_priv(sh);  in virtscsi_target_alloc()  local
747 	tgt->req_vq = &vscsi->req_vqs[0];  in virtscsi_target_alloc()
810 static void __virtscsi_set_affinity(struct virtio_scsi *vscsi, bool affinity)  in __virtscsi_set_affinity()  argument
820 	if ((vscsi->num_queues == 1 ||  in __virtscsi_set_affinity()
821 	     vscsi->num_queues != num_online_cpus()) && affinity) {  in __virtscsi_set_affinity()
822 		if (vscsi->affinity_hint_set)  in __virtscsi_set_affinity()
831 			virtqueue_set_affinity(vscsi->req_vqs[i].vq, cpu);  in __virtscsi_set_affinity()
835 		vscsi->affinity_hint_set = true;  in __virtscsi_set_affinity()
837 		for (i = 0; i < vscsi->num_queues; i++) {  in __virtscsi_set_affinity()
838 			if (!vscsi->req_vqs[i].vq)  in __virtscsi_set_affinity()
841 			virtqueue_set_affinity(vscsi->req_vqs[i].vq, -1);  in __virtscsi_set_affinity()
844 		vscsi->affinity_hint_set = false;  in __virtscsi_set_affinity()
848 static void virtscsi_set_affinity(struct virtio_scsi *vscsi, bool affinity)  in virtscsi_set_affinity()  argument
851 	__virtscsi_set_affinity(vscsi, affinity);  in virtscsi_set_affinity()
858 	struct virtio_scsi *vscsi = container_of(nfb, struct virtio_scsi, nb);  in virtscsi_cpu_callback()  local
864 		__virtscsi_set_affinity(vscsi, true);  in virtscsi_cpu_callback()
882 	struct virtio_scsi *vscsi = shost_priv(sh);  in virtscsi_remove_vqs()  local
884 	virtscsi_set_affinity(vscsi, false);  in virtscsi_remove_vqs()
893 			 struct virtio_scsi *vscsi)  in virtscsi_init()  argument
902 	num_vqs = vscsi->num_queues + VIRTIO_SCSI_VQ_BASE;  in virtscsi_init()
926 	virtscsi_init_vq(&vscsi->ctrl_vq, vqs[0]);  in virtscsi_init()
927 	virtscsi_init_vq(&vscsi->event_vq, vqs[1]);  in virtscsi_init()
929 		virtscsi_init_vq(&vscsi->req_vqs[i - VIRTIO_SCSI_VQ_BASE],  in virtscsi_init()
932 	virtscsi_set_affinity(vscsi, true);  in virtscsi_init()
951 	struct virtio_scsi *vscsi;  in virtscsi_probe()  local
975 		sizeof(*vscsi) + sizeof(vscsi->req_vqs[0]) * num_queues);  in virtscsi_probe()
981 	vscsi = shost_priv(shost);  in virtscsi_probe()
982 	vscsi->vdev = vdev;  in virtscsi_probe()
983 	vscsi->num_queues = num_queues;  in virtscsi_probe()
986 	err = virtscsi_init(vdev, vscsi);  in virtscsi_probe()
990 	vscsi->nb.notifier_call = &virtscsi_cpu_callback;  in virtscsi_probe()
991 	err = register_hotcpu_notifier(&vscsi->nb);  in virtscsi_probe()
1030 		virtscsi_kick_event_all(vscsi);  in virtscsi_probe()
1045 	struct virtio_scsi *vscsi = shost_priv(shost);  in virtscsi_remove()  local
1048 		virtscsi_cancel_event_work(vscsi);  in virtscsi_remove()
1052 	unregister_hotcpu_notifier(&vscsi->nb);  in virtscsi_remove()
1062 	struct virtio_scsi *vscsi = shost_priv(sh);  in virtscsi_freeze()  local
1064 	unregister_hotcpu_notifier(&vscsi->nb);  in virtscsi_freeze()
1072 	struct virtio_scsi *vscsi = shost_priv(sh);  in virtscsi_restore()  local
1075 	err = virtscsi_init(vdev, vscsi);  in virtscsi_restore()
1079 	err = register_hotcpu_notifier(&vscsi->nb);  in virtscsi_restore()
1088 		virtscsi_kick_event_all(vscsi);  in virtscsi_restore()