163 csio_scsi_fcp_cmnd(struct csio_ioreq *req, void *addr)  in csio_scsi_fcp_cmnd()  argument
166 	struct scsi_cmnd *scmnd = csio_scsi_cmnd(req);  in csio_scsi_fcp_cmnd()
178 		if (req->nsge)  in csio_scsi_fcp_cmnd()
179 			if (req->datadir == DMA_TO_DEVICE)  in csio_scsi_fcp_cmnd()
201 csio_scsi_init_cmd_wr(struct csio_ioreq *req, void *addr, uint32_t size)  in csio_scsi_init_cmd_wr()  argument
203 	struct csio_hw *hw = req->lnode->hwp;  in csio_scsi_init_cmd_wr()
204 	struct csio_rnode *rn = req->rnode;  in csio_scsi_init_cmd_wr()
215 	wr->cookie = (uintptr_t) req;  in csio_scsi_init_cmd_wr()
216 	wr->iqid = cpu_to_be16(csio_q_physiqid(hw, req->iq_idx));  in csio_scsi_init_cmd_wr()
217 	wr->tmo_val = (uint8_t) req->tmo;  in csio_scsi_init_cmd_wr()
222 	dma_buf = &req->dma_buf;  in csio_scsi_init_cmd_wr()
236 	csio_scsi_fcp_cmnd(req, (void *)((uintptr_t)addr +  in csio_scsi_init_cmd_wr()
255 csio_scsi_cmd(struct csio_ioreq *req)  in csio_scsi_cmd()  argument
258 	struct csio_hw *hw = req->lnode->hwp;  in csio_scsi_cmd()
262 	req->drv_status = csio_wr_get(hw, req->eq_idx, size, &wrp);  in csio_scsi_cmd()
263 	if (unlikely(req->drv_status != 0))  in csio_scsi_cmd()
268 		csio_scsi_init_cmd_wr(req, wrp.addr1, size);  in csio_scsi_cmd()
270 		uint8_t *tmpwr = csio_q_eq_wrap(hw, req->eq_idx);  in csio_scsi_cmd()
276 		csio_scsi_init_cmd_wr(req, (void *)tmpwr, size);  in csio_scsi_cmd()
290 csio_scsi_init_ultptx_dsgl(struct csio_hw *hw, struct csio_ioreq *req,  in csio_scsi_init_ultptx_dsgl()  argument
299 	struct scsi_cmnd *scmnd = csio_scsi_cmnd(req);  in csio_scsi_init_ultptx_dsgl()
302 				     ULPTX_NSGE_V(req->nsge));  in csio_scsi_init_ultptx_dsgl()
304 	if (likely(!req->dcopy)) {  in csio_scsi_init_ultptx_dsgl()
305 		scsi_for_each_sg(scmnd, sgel, req->nsge, i) {  in csio_scsi_init_ultptx_dsgl()
328 		list_for_each(tmp, &req->gen_list) {  in csio_scsi_init_ultptx_dsgl()
360 csio_scsi_init_read_wr(struct csio_ioreq *req, void *wrp, uint32_t size)  in csio_scsi_init_read_wr()  argument
362 	struct csio_hw *hw = req->lnode->hwp;  in csio_scsi_init_read_wr()
363 	struct csio_rnode *rn = req->rnode;  in csio_scsi_init_read_wr()
368 	struct scsi_cmnd *scmnd = csio_scsi_cmnd(req);  in csio_scsi_init_read_wr()
374 	wr->cookie = (uintptr_t)req;  in csio_scsi_init_read_wr()
375 	wr->iqid = cpu_to_be16(csio_q_physiqid(hw, req->iq_idx));  in csio_scsi_init_read_wr()
376 	wr->tmo_val = (uint8_t)(req->tmo);  in csio_scsi_init_read_wr()
381 	dma_buf = &req->dma_buf;  in csio_scsi_init_read_wr()
393 	csio_scsi_fcp_cmnd(req, (void *)((uintptr_t)wrp +  in csio_scsi_init_read_wr()
401 	csio_scsi_init_ultptx_dsgl(hw, req, sgl);  in csio_scsi_init_read_wr()
413 csio_scsi_init_write_wr(struct csio_ioreq *req, void *wrp, uint32_t size)  in csio_scsi_init_write_wr()  argument
415 	struct csio_hw *hw = req->lnode->hwp;  in csio_scsi_init_write_wr()
416 	struct csio_rnode *rn = req->rnode;  in csio_scsi_init_write_wr()
421 	struct scsi_cmnd *scmnd = csio_scsi_cmnd(req);  in csio_scsi_init_write_wr()
427 	wr->cookie = (uintptr_t)req;  in csio_scsi_init_write_wr()
428 	wr->iqid = cpu_to_be16(csio_q_physiqid(hw, req->iq_idx));  in csio_scsi_init_write_wr()
429 	wr->tmo_val = (uint8_t)(req->tmo);  in csio_scsi_init_write_wr()
434 	dma_buf = &req->dma_buf;  in csio_scsi_init_write_wr()
446 	csio_scsi_fcp_cmnd(req, (void *)((uintptr_t)wrp +  in csio_scsi_init_write_wr()
454 	csio_scsi_init_ultptx_dsgl(hw, req, sgl);  in csio_scsi_init_write_wr()
458 #define CSIO_SCSI_DATA_WRSZ(req, oper, sz, imm)				       \  argument
464 	if (unlikely((req)->nsge > 1))				               \
466 				(ALIGN(((req)->nsge - 1), 2) / 2));            \
479 csio_scsi_read(struct csio_ioreq *req)  in csio_scsi_read()  argument
483 	struct csio_hw *hw = req->lnode->hwp;  in csio_scsi_read()
486 	CSIO_SCSI_DATA_WRSZ(req, read, size, scsim->proto_cmd_len);  in csio_scsi_read()
489 	req->drv_status = csio_wr_get(hw, req->eq_idx, size, &wrp);  in csio_scsi_read()
490 	if (likely(req->drv_status == 0)) {  in csio_scsi_read()
493 			csio_scsi_init_read_wr(req, wrp.addr1, size);  in csio_scsi_read()
495 			uint8_t *tmpwr = csio_q_eq_wrap(hw, req->eq_idx);  in csio_scsi_read()
500 			csio_scsi_init_read_wr(req, (void *)tmpwr, size);  in csio_scsi_read()
516 csio_scsi_write(struct csio_ioreq *req)  in csio_scsi_write()  argument
520 	struct csio_hw *hw = req->lnode->hwp;  in csio_scsi_write()
523 	CSIO_SCSI_DATA_WRSZ(req, write, size, scsim->proto_cmd_len);  in csio_scsi_write()
526 	req->drv_status = csio_wr_get(hw, req->eq_idx, size, &wrp);  in csio_scsi_write()
527 	if (likely(req->drv_status == 0)) {  in csio_scsi_write()
530 			csio_scsi_init_write_wr(req, wrp.addr1, size);  in csio_scsi_write()
532 			uint8_t *tmpwr = csio_q_eq_wrap(hw, req->eq_idx);  in csio_scsi_write()
537 			csio_scsi_init_write_wr(req, (void *)tmpwr, size);  in csio_scsi_write()
553 csio_setup_ddp(struct csio_scsim *scsim, struct csio_ioreq *req)  in csio_setup_ddp()  argument
556 	struct csio_hw *hw = req->lnode->hwp;  in csio_setup_ddp()
559 	struct scsi_cmnd *scmnd = csio_scsi_cmnd(req);  in csio_setup_ddp()
569 	scsi_for_each_sg(scmnd, sgel, req->nsge, i) {  in csio_setup_ddp()
583 		if ((i != (req->nsge - 1)) &&  in csio_setup_ddp()
593 	req->dcopy = 0;  in csio_setup_ddp()
594 	csio_scsi_read(req);  in csio_setup_ddp()
604 	req->dcopy = 1;  in csio_setup_ddp()
607 	INIT_LIST_HEAD(&req->gen_list);  in csio_setup_ddp()
615 			req->drv_status = -EBUSY;  in csio_setup_ddp()
620 		list_add_tail(&dma_buf->list, &req->gen_list);  in csio_setup_ddp()
624 	if (!req->drv_status) {  in csio_setup_ddp()
626 		req->nsge = i;  in csio_setup_ddp()
627 		csio_scsi_read(req);  in csio_setup_ddp()
633 		csio_put_scsi_ddp_list(scsim, &req->gen_list, i);  in csio_setup_ddp()
646 csio_scsi_init_abrt_cls_wr(struct csio_ioreq *req, void *addr, uint32_t size,  in csio_scsi_init_abrt_cls_wr()  argument
649 	struct csio_hw *hw = req->lnode->hwp;  in csio_scsi_init_abrt_cls_wr()
650 	struct csio_rnode *rn = req->rnode;  in csio_scsi_init_abrt_cls_wr()
658 	wr->cookie = (uintptr_t) req;  in csio_scsi_init_abrt_cls_wr()
659 	wr->iqid = cpu_to_be16(csio_q_physiqid(hw, req->iq_idx));  in csio_scsi_init_abrt_cls_wr()
660 	wr->tmo_val = (uint8_t) req->tmo;  in csio_scsi_init_abrt_cls_wr()
670 	wr->t_cookie = (uintptr_t) req;  in csio_scsi_init_abrt_cls_wr()
674 csio_scsi_abrt_cls(struct csio_ioreq *req, bool abort)  in csio_scsi_abrt_cls()  argument
677 	struct csio_hw *hw = req->lnode->hwp;  in csio_scsi_abrt_cls()
680 	req->drv_status = csio_wr_get(hw, req->eq_idx, size, &wrp);  in csio_scsi_abrt_cls()
681 	if (req->drv_status != 0)  in csio_scsi_abrt_cls()
686 		csio_scsi_init_abrt_cls_wr(req, wrp.addr1, size, abort);  in csio_scsi_abrt_cls()
688 		uint8_t *tmpwr = csio_q_eq_wrap(hw, req->eq_idx);  in csio_scsi_abrt_cls()
693 		csio_scsi_init_abrt_cls_wr(req, (void *)tmpwr, size, abort);  in csio_scsi_abrt_cls()
703 csio_scsis_uninit(struct csio_ioreq *req, enum csio_scsi_ev evt)  in csio_scsis_uninit()  argument
705 	struct csio_hw *hw = req->lnode->hwp;  in csio_scsis_uninit()
711 		if (req->nsge) {  in csio_scsis_uninit()
712 			if (req->datadir == DMA_TO_DEVICE) {  in csio_scsis_uninit()
713 				req->dcopy = 0;  in csio_scsis_uninit()
714 				csio_scsi_write(req);  in csio_scsis_uninit()
716 				csio_setup_ddp(scsim, req);  in csio_scsis_uninit()
718 			csio_scsi_cmd(req);  in csio_scsis_uninit()
721 		if (likely(req->drv_status == 0)) {  in csio_scsis_uninit()
723 			csio_set_state(&req->sm, csio_scsis_io_active);  in csio_scsis_uninit()
724 			list_add_tail(&req->sm.sm_list, &scsim->active_q);  in csio_scsis_uninit()
725 			csio_wr_issue(hw, req->eq_idx, false);  in csio_scsis_uninit()
733 		csio_scsi_cmd(req);  in csio_scsis_uninit()
734 		if (req->drv_status == 0) {  in csio_scsis_uninit()
743 			csio_set_state(&req->sm, csio_scsis_tm_active);  in csio_scsis_uninit()
744 			list_add_tail(&req->sm.sm_list, &scsim->active_q);  in csio_scsis_uninit()
745 			csio_wr_issue(hw, req->eq_idx, false);  in csio_scsis_uninit()
762 		req->drv_status = -EINVAL;  in csio_scsis_uninit()
763 		csio_warn(hw, "Trying to abort/close completed IO:%p!\n", req);  in csio_scsis_uninit()
767 		csio_dbg(hw, "Unhandled event:%d sent to req:%p\n", evt, req);  in csio_scsis_uninit()
773 csio_scsis_io_active(struct csio_ioreq *req, enum csio_scsi_ev evt)  in csio_scsis_io_active()  argument
775 	struct csio_hw *hw = req->lnode->hwp;  in csio_scsis_io_active()
782 		list_del_init(&req->sm.sm_list);  in csio_scsis_io_active()
783 		csio_set_state(&req->sm, csio_scsis_uninit);  in csio_scsis_io_active()
799 		if (unlikely(req->wr_status != FW_SUCCESS)) {  in csio_scsis_io_active()
800 			rn = req->rnode;  in csio_scsis_io_active()
805 			if (csio_scsi_itnexus_loss_error(req->wr_status) &&  in csio_scsis_io_active()
807 				csio_set_state(&req->sm,  in csio_scsis_io_active()
809 				list_add_tail(&req->sm.sm_list,  in csio_scsis_io_active()
817 		csio_scsi_abrt_cls(req, SCSI_ABORT);  in csio_scsis_io_active()
818 		if (req->drv_status == 0) {  in csio_scsis_io_active()
819 			csio_wr_issue(hw, req->eq_idx, false);  in csio_scsis_io_active()
820 			csio_set_state(&req->sm, csio_scsis_aborting);  in csio_scsis_io_active()
825 		csio_scsi_abrt_cls(req, SCSI_CLOSE);  in csio_scsis_io_active()
826 		if (req->drv_status == 0) {  in csio_scsis_io_active()
827 			csio_wr_issue(hw, req->eq_idx, false);  in csio_scsis_io_active()
828 			csio_set_state(&req->sm, csio_scsis_closing);  in csio_scsis_io_active()
833 		req->wr_status = FW_HOSTERROR;  in csio_scsis_io_active()
835 		csio_set_state(&req->sm, csio_scsis_uninit);  in csio_scsis_io_active()
839 		csio_dbg(hw, "Unhandled event:%d sent to req:%p\n", evt, req);  in csio_scsis_io_active()
845 csio_scsis_tm_active(struct csio_ioreq *req, enum csio_scsi_ev evt)  in csio_scsis_tm_active()  argument
847 	struct csio_hw *hw = req->lnode->hwp;  in csio_scsis_tm_active()
853 		list_del_init(&req->sm.sm_list);  in csio_scsis_tm_active()
854 		csio_set_state(&req->sm, csio_scsis_uninit);  in csio_scsis_tm_active()
859 		csio_scsi_abrt_cls(req, SCSI_ABORT);  in csio_scsis_tm_active()
860 		if (req->drv_status == 0) {  in csio_scsis_tm_active()
861 			csio_wr_issue(hw, req->eq_idx, false);  in csio_scsis_tm_active()
862 			csio_set_state(&req->sm, csio_scsis_aborting);  in csio_scsis_tm_active()
868 		csio_scsi_abrt_cls(req, SCSI_CLOSE);  in csio_scsis_tm_active()
869 		if (req->drv_status == 0) {  in csio_scsis_tm_active()
870 			csio_wr_issue(hw, req->eq_idx, false);  in csio_scsis_tm_active()
871 			csio_set_state(&req->sm, csio_scsis_closing);  in csio_scsis_tm_active()
876 		req->wr_status = FW_HOSTERROR;  in csio_scsis_tm_active()
878 		csio_set_state(&req->sm, csio_scsis_uninit);  in csio_scsis_tm_active()
882 		csio_dbg(hw, "Unhandled event:%d sent to req:%p\n", evt, req);  in csio_scsis_tm_active()
888 csio_scsis_aborting(struct csio_ioreq *req, enum csio_scsi_ev evt)  in csio_scsis_aborting()  argument
890 	struct csio_hw *hw = req->lnode->hwp;  in csio_scsis_aborting()
897 			 "in aborting st\n", req, req->wr_status);  in csio_scsis_aborting()
907 		req->drv_status = -ECANCELED;  in csio_scsis_aborting()
917 			 req, req->wr_status, req->drv_status);  in csio_scsis_aborting()
922 		if (req->drv_status != -ECANCELED) {  in csio_scsis_aborting()
925 				   " req:%p\n", req);  in csio_scsis_aborting()
952 		if ((req->wr_status == FW_SUCCESS) ||  in csio_scsis_aborting()
953 		    (req->wr_status == FW_EINVAL) ||  in csio_scsis_aborting()
954 		    csio_scsi_itnexus_loss_error(req->wr_status))  in csio_scsis_aborting()
955 			req->wr_status = FW_SCSI_ABORT_REQUESTED;  in csio_scsis_aborting()
958 		list_del_init(&req->sm.sm_list);  in csio_scsis_aborting()
959 		csio_set_state(&req->sm, csio_scsis_uninit);  in csio_scsis_aborting()
963 		req->wr_status = FW_HOSTERROR;  in csio_scsis_aborting()
965 		csio_set_state(&req->sm, csio_scsis_uninit);  in csio_scsis_aborting()
979 		csio_dbg(hw, "Unhandled event:%d sent to req:%p\n", evt, req);  in csio_scsis_aborting()
985 csio_scsis_closing(struct csio_ioreq *req, enum csio_scsi_ev evt)  in csio_scsis_closing()  argument
987 	struct csio_hw *hw = req->lnode->hwp;  in csio_scsis_closing()
994 			 "in closing st\n", req, req->wr_status);  in csio_scsis_closing()
1004 		req->drv_status = -ECANCELED;  in csio_scsis_closing()
1012 		if (req->drv_status != -ECANCELED) {  in csio_scsis_closing()
1015 				   " req:%p\n", req);  in csio_scsis_closing()
1024 		CSIO_DB_ASSERT((req->wr_status == FW_SUCCESS) ||  in csio_scsis_closing()
1025 					(req->wr_status == FW_EINVAL));  in csio_scsis_closing()
1026 		req->wr_status = FW_SCSI_CLOSE_REQUESTED;  in csio_scsis_closing()
1029 		list_del_init(&req->sm.sm_list);  in csio_scsis_closing()
1030 		csio_set_state(&req->sm, csio_scsis_uninit);  in csio_scsis_closing()
1037 		req->wr_status = FW_HOSTERROR;  in csio_scsis_closing()
1039 		csio_set_state(&req->sm, csio_scsis_uninit);  in csio_scsis_closing()
1043 		csio_dbg(hw, "Unhandled event:%d sent to req:%p\n", evt, req);  in csio_scsis_closing()
1049 csio_scsis_shost_cmpl_await(struct csio_ioreq *req, enum csio_scsi_ev evt)  in csio_scsis_shost_cmpl_await()  argument
1070 		req->drv_status = 0;  in csio_scsis_shost_cmpl_await()
1073 		csio_set_state(&req->sm, csio_scsis_uninit);  in csio_scsis_shost_cmpl_await()
1076 		csio_dbg(req->lnode->hwp, "Unhandled event:%d sent to req:%p\n",  in csio_scsis_shost_cmpl_await()
1077 			 evt, req);  in csio_scsis_shost_cmpl_await()
1489 csio_scsi_copy_to_sgl(struct csio_hw *hw, struct csio_ioreq *req)  in csio_scsi_copy_to_sgl()  argument
1491 	struct scsi_cmnd *scmnd  = (struct scsi_cmnd *)csio_scsi_cmnd(req);  in csio_scsi_copy_to_sgl()
1504 	dma_buf = (struct csio_dma_buf *)csio_list_next(&req->gen_list);  in csio_scsi_copy_to_sgl()
1531 				sg, req);  in csio_scsi_copy_to_sgl()
1558 csio_scsi_err_handler(struct csio_hw *hw, struct csio_ioreq *req)  in csio_scsi_err_handler()  argument
1560 	struct scsi_cmnd *cmnd  = (struct scsi_cmnd *)csio_scsi_cmnd(req);  in csio_scsi_err_handler()
1571 	switch (req->wr_status) {  in csio_scsi_err_handler()
1581 		dma_buf = &req->dma_buf;  in csio_scsi_err_handler()
1647 		csio_dbg(hw, "Req %p cmd:%p op:%x %s\n", req, cmnd,  in csio_scsi_err_handler()
1649 			    (req->wr_status == FW_SCSI_CLOSE_REQUESTED) ?  in csio_scsi_err_handler()
1656 		if (req->wr_status == FW_SCSI_CLOSE_REQUESTED)  in csio_scsi_err_handler()
1665 			 req, cmnd, req->wr_status);  in csio_scsi_err_handler()
1707 			    req->wr_status, req, cmnd);  in csio_scsi_err_handler()
1716 	if (req->nsge > 0)  in csio_scsi_err_handler()
1723 	csio_scsi_cmnd(req) = NULL;  in csio_scsi_err_handler()
1724 	complete_all(&req->cmplobj);  in csio_scsi_err_handler()
1734 csio_scsi_cbfn(struct csio_hw *hw, struct csio_ioreq *req)  in csio_scsi_cbfn()  argument
1736 	struct scsi_cmnd *cmnd  = (struct scsi_cmnd *)csio_scsi_cmnd(req);  in csio_scsi_cbfn()
1740 	if (likely(req->wr_status == FW_SUCCESS)) {  in csio_scsi_cbfn()
1741 		if (req->nsge > 0) {  in csio_scsi_cbfn()
1743 			if (req->dcopy)  in csio_scsi_cbfn()
1744 				host_status = csio_scsi_copy_to_sgl(hw, req);  in csio_scsi_cbfn()
1749 		csio_scsi_cmnd(req) = NULL;  in csio_scsi_cbfn()
1753 		csio_scsi_err_handler(hw, req);  in csio_scsi_cbfn()
2017 csio_tm_cbfn(struct csio_hw *hw, struct csio_ioreq *req)  in csio_tm_cbfn()  argument
2019 	struct scsi_cmnd *cmnd  = (struct scsi_cmnd *)csio_scsi_cmnd(req);  in csio_tm_cbfn()
2026 		      req, req->wr_status);  in csio_tm_cbfn()
2029 	cmnd->SCp.Status = req->wr_status;  in csio_tm_cbfn()
2042 	if (req->wr_status == FW_SCSI_RSP_ERR) {  in csio_tm_cbfn()
2043 		dma_buf = &req->dma_buf;  in csio_tm_cbfn()
2058 	csio_scsi_cmnd(req) = NULL;  in csio_tm_cbfn()