104 static void fnic_cleanup_io(struct fnic *fnic, int exclude_id);
106 static inline spinlock_t *fnic_io_lock_hash(struct fnic *fnic,  in fnic_io_lock_hash()  argument
111 	return &fnic->io_req_lock[hash];  in fnic_io_lock_hash()
114 static inline spinlock_t *fnic_io_lock_tag(struct fnic *fnic,  in fnic_io_lock_tag()  argument
117 	return &fnic->io_req_lock[tag & (FNIC_IO_LOCKS - 1)];  in fnic_io_lock_tag()
124 static void fnic_release_ioreq_buf(struct fnic *fnic,  in fnic_release_ioreq_buf()  argument
129 		pci_unmap_single(fnic->pdev, io_req->sgl_list_pa,  in fnic_release_ioreq_buf()
136 			     fnic->io_sgl_pool[io_req->sgl_type]);  in fnic_release_ioreq_buf()
138 		pci_unmap_single(fnic->pdev, io_req->sense_buf_pa,  in fnic_release_ioreq_buf()
143 static int free_wq_copy_descs(struct fnic *fnic, struct vnic_wq_copy *wq)  in free_wq_copy_descs()  argument
146 	if (!fnic->fw_ack_recd[0])  in free_wq_copy_descs()
153 	if (wq->to_clean_index <= fnic->fw_ack_index[0])  in free_wq_copy_descs()
154 		wq->ring.desc_avail += (fnic->fw_ack_index[0]  in free_wq_copy_descs()
159 					+ fnic->fw_ack_index[0] + 1);  in free_wq_copy_descs()
167 		(fnic->fw_ack_index[0] + 1) % wq->ring.desc_count;  in free_wq_copy_descs()
170 	fnic->fw_ack_recd[0] = 0;  in free_wq_copy_descs()
180 __fnic_set_state_flags(struct fnic *fnic, unsigned long st_flags,  in __fnic_set_state_flags()  argument
183 	struct Scsi_Host *host = fnic->lport->host;  in __fnic_set_state_flags()
191 		fnic->state_flags &= ~st_flags;  in __fnic_set_state_flags()
193 		fnic->state_flags |= st_flags;  in __fnic_set_state_flags()
206 int fnic_fw_reset_handler(struct fnic *fnic)  in fnic_fw_reset_handler()  argument
208 	struct vnic_wq_copy *wq = &fnic->wq_copy[0];  in fnic_fw_reset_handler()
213 	fnic_set_state_flags(fnic, FNIC_FLAGS_FWRESET);  in fnic_fw_reset_handler()
215 	skb_queue_purge(&fnic->frame_queue);  in fnic_fw_reset_handler()
216 	skb_queue_purge(&fnic->tx_queue);  in fnic_fw_reset_handler()
219 	while (atomic_read(&fnic->in_flight))  in fnic_fw_reset_handler()
222 	spin_lock_irqsave(&fnic->wq_copy_lock[0], flags);  in fnic_fw_reset_handler()
224 	if (vnic_wq_copy_desc_avail(wq) <= fnic->wq_copy_desc_low[0])  in fnic_fw_reset_handler()
225 		free_wq_copy_descs(fnic, wq);  in fnic_fw_reset_handler()
231 		atomic64_inc(&fnic->fnic_stats.fw_stats.active_fw_reqs);  in fnic_fw_reset_handler()
232 		if (atomic64_read(&fnic->fnic_stats.fw_stats.active_fw_reqs) >  in fnic_fw_reset_handler()
233 			  atomic64_read(&fnic->fnic_stats.fw_stats.max_fw_reqs))  in fnic_fw_reset_handler()
234 			atomic64_set(&fnic->fnic_stats.fw_stats.max_fw_reqs,  in fnic_fw_reset_handler()
236 				  &fnic->fnic_stats.fw_stats.active_fw_reqs));  in fnic_fw_reset_handler()
239 	spin_unlock_irqrestore(&fnic->wq_copy_lock[0], flags);  in fnic_fw_reset_handler()
242 		atomic64_inc(&fnic->fnic_stats.reset_stats.fw_resets);  in fnic_fw_reset_handler()
243 		FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host,  in fnic_fw_reset_handler()
246 		fnic_clear_state_flags(fnic, FNIC_FLAGS_FWRESET);  in fnic_fw_reset_handler()
247 		FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host,  in fnic_fw_reset_handler()
259 int fnic_flogi_reg_handler(struct fnic *fnic, u32 fc_id)  in fnic_flogi_reg_handler()  argument
261 	struct vnic_wq_copy *wq = &fnic->wq_copy[0];  in fnic_flogi_reg_handler()
263 	struct fc_lport *lp = fnic->lport;  in fnic_flogi_reg_handler()
268 	spin_lock_irqsave(&fnic->wq_copy_lock[0], flags);  in fnic_flogi_reg_handler()
270 	if (vnic_wq_copy_desc_avail(wq) <= fnic->wq_copy_desc_low[0])  in fnic_flogi_reg_handler()
271 		free_wq_copy_descs(fnic, wq);  in fnic_flogi_reg_handler()
278 	if (fnic->ctlr.map_dest) {  in fnic_flogi_reg_handler()
282 		memcpy(gw_mac, fnic->ctlr.dest_addr, ETH_ALEN);  in fnic_flogi_reg_handler()
286 	if ((fnic->config.flags & VFCF_FIP_CAPABLE) && !fnic->ctlr.map_dest) {  in fnic_flogi_reg_handler()
289 						fnic->data_src_addr,  in fnic_flogi_reg_handler()
291 		FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host,  in fnic_flogi_reg_handler()
293 			      fc_id, fnic->data_src_addr, gw_mac);  in fnic_flogi_reg_handler()
297 		FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host,  in fnic_flogi_reg_handler()
299 			      fc_id, fnic->ctlr.map_dest, gw_mac);  in fnic_flogi_reg_handler()
302 	atomic64_inc(&fnic->fnic_stats.fw_stats.active_fw_reqs);  in fnic_flogi_reg_handler()
303 	if (atomic64_read(&fnic->fnic_stats.fw_stats.active_fw_reqs) >  in fnic_flogi_reg_handler()
304 		  atomic64_read(&fnic->fnic_stats.fw_stats.max_fw_reqs))  in fnic_flogi_reg_handler()
305 		atomic64_set(&fnic->fnic_stats.fw_stats.max_fw_reqs,  in fnic_flogi_reg_handler()
306 		  atomic64_read(&fnic->fnic_stats.fw_stats.active_fw_reqs));  in fnic_flogi_reg_handler()
309 	spin_unlock_irqrestore(&fnic->wq_copy_lock[0], flags);  in fnic_flogi_reg_handler()
317 static inline int fnic_queue_wq_copy_desc(struct fnic *fnic,  in fnic_queue_wq_copy_desc()  argument
327 	struct misc_stats *misc_stats = &fnic->fnic_stats.misc_stats;  in fnic_queue_wq_copy_desc()
346 			(fnic->pdev,  in fnic_queue_wq_copy_desc()
351 		r = pci_dma_mapping_error(fnic->pdev, io_req->sgl_list_pa);  in fnic_queue_wq_copy_desc()
358 	io_req->sense_buf_pa = pci_map_single(fnic->pdev,  in fnic_queue_wq_copy_desc()
363 	r = pci_dma_mapping_error(fnic->pdev, io_req->sense_buf_pa);  in fnic_queue_wq_copy_desc()
365 		pci_unmap_single(fnic->pdev, io_req->sgl_list_pa,  in fnic_queue_wq_copy_desc()
375 	spin_lock_irqsave(&fnic->wq_copy_lock[0], intr_flags);  in fnic_queue_wq_copy_desc()
377 	if (vnic_wq_copy_desc_avail(wq) <= fnic->wq_copy_desc_low[0])  in fnic_queue_wq_copy_desc()
378 		free_wq_copy_descs(fnic, wq);  in fnic_queue_wq_copy_desc()
381 		spin_unlock_irqrestore(&fnic->wq_copy_lock[0], intr_flags);  in fnic_queue_wq_copy_desc()
382 		FNIC_SCSI_DBG(KERN_INFO, fnic->lport->host,  in fnic_queue_wq_copy_desc()
395 	if ((fnic->config.flags & VFCF_FCP_SEQ_LVL_ERR) &&  in fnic_queue_wq_copy_desc()
414 	atomic64_inc(&fnic->fnic_stats.fw_stats.active_fw_reqs);  in fnic_queue_wq_copy_desc()
415 	if (atomic64_read(&fnic->fnic_stats.fw_stats.active_fw_reqs) >  in fnic_queue_wq_copy_desc()
416 		  atomic64_read(&fnic->fnic_stats.fw_stats.max_fw_reqs))  in fnic_queue_wq_copy_desc()
417 		atomic64_set(&fnic->fnic_stats.fw_stats.max_fw_reqs,  in fnic_queue_wq_copy_desc()
418 		  atomic64_read(&fnic->fnic_stats.fw_stats.active_fw_reqs));  in fnic_queue_wq_copy_desc()
420 	spin_unlock_irqrestore(&fnic->wq_copy_lock[0], intr_flags);  in fnic_queue_wq_copy_desc()
434 	struct fnic *fnic = lport_priv(lp);  in fnic_queuecommand_lck()  local
435 	struct fnic_stats *fnic_stats = &fnic->fnic_stats;  in fnic_queuecommand_lck()
446 	if (unlikely(fnic_chk_state_flags_locked(fnic, FNIC_FLAGS_IO_BLOCKED)))  in fnic_queuecommand_lck()
460 		FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host,  in fnic_queuecommand_lck()
471 	atomic_inc(&fnic->in_flight);  in fnic_queuecommand_lck()
483 	io_req = mempool_alloc(fnic->io_req_pool, GFP_ATOMIC);  in fnic_queuecommand_lck()
497 		mempool_free(io_req, fnic->io_req_pool);  in fnic_queuecommand_lck()
509 			mempool_alloc(fnic->io_sgl_pool[io_req->sgl_type],  in fnic_queuecommand_lck()
515 			mempool_free(io_req, fnic->io_req_pool);  in fnic_queuecommand_lck()
534 	io_lock = fnic_io_lock_hash(fnic, sc);  in fnic_queuecommand_lck()
547 	wq = &fnic->wq_copy[0];  in fnic_queuecommand_lck()
548 	ret = fnic_queue_wq_copy_desc(fnic, wq, io_req, sc, sg_count);  in fnic_queuecommand_lck()
562 			fnic_release_ioreq_buf(fnic, io_req, sc);  in fnic_queuecommand_lck()
563 			mempool_free(io_req, fnic->io_req_pool);  in fnic_queuecommand_lck()
565 		atomic_dec(&fnic->in_flight);  in fnic_queuecommand_lck()
595 	atomic_dec(&fnic->in_flight);  in fnic_queuecommand_lck()
607 static int fnic_fcpio_fw_reset_cmpl_handler(struct fnic *fnic,  in DEF_SCSI_QCMD()
615 	struct reset_stats *reset_stats = &fnic->fnic_stats.reset_stats;  in DEF_SCSI_QCMD()
622 	fnic_cleanup_io(fnic, SCSI_NO_TAG);  in DEF_SCSI_QCMD()
624 	atomic64_set(&fnic->fnic_stats.fw_stats.active_fw_reqs, 0);  in DEF_SCSI_QCMD()
625 	atomic64_set(&fnic->fnic_stats.io_stats.active_ios, 0);  in DEF_SCSI_QCMD()
627 	spin_lock_irqsave(&fnic->fnic_lock, flags);  in DEF_SCSI_QCMD()
630 	if (fnic->state == FNIC_IN_FC_TRANS_ETH_MODE) {  in DEF_SCSI_QCMD()
633 			FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host,  in DEF_SCSI_QCMD()
636 			fnic->state = FNIC_IN_ETH_MODE;  in DEF_SCSI_QCMD()
639 				      fnic->lport->host,  in DEF_SCSI_QCMD()
649 			fnic->state = FNIC_IN_FC_MODE;  in DEF_SCSI_QCMD()
655 			      fnic->lport->host,  in DEF_SCSI_QCMD()
657 			      " reset cmpl\n", fnic_state_to_str(fnic->state));  in DEF_SCSI_QCMD()
663 	if (fnic->remove_wait)  in DEF_SCSI_QCMD()
664 		complete(fnic->remove_wait);  in DEF_SCSI_QCMD()
670 	if (fnic->remove_wait || ret) {  in DEF_SCSI_QCMD()
671 		spin_unlock_irqrestore(&fnic->fnic_lock, flags);  in DEF_SCSI_QCMD()
672 		skb_queue_purge(&fnic->tx_queue);  in DEF_SCSI_QCMD()
676 	spin_unlock_irqrestore(&fnic->fnic_lock, flags);  in DEF_SCSI_QCMD()
678 	fnic_flush_tx(fnic);  in DEF_SCSI_QCMD()
681 	fnic_clear_state_flags(fnic, FNIC_FLAGS_FWRESET);  in DEF_SCSI_QCMD()
690 static int fnic_fcpio_flogi_reg_cmpl_handler(struct fnic *fnic,  in fnic_fcpio_flogi_reg_cmpl_handler()  argument
702 	spin_lock_irqsave(&fnic->fnic_lock, flags);  in fnic_fcpio_flogi_reg_cmpl_handler()
704 	if (fnic->state == FNIC_IN_ETH_TRANS_FC_MODE) {  in fnic_fcpio_flogi_reg_cmpl_handler()
708 			FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host,  in fnic_fcpio_flogi_reg_cmpl_handler()
710 			fnic->state = FNIC_IN_FC_MODE;  in fnic_fcpio_flogi_reg_cmpl_handler()
713 				      fnic->lport->host,  in fnic_fcpio_flogi_reg_cmpl_handler()
716 			fnic->state = FNIC_IN_ETH_MODE;  in fnic_fcpio_flogi_reg_cmpl_handler()
720 		FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host,  in fnic_fcpio_flogi_reg_cmpl_handler()
723 			      fnic_state_to_str(fnic->state));  in fnic_fcpio_flogi_reg_cmpl_handler()
728 		if (fnic->stop_rx_link_events) {  in fnic_fcpio_flogi_reg_cmpl_handler()
729 			spin_unlock_irqrestore(&fnic->fnic_lock, flags);  in fnic_fcpio_flogi_reg_cmpl_handler()
732 		spin_unlock_irqrestore(&fnic->fnic_lock, flags);  in fnic_fcpio_flogi_reg_cmpl_handler()
734 		fnic_flush_tx(fnic);  in fnic_fcpio_flogi_reg_cmpl_handler()
735 		queue_work(fnic_event_queue, &fnic->frame_work);  in fnic_fcpio_flogi_reg_cmpl_handler()
737 		spin_unlock_irqrestore(&fnic->fnic_lock, flags);  in fnic_fcpio_flogi_reg_cmpl_handler()
769 static inline void fnic_fcpio_ack_handler(struct fnic *fnic,  in fnic_fcpio_ack_handler()  argument
779 	wq = &fnic->wq_copy[cq_index - fnic->raw_wq_count - fnic->rq_count];  in fnic_fcpio_ack_handler()
780 	spin_lock_irqsave(&fnic->wq_copy_lock[0], flags);  in fnic_fcpio_ack_handler()
782 	fnic->fnic_stats.misc_stats.last_ack_time = jiffies;  in fnic_fcpio_ack_handler()
784 		fnic->fw_ack_index[0] = request_out;  in fnic_fcpio_ack_handler()
785 		fnic->fw_ack_recd[0] = 1;  in fnic_fcpio_ack_handler()
788 			&fnic->fnic_stats.misc_stats.ack_index_out_of_range);  in fnic_fcpio_ack_handler()
790 	spin_unlock_irqrestore(&fnic->wq_copy_lock[0], flags);  in fnic_fcpio_ack_handler()
792 		  fnic->lport->host->host_no, 0, 0, ox_id_tag[2], ox_id_tag[3],  in fnic_fcpio_ack_handler()
800 static void fnic_fcpio_icmnd_cmpl_handler(struct fnic *fnic,  in fnic_fcpio_icmnd_cmpl_handler()  argument
811 	struct fnic_stats *fnic_stats = &fnic->fnic_stats;  in fnic_fcpio_icmnd_cmpl_handler()
822 	if (id >= fnic->fnic_max_tag_id) {  in fnic_fcpio_icmnd_cmpl_handler()
823 		shost_printk(KERN_ERR, fnic->lport->host,  in fnic_fcpio_icmnd_cmpl_handler()
829 	sc = scsi_host_find_tag(fnic->lport->host, id);  in fnic_fcpio_icmnd_cmpl_handler()
833 		shost_printk(KERN_ERR, fnic->lport->host,  in fnic_fcpio_icmnd_cmpl_handler()
838 			  fnic->lport->host->host_no, id,  in fnic_fcpio_icmnd_cmpl_handler()
848 	io_lock = fnic_io_lock_hash(fnic, sc);  in fnic_fcpio_icmnd_cmpl_handler()
856 		shost_printk(KERN_ERR, fnic->lport->host,  in fnic_fcpio_icmnd_cmpl_handler()
877 			FNIC_SCSI_DBG(KERN_INFO, fnic->lport->host,  in fnic_fcpio_icmnd_cmpl_handler()
888 			FNIC_SCSI_DBG(KERN_INFO, fnic->lport->host,  in fnic_fcpio_icmnd_cmpl_handler()
961 		shost_printk(KERN_ERR, fnic->lport->host, "hdr status = %s\n",  in fnic_fcpio_icmnd_cmpl_handler()
969 		shost_printk(KERN_ERR, fnic->lport->host, "hdr status = %s\n",  in fnic_fcpio_icmnd_cmpl_handler()
978 	fnic_release_ioreq_buf(fnic, io_req, sc);  in fnic_fcpio_icmnd_cmpl_handler()
980 	mempool_free(io_req, fnic->io_req_pool);  in fnic_fcpio_icmnd_cmpl_handler()
997 		fnic->lport->host_stats.fcp_input_requests++;  in fnic_fcpio_icmnd_cmpl_handler()
998 		fnic->fcp_input_bytes += xfer_len;  in fnic_fcpio_icmnd_cmpl_handler()
1000 		fnic->lport->host_stats.fcp_output_requests++;  in fnic_fcpio_icmnd_cmpl_handler()
1001 		fnic->fcp_output_bytes += xfer_len;  in fnic_fcpio_icmnd_cmpl_handler()
1003 		fnic->lport->host_stats.fcp_control_requests++;  in fnic_fcpio_icmnd_cmpl_handler()
1006 	if (atomic64_read(&fnic->io_cmpl_skip))  in fnic_fcpio_icmnd_cmpl_handler()
1007 		atomic64_dec(&fnic->io_cmpl_skip);  in fnic_fcpio_icmnd_cmpl_handler()
1019 static void fnic_fcpio_itmf_cmpl_handler(struct fnic *fnic,  in fnic_fcpio_itmf_cmpl_handler()  argument
1028 	struct fnic_stats *fnic_stats = &fnic->fnic_stats;  in fnic_fcpio_itmf_cmpl_handler()
1029 	struct abort_stats *abts_stats = &fnic->fnic_stats.abts_stats;  in fnic_fcpio_itmf_cmpl_handler()
1030 	struct terminate_stats *term_stats = &fnic->fnic_stats.term_stats;  in fnic_fcpio_itmf_cmpl_handler()
1031 	struct misc_stats *misc_stats = &fnic->fnic_stats.misc_stats;  in fnic_fcpio_itmf_cmpl_handler()
1039 	if ((id & FNIC_TAG_MASK) >= fnic->fnic_max_tag_id) {  in fnic_fcpio_itmf_cmpl_handler()
1040 		shost_printk(KERN_ERR, fnic->lport->host,  in fnic_fcpio_itmf_cmpl_handler()
1046 	sc = scsi_host_find_tag(fnic->lport->host, id & FNIC_TAG_MASK);  in fnic_fcpio_itmf_cmpl_handler()
1050 		shost_printk(KERN_ERR, fnic->lport->host,  in fnic_fcpio_itmf_cmpl_handler()
1055 	io_lock = fnic_io_lock_hash(fnic, sc);  in fnic_fcpio_itmf_cmpl_handler()
1063 		shost_printk(KERN_ERR, fnic->lport->host,  in fnic_fcpio_itmf_cmpl_handler()
1074 		FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host,  in fnic_fcpio_itmf_cmpl_handler()
1119 		if (atomic64_read(&fnic->io_cmpl_skip))  in fnic_fcpio_itmf_cmpl_handler()
1120 			atomic64_dec(&fnic->io_cmpl_skip);  in fnic_fcpio_itmf_cmpl_handler()
1127 		FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host,  in fnic_fcpio_itmf_cmpl_handler()
1141 			FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host,  in fnic_fcpio_itmf_cmpl_handler()
1148 			fnic_release_ioreq_buf(fnic, io_req, sc);  in fnic_fcpio_itmf_cmpl_handler()
1149 			mempool_free(io_req, fnic->io_req_pool);  in fnic_fcpio_itmf_cmpl_handler()
1178 			FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host,  in fnic_fcpio_itmf_cmpl_handler()
1193 			FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host,  in fnic_fcpio_itmf_cmpl_handler()
1202 		FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host,  in fnic_fcpio_itmf_cmpl_handler()
1211 		shost_printk(KERN_ERR, fnic->lport->host,  in fnic_fcpio_itmf_cmpl_handler()
1227 	struct fnic *fnic = vnic_dev_priv(vdev);  in fnic_fcpio_cmpl_handler()  local
1235 		atomic64_dec(&fnic->fnic_stats.fw_stats.active_fw_reqs);  in fnic_fcpio_cmpl_handler()
1243 		fnic_fcpio_ack_handler(fnic, cq_index, desc);  in fnic_fcpio_cmpl_handler()
1247 		fnic_fcpio_icmnd_cmpl_handler(fnic, desc);  in fnic_fcpio_cmpl_handler()
1251 		fnic_fcpio_itmf_cmpl_handler(fnic, desc);  in fnic_fcpio_cmpl_handler()
1256 		fnic_fcpio_flogi_reg_cmpl_handler(fnic, desc);  in fnic_fcpio_cmpl_handler()
1260 		fnic_fcpio_fw_reset_cmpl_handler(fnic, desc);  in fnic_fcpio_cmpl_handler()
1264 		FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host,  in fnic_fcpio_cmpl_handler()
1277 int fnic_wq_copy_cmpl_handler(struct fnic *fnic, int copy_work_to_do)  in fnic_wq_copy_cmpl_handler()  argument
1283 	for (i = 0; i < fnic->wq_copy_count; i++) {  in fnic_wq_copy_cmpl_handler()
1284 		cq_index = i + fnic->raw_wq_count + fnic->rq_count;  in fnic_wq_copy_cmpl_handler()
1285 		cur_work_done = vnic_cq_copy_service(&fnic->cq[cq_index],  in fnic_wq_copy_cmpl_handler()
1293 static void fnic_cleanup_io(struct fnic *fnic, int exclude_id)  in fnic_cleanup_io()  argument
1301 	struct fnic_stats *fnic_stats = &fnic->fnic_stats;  in fnic_cleanup_io()
1303 	for (i = 0; i < fnic->fnic_max_tag_id; i++) {  in fnic_cleanup_io()
1307 		io_lock = fnic_io_lock_tag(fnic, i);  in fnic_cleanup_io()
1309 		sc = scsi_host_find_tag(fnic->lport->host, i);  in fnic_cleanup_io()
1347 		fnic_release_ioreq_buf(fnic, io_req, sc);  in fnic_cleanup_io()
1348 		mempool_free(io_req, fnic->io_req_pool);  in fnic_cleanup_io()
1352 		FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host,  in fnic_cleanup_io()
1356 		if (atomic64_read(&fnic->io_cmpl_skip))  in fnic_cleanup_io()
1357 			atomic64_dec(&fnic->io_cmpl_skip);  in fnic_cleanup_io()
1381 	struct fnic *fnic = vnic_dev_priv(wq->vdev);  in fnic_wq_copy_cleanup_handler()  local
1392 	if (id >= fnic->fnic_max_tag_id)  in fnic_wq_copy_cleanup_handler()
1395 	sc = scsi_host_find_tag(fnic->lport->host, id);  in fnic_wq_copy_cleanup_handler()
1399 	io_lock = fnic_io_lock_hash(fnic, sc);  in fnic_wq_copy_cleanup_handler()
1417 	fnic_release_ioreq_buf(fnic, io_req, sc);  in fnic_wq_copy_cleanup_handler()
1418 	mempool_free(io_req, fnic->io_req_pool);  in fnic_wq_copy_cleanup_handler()
1422 	FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host, "wq_copy_cleanup_handler:"  in fnic_wq_copy_cleanup_handler()
1438 static inline int fnic_queue_abort_io_req(struct fnic *fnic, int tag,  in fnic_queue_abort_io_req()  argument
1442 	struct vnic_wq_copy *wq = &fnic->wq_copy[0];  in fnic_queue_abort_io_req()
1443 	struct Scsi_Host *host = fnic->lport->host;  in fnic_queue_abort_io_req()
1444 	struct misc_stats *misc_stats = &fnic->fnic_stats.misc_stats;  in fnic_queue_abort_io_req()
1448 	if (unlikely(fnic_chk_state_flags_locked(fnic,  in fnic_queue_abort_io_req()
1453 		atomic_inc(&fnic->in_flight);  in fnic_queue_abort_io_req()
1456 	spin_lock_irqsave(&fnic->wq_copy_lock[0], flags);  in fnic_queue_abort_io_req()
1458 	if (vnic_wq_copy_desc_avail(wq) <= fnic->wq_copy_desc_low[0])  in fnic_queue_abort_io_req()
1459 		free_wq_copy_descs(fnic, wq);  in fnic_queue_abort_io_req()
1462 		spin_unlock_irqrestore(&fnic->wq_copy_lock[0], flags);  in fnic_queue_abort_io_req()
1463 		atomic_dec(&fnic->in_flight);  in fnic_queue_abort_io_req()
1464 		FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host,  in fnic_queue_abort_io_req()
1471 				     fnic->config.ra_tov, fnic->config.ed_tov);  in fnic_queue_abort_io_req()
1473 	atomic64_inc(&fnic->fnic_stats.fw_stats.active_fw_reqs);  in fnic_queue_abort_io_req()
1474 	if (atomic64_read(&fnic->fnic_stats.fw_stats.active_fw_reqs) >  in fnic_queue_abort_io_req()
1475 		  atomic64_read(&fnic->fnic_stats.fw_stats.max_fw_reqs))  in fnic_queue_abort_io_req()
1476 		atomic64_set(&fnic->fnic_stats.fw_stats.max_fw_reqs,  in fnic_queue_abort_io_req()
1477 		  atomic64_read(&fnic->fnic_stats.fw_stats.active_fw_reqs));  in fnic_queue_abort_io_req()
1479 	spin_unlock_irqrestore(&fnic->wq_copy_lock[0], flags);  in fnic_queue_abort_io_req()
1480 	atomic_dec(&fnic->in_flight);  in fnic_queue_abort_io_req()
1485 static void fnic_rport_exch_reset(struct fnic *fnic, u32 port_id)  in fnic_rport_exch_reset()  argument
1494 	struct reset_stats *reset_stats = &fnic->fnic_stats.reset_stats;  in fnic_rport_exch_reset()
1495 	struct terminate_stats *term_stats = &fnic->fnic_stats.term_stats;  in fnic_rport_exch_reset()
1500 		      fnic->lport->host,  in fnic_rport_exch_reset()
1504 	if (fnic->in_remove)  in fnic_rport_exch_reset()
1507 	for (tag = 0; tag < fnic->fnic_max_tag_id; tag++) {  in fnic_rport_exch_reset()
1509 		io_lock = fnic_io_lock_tag(fnic, tag);  in fnic_rport_exch_reset()
1511 		sc = scsi_host_find_tag(fnic->lport->host, tag);  in fnic_rport_exch_reset()
1526 			FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host,  in fnic_rport_exch_reset()
1542 			shost_printk(KERN_ERR, fnic->lport->host,  in fnic_rport_exch_reset()
1549 			shost_printk(KERN_ERR, fnic->lport->host,  in fnic_rport_exch_reset()
1561 			FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host,  in fnic_rport_exch_reset()
1568 		FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host,  in fnic_rport_exch_reset()
1576 		if (fnic_queue_abort_io_req(fnic, abt_tag,  in fnic_rport_exch_reset()
1617 	struct fnic *fnic;  in fnic_terminate_rport_io()  local
1639 	fnic = lport_priv(lport);  in fnic_terminate_rport_io()
1641 		      fnic->lport->host, "fnic_terminate_rport_io called"  in fnic_terminate_rport_io()
1646 	if (fnic->in_remove)  in fnic_terminate_rport_io()
1649 	reset_stats = &fnic->fnic_stats.reset_stats;  in fnic_terminate_rport_io()
1650 	term_stats = &fnic->fnic_stats.term_stats;  in fnic_terminate_rport_io()
1652 	for (tag = 0; tag < fnic->fnic_max_tag_id; tag++) {  in fnic_terminate_rport_io()
1654 		io_lock = fnic_io_lock_tag(fnic, tag);  in fnic_terminate_rport_io()
1656 		sc = scsi_host_find_tag(fnic->lport->host, tag);  in fnic_terminate_rport_io()
1677 			FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host,  in fnic_terminate_rport_io()
1692 			shost_printk(KERN_ERR, fnic->lport->host,  in fnic_terminate_rport_io()
1698 			FNIC_SCSI_DBG(KERN_INFO, fnic->lport->host,  in fnic_terminate_rport_io()
1710 			FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host,  in fnic_terminate_rport_io()
1717 			      fnic->lport->host,  in fnic_terminate_rport_io()
1725 		if (fnic_queue_abort_io_req(fnic, abt_tag,  in fnic_terminate_rport_io()
1762 	struct fnic *fnic;  in fnic_abort_cmd()  local
1784 	fnic = lport_priv(lp);  in fnic_abort_cmd()
1785 	fnic_stats = &fnic->fnic_stats;  in fnic_abort_cmd()
1786 	abts_stats = &fnic->fnic_stats.abts_stats;  in fnic_abort_cmd()
1787 	term_stats = &fnic->fnic_stats.term_stats;  in fnic_abort_cmd()
1792 		fnic->lport->host,  in fnic_abort_cmd()
1815 	io_lock = fnic_io_lock_hash(fnic, sc);  in fnic_abort_cmd()
1856 	if (fnic_queue_abort_io_req(fnic, sc->request->tag, task_req,  in fnic_abort_cmd()
1884 				    (2 * fnic->config.ra_tov +  in fnic_abort_cmd()
1885 				     fnic->config.ed_tov));  in fnic_abort_cmd()
1917 		FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host,  in fnic_abort_cmd()
1921 			FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host,  in fnic_abort_cmd()
1942 	fnic_release_ioreq_buf(fnic, io_req, sc);  in fnic_abort_cmd()
1943 	mempool_free(io_req, fnic->io_req_pool);  in fnic_abort_cmd()
1954 	FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host,  in fnic_abort_cmd()
1961 static inline int fnic_queue_dr_io_req(struct fnic *fnic,  in fnic_queue_dr_io_req()  argument
1965 	struct vnic_wq_copy *wq = &fnic->wq_copy[0];  in fnic_queue_dr_io_req()
1966 	struct Scsi_Host *host = fnic->lport->host;  in fnic_queue_dr_io_req()
1967 	struct misc_stats *misc_stats = &fnic->fnic_stats.misc_stats;  in fnic_queue_dr_io_req()
1973 	if (unlikely(fnic_chk_state_flags_locked(fnic,  in fnic_queue_dr_io_req()
1978 		atomic_inc(&fnic->in_flight);  in fnic_queue_dr_io_req()
1981 	spin_lock_irqsave(&fnic->wq_copy_lock[0], intr_flags);  in fnic_queue_dr_io_req()
1983 	if (vnic_wq_copy_desc_avail(wq) <= fnic->wq_copy_desc_low[0])  in fnic_queue_dr_io_req()
1984 		free_wq_copy_descs(fnic, wq);  in fnic_queue_dr_io_req()
1987 		FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host,  in fnic_queue_dr_io_req()
2000 				     fnic->config.ra_tov, fnic->config.ed_tov);  in fnic_queue_dr_io_req()
2002 	atomic64_inc(&fnic->fnic_stats.fw_stats.active_fw_reqs);  in fnic_queue_dr_io_req()
2003 	if (atomic64_read(&fnic->fnic_stats.fw_stats.active_fw_reqs) >  in fnic_queue_dr_io_req()
2004 		  atomic64_read(&fnic->fnic_stats.fw_stats.max_fw_reqs))  in fnic_queue_dr_io_req()
2005 		atomic64_set(&fnic->fnic_stats.fw_stats.max_fw_reqs,  in fnic_queue_dr_io_req()
2006 		  atomic64_read(&fnic->fnic_stats.fw_stats.active_fw_reqs));  in fnic_queue_dr_io_req()
2009 	spin_unlock_irqrestore(&fnic->wq_copy_lock[0], intr_flags);  in fnic_queue_dr_io_req()
2010 	atomic_dec(&fnic->in_flight);  in fnic_queue_dr_io_req()
2021 static int fnic_clean_pending_aborts(struct fnic *fnic,  in fnic_clean_pending_aborts()  argument
2035 	for (tag = 0; tag < fnic->fnic_max_tag_id; tag++) {  in fnic_clean_pending_aborts()
2036 		io_lock = fnic_io_lock_tag(fnic, tag);  in fnic_clean_pending_aborts()
2038 		sc = scsi_host_find_tag(fnic->lport->host, tag);  in fnic_clean_pending_aborts()
2059 		FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host,  in fnic_clean_pending_aborts()
2069 			FNIC_SCSI_DBG(KERN_INFO, fnic->lport->host,  in fnic_clean_pending_aborts()
2077 			shost_printk(KERN_ERR, fnic->lport->host,  in fnic_clean_pending_aborts()
2095 			FNIC_SCSI_DBG(KERN_INFO, fnic->lport->host,  in fnic_clean_pending_aborts()
2106 		if (fnic_queue_abort_io_req(fnic, abt_tag,  in fnic_clean_pending_aborts()
2128 					    (fnic->config.ed_tov));  in fnic_clean_pending_aborts()
2152 		fnic_release_ioreq_buf(fnic, io_req, sc);  in fnic_clean_pending_aborts()
2153 		mempool_free(io_req, fnic->io_req_pool);  in fnic_clean_pending_aborts()
2156 	schedule_timeout(msecs_to_jiffies(2 * fnic->config.ed_tov));  in fnic_clean_pending_aborts()
2159 	if (fnic_is_abts_pending(fnic, lr_sc))  in fnic_clean_pending_aborts()
2171 fnic_scsi_host_start_tag(struct fnic *fnic, struct scsi_cmnd *sc)  in fnic_scsi_host_start_tag()  argument
2173 	struct blk_queue_tag *bqt = fnic->lport->host->bqt;  in fnic_scsi_host_start_tag()
2207 fnic_scsi_host_end_tag(struct fnic *fnic, struct scsi_cmnd *sc)  in fnic_scsi_host_end_tag()  argument
2209 	struct blk_queue_tag *bqt = fnic->lport->host->bqt;  in fnic_scsi_host_end_tag()
2233 	struct fnic *fnic;  in fnic_device_reset()  local
2254 	fnic = lport_priv(lp);  in fnic_device_reset()
2255 	fnic_stats = &fnic->fnic_stats;  in fnic_device_reset()
2256 	reset_stats = &fnic->fnic_stats.reset_stats;  in fnic_device_reset()
2261 	FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host,  in fnic_device_reset()
2295 		tag = fnic_scsi_host_start_tag(fnic, sc);  in fnic_device_reset()
2300 	io_lock = fnic_io_lock_hash(fnic, sc);  in fnic_device_reset()
2309 		io_req = mempool_alloc(fnic->io_req_pool, GFP_ATOMIC);  in fnic_device_reset()
2323 	FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host, "TAG %x\n", tag);  in fnic_device_reset()
2329 	if (fnic_queue_dr_io_req(fnic, sc, io_req)) {  in fnic_device_reset()
2351 		FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host,  in fnic_device_reset()
2365 		FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host,  in fnic_device_reset()
2381 			if (fnic_queue_abort_io_req(fnic,  in fnic_device_reset()
2393 				FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host,  in fnic_device_reset()
2420 			      fnic->lport->host,  in fnic_device_reset()
2433 	if (fnic_clean_pending_aborts(fnic, sc)) {  in fnic_device_reset()
2436 		FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host,  in fnic_device_reset()
2457 		fnic_release_ioreq_buf(fnic, io_req, sc);  in fnic_device_reset()
2458 		mempool_free(io_req, fnic->io_req_pool);  in fnic_device_reset()
2472 		fnic_scsi_host_end_tag(fnic, sc);  in fnic_device_reset()
2474 	FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host,  in fnic_device_reset()
2489 	struct fnic *fnic;  in fnic_reset()  local
2494 	fnic = lport_priv(lp);  in fnic_reset()
2495 	reset_stats = &fnic->fnic_stats.reset_stats;  in fnic_reset()
2497 	FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host,  in fnic_reset()
2508 	FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host,  in fnic_reset()
2567 	struct fnic *fnic = lport_priv(lp);  in fnic_scsi_abort_io()  local
2572 	spin_lock_irqsave(&fnic->fnic_lock, flags);  in fnic_scsi_abort_io()
2573 	if (unlikely(fnic->state == FNIC_IN_FC_TRANS_ETH_MODE)) {  in fnic_scsi_abort_io()
2575 		spin_unlock_irqrestore(&fnic->fnic_lock, flags);  in fnic_scsi_abort_io()
2580 	fnic->remove_wait = &remove_wait;  in fnic_scsi_abort_io()
2581 	old_state = fnic->state;  in fnic_scsi_abort_io()
2582 	fnic->state = FNIC_IN_FC_TRANS_ETH_MODE;  in fnic_scsi_abort_io()
2583 	fnic_update_mac_locked(fnic, fnic->ctlr.ctl_src_addr);  in fnic_scsi_abort_io()
2584 	spin_unlock_irqrestore(&fnic->fnic_lock, flags);  in fnic_scsi_abort_io()
2586 	err = fnic_fw_reset_handler(fnic);  in fnic_scsi_abort_io()
2588 		spin_lock_irqsave(&fnic->fnic_lock, flags);  in fnic_scsi_abort_io()
2589 		if (fnic->state == FNIC_IN_FC_TRANS_ETH_MODE)  in fnic_scsi_abort_io()
2590 			fnic->state = old_state;  in fnic_scsi_abort_io()
2591 		fnic->remove_wait = NULL;  in fnic_scsi_abort_io()
2592 		spin_unlock_irqrestore(&fnic->fnic_lock, flags);  in fnic_scsi_abort_io()
2600 	spin_lock_irqsave(&fnic->fnic_lock, flags);  in fnic_scsi_abort_io()
2601 	fnic->remove_wait = NULL;  in fnic_scsi_abort_io()
2602 	FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host,  in fnic_scsi_abort_io()
2604 		      (fnic->state == FNIC_IN_ETH_MODE) ?  in fnic_scsi_abort_io()
2606 	spin_unlock_irqrestore(&fnic->fnic_lock, flags);  in fnic_scsi_abort_io()
2617 	struct fnic *fnic = lport_priv(lp);  in fnic_scsi_cleanup()  local
2621 	spin_lock_irqsave(&fnic->fnic_lock, flags);  in fnic_scsi_cleanup()
2622 	if (unlikely(fnic->state == FNIC_IN_FC_TRANS_ETH_MODE)) {  in fnic_scsi_cleanup()
2624 		spin_unlock_irqrestore(&fnic->fnic_lock, flags);  in fnic_scsi_cleanup()
2628 	old_state = fnic->state;  in fnic_scsi_cleanup()
2629 	fnic->state = FNIC_IN_FC_TRANS_ETH_MODE;  in fnic_scsi_cleanup()
2630 	fnic_update_mac_locked(fnic, fnic->ctlr.ctl_src_addr);  in fnic_scsi_cleanup()
2631 	spin_unlock_irqrestore(&fnic->fnic_lock, flags);  in fnic_scsi_cleanup()
2633 	if (fnic_fw_reset_handler(fnic)) {  in fnic_scsi_cleanup()
2634 		spin_lock_irqsave(&fnic->fnic_lock, flags);  in fnic_scsi_cleanup()
2635 		if (fnic->state == FNIC_IN_FC_TRANS_ETH_MODE)  in fnic_scsi_cleanup()
2636 			fnic->state = old_state;  in fnic_scsi_cleanup()
2637 		spin_unlock_irqrestore(&fnic->fnic_lock, flags);  in fnic_scsi_cleanup()
2648 	struct fnic *fnic = lport_priv(lp);  in fnic_exch_mgr_reset()  local
2655 		fnic_rport_exch_reset(fnic, did);  in fnic_exch_mgr_reset()
2663 	if (!fnic->in_remove)  in fnic_exch_mgr_reset()
2681 int fnic_is_abts_pending(struct fnic *fnic, struct scsi_cmnd *lr_sc)  in fnic_is_abts_pending()  argument
2695 	for (tag = 0; tag < fnic->fnic_max_tag_id; tag++) {  in fnic_is_abts_pending()
2696 		sc = scsi_host_find_tag(fnic->lport->host, tag);  in fnic_is_abts_pending()
2704 		io_lock = fnic_io_lock_hash(fnic, sc);  in fnic_is_abts_pending()
2718 		FNIC_SCSI_DBG(KERN_INFO, fnic->lport->host,  in fnic_is_abts_pending()