247 static int build_rdma_recv(struct iwch_qp *qhp, union t3_wr *wqe,  in build_rdma_recv()  argument
254 	err = iwch_sgl2pbl_map(qhp->rhp, wr->sg_list, wr->num_sge, pbl_addr,  in build_rdma_recv()
280 	qhp->wq.rq[Q_PTR2IDX(qhp->wq.rq_wptr,  in build_rdma_recv()
281 			     qhp->wq.rq_size_log2)].wr_id = wr->wr_id;  in build_rdma_recv()
282 	qhp->wq.rq[Q_PTR2IDX(qhp->wq.rq_wptr,  in build_rdma_recv()
283 			     qhp->wq.rq_size_log2)].pbl_addr = 0;  in build_rdma_recv()
287 static int build_zero_stag_recv(struct iwch_qp *qhp, union t3_wr *wqe,  in build_zero_stag_recv()  argument
301 	pbl_addr = cxio_hal_pblpool_alloc(&qhp->rhp->rdev, T3_STAG0_PBL_SIZE);  in build_zero_stag_recv()
308 	pbl_offset = (pbl_addr - qhp->rhp->rdev.rnic_info.pbl_base) >> 3;  in build_zero_stag_recv()
343 	qhp->wq.rq[Q_PTR2IDX(qhp->wq.rq_wptr,  in build_zero_stag_recv()
344 			     qhp->wq.rq_size_log2)].wr_id = wr->wr_id;  in build_zero_stag_recv()
345 	qhp->wq.rq[Q_PTR2IDX(qhp->wq.rq_wptr,  in build_zero_stag_recv()
346 			     qhp->wq.rq_size_log2)].pbl_addr = pbl_addr;  in build_zero_stag_recv()
357 	struct iwch_qp *qhp;  in iwch_post_send()  local
365 	qhp = to_iwch_qp(ibqp);  in iwch_post_send()
366 	spin_lock_irqsave(&qhp->lock, flag);  in iwch_post_send()
367 	if (qhp->attr.state > IWCH_QP_STATE_RTS) {  in iwch_post_send()
368 		spin_unlock_irqrestore(&qhp->lock, flag);  in iwch_post_send()
372 	num_wrs = Q_FREECNT(qhp->wq.sq_rptr, qhp->wq.sq_wptr,  in iwch_post_send()
373 		  qhp->wq.sq_size_log2);  in iwch_post_send()
375 		spin_unlock_irqrestore(&qhp->lock, flag);  in iwch_post_send()
384 		idx = Q_PTR2IDX(qhp->wq.wptr, qhp->wq.size_log2);  in iwch_post_send()
385 		wqe = (union t3_wr *) (qhp->wq.queue + idx);  in iwch_post_send()
391 		sqp = qhp->wq.sq +  in iwch_post_send()
392 		      Q_PTR2IDX(qhp->wq.sq_wptr, qhp->wq.sq_size_log2);  in iwch_post_send()
414 			if (!qhp->wq.oldest_read)  in iwch_post_send()
415 				qhp->wq.oldest_read = sqp;  in iwch_post_send()
420 						 &wr_cnt, &qhp->wq);  in iwch_post_send()
435 		wqe->send.wrid.id0.hi = qhp->wq.sq_wptr;  in iwch_post_send()
438 		sqp->sq_wptr = qhp->wq.sq_wptr;  in iwch_post_send()
443 			       Q_GENBIT(qhp->wq.wptr, qhp->wq.size_log2),  in iwch_post_send()
448 		     Q_PTR2IDX(qhp->wq.sq_wptr, qhp->wq.sq_size_log2),  in iwch_post_send()
452 		qhp->wq.wptr += wr_cnt;  in iwch_post_send()
453 		++(qhp->wq.sq_wptr);  in iwch_post_send()
455 	spin_unlock_irqrestore(&qhp->lock, flag);  in iwch_post_send()
456 	if (cxio_wq_db_enabled(&qhp->wq))  in iwch_post_send()
457 		ring_doorbell(qhp->wq.doorbell, qhp->wq.qpid);  in iwch_post_send()
469 	struct iwch_qp *qhp;  in iwch_post_receive()  local
475 	qhp = to_iwch_qp(ibqp);  in iwch_post_receive()
476 	spin_lock_irqsave(&qhp->lock, flag);  in iwch_post_receive()
477 	if (qhp->attr.state > IWCH_QP_STATE_RTS) {  in iwch_post_receive()
478 		spin_unlock_irqrestore(&qhp->lock, flag);  in iwch_post_receive()
482 	num_wrs = Q_FREECNT(qhp->wq.rq_rptr, qhp->wq.rq_wptr,  in iwch_post_receive()
483 			    qhp->wq.rq_size_log2) - 1;  in iwch_post_receive()
485 		spin_unlock_irqrestore(&qhp->lock, flag);  in iwch_post_receive()
494 		idx = Q_PTR2IDX(qhp->wq.wptr, qhp->wq.size_log2);  in iwch_post_receive()
495 		wqe = (union t3_wr *) (qhp->wq.queue + idx);  in iwch_post_receive()
498 				err = build_rdma_recv(qhp, wqe, wr);  in iwch_post_receive()
500 				err = build_zero_stag_recv(qhp, wqe, wr);  in iwch_post_receive()
508 			       Q_GENBIT(qhp->wq.wptr, qhp->wq.size_log2),  in iwch_post_receive()
512 		     idx, qhp->wq.rq_wptr, qhp->wq.rq_rptr, wqe);  in iwch_post_receive()
513 		++(qhp->wq.rq_wptr);  in iwch_post_receive()
514 		++(qhp->wq.wptr);  in iwch_post_receive()
518 	spin_unlock_irqrestore(&qhp->lock, flag);  in iwch_post_receive()
519 	if (cxio_wq_db_enabled(&qhp->wq))  in iwch_post_receive()
520 		ring_doorbell(qhp->wq.doorbell, qhp->wq.qpid);  in iwch_post_receive()
534 	struct iwch_qp *qhp;  in iwch_bind_mw()  local
546 	qhp = to_iwch_qp(qp);  in iwch_bind_mw()
548 	rhp = qhp->rhp;  in iwch_bind_mw()
550 	spin_lock_irqsave(&qhp->lock, flag);  in iwch_bind_mw()
551 	if (qhp->attr.state > IWCH_QP_STATE_RTS) {  in iwch_bind_mw()
552 		spin_unlock_irqrestore(&qhp->lock, flag);  in iwch_bind_mw()
555 	num_wrs = Q_FREECNT(qhp->wq.sq_rptr, qhp->wq.sq_wptr,  in iwch_bind_mw()
556 			    qhp->wq.sq_size_log2);  in iwch_bind_mw()
558 		spin_unlock_irqrestore(&qhp->lock, flag);  in iwch_bind_mw()
561 	idx = Q_PTR2IDX(qhp->wq.wptr, qhp->wq.size_log2);  in iwch_bind_mw()
564 	wqe = (union t3_wr *) (qhp->wq.queue + idx);  in iwch_bind_mw()
585 		spin_unlock_irqrestore(&qhp->lock, flag);  in iwch_bind_mw()
588 	wqe->send.wrid.id0.hi = qhp->wq.sq_wptr;  in iwch_bind_mw()
589 	sqp = qhp->wq.sq + Q_PTR2IDX(qhp->wq.sq_wptr, qhp->wq.sq_size_log2);  in iwch_bind_mw()
592 	sqp->sq_wptr = qhp->wq.sq_wptr;  in iwch_bind_mw()
598 		       Q_GENBIT(qhp->wq.wptr, qhp->wq.size_log2), 0,  in iwch_bind_mw()
600 	++(qhp->wq.wptr);  in iwch_bind_mw()
601 	++(qhp->wq.sq_wptr);  in iwch_bind_mw()
602 	spin_unlock_irqrestore(&qhp->lock, flag);  in iwch_bind_mw()
604 	if (cxio_wq_db_enabled(&qhp->wq))  in iwch_bind_mw()
605 		ring_doorbell(qhp->wq.doorbell, qhp->wq.qpid);  in iwch_bind_mw()
774 int iwch_post_terminate(struct iwch_qp *qhp, struct respQ_msg_t *rsp_msg)  in iwch_post_terminate()  argument
798 	wqe->send.wrh.gen_tid_len = cpu_to_be32(V_FW_RIWR_TID(qhp->ep->hwtid));  in iwch_post_terminate()
800 	return iwch_cxgb3_ofld_send(qhp->rhp->rdev.t3cdev_p, skb);  in iwch_post_terminate()
806 static void __flush_qp(struct iwch_qp *qhp, struct iwch_cq *rchp,  in __flush_qp()  argument
813 	PDBG("%s qhp %p rchp %p schp %p\n", __func__, qhp, rchp, schp);  in __flush_qp()
815 	atomic_inc(&qhp->refcnt);  in __flush_qp()
816 	spin_unlock(&qhp->lock);  in __flush_qp()
820 	spin_lock(&qhp->lock);  in __flush_qp()
822 	cxio_count_rcqes(&rchp->cq, &qhp->wq, &count);  in __flush_qp()
823 	flushed = cxio_flush_rq(&qhp->wq, &rchp->cq, count);  in __flush_qp()
824 	spin_unlock(&qhp->lock);  in __flush_qp()
834 	spin_lock(&qhp->lock);  in __flush_qp()
836 	cxio_count_scqes(&schp->cq, &qhp->wq, &count);  in __flush_qp()
837 	flushed = cxio_flush_sq(&qhp->wq, &schp->cq, count);  in __flush_qp()
838 	spin_unlock(&qhp->lock);  in __flush_qp()
847 	if (atomic_dec_and_test(&qhp->refcnt))  in __flush_qp()
848 	        wake_up(&qhp->wait);  in __flush_qp()
850 	spin_lock(&qhp->lock);  in __flush_qp()
853 static void flush_qp(struct iwch_qp *qhp)  in flush_qp()  argument
857 	rchp = get_chp(qhp->rhp, qhp->attr.rcq);  in flush_qp()
858 	schp = get_chp(qhp->rhp, qhp->attr.scq);  in flush_qp()
860 	if (qhp->ibqp.uobject) {  in flush_qp()
861 		cxio_set_wq_in_error(&qhp->wq);  in flush_qp()
875 	__flush_qp(qhp, rchp, schp);  in flush_qp()
882 u16 iwch_rqes_posted(struct iwch_qp *qhp)  in iwch_rqes_posted()  argument
884 	union t3_wr *wqe = qhp->wq.queue;  in iwch_rqes_posted()
891 	PDBG("%s qhp %p count %u\n", __func__, qhp, count);  in iwch_rqes_posted()
895 static int rdma_init(struct iwch_dev *rhp, struct iwch_qp *qhp,  in rdma_init()  argument
902 	init_attr.tid = qhp->ep->hwtid;  in rdma_init()
903 	init_attr.qpid = qhp->wq.qpid;  in rdma_init()
904 	init_attr.pdid = qhp->attr.pd;  in rdma_init()
905 	init_attr.scqid = qhp->attr.scq;  in rdma_init()
906 	init_attr.rcqid = qhp->attr.rcq;  in rdma_init()
907 	init_attr.rq_addr = qhp->wq.rq_addr;  in rdma_init()
908 	init_attr.rq_size = 1 << qhp->wq.rq_size_log2;  in rdma_init()
910 		qhp->attr.mpa_attr.recv_marker_enabled |  in rdma_init()
911 		(qhp->attr.mpa_attr.xmit_marker_enabled << 1) |  in rdma_init()
912 		(qhp->attr.mpa_attr.crc_enabled << 2);  in rdma_init()
917 	if (!qhp->ibqp.uobject)  in rdma_init()
921 	init_attr.tcp_emss = qhp->ep->emss;  in rdma_init()
922 	init_attr.ord = qhp->attr.max_ord;  in rdma_init()
923 	init_attr.ird = qhp->attr.max_ird;  in rdma_init()
924 	init_attr.qp_dma_addr = qhp->wq.dma_addr;  in rdma_init()
925 	init_attr.qp_dma_size = (1UL << qhp->wq.size_log2);  in rdma_init()
926 	init_attr.rqe_count = iwch_rqes_posted(qhp);  in rdma_init()
927 	init_attr.flags = qhp->attr.mpa_attr.initiator ? MPA_INITIATOR : 0;  in rdma_init()
928 	init_attr.chan = qhp->ep->l2t->smt_idx;  in rdma_init()
931 		if (init_attr.ord == 0 && qhp->attr.mpa_attr.initiator)  in rdma_init()
933 		if (init_attr.ird == 0 && !qhp->attr.mpa_attr.initiator)  in rdma_init()
937 	init_attr.irs = qhp->ep->rcv_seq;  in rdma_init()
947 int iwch_modify_qp(struct iwch_dev *rhp, struct iwch_qp *qhp,  in iwch_modify_qp()  argument
953 	struct iwch_qp_attributes newattr = qhp->attr;  in iwch_modify_qp()
962 	     qhp, qhp->wq.qpid, qhp->ep, qhp->attr.state,  in iwch_modify_qp()
965 	spin_lock_irqsave(&qhp->lock, flag);  in iwch_modify_qp()
969 		if (qhp->attr.state != IWCH_QP_STATE_IDLE) {  in iwch_modify_qp()
995 		qhp->attr = newattr;  in iwch_modify_qp()
1000 	if (qhp->attr.state == attrs->next_state)  in iwch_modify_qp()
1003 	switch (qhp->attr.state) {  in iwch_modify_qp()
1015 			qhp->attr.mpa_attr = attrs->mpa_attr;  in iwch_modify_qp()
1016 			qhp->attr.llp_stream_handle = attrs->llp_stream_handle;  in iwch_modify_qp()
1017 			qhp->ep = qhp->attr.llp_stream_handle;  in iwch_modify_qp()
1018 			qhp->attr.state = IWCH_QP_STATE_RTS;  in iwch_modify_qp()
1026 			get_ep(&qhp->ep->com);  in iwch_modify_qp()
1027 			spin_unlock_irqrestore(&qhp->lock, flag);  in iwch_modify_qp()
1028 			ret = rdma_init(rhp, qhp, mask, attrs);  in iwch_modify_qp()
1029 			spin_lock_irqsave(&qhp->lock, flag);  in iwch_modify_qp()
1034 			qhp->attr.state = IWCH_QP_STATE_ERROR;  in iwch_modify_qp()
1035 			flush_qp(qhp);  in iwch_modify_qp()
1045 			BUG_ON(atomic_read(&qhp->ep->com.kref.refcount) < 2);  in iwch_modify_qp()
1046 			qhp->attr.state = IWCH_QP_STATE_CLOSING;  in iwch_modify_qp()
1050 				ep = qhp->ep;  in iwch_modify_qp()
1055 			qhp->attr.state = IWCH_QP_STATE_TERMINATE;  in iwch_modify_qp()
1056 			if (qhp->ibqp.uobject)  in iwch_modify_qp()
1057 				cxio_set_wq_in_error(&qhp->wq);  in iwch_modify_qp()
1062 			qhp->attr.state = IWCH_QP_STATE_ERROR;  in iwch_modify_qp()
1066 				ep = qhp->ep;  in iwch_modify_qp()
1083 				flush_qp(qhp);  in iwch_modify_qp()
1084 				qhp->attr.state = IWCH_QP_STATE_IDLE;  in iwch_modify_qp()
1085 				qhp->attr.llp_stream_handle = NULL;  in iwch_modify_qp()
1086 				put_ep(&qhp->ep->com);  in iwch_modify_qp()
1087 				qhp->ep = NULL;  in iwch_modify_qp()
1088 				wake_up(&qhp->wait);  in iwch_modify_qp()
1103 		if (!Q_EMPTY(qhp->wq.sq_rptr, qhp->wq.sq_wptr) ||  in iwch_modify_qp()
1104 		    !Q_EMPTY(qhp->wq.rq_rptr, qhp->wq.rq_wptr)) {  in iwch_modify_qp()
1108 		qhp->attr.state = IWCH_QP_STATE_IDLE;  in iwch_modify_qp()
1119 		       __func__, qhp->attr.state);  in iwch_modify_qp()
1126 	PDBG("%s disassociating ep %p qpid 0x%x\n", __func__, qhp->ep,  in iwch_modify_qp()
1127 	     qhp->wq.qpid);  in iwch_modify_qp()
1130 	qhp->attr.llp_stream_handle = NULL;  in iwch_modify_qp()
1131 	ep = qhp->ep;  in iwch_modify_qp()
1132 	qhp->ep = NULL;  in iwch_modify_qp()
1133 	qhp->attr.state = IWCH_QP_STATE_ERROR;  in iwch_modify_qp()
1135 	wake_up(&qhp->wait);  in iwch_modify_qp()
1137 	flush_qp(qhp);  in iwch_modify_qp()
1139 	spin_unlock_irqrestore(&qhp->lock, flag);  in iwch_modify_qp()
1142 		iwch_post_terminate(qhp, NULL);  in iwch_modify_qp()
1161 	PDBG("%s exit state %d\n", __func__, qhp->attr.state);  in iwch_modify_qp()