248 static int build_rdma_recv(struct iwch_qp *qhp, union t3_wr *wqe,  in build_rdma_recv()  argument
255 	err = iwch_sgl2pbl_map(qhp->rhp, wr->sg_list, wr->num_sge, pbl_addr,  in build_rdma_recv()
281 	qhp->wq.rq[Q_PTR2IDX(qhp->wq.rq_wptr,  in build_rdma_recv()
282 			     qhp->wq.rq_size_log2)].wr_id = wr->wr_id;  in build_rdma_recv()
283 	qhp->wq.rq[Q_PTR2IDX(qhp->wq.rq_wptr,  in build_rdma_recv()
284 			     qhp->wq.rq_size_log2)].pbl_addr = 0;  in build_rdma_recv()
288 static int build_zero_stag_recv(struct iwch_qp *qhp, union t3_wr *wqe,  in build_zero_stag_recv()  argument
302 	pbl_addr = cxio_hal_pblpool_alloc(&qhp->rhp->rdev, T3_STAG0_PBL_SIZE);  in build_zero_stag_recv()
309 	pbl_offset = (pbl_addr - qhp->rhp->rdev.rnic_info.pbl_base) >> 3;  in build_zero_stag_recv()
344 	qhp->wq.rq[Q_PTR2IDX(qhp->wq.rq_wptr,  in build_zero_stag_recv()
345 			     qhp->wq.rq_size_log2)].wr_id = wr->wr_id;  in build_zero_stag_recv()
346 	qhp->wq.rq[Q_PTR2IDX(qhp->wq.rq_wptr,  in build_zero_stag_recv()
347 			     qhp->wq.rq_size_log2)].pbl_addr = pbl_addr;  in build_zero_stag_recv()
358 	struct iwch_qp *qhp;  in iwch_post_send()  local
366 	qhp = to_iwch_qp(ibqp);  in iwch_post_send()
367 	spin_lock_irqsave(&qhp->lock, flag);  in iwch_post_send()
368 	if (qhp->attr.state > IWCH_QP_STATE_RTS) {  in iwch_post_send()
369 		spin_unlock_irqrestore(&qhp->lock, flag);  in iwch_post_send()
373 	num_wrs = Q_FREECNT(qhp->wq.sq_rptr, qhp->wq.sq_wptr,  in iwch_post_send()
374 		  qhp->wq.sq_size_log2);  in iwch_post_send()
376 		spin_unlock_irqrestore(&qhp->lock, flag);  in iwch_post_send()
385 		idx = Q_PTR2IDX(qhp->wq.wptr, qhp->wq.size_log2);  in iwch_post_send()
386 		wqe = (union t3_wr *) (qhp->wq.queue + idx);  in iwch_post_send()
392 		sqp = qhp->wq.sq +  in iwch_post_send()
393 		      Q_PTR2IDX(qhp->wq.sq_wptr, qhp->wq.sq_size_log2);  in iwch_post_send()
415 			if (!qhp->wq.oldest_read)  in iwch_post_send()
416 				qhp->wq.oldest_read = sqp;  in iwch_post_send()
421 					   &wr_cnt, &qhp->wq);  in iwch_post_send()
436 		wqe->send.wrid.id0.hi = qhp->wq.sq_wptr;  in iwch_post_send()
439 		sqp->sq_wptr = qhp->wq.sq_wptr;  in iwch_post_send()
444 			       Q_GENBIT(qhp->wq.wptr, qhp->wq.size_log2),  in iwch_post_send()
449 		     Q_PTR2IDX(qhp->wq.sq_wptr, qhp->wq.sq_size_log2),  in iwch_post_send()
453 		qhp->wq.wptr += wr_cnt;  in iwch_post_send()
454 		++(qhp->wq.sq_wptr);  in iwch_post_send()
456 	spin_unlock_irqrestore(&qhp->lock, flag);  in iwch_post_send()
457 	if (cxio_wq_db_enabled(&qhp->wq))  in iwch_post_send()
458 		ring_doorbell(qhp->wq.doorbell, qhp->wq.qpid);  in iwch_post_send()
470 	struct iwch_qp *qhp;  in iwch_post_receive()  local
476 	qhp = to_iwch_qp(ibqp);  in iwch_post_receive()
477 	spin_lock_irqsave(&qhp->lock, flag);  in iwch_post_receive()
478 	if (qhp->attr.state > IWCH_QP_STATE_RTS) {  in iwch_post_receive()
479 		spin_unlock_irqrestore(&qhp->lock, flag);  in iwch_post_receive()
483 	num_wrs = Q_FREECNT(qhp->wq.rq_rptr, qhp->wq.rq_wptr,  in iwch_post_receive()
484 			    qhp->wq.rq_size_log2) - 1;  in iwch_post_receive()
486 		spin_unlock_irqrestore(&qhp->lock, flag);  in iwch_post_receive()
495 		idx = Q_PTR2IDX(qhp->wq.wptr, qhp->wq.size_log2);  in iwch_post_receive()
496 		wqe = (union t3_wr *) (qhp->wq.queue + idx);  in iwch_post_receive()
499 				err = build_rdma_recv(qhp, wqe, wr);  in iwch_post_receive()
501 				err = build_zero_stag_recv(qhp, wqe, wr);  in iwch_post_receive()
509 			       Q_GENBIT(qhp->wq.wptr, qhp->wq.size_log2),  in iwch_post_receive()
513 		     idx, qhp->wq.rq_wptr, qhp->wq.rq_rptr, wqe);  in iwch_post_receive()
514 		++(qhp->wq.rq_wptr);  in iwch_post_receive()
515 		++(qhp->wq.wptr);  in iwch_post_receive()
519 	spin_unlock_irqrestore(&qhp->lock, flag);  in iwch_post_receive()
520 	if (cxio_wq_db_enabled(&qhp->wq))  in iwch_post_receive()
521 		ring_doorbell(qhp->wq.doorbell, qhp->wq.qpid);  in iwch_post_receive()
535 	struct iwch_qp *qhp;  in iwch_bind_mw()  local
547 	qhp = to_iwch_qp(qp);  in iwch_bind_mw()
549 	rhp = qhp->rhp;  in iwch_bind_mw()
551 	spin_lock_irqsave(&qhp->lock, flag);  in iwch_bind_mw()
552 	if (qhp->attr.state > IWCH_QP_STATE_RTS) {  in iwch_bind_mw()
553 		spin_unlock_irqrestore(&qhp->lock, flag);  in iwch_bind_mw()
556 	num_wrs = Q_FREECNT(qhp->wq.sq_rptr, qhp->wq.sq_wptr,  in iwch_bind_mw()
557 			    qhp->wq.sq_size_log2);  in iwch_bind_mw()
559 		spin_unlock_irqrestore(&qhp->lock, flag);  in iwch_bind_mw()
562 	idx = Q_PTR2IDX(qhp->wq.wptr, qhp->wq.size_log2);  in iwch_bind_mw()
565 	wqe = (union t3_wr *) (qhp->wq.queue + idx);  in iwch_bind_mw()
586 		spin_unlock_irqrestore(&qhp->lock, flag);  in iwch_bind_mw()
589 	wqe->send.wrid.id0.hi = qhp->wq.sq_wptr;  in iwch_bind_mw()
590 	sqp = qhp->wq.sq + Q_PTR2IDX(qhp->wq.sq_wptr, qhp->wq.sq_size_log2);  in iwch_bind_mw()
593 	sqp->sq_wptr = qhp->wq.sq_wptr;  in iwch_bind_mw()
599 		       Q_GENBIT(qhp->wq.wptr, qhp->wq.size_log2), 0,  in iwch_bind_mw()
601 	++(qhp->wq.wptr);  in iwch_bind_mw()
602 	++(qhp->wq.sq_wptr);  in iwch_bind_mw()
603 	spin_unlock_irqrestore(&qhp->lock, flag);  in iwch_bind_mw()
605 	if (cxio_wq_db_enabled(&qhp->wq))  in iwch_bind_mw()
606 		ring_doorbell(qhp->wq.doorbell, qhp->wq.qpid);  in iwch_bind_mw()
775 int iwch_post_terminate(struct iwch_qp *qhp, struct respQ_msg_t *rsp_msg)  in iwch_post_terminate()  argument
799 	wqe->send.wrh.gen_tid_len = cpu_to_be32(V_FW_RIWR_TID(qhp->ep->hwtid));  in iwch_post_terminate()
801 	return iwch_cxgb3_ofld_send(qhp->rhp->rdev.t3cdev_p, skb);  in iwch_post_terminate()
807 static void __flush_qp(struct iwch_qp *qhp, struct iwch_cq *rchp,  in __flush_qp()  argument
814 	PDBG("%s qhp %p rchp %p schp %p\n", __func__, qhp, rchp, schp);  in __flush_qp()
816 	atomic_inc(&qhp->refcnt);  in __flush_qp()
817 	spin_unlock(&qhp->lock);  in __flush_qp()
821 	spin_lock(&qhp->lock);  in __flush_qp()
823 	cxio_count_rcqes(&rchp->cq, &qhp->wq, &count);  in __flush_qp()
824 	flushed = cxio_flush_rq(&qhp->wq, &rchp->cq, count);  in __flush_qp()
825 	spin_unlock(&qhp->lock);  in __flush_qp()
835 	spin_lock(&qhp->lock);  in __flush_qp()
837 	cxio_count_scqes(&schp->cq, &qhp->wq, &count);  in __flush_qp()
838 	flushed = cxio_flush_sq(&qhp->wq, &schp->cq, count);  in __flush_qp()
839 	spin_unlock(&qhp->lock);  in __flush_qp()
848 	if (atomic_dec_and_test(&qhp->refcnt))  in __flush_qp()
849 	        wake_up(&qhp->wait);  in __flush_qp()
851 	spin_lock(&qhp->lock);  in __flush_qp()
854 static void flush_qp(struct iwch_qp *qhp)  in flush_qp()  argument
858 	rchp = get_chp(qhp->rhp, qhp->attr.rcq);  in flush_qp()
859 	schp = get_chp(qhp->rhp, qhp->attr.scq);  in flush_qp()
861 	if (qhp->ibqp.uobject) {  in flush_qp()
862 		cxio_set_wq_in_error(&qhp->wq);  in flush_qp()
876 	__flush_qp(qhp, rchp, schp);  in flush_qp()
883 u16 iwch_rqes_posted(struct iwch_qp *qhp)  in iwch_rqes_posted()  argument
885 	union t3_wr *wqe = qhp->wq.queue;  in iwch_rqes_posted()
892 	PDBG("%s qhp %p count %u\n", __func__, qhp, count);  in iwch_rqes_posted()
896 static int rdma_init(struct iwch_dev *rhp, struct iwch_qp *qhp,  in rdma_init()  argument
903 	init_attr.tid = qhp->ep->hwtid;  in rdma_init()
904 	init_attr.qpid = qhp->wq.qpid;  in rdma_init()
905 	init_attr.pdid = qhp->attr.pd;  in rdma_init()
906 	init_attr.scqid = qhp->attr.scq;  in rdma_init()
907 	init_attr.rcqid = qhp->attr.rcq;  in rdma_init()
908 	init_attr.rq_addr = qhp->wq.rq_addr;  in rdma_init()
909 	init_attr.rq_size = 1 << qhp->wq.rq_size_log2;  in rdma_init()
911 		qhp->attr.mpa_attr.recv_marker_enabled |  in rdma_init()
912 		(qhp->attr.mpa_attr.xmit_marker_enabled << 1) |  in rdma_init()
913 		(qhp->attr.mpa_attr.crc_enabled << 2);  in rdma_init()
918 	if (!qhp->ibqp.uobject)  in rdma_init()
922 	init_attr.tcp_emss = qhp->ep->emss;  in rdma_init()
923 	init_attr.ord = qhp->attr.max_ord;  in rdma_init()
924 	init_attr.ird = qhp->attr.max_ird;  in rdma_init()
925 	init_attr.qp_dma_addr = qhp->wq.dma_addr;  in rdma_init()
926 	init_attr.qp_dma_size = (1UL << qhp->wq.size_log2);  in rdma_init()
927 	init_attr.rqe_count = iwch_rqes_posted(qhp);  in rdma_init()
928 	init_attr.flags = qhp->attr.mpa_attr.initiator ? MPA_INITIATOR : 0;  in rdma_init()
929 	init_attr.chan = qhp->ep->l2t->smt_idx;  in rdma_init()
932 		if (init_attr.ord == 0 && qhp->attr.mpa_attr.initiator)  in rdma_init()
934 		if (init_attr.ird == 0 && !qhp->attr.mpa_attr.initiator)  in rdma_init()
938 	init_attr.irs = qhp->ep->rcv_seq;  in rdma_init()
948 int iwch_modify_qp(struct iwch_dev *rhp, struct iwch_qp *qhp,  in iwch_modify_qp()  argument
954 	struct iwch_qp_attributes newattr = qhp->attr;  in iwch_modify_qp()
963 	     qhp, qhp->wq.qpid, qhp->ep, qhp->attr.state,  in iwch_modify_qp()
966 	spin_lock_irqsave(&qhp->lock, flag);  in iwch_modify_qp()
970 		if (qhp->attr.state != IWCH_QP_STATE_IDLE) {  in iwch_modify_qp()
996 		qhp->attr = newattr;  in iwch_modify_qp()
1001 	if (qhp->attr.state == attrs->next_state)  in iwch_modify_qp()
1004 	switch (qhp->attr.state) {  in iwch_modify_qp()
1016 			qhp->attr.mpa_attr = attrs->mpa_attr;  in iwch_modify_qp()
1017 			qhp->attr.llp_stream_handle = attrs->llp_stream_handle;  in iwch_modify_qp()
1018 			qhp->ep = qhp->attr.llp_stream_handle;  in iwch_modify_qp()
1019 			qhp->attr.state = IWCH_QP_STATE_RTS;  in iwch_modify_qp()
1027 			get_ep(&qhp->ep->com);  in iwch_modify_qp()
1028 			spin_unlock_irqrestore(&qhp->lock, flag);  in iwch_modify_qp()
1029 			ret = rdma_init(rhp, qhp, mask, attrs);  in iwch_modify_qp()
1030 			spin_lock_irqsave(&qhp->lock, flag);  in iwch_modify_qp()
1035 			qhp->attr.state = IWCH_QP_STATE_ERROR;  in iwch_modify_qp()
1036 			flush_qp(qhp);  in iwch_modify_qp()
1046 			BUG_ON(atomic_read(&qhp->ep->com.kref.refcount) < 2);  in iwch_modify_qp()
1047 			qhp->attr.state = IWCH_QP_STATE_CLOSING;  in iwch_modify_qp()
1051 				ep = qhp->ep;  in iwch_modify_qp()
1056 			qhp->attr.state = IWCH_QP_STATE_TERMINATE;  in iwch_modify_qp()
1057 			if (qhp->ibqp.uobject)  in iwch_modify_qp()
1058 				cxio_set_wq_in_error(&qhp->wq);  in iwch_modify_qp()
1063 			qhp->attr.state = IWCH_QP_STATE_ERROR;  in iwch_modify_qp()
1067 				ep = qhp->ep;  in iwch_modify_qp()
1084 				flush_qp(qhp);  in iwch_modify_qp()
1085 				qhp->attr.state = IWCH_QP_STATE_IDLE;  in iwch_modify_qp()
1086 				qhp->attr.llp_stream_handle = NULL;  in iwch_modify_qp()
1087 				put_ep(&qhp->ep->com);  in iwch_modify_qp()
1088 				qhp->ep = NULL;  in iwch_modify_qp()
1089 				wake_up(&qhp->wait);  in iwch_modify_qp()
1104 		if (!Q_EMPTY(qhp->wq.sq_rptr, qhp->wq.sq_wptr) ||  in iwch_modify_qp()
1105 		    !Q_EMPTY(qhp->wq.rq_rptr, qhp->wq.rq_wptr)) {  in iwch_modify_qp()
1109 		qhp->attr.state = IWCH_QP_STATE_IDLE;  in iwch_modify_qp()
1120 		       __func__, qhp->attr.state);  in iwch_modify_qp()
1127 	PDBG("%s disassociating ep %p qpid 0x%x\n", __func__, qhp->ep,  in iwch_modify_qp()
1128 	     qhp->wq.qpid);  in iwch_modify_qp()
1131 	qhp->attr.llp_stream_handle = NULL;  in iwch_modify_qp()
1132 	ep = qhp->ep;  in iwch_modify_qp()
1133 	qhp->ep = NULL;  in iwch_modify_qp()
1134 	qhp->attr.state = IWCH_QP_STATE_ERROR;  in iwch_modify_qp()
1136 	wake_up(&qhp->wait);  in iwch_modify_qp()
1138 	flush_qp(qhp);  in iwch_modify_qp()
1140 	spin_unlock_irqrestore(&qhp->lock, flag);  in iwch_modify_qp()
1143 		iwch_post_terminate(qhp, NULL);  in iwch_modify_qp()
1162 	PDBG("%s exit state %d\n", __func__, qhp->attr.state);  in iwch_modify_qp()