Lines Matching refs:qp
57 static void start_timer(struct qib_qp *qp) in start_timer() argument
59 qp->s_flags |= QIB_S_TIMER; in start_timer()
60 qp->s_timer.function = rc_timeout; in start_timer()
62 qp->s_timer.expires = jiffies + qp->timeout_jiffies; in start_timer()
63 add_timer(&qp->s_timer); in start_timer()
77 static int qib_make_rc_ack(struct qib_ibdev *dev, struct qib_qp *qp, in qib_make_rc_ack() argument
87 if (!(ib_qib_state_ops[qp->state] & QIB_PROCESS_RECV_OK)) in qib_make_rc_ack()
93 switch (qp->s_ack_state) { in qib_make_rc_ack()
96 e = &qp->s_ack_queue[qp->s_tail_ack_queue]; in qib_make_rc_ack()
108 if (++qp->s_tail_ack_queue > QIB_MAX_RDMA_ATOMIC) in qib_make_rc_ack()
109 qp->s_tail_ack_queue = 0; in qib_make_rc_ack()
114 if (qp->r_head_ack_queue == qp->s_tail_ack_queue) { in qib_make_rc_ack()
115 if (qp->s_flags & QIB_S_ACK_PENDING) in qib_make_rc_ack()
120 e = &qp->s_ack_queue[qp->s_tail_ack_queue]; in qib_make_rc_ack()
130 qp->s_tail_ack_queue = qp->r_head_ack_queue; in qib_make_rc_ack()
134 qp->s_rdma_mr = e->rdma_sge.mr; in qib_make_rc_ack()
135 if (qp->s_rdma_mr) in qib_make_rc_ack()
136 qib_get_mr(qp->s_rdma_mr); in qib_make_rc_ack()
137 qp->s_ack_rdma_sge.sge = e->rdma_sge; in qib_make_rc_ack()
138 qp->s_ack_rdma_sge.num_sge = 1; in qib_make_rc_ack()
139 qp->s_cur_sge = &qp->s_ack_rdma_sge; in qib_make_rc_ack()
142 qp->s_ack_state = OP(RDMA_READ_RESPONSE_FIRST); in qib_make_rc_ack()
144 qp->s_ack_state = OP(RDMA_READ_RESPONSE_ONLY); in qib_make_rc_ack()
147 ohdr->u.aeth = qib_compute_aeth(qp); in qib_make_rc_ack()
149 qp->s_ack_rdma_psn = e->psn; in qib_make_rc_ack()
150 bth2 = qp->s_ack_rdma_psn++ & QIB_PSN_MASK; in qib_make_rc_ack()
153 qp->s_cur_sge = NULL; in qib_make_rc_ack()
155 qp->s_ack_state = OP(ATOMIC_ACKNOWLEDGE); in qib_make_rc_ack()
156 ohdr->u.at.aeth = qib_compute_aeth(qp); in qib_make_rc_ack()
165 bth0 = qp->s_ack_state << 24; in qib_make_rc_ack()
169 qp->s_ack_state = OP(RDMA_READ_RESPONSE_MIDDLE); in qib_make_rc_ack()
172 qp->s_cur_sge = &qp->s_ack_rdma_sge; in qib_make_rc_ack()
173 qp->s_rdma_mr = qp->s_ack_rdma_sge.sge.mr; in qib_make_rc_ack()
174 if (qp->s_rdma_mr) in qib_make_rc_ack()
175 qib_get_mr(qp->s_rdma_mr); in qib_make_rc_ack()
176 len = qp->s_ack_rdma_sge.sge.sge_length; in qib_make_rc_ack()
180 ohdr->u.aeth = qib_compute_aeth(qp); in qib_make_rc_ack()
182 qp->s_ack_state = OP(RDMA_READ_RESPONSE_LAST); in qib_make_rc_ack()
183 e = &qp->s_ack_queue[qp->s_tail_ack_queue]; in qib_make_rc_ack()
186 bth0 = qp->s_ack_state << 24; in qib_make_rc_ack()
187 bth2 = qp->s_ack_rdma_psn++ & QIB_PSN_MASK; in qib_make_rc_ack()
198 qp->s_ack_state = OP(SEND_ONLY); in qib_make_rc_ack()
199 qp->s_flags &= ~QIB_S_ACK_PENDING; in qib_make_rc_ack()
200 qp->s_cur_sge = NULL; in qib_make_rc_ack()
201 if (qp->s_nak_state) in qib_make_rc_ack()
203 cpu_to_be32((qp->r_msn & QIB_MSN_MASK) | in qib_make_rc_ack()
204 (qp->s_nak_state << in qib_make_rc_ack()
207 ohdr->u.aeth = qib_compute_aeth(qp); in qib_make_rc_ack()
211 bth2 = qp->s_ack_psn & QIB_PSN_MASK; in qib_make_rc_ack()
213 qp->s_rdma_ack_cnt++; in qib_make_rc_ack()
214 qp->s_hdrwords = hwords; in qib_make_rc_ack()
215 qp->s_cur_size = len; in qib_make_rc_ack()
216 qib_make_ruc_header(qp, ohdr, bth0, bth2); in qib_make_rc_ack()
220 qp->s_ack_state = OP(ACKNOWLEDGE); in qib_make_rc_ack()
221 qp->s_flags &= ~(QIB_S_RESP_PENDING | QIB_S_ACK_PENDING); in qib_make_rc_ack()
231 int qib_make_rc_req(struct qib_qp *qp) in qib_make_rc_req() argument
233 struct qib_ibdev *dev = to_idev(qp->ibqp.device); in qib_make_rc_req()
241 u32 pmtu = qp->pmtu; in qib_make_rc_req()
247 ohdr = &qp->s_hdr->u.oth; in qib_make_rc_req()
248 if (qp->remote_ah_attr.ah_flags & IB_AH_GRH) in qib_make_rc_req()
249 ohdr = &qp->s_hdr->u.l.oth; in qib_make_rc_req()
255 spin_lock_irqsave(&qp->s_lock, flags); in qib_make_rc_req()
258 if ((qp->s_flags & QIB_S_RESP_PENDING) && in qib_make_rc_req()
259 qib_make_rc_ack(dev, qp, ohdr, pmtu)) in qib_make_rc_req()
262 if (!(ib_qib_state_ops[qp->state] & QIB_PROCESS_SEND_OK)) { in qib_make_rc_req()
263 if (!(ib_qib_state_ops[qp->state] & QIB_FLUSH_SEND)) in qib_make_rc_req()
266 if (qp->s_last == qp->s_head) in qib_make_rc_req()
269 if (atomic_read(&qp->s_dma_busy)) { in qib_make_rc_req()
270 qp->s_flags |= QIB_S_WAIT_DMA; in qib_make_rc_req()
273 wqe = get_swqe_ptr(qp, qp->s_last); in qib_make_rc_req()
274 qib_send_complete(qp, wqe, qp->s_last != qp->s_acked ? in qib_make_rc_req()
280 if (qp->s_flags & (QIB_S_WAIT_RNR | QIB_S_WAIT_ACK)) in qib_make_rc_req()
283 if (qib_cmp24(qp->s_psn, qp->s_sending_hpsn) <= 0) { in qib_make_rc_req()
284 if (qib_cmp24(qp->s_sending_psn, qp->s_sending_hpsn) <= 0) { in qib_make_rc_req()
285 qp->s_flags |= QIB_S_WAIT_PSN; in qib_make_rc_req()
288 qp->s_sending_psn = qp->s_psn; in qib_make_rc_req()
289 qp->s_sending_hpsn = qp->s_psn - 1; in qib_make_rc_req()
297 wqe = get_swqe_ptr(qp, qp->s_cur); in qib_make_rc_req()
298 switch (qp->s_state) { in qib_make_rc_req()
300 if (!(ib_qib_state_ops[qp->state] & QIB_PROCESS_NEXT_SEND_OK)) in qib_make_rc_req()
310 if (qp->s_cur == qp->s_tail) { in qib_make_rc_req()
312 if (qp->s_tail == qp->s_head) in qib_make_rc_req()
319 qp->s_num_rd_atomic) { in qib_make_rc_req()
320 qp->s_flags |= QIB_S_WAIT_FENCE; in qib_make_rc_req()
323 wqe->psn = qp->s_next_psn; in qib_make_rc_req()
332 ss = &qp->s_sge; in qib_make_rc_req()
333 bth2 = qp->s_psn & QIB_PSN_MASK; in qib_make_rc_req()
338 if (!(qp->s_flags & QIB_S_UNLIMITED_CREDIT) && in qib_make_rc_req()
339 qib_cmp24(wqe->ssn, qp->s_lsn + 1) > 0) { in qib_make_rc_req()
340 qp->s_flags |= QIB_S_WAIT_SSN_CREDIT; in qib_make_rc_req()
346 qp->s_state = OP(SEND_FIRST); in qib_make_rc_req()
351 qp->s_state = OP(SEND_ONLY); in qib_make_rc_req()
353 qp->s_state = OP(SEND_ONLY_WITH_IMMEDIATE); in qib_make_rc_req()
361 if (++qp->s_cur == qp->s_size) in qib_make_rc_req()
362 qp->s_cur = 0; in qib_make_rc_req()
366 if (newreq && !(qp->s_flags & QIB_S_UNLIMITED_CREDIT)) in qib_make_rc_req()
367 qp->s_lsn++; in qib_make_rc_req()
371 if (!(qp->s_flags & QIB_S_UNLIMITED_CREDIT) && in qib_make_rc_req()
372 qib_cmp24(wqe->ssn, qp->s_lsn + 1) > 0) { in qib_make_rc_req()
373 qp->s_flags |= QIB_S_WAIT_SSN_CREDIT; in qib_make_rc_req()
386 qp->s_state = OP(RDMA_WRITE_FIRST); in qib_make_rc_req()
391 qp->s_state = OP(RDMA_WRITE_ONLY); in qib_make_rc_req()
393 qp->s_state = OP(RDMA_WRITE_ONLY_WITH_IMMEDIATE); in qib_make_rc_req()
402 if (++qp->s_cur == qp->s_size) in qib_make_rc_req()
403 qp->s_cur = 0; in qib_make_rc_req()
412 if (qp->s_num_rd_atomic >= in qib_make_rc_req()
413 qp->s_max_rd_atomic) { in qib_make_rc_req()
414 qp->s_flags |= QIB_S_WAIT_RDMAR; in qib_make_rc_req()
417 qp->s_num_rd_atomic++; in qib_make_rc_req()
418 if (!(qp->s_flags & QIB_S_UNLIMITED_CREDIT)) in qib_make_rc_req()
419 qp->s_lsn++; in qib_make_rc_req()
425 qp->s_next_psn += (len - 1) / pmtu; in qib_make_rc_req()
426 wqe->lpsn = qp->s_next_psn++; in qib_make_rc_req()
434 qp->s_state = OP(RDMA_READ_REQUEST); in qib_make_rc_req()
439 if (++qp->s_cur == qp->s_size) in qib_make_rc_req()
440 qp->s_cur = 0; in qib_make_rc_req()
450 if (qp->s_num_rd_atomic >= in qib_make_rc_req()
451 qp->s_max_rd_atomic) { in qib_make_rc_req()
452 qp->s_flags |= QIB_S_WAIT_RDMAR; in qib_make_rc_req()
455 qp->s_num_rd_atomic++; in qib_make_rc_req()
456 if (!(qp->s_flags & QIB_S_UNLIMITED_CREDIT)) in qib_make_rc_req()
457 qp->s_lsn++; in qib_make_rc_req()
461 qp->s_state = OP(COMPARE_SWAP); in qib_make_rc_req()
467 qp->s_state = OP(FETCH_ADD); in qib_make_rc_req()
482 if (++qp->s_cur == qp->s_size) in qib_make_rc_req()
483 qp->s_cur = 0; in qib_make_rc_req()
489 qp->s_sge.sge = wqe->sg_list[0]; in qib_make_rc_req()
490 qp->s_sge.sg_list = wqe->sg_list + 1; in qib_make_rc_req()
491 qp->s_sge.num_sge = wqe->wr.num_sge; in qib_make_rc_req()
492 qp->s_sge.total_len = wqe->length; in qib_make_rc_req()
493 qp->s_len = wqe->length; in qib_make_rc_req()
495 qp->s_tail++; in qib_make_rc_req()
496 if (qp->s_tail >= qp->s_size) in qib_make_rc_req()
497 qp->s_tail = 0; in qib_make_rc_req()
500 qp->s_psn = wqe->lpsn + 1; in qib_make_rc_req()
502 qp->s_psn++; in qib_make_rc_req()
503 if (qib_cmp24(qp->s_psn, qp->s_next_psn) > 0) in qib_make_rc_req()
504 qp->s_next_psn = qp->s_psn; in qib_make_rc_req()
518 qp->s_len = restart_sge(&qp->s_sge, wqe, qp->s_psn, pmtu); in qib_make_rc_req()
521 qp->s_state = OP(SEND_MIDDLE); in qib_make_rc_req()
524 bth2 = qp->s_psn++ & QIB_PSN_MASK; in qib_make_rc_req()
525 if (qib_cmp24(qp->s_psn, qp->s_next_psn) > 0) in qib_make_rc_req()
526 qp->s_next_psn = qp->s_psn; in qib_make_rc_req()
527 ss = &qp->s_sge; in qib_make_rc_req()
528 len = qp->s_len; in qib_make_rc_req()
534 qp->s_state = OP(SEND_LAST); in qib_make_rc_req()
536 qp->s_state = OP(SEND_LAST_WITH_IMMEDIATE); in qib_make_rc_req()
544 qp->s_cur++; in qib_make_rc_req()
545 if (qp->s_cur >= qp->s_size) in qib_make_rc_req()
546 qp->s_cur = 0; in qib_make_rc_req()
559 qp->s_len = restart_sge(&qp->s_sge, wqe, qp->s_psn, pmtu); in qib_make_rc_req()
562 qp->s_state = OP(RDMA_WRITE_MIDDLE); in qib_make_rc_req()
565 bth2 = qp->s_psn++ & QIB_PSN_MASK; in qib_make_rc_req()
566 if (qib_cmp24(qp->s_psn, qp->s_next_psn) > 0) in qib_make_rc_req()
567 qp->s_next_psn = qp->s_psn; in qib_make_rc_req()
568 ss = &qp->s_sge; in qib_make_rc_req()
569 len = qp->s_len; in qib_make_rc_req()
575 qp->s_state = OP(RDMA_WRITE_LAST); in qib_make_rc_req()
577 qp->s_state = OP(RDMA_WRITE_LAST_WITH_IMMEDIATE); in qib_make_rc_req()
585 qp->s_cur++; in qib_make_rc_req()
586 if (qp->s_cur >= qp->s_size) in qib_make_rc_req()
587 qp->s_cur = 0; in qib_make_rc_req()
600 len = ((qp->s_psn - wqe->psn) & QIB_PSN_MASK) * pmtu; in qib_make_rc_req()
606 qp->s_state = OP(RDMA_READ_REQUEST); in qib_make_rc_req()
608 bth2 = (qp->s_psn & QIB_PSN_MASK) | IB_BTH_REQ_ACK; in qib_make_rc_req()
609 qp->s_psn = wqe->lpsn + 1; in qib_make_rc_req()
612 qp->s_cur++; in qib_make_rc_req()
613 if (qp->s_cur == qp->s_size) in qib_make_rc_req()
614 qp->s_cur = 0; in qib_make_rc_req()
617 qp->s_sending_hpsn = bth2; in qib_make_rc_req()
621 if (qp->s_flags & QIB_S_SEND_ONE) { in qib_make_rc_req()
622 qp->s_flags &= ~QIB_S_SEND_ONE; in qib_make_rc_req()
623 qp->s_flags |= QIB_S_WAIT_ACK; in qib_make_rc_req()
626 qp->s_len -= len; in qib_make_rc_req()
627 qp->s_hdrwords = hwords; in qib_make_rc_req()
628 qp->s_cur_sge = ss; in qib_make_rc_req()
629 qp->s_cur_size = len; in qib_make_rc_req()
630 qib_make_ruc_header(qp, ohdr, bth0 | (qp->s_state << 24), bth2); in qib_make_rc_req()
636 qp->s_flags &= ~QIB_S_BUSY; in qib_make_rc_req()
638 spin_unlock_irqrestore(&qp->s_lock, flags); in qib_make_rc_req()
650 void qib_send_rc_ack(struct qib_qp *qp) in qib_send_rc_ack() argument
652 struct qib_devdata *dd = dd_from_ibdev(qp->ibqp.device); in qib_send_rc_ack()
653 struct qib_ibport *ibp = to_iport(qp->ibqp.device, qp->port_num); in qib_send_rc_ack()
666 spin_lock_irqsave(&qp->s_lock, flags); in qib_send_rc_ack()
668 if (!(ib_qib_state_ops[qp->state] & QIB_PROCESS_RECV_OK)) in qib_send_rc_ack()
672 if ((qp->s_flags & QIB_S_RESP_PENDING) || qp->s_rdma_ack_cnt) in qib_send_rc_ack()
680 if (unlikely(qp->remote_ah_attr.ah_flags & IB_AH_GRH)) { in qib_send_rc_ack()
682 &qp->remote_ah_attr.grh, hwords, 0); in qib_send_rc_ack()
687 bth0 = qib_get_pkey(ibp, qp->s_pkey_index) | (OP(ACKNOWLEDGE) << 24); in qib_send_rc_ack()
688 if (qp->s_mig_state == IB_MIG_MIGRATED) in qib_send_rc_ack()
690 if (qp->r_nak_state) in qib_send_rc_ack()
691 ohdr->u.aeth = cpu_to_be32((qp->r_msn & QIB_MSN_MASK) | in qib_send_rc_ack()
692 (qp->r_nak_state << in qib_send_rc_ack()
695 ohdr->u.aeth = qib_compute_aeth(qp); in qib_send_rc_ack()
696 lrh0 |= ibp->sl_to_vl[qp->remote_ah_attr.sl] << 12 | in qib_send_rc_ack()
697 qp->remote_ah_attr.sl << 4; in qib_send_rc_ack()
699 hdr.lrh[1] = cpu_to_be16(qp->remote_ah_attr.dlid); in qib_send_rc_ack()
701 hdr.lrh[3] = cpu_to_be16(ppd->lid | qp->remote_ah_attr.src_path_bits); in qib_send_rc_ack()
703 ohdr->bth[1] = cpu_to_be32(qp->remote_qpn); in qib_send_rc_ack()
704 ohdr->bth[2] = cpu_to_be32(qp->r_ack_psn & QIB_PSN_MASK); in qib_send_rc_ack()
706 spin_unlock_irqrestore(&qp->s_lock, flags); in qib_send_rc_ack()
713 qp->s_srate, lrh0 >> 12); in qib_send_rc_ack()
726 spin_lock_irqsave(&qp->s_lock, flags); in qib_send_rc_ack()
761 if (ib_qib_state_ops[qp->state] & QIB_PROCESS_RECV_OK) { in qib_send_rc_ack()
763 qp->s_flags |= QIB_S_ACK_PENDING | QIB_S_RESP_PENDING; in qib_send_rc_ack()
764 qp->s_nak_state = qp->r_nak_state; in qib_send_rc_ack()
765 qp->s_ack_psn = qp->r_ack_psn; in qib_send_rc_ack()
768 qib_schedule_send(qp); in qib_send_rc_ack()
771 spin_unlock_irqrestore(&qp->s_lock, flags); in qib_send_rc_ack()
785 static void reset_psn(struct qib_qp *qp, u32 psn) in reset_psn() argument
787 u32 n = qp->s_acked; in reset_psn()
788 struct qib_swqe *wqe = get_swqe_ptr(qp, n); in reset_psn()
791 qp->s_cur = n; in reset_psn()
798 qp->s_state = OP(SEND_LAST); in reset_psn()
807 if (++n == qp->s_size) in reset_psn()
809 if (n == qp->s_tail) in reset_psn()
811 wqe = get_swqe_ptr(qp, n); in reset_psn()
815 qp->s_cur = n; in reset_psn()
821 qp->s_state = OP(SEND_LAST); in reset_psn()
835 qp->s_state = OP(RDMA_READ_RESPONSE_FIRST); in reset_psn()
840 qp->s_state = OP(RDMA_READ_RESPONSE_LAST); in reset_psn()
844 qp->s_state = OP(RDMA_READ_RESPONSE_MIDDLE); in reset_psn()
852 qp->s_state = OP(SEND_LAST); in reset_psn()
855 qp->s_psn = psn; in reset_psn()
861 if ((qib_cmp24(qp->s_psn, qp->s_sending_hpsn) <= 0) && in reset_psn()
862 (qib_cmp24(qp->s_sending_psn, qp->s_sending_hpsn) <= 0)) in reset_psn()
863 qp->s_flags |= QIB_S_WAIT_PSN; in reset_psn()
870 static void qib_restart_rc(struct qib_qp *qp, u32 psn, int wait) in qib_restart_rc() argument
872 struct qib_swqe *wqe = get_swqe_ptr(qp, qp->s_acked); in qib_restart_rc()
875 if (qp->s_retry == 0) { in qib_restart_rc()
876 if (qp->s_mig_state == IB_MIG_ARMED) { in qib_restart_rc()
877 qib_migrate_qp(qp); in qib_restart_rc()
878 qp->s_retry = qp->s_retry_cnt; in qib_restart_rc()
879 } else if (qp->s_last == qp->s_acked) { in qib_restart_rc()
880 qib_send_complete(qp, wqe, IB_WC_RETRY_EXC_ERR); in qib_restart_rc()
881 qib_error_qp(qp, IB_WC_WR_FLUSH_ERR); in qib_restart_rc()
886 qp->s_retry--; in qib_restart_rc()
888 ibp = to_iport(qp->ibqp.device, qp->port_num); in qib_restart_rc()
892 ibp->n_rc_resends += (qp->s_psn - psn) & QIB_PSN_MASK; in qib_restart_rc()
894 qp->s_flags &= ~(QIB_S_WAIT_FENCE | QIB_S_WAIT_RDMAR | in qib_restart_rc()
898 qp->s_flags |= QIB_S_SEND_ONE; in qib_restart_rc()
899 reset_psn(qp, psn); in qib_restart_rc()
907 struct qib_qp *qp = (struct qib_qp *)arg; in rc_timeout() local
911 spin_lock_irqsave(&qp->r_lock, flags); in rc_timeout()
912 spin_lock(&qp->s_lock); in rc_timeout()
913 if (qp->s_flags & QIB_S_TIMER) { in rc_timeout()
914 ibp = to_iport(qp->ibqp.device, qp->port_num); in rc_timeout()
916 qp->s_flags &= ~QIB_S_TIMER; in rc_timeout()
917 del_timer(&qp->s_timer); in rc_timeout()
918 qib_restart_rc(qp, qp->s_last_psn + 1, 1); in rc_timeout()
919 qib_schedule_send(qp); in rc_timeout()
921 spin_unlock(&qp->s_lock); in rc_timeout()
922 spin_unlock_irqrestore(&qp->r_lock, flags); in rc_timeout()
930 struct qib_qp *qp = (struct qib_qp *)arg; in qib_rc_rnr_retry() local
933 spin_lock_irqsave(&qp->s_lock, flags); in qib_rc_rnr_retry()
934 if (qp->s_flags & QIB_S_WAIT_RNR) { in qib_rc_rnr_retry()
935 qp->s_flags &= ~QIB_S_WAIT_RNR; in qib_rc_rnr_retry()
936 del_timer(&qp->s_timer); in qib_rc_rnr_retry()
937 qib_schedule_send(qp); in qib_rc_rnr_retry()
939 spin_unlock_irqrestore(&qp->s_lock, flags); in qib_rc_rnr_retry()
946 static void reset_sending_psn(struct qib_qp *qp, u32 psn) in reset_sending_psn() argument
949 u32 n = qp->s_last; in reset_sending_psn()
953 wqe = get_swqe_ptr(qp, n); in reset_sending_psn()
956 qp->s_sending_psn = wqe->lpsn + 1; in reset_sending_psn()
958 qp->s_sending_psn = psn + 1; in reset_sending_psn()
961 if (++n == qp->s_size) in reset_sending_psn()
963 if (n == qp->s_tail) in reset_sending_psn()
971 void qib_rc_send_complete(struct qib_qp *qp, struct qib_ib_header *hdr) in qib_rc_send_complete() argument
980 if (!(ib_qib_state_ops[qp->state] & QIB_PROCESS_OR_FLUSH_SEND)) in qib_rc_send_complete()
992 WARN_ON(!qp->s_rdma_ack_cnt); in qib_rc_send_complete()
993 qp->s_rdma_ack_cnt--; in qib_rc_send_complete()
998 reset_sending_psn(qp, psn); in qib_rc_send_complete()
1004 if ((psn & IB_BTH_REQ_ACK) && qp->s_acked != qp->s_tail && in qib_rc_send_complete()
1005 !(qp->s_flags & (QIB_S_TIMER | QIB_S_WAIT_RNR | QIB_S_WAIT_PSN)) && in qib_rc_send_complete()
1006 (ib_qib_state_ops[qp->state] & QIB_PROCESS_RECV_OK)) in qib_rc_send_complete()
1007 start_timer(qp); in qib_rc_send_complete()
1009 while (qp->s_last != qp->s_acked) { in qib_rc_send_complete()
1010 wqe = get_swqe_ptr(qp, qp->s_last); in qib_rc_send_complete()
1011 if (qib_cmp24(wqe->lpsn, qp->s_sending_psn) >= 0 && in qib_rc_send_complete()
1012 qib_cmp24(qp->s_sending_psn, qp->s_sending_hpsn) <= 0) in qib_rc_send_complete()
1020 if (!(qp->s_flags & QIB_S_SIGNAL_REQ_WR) || in qib_rc_send_complete()
1027 wc.qp = &qp->ibqp; in qib_rc_send_complete()
1028 qib_cq_enter(to_icq(qp->ibqp.send_cq), &wc, 0); in qib_rc_send_complete()
1030 if (++qp->s_last >= qp->s_size) in qib_rc_send_complete()
1031 qp->s_last = 0; in qib_rc_send_complete()
1037 if (qp->s_flags & QIB_S_WAIT_PSN && in qib_rc_send_complete()
1038 qib_cmp24(qp->s_sending_psn, qp->s_sending_hpsn) > 0) { in qib_rc_send_complete()
1039 qp->s_flags &= ~QIB_S_WAIT_PSN; in qib_rc_send_complete()
1040 qp->s_sending_psn = qp->s_psn; in qib_rc_send_complete()
1041 qp->s_sending_hpsn = qp->s_psn - 1; in qib_rc_send_complete()
1042 qib_schedule_send(qp); in qib_rc_send_complete()
1046 static inline void update_last_psn(struct qib_qp *qp, u32 psn) in update_last_psn() argument
1048 qp->s_last_psn = psn; in update_last_psn()
1056 static struct qib_swqe *do_rc_completion(struct qib_qp *qp, in do_rc_completion() argument
1068 if (qib_cmp24(wqe->lpsn, qp->s_sending_psn) < 0 || in do_rc_completion()
1069 qib_cmp24(qp->s_sending_psn, qp->s_sending_hpsn) > 0) { in do_rc_completion()
1076 if (!(qp->s_flags & QIB_S_SIGNAL_REQ_WR) || in do_rc_completion()
1083 wc.qp = &qp->ibqp; in do_rc_completion()
1084 qib_cq_enter(to_icq(qp->ibqp.send_cq), &wc, 0); in do_rc_completion()
1086 if (++qp->s_last >= qp->s_size) in do_rc_completion()
1087 qp->s_last = 0; in do_rc_completion()
1091 qp->s_retry = qp->s_retry_cnt; in do_rc_completion()
1092 update_last_psn(qp, wqe->lpsn); in do_rc_completion()
1099 if (qp->s_acked == qp->s_cur) { in do_rc_completion()
1100 if (++qp->s_cur >= qp->s_size) in do_rc_completion()
1101 qp->s_cur = 0; in do_rc_completion()
1102 qp->s_acked = qp->s_cur; in do_rc_completion()
1103 wqe = get_swqe_ptr(qp, qp->s_cur); in do_rc_completion()
1104 if (qp->s_acked != qp->s_tail) { in do_rc_completion()
1105 qp->s_state = OP(SEND_LAST); in do_rc_completion()
1106 qp->s_psn = wqe->psn; in do_rc_completion()
1109 if (++qp->s_acked >= qp->s_size) in do_rc_completion()
1110 qp->s_acked = 0; in do_rc_completion()
1111 if (qp->state == IB_QPS_SQD && qp->s_acked == qp->s_cur) in do_rc_completion()
1112 qp->s_draining = 0; in do_rc_completion()
1113 wqe = get_swqe_ptr(qp, qp->s_acked); in do_rc_completion()
1129 static int do_rc_ack(struct qib_qp *qp, u32 aeth, u32 psn, int opcode, in do_rc_ack() argument
1140 if (qp->s_flags & (QIB_S_TIMER | QIB_S_WAIT_RNR)) { in do_rc_ack()
1141 qp->s_flags &= ~(QIB_S_TIMER | QIB_S_WAIT_RNR); in do_rc_ack()
1142 del_timer(&qp->s_timer); in do_rc_ack()
1154 wqe = get_swqe_ptr(qp, qp->s_acked); in do_rc_ack()
1155 ibp = to_iport(qp->ibqp.device, qp->port_num); in do_rc_ack()
1189 if (!(qp->r_flags & QIB_R_RDMAR_SEQ)) { in do_rc_ack()
1190 qp->r_flags |= QIB_R_RDMAR_SEQ; in do_rc_ack()
1191 qib_restart_rc(qp, qp->s_last_psn + 1, 0); in do_rc_ack()
1192 if (list_empty(&qp->rspwait)) { in do_rc_ack()
1193 qp->r_flags |= QIB_R_RSP_SEND; in do_rc_ack()
1194 atomic_inc(&qp->refcount); in do_rc_ack()
1195 list_add_tail(&qp->rspwait, in do_rc_ack()
1210 if (qp->s_num_rd_atomic && in do_rc_ack()
1214 qp->s_num_rd_atomic--; in do_rc_ack()
1216 if ((qp->s_flags & QIB_S_WAIT_FENCE) && in do_rc_ack()
1217 !qp->s_num_rd_atomic) { in do_rc_ack()
1218 qp->s_flags &= ~(QIB_S_WAIT_FENCE | in do_rc_ack()
1220 qib_schedule_send(qp); in do_rc_ack()
1221 } else if (qp->s_flags & QIB_S_WAIT_RDMAR) { in do_rc_ack()
1222 qp->s_flags &= ~(QIB_S_WAIT_RDMAR | in do_rc_ack()
1224 qib_schedule_send(qp); in do_rc_ack()
1227 wqe = do_rc_completion(qp, wqe, ibp); in do_rc_ack()
1228 if (qp->s_acked == qp->s_tail) in do_rc_ack()
1235 if (qp->s_acked != qp->s_tail) { in do_rc_ack()
1240 start_timer(qp); in do_rc_ack()
1245 if (qib_cmp24(qp->s_psn, psn) <= 0) in do_rc_ack()
1246 reset_psn(qp, psn + 1); in do_rc_ack()
1247 } else if (qib_cmp24(qp->s_psn, psn) <= 0) { in do_rc_ack()
1248 qp->s_state = OP(SEND_LAST); in do_rc_ack()
1249 qp->s_psn = psn + 1; in do_rc_ack()
1251 if (qp->s_flags & QIB_S_WAIT_ACK) { in do_rc_ack()
1252 qp->s_flags &= ~QIB_S_WAIT_ACK; in do_rc_ack()
1253 qib_schedule_send(qp); in do_rc_ack()
1255 qib_get_credit(qp, aeth); in do_rc_ack()
1256 qp->s_rnr_retry = qp->s_rnr_retry_cnt; in do_rc_ack()
1257 qp->s_retry = qp->s_retry_cnt; in do_rc_ack()
1258 update_last_psn(qp, psn); in do_rc_ack()
1264 if (qp->s_acked == qp->s_tail) in do_rc_ack()
1266 if (qp->s_flags & QIB_S_WAIT_RNR) in do_rc_ack()
1268 if (qp->s_rnr_retry == 0) { in do_rc_ack()
1272 if (qp->s_rnr_retry_cnt < 7) in do_rc_ack()
1273 qp->s_rnr_retry--; in do_rc_ack()
1276 update_last_psn(qp, psn - 1); in do_rc_ack()
1278 ibp->n_rc_resends += (qp->s_psn - psn) & QIB_PSN_MASK; in do_rc_ack()
1280 reset_psn(qp, psn); in do_rc_ack()
1282 qp->s_flags &= ~(QIB_S_WAIT_SSN_CREDIT | QIB_S_WAIT_ACK); in do_rc_ack()
1283 qp->s_flags |= QIB_S_WAIT_RNR; in do_rc_ack()
1284 qp->s_timer.function = qib_rc_rnr_retry; in do_rc_ack()
1285 qp->s_timer.expires = jiffies + usecs_to_jiffies( in do_rc_ack()
1288 add_timer(&qp->s_timer); in do_rc_ack()
1292 if (qp->s_acked == qp->s_tail) in do_rc_ack()
1295 update_last_psn(qp, psn - 1); in do_rc_ack()
1306 qib_restart_rc(qp, psn, 0); in do_rc_ack()
1307 qib_schedule_send(qp); in do_rc_ack()
1324 if (qp->s_last == qp->s_acked) { in do_rc_ack()
1325 qib_send_complete(qp, wqe, status); in do_rc_ack()
1326 qib_error_qp(qp, IB_WC_WR_FLUSH_ERR); in do_rc_ack()
1334 qp->s_retry = qp->s_retry_cnt; in do_rc_ack()
1335 qp->s_rnr_retry = qp->s_rnr_retry_cnt; in do_rc_ack()
1352 static void rdma_seq_err(struct qib_qp *qp, struct qib_ibport *ibp, u32 psn, in rdma_seq_err() argument
1358 if (qp->s_flags & (QIB_S_TIMER | QIB_S_WAIT_RNR)) { in rdma_seq_err()
1359 qp->s_flags &= ~(QIB_S_TIMER | QIB_S_WAIT_RNR); in rdma_seq_err()
1360 del_timer(&qp->s_timer); in rdma_seq_err()
1363 wqe = get_swqe_ptr(qp, qp->s_acked); in rdma_seq_err()
1370 wqe = do_rc_completion(qp, wqe, ibp); in rdma_seq_err()
1374 qp->r_flags |= QIB_R_RDMAR_SEQ; in rdma_seq_err()
1375 qib_restart_rc(qp, qp->s_last_psn + 1, 0); in rdma_seq_err()
1376 if (list_empty(&qp->rspwait)) { in rdma_seq_err()
1377 qp->r_flags |= QIB_R_RSP_SEND; in rdma_seq_err()
1378 atomic_inc(&qp->refcount); in rdma_seq_err()
1379 list_add_tail(&qp->rspwait, &rcd->qp_wait_list); in rdma_seq_err()
1402 struct qib_qp *qp, in qib_rc_rcv_resp() argument
1421 if ((qib_cmp24(psn, qp->s_sending_psn) >= 0) && in qib_rc_rcv_resp()
1422 (qib_cmp24(qp->s_sending_psn, qp->s_sending_hpsn) <= 0)) { in qib_rc_rcv_resp()
1428 if (!(qp->s_flags & QIB_S_BUSY)) { in qib_rc_rcv_resp()
1439 spin_lock_irqsave(&qp->s_lock, flags); in qib_rc_rcv_resp()
1440 if (!(ib_qib_state_ops[qp->state] & QIB_PROCESS_RECV_OK)) in qib_rc_rcv_resp()
1444 if (qib_cmp24(psn, qp->s_next_psn) >= 0) in qib_rc_rcv_resp()
1448 diff = qib_cmp24(psn, qp->s_last_psn); in qib_rc_rcv_resp()
1454 qib_get_credit(qp, aeth); in qib_rc_rcv_resp()
1463 if (qp->r_flags & QIB_R_RDMAR_SEQ) { in qib_rc_rcv_resp()
1464 if (qib_cmp24(psn, qp->s_last_psn + 1) != 0) in qib_rc_rcv_resp()
1466 qp->r_flags &= ~QIB_R_RDMAR_SEQ; in qib_rc_rcv_resp()
1469 if (unlikely(qp->s_acked == qp->s_tail)) in qib_rc_rcv_resp()
1471 wqe = get_swqe_ptr(qp, qp->s_acked); in qib_rc_rcv_resp()
1486 if (!do_rc_ack(qp, aeth, psn, opcode, val, rcd) || in qib_rc_rcv_resp()
1490 wqe = get_swqe_ptr(qp, qp->s_acked); in qib_rc_rcv_resp()
1498 qp->s_rdma_read_len = restart_sge(&qp->s_rdma_read_sge, in qib_rc_rcv_resp()
1504 if (unlikely(qib_cmp24(psn, qp->s_last_psn + 1))) in qib_rc_rcv_resp()
1511 if (unlikely(pmtu >= qp->s_rdma_read_len)) in qib_rc_rcv_resp()
1518 qp->s_flags |= QIB_S_TIMER; in qib_rc_rcv_resp()
1519 mod_timer(&qp->s_timer, jiffies + qp->timeout_jiffies); in qib_rc_rcv_resp()
1520 if (qp->s_flags & QIB_S_WAIT_ACK) { in qib_rc_rcv_resp()
1521 qp->s_flags &= ~QIB_S_WAIT_ACK; in qib_rc_rcv_resp()
1522 qib_schedule_send(qp); in qib_rc_rcv_resp()
1526 qp->s_retry = qp->s_retry_cnt; in qib_rc_rcv_resp()
1532 qp->s_rdma_read_len -= pmtu; in qib_rc_rcv_resp()
1533 update_last_psn(qp, psn); in qib_rc_rcv_resp()
1534 spin_unlock_irqrestore(&qp->s_lock, flags); in qib_rc_rcv_resp()
1535 qib_copy_sge(&qp->s_rdma_read_sge, data, pmtu, 0); in qib_rc_rcv_resp()
1540 if (!do_rc_ack(qp, aeth, psn, opcode, 0, rcd)) in qib_rc_rcv_resp()
1556 wqe = get_swqe_ptr(qp, qp->s_acked); in qib_rc_rcv_resp()
1557 qp->s_rdma_read_len = restart_sge(&qp->s_rdma_read_sge, in qib_rc_rcv_resp()
1563 if (unlikely(qib_cmp24(psn, qp->s_last_psn + 1))) in qib_rc_rcv_resp()
1578 if (unlikely(tlen != qp->s_rdma_read_len)) in qib_rc_rcv_resp()
1581 qib_copy_sge(&qp->s_rdma_read_sge, data, tlen, 0); in qib_rc_rcv_resp()
1582 WARN_ON(qp->s_rdma_read_sge.num_sge); in qib_rc_rcv_resp()
1583 (void) do_rc_ack(qp, aeth, psn, in qib_rc_rcv_resp()
1593 rdma_seq_err(qp, ibp, psn, rcd); in qib_rc_rcv_resp()
1599 if (qp->s_last == qp->s_acked) { in qib_rc_rcv_resp()
1600 qib_send_complete(qp, wqe, status); in qib_rc_rcv_resp()
1601 qib_error_qp(qp, IB_WC_WR_FLUSH_ERR); in qib_rc_rcv_resp()
1604 spin_unlock_irqrestore(&qp->s_lock, flags); in qib_rc_rcv_resp()
1626 struct qib_qp *qp, in qib_rc_rcv_error() argument
1632 struct qib_ibport *ibp = to_iport(qp->ibqp.device, qp->port_num); in qib_rc_rcv_error()
1644 if (!qp->r_nak_state) { in qib_rc_rcv_error()
1646 qp->r_nak_state = IB_NAK_PSN_ERROR; in qib_rc_rcv_error()
1648 qp->r_ack_psn = qp->r_psn; in qib_rc_rcv_error()
1654 if (list_empty(&qp->rspwait)) { in qib_rc_rcv_error()
1655 qp->r_flags |= QIB_R_RSP_NAK; in qib_rc_rcv_error()
1656 atomic_inc(&qp->refcount); in qib_rc_rcv_error()
1657 list_add_tail(&qp->rspwait, &rcd->qp_wait_list); in qib_rc_rcv_error()
1683 spin_lock_irqsave(&qp->s_lock, flags); in qib_rc_rcv_error()
1685 for (i = qp->r_head_ack_queue; ; i = prev) { in qib_rc_rcv_error()
1686 if (i == qp->s_tail_ack_queue) in qib_rc_rcv_error()
1692 if (prev == qp->r_head_ack_queue) { in qib_rc_rcv_error()
1696 e = &qp->s_ack_queue[prev]; in qib_rc_rcv_error()
1702 if (prev == qp->s_tail_ack_queue && in qib_rc_rcv_error()
1730 qp->pmtu; in qib_rc_rcv_error()
1743 ok = qib_rkey_ok(qp, &e->rdma_sge, len, vaddr, rkey, in qib_rc_rcv_error()
1755 qp->s_tail_ack_queue = prev; in qib_rc_rcv_error()
1768 qp->s_tail_ack_queue = prev; in qib_rc_rcv_error()
1783 if (i == qp->r_head_ack_queue) { in qib_rc_rcv_error()
1784 spin_unlock_irqrestore(&qp->s_lock, flags); in qib_rc_rcv_error()
1785 qp->r_nak_state = 0; in qib_rc_rcv_error()
1786 qp->r_ack_psn = qp->r_psn - 1; in qib_rc_rcv_error()
1794 if (!(qp->s_flags & QIB_S_RESP_PENDING)) { in qib_rc_rcv_error()
1795 spin_unlock_irqrestore(&qp->s_lock, flags); in qib_rc_rcv_error()
1796 qp->r_nak_state = 0; in qib_rc_rcv_error()
1797 qp->r_ack_psn = qp->s_ack_queue[i].psn - 1; in qib_rc_rcv_error()
1804 qp->s_tail_ack_queue = i; in qib_rc_rcv_error()
1807 qp->s_ack_state = OP(ACKNOWLEDGE); in qib_rc_rcv_error()
1808 qp->s_flags |= QIB_S_RESP_PENDING; in qib_rc_rcv_error()
1809 qp->r_nak_state = 0; in qib_rc_rcv_error()
1810 qib_schedule_send(qp); in qib_rc_rcv_error()
1813 spin_unlock_irqrestore(&qp->s_lock, flags); in qib_rc_rcv_error()
1821 void qib_rc_error(struct qib_qp *qp, enum ib_wc_status err) in qib_rc_error() argument
1826 spin_lock_irqsave(&qp->s_lock, flags); in qib_rc_error()
1827 lastwqe = qib_error_qp(qp, err); in qib_rc_error()
1828 spin_unlock_irqrestore(&qp->s_lock, flags); in qib_rc_error()
1833 ev.device = qp->ibqp.device; in qib_rc_error()
1834 ev.element.qp = &qp->ibqp; in qib_rc_error()
1836 qp->ibqp.event_handler(&ev, qp->ibqp.qp_context); in qib_rc_error()
1840 static inline void qib_update_ack_queue(struct qib_qp *qp, unsigned n) in qib_update_ack_queue() argument
1847 qp->s_tail_ack_queue = next; in qib_update_ack_queue()
1848 qp->s_ack_state = OP(ACKNOWLEDGE); in qib_update_ack_queue()
1865 int has_grh, void *data, u32 tlen, struct qib_qp *qp) in qib_rc_rcv() argument
1874 u32 pmtu = qp->pmtu; in qib_rc_rcv()
1890 if (qib_ruc_check_hdr(ibp, hdr, has_grh, qp, opcode)) in qib_rc_rcv()
1904 qib_rc_rcv_resp(ibp, ohdr, data, tlen, qp, opcode, psn, in qib_rc_rcv()
1910 diff = qib_cmp24(psn, qp->r_psn); in qib_rc_rcv()
1912 if (qib_rc_rcv_error(ohdr, data, qp, opcode, psn, diff, rcd)) in qib_rc_rcv()
1918 switch (qp->r_state) { in qib_rc_rcv()
1951 if (qp->state == IB_QPS_RTR && !(qp->r_flags & QIB_R_COMM_EST)) { in qib_rc_rcv()
1952 qp->r_flags |= QIB_R_COMM_EST; in qib_rc_rcv()
1953 if (qp->ibqp.event_handler) { in qib_rc_rcv()
1956 ev.device = qp->ibqp.device; in qib_rc_rcv()
1957 ev.element.qp = &qp->ibqp; in qib_rc_rcv()
1959 qp->ibqp.event_handler(&ev, qp->ibqp.qp_context); in qib_rc_rcv()
1966 ret = qib_get_rwqe(qp, 0); in qib_rc_rcv()
1971 qp->r_rcv_len = 0; in qib_rc_rcv()
1979 qp->r_rcv_len += pmtu; in qib_rc_rcv()
1980 if (unlikely(qp->r_rcv_len > qp->r_len)) in qib_rc_rcv()
1982 qib_copy_sge(&qp->r_sge, data, pmtu, 1); in qib_rc_rcv()
1987 ret = qib_get_rwqe(qp, 1); in qib_rc_rcv()
1996 ret = qib_get_rwqe(qp, 0); in qib_rc_rcv()
2001 qp->r_rcv_len = 0; in qib_rc_rcv()
2025 wc.byte_len = tlen + qp->r_rcv_len; in qib_rc_rcv()
2026 if (unlikely(wc.byte_len > qp->r_len)) in qib_rc_rcv()
2028 qib_copy_sge(&qp->r_sge, data, tlen, 1); in qib_rc_rcv()
2029 qib_put_ss(&qp->r_sge); in qib_rc_rcv()
2030 qp->r_msn++; in qib_rc_rcv()
2031 if (!test_and_clear_bit(QIB_R_WRID_VALID, &qp->r_aflags)) in qib_rc_rcv()
2033 wc.wr_id = qp->r_wr_id; in qib_rc_rcv()
2040 wc.qp = &qp->ibqp; in qib_rc_rcv()
2041 wc.src_qp = qp->remote_qpn; in qib_rc_rcv()
2042 wc.slid = qp->remote_ah_attr.dlid; in qib_rc_rcv()
2043 wc.sl = qp->remote_ah_attr.sl; in qib_rc_rcv()
2050 qib_cq_enter(to_icq(qp->ibqp.recv_cq), &wc, in qib_rc_rcv()
2058 if (unlikely(!(qp->qp_access_flags & IB_ACCESS_REMOTE_WRITE))) in qib_rc_rcv()
2063 qp->r_len = be32_to_cpu(reth->length); in qib_rc_rcv()
2064 qp->r_rcv_len = 0; in qib_rc_rcv()
2065 qp->r_sge.sg_list = NULL; in qib_rc_rcv()
2066 if (qp->r_len != 0) { in qib_rc_rcv()
2072 ok = qib_rkey_ok(qp, &qp->r_sge.sge, qp->r_len, vaddr, in qib_rc_rcv()
2076 qp->r_sge.num_sge = 1; in qib_rc_rcv()
2078 qp->r_sge.num_sge = 0; in qib_rc_rcv()
2079 qp->r_sge.sge.mr = NULL; in qib_rc_rcv()
2080 qp->r_sge.sge.vaddr = NULL; in qib_rc_rcv()
2081 qp->r_sge.sge.length = 0; in qib_rc_rcv()
2082 qp->r_sge.sge.sge_length = 0; in qib_rc_rcv()
2088 ret = qib_get_rwqe(qp, 1); in qib_rc_rcv()
2103 if (unlikely(!(qp->qp_access_flags & IB_ACCESS_REMOTE_READ))) in qib_rc_rcv()
2105 next = qp->r_head_ack_queue + 1; in qib_rc_rcv()
2109 spin_lock_irqsave(&qp->s_lock, flags); in qib_rc_rcv()
2110 if (unlikely(next == qp->s_tail_ack_queue)) { in qib_rc_rcv()
2111 if (!qp->s_ack_queue[next].sent) in qib_rc_rcv()
2113 qib_update_ack_queue(qp, next); in qib_rc_rcv()
2115 e = &qp->s_ack_queue[qp->r_head_ack_queue]; in qib_rc_rcv()
2128 ok = qib_rkey_ok(qp, &e->rdma_sge, len, vaddr, in qib_rc_rcv()
2137 qp->r_psn += (len - 1) / pmtu; in qib_rc_rcv()
2147 e->lpsn = qp->r_psn; in qib_rc_rcv()
2153 qp->r_msn++; in qib_rc_rcv()
2154 qp->r_psn++; in qib_rc_rcv()
2155 qp->r_state = opcode; in qib_rc_rcv()
2156 qp->r_nak_state = 0; in qib_rc_rcv()
2157 qp->r_head_ack_queue = next; in qib_rc_rcv()
2160 qp->s_flags |= QIB_S_RESP_PENDING; in qib_rc_rcv()
2161 qib_schedule_send(qp); in qib_rc_rcv()
2176 if (unlikely(!(qp->qp_access_flags & IB_ACCESS_REMOTE_ATOMIC))) in qib_rc_rcv()
2178 next = qp->r_head_ack_queue + 1; in qib_rc_rcv()
2181 spin_lock_irqsave(&qp->s_lock, flags); in qib_rc_rcv()
2182 if (unlikely(next == qp->s_tail_ack_queue)) { in qib_rc_rcv()
2183 if (!qp->s_ack_queue[next].sent) in qib_rc_rcv()
2185 qib_update_ack_queue(qp, next); in qib_rc_rcv()
2187 e = &qp->s_ack_queue[qp->r_head_ack_queue]; in qib_rc_rcv()
2199 if (unlikely(!qib_rkey_ok(qp, &qp->r_sge.sge, sizeof(u64), in qib_rc_rcv()
2204 maddr = (atomic64_t *) qp->r_sge.sge.vaddr; in qib_rc_rcv()
2208 (u64) cmpxchg((u64 *) qp->r_sge.sge.vaddr, in qib_rc_rcv()
2211 qib_put_mr(qp->r_sge.sge.mr); in qib_rc_rcv()
2212 qp->r_sge.num_sge = 0; in qib_rc_rcv()
2217 qp->r_msn++; in qib_rc_rcv()
2218 qp->r_psn++; in qib_rc_rcv()
2219 qp->r_state = opcode; in qib_rc_rcv()
2220 qp->r_nak_state = 0; in qib_rc_rcv()
2221 qp->r_head_ack_queue = next; in qib_rc_rcv()
2224 qp->s_flags |= QIB_S_RESP_PENDING; in qib_rc_rcv()
2225 qib_schedule_send(qp); in qib_rc_rcv()
2234 qp->r_psn++; in qib_rc_rcv()
2235 qp->r_state = opcode; in qib_rc_rcv()
2236 qp->r_ack_psn = psn; in qib_rc_rcv()
2237 qp->r_nak_state = 0; in qib_rc_rcv()
2244 qp->r_nak_state = IB_RNR_NAK | qp->r_min_rnr_timer; in qib_rc_rcv()
2245 qp->r_ack_psn = qp->r_psn; in qib_rc_rcv()
2247 if (list_empty(&qp->rspwait)) { in qib_rc_rcv()
2248 qp->r_flags |= QIB_R_RSP_NAK; in qib_rc_rcv()
2249 atomic_inc(&qp->refcount); in qib_rc_rcv()
2250 list_add_tail(&qp->rspwait, &rcd->qp_wait_list); in qib_rc_rcv()
2255 qib_rc_error(qp, IB_WC_LOC_QP_OP_ERR); in qib_rc_rcv()
2256 qp->r_nak_state = IB_NAK_REMOTE_OPERATIONAL_ERROR; in qib_rc_rcv()
2257 qp->r_ack_psn = qp->r_psn; in qib_rc_rcv()
2259 if (list_empty(&qp->rspwait)) { in qib_rc_rcv()
2260 qp->r_flags |= QIB_R_RSP_NAK; in qib_rc_rcv()
2261 atomic_inc(&qp->refcount); in qib_rc_rcv()
2262 list_add_tail(&qp->rspwait, &rcd->qp_wait_list); in qib_rc_rcv()
2267 spin_unlock_irqrestore(&qp->s_lock, flags); in qib_rc_rcv()
2269 qib_rc_error(qp, IB_WC_LOC_QP_OP_ERR); in qib_rc_rcv()
2270 qp->r_nak_state = IB_NAK_INVALID_REQUEST; in qib_rc_rcv()
2271 qp->r_ack_psn = qp->r_psn; in qib_rc_rcv()
2273 if (list_empty(&qp->rspwait)) { in qib_rc_rcv()
2274 qp->r_flags |= QIB_R_RSP_NAK; in qib_rc_rcv()
2275 atomic_inc(&qp->refcount); in qib_rc_rcv()
2276 list_add_tail(&qp->rspwait, &rcd->qp_wait_list); in qib_rc_rcv()
2281 spin_unlock_irqrestore(&qp->s_lock, flags); in qib_rc_rcv()
2283 qib_rc_error(qp, IB_WC_LOC_PROT_ERR); in qib_rc_rcv()
2284 qp->r_nak_state = IB_NAK_REMOTE_ACCESS_ERROR; in qib_rc_rcv()
2285 qp->r_ack_psn = qp->r_psn; in qib_rc_rcv()
2287 qib_send_rc_ack(qp); in qib_rc_rcv()
2291 spin_unlock_irqrestore(&qp->s_lock, flags); in qib_rc_rcv()