1/* 2 * Copyright (c) 2004 Topspin Communications. All rights reserved. 3 * Copyright (c) 2005 Cisco Systems. All rights reserved. 4 * Copyright (c) 2005 Mellanox Technologies. All rights reserved. 5 * Copyright (c) 2004 Voltaire, Inc. All rights reserved. 6 * Copyright (c) 2005 Open Grid Computing, Inc. All rights reserved. 7 * 8 * This software is available to you under a choice of one of two 9 * licenses. You may choose to be licensed under the terms of the GNU 10 * General Public License (GPL) Version 2, available from the file 11 * COPYING in the main directory of this source tree, or the 12 * OpenIB.org BSD license below: 13 * 14 * Redistribution and use in source and binary forms, with or 15 * without modification, are permitted provided that the following 16 * conditions are met: 17 * 18 * - Redistributions of source code must retain the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer. 21 * 22 * - Redistributions in binary form must reproduce the above 23 * copyright notice, this list of conditions and the following 24 * disclaimer in the documentation and/or other materials 25 * provided with the distribution. 26 * 27 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 28 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 29 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 30 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 31 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 32 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 33 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 34 * SOFTWARE. 35 * 36 */ 37 38#include <linux/delay.h> 39#include <linux/gfp.h> 40 41#include "c2.h" 42#include "c2_vq.h" 43#include "c2_status.h" 44 45#define C2_MAX_ORD_PER_QP 128 46#define C2_MAX_IRD_PER_QP 128 47 48#define C2_HINT_MAKE(q_index, hint_count) (((q_index) << 16) | hint_count) 49#define C2_HINT_GET_INDEX(hint) (((hint) & 0x7FFF0000) >> 16) 50#define C2_HINT_GET_COUNT(hint) ((hint) & 0x0000FFFF) 51 52#define NO_SUPPORT -1 53static const u8 c2_opcode[] = { 54 [IB_WR_SEND] = C2_WR_TYPE_SEND, 55 [IB_WR_SEND_WITH_IMM] = NO_SUPPORT, 56 [IB_WR_RDMA_WRITE] = C2_WR_TYPE_RDMA_WRITE, 57 [IB_WR_RDMA_WRITE_WITH_IMM] = NO_SUPPORT, 58 [IB_WR_RDMA_READ] = C2_WR_TYPE_RDMA_READ, 59 [IB_WR_ATOMIC_CMP_AND_SWP] = NO_SUPPORT, 60 [IB_WR_ATOMIC_FETCH_AND_ADD] = NO_SUPPORT, 61}; 62 63static int to_c2_state(enum ib_qp_state ib_state) 64{ 65 switch (ib_state) { 66 case IB_QPS_RESET: 67 return C2_QP_STATE_IDLE; 68 case IB_QPS_RTS: 69 return C2_QP_STATE_RTS; 70 case IB_QPS_SQD: 71 return C2_QP_STATE_CLOSING; 72 case IB_QPS_SQE: 73 return C2_QP_STATE_CLOSING; 74 case IB_QPS_ERR: 75 return C2_QP_STATE_ERROR; 76 default: 77 return -1; 78 } 79} 80 81static int to_ib_state(enum c2_qp_state c2_state) 82{ 83 switch (c2_state) { 84 case C2_QP_STATE_IDLE: 85 return IB_QPS_RESET; 86 case C2_QP_STATE_CONNECTING: 87 return IB_QPS_RTR; 88 case C2_QP_STATE_RTS: 89 return IB_QPS_RTS; 90 case C2_QP_STATE_CLOSING: 91 return IB_QPS_SQD; 92 case C2_QP_STATE_ERROR: 93 return IB_QPS_ERR; 94 case C2_QP_STATE_TERMINATE: 95 return IB_QPS_SQE; 96 default: 97 return -1; 98 } 99} 100 101static const char *to_ib_state_str(int ib_state) 102{ 103 static const char *state_str[] = { 104 "IB_QPS_RESET", 105 "IB_QPS_INIT", 106 "IB_QPS_RTR", 107 "IB_QPS_RTS", 108 "IB_QPS_SQD", 109 "IB_QPS_SQE", 110 "IB_QPS_ERR" 111 }; 112 if (ib_state < IB_QPS_RESET || 113 ib_state > IB_QPS_ERR) 114 return "<invalid IB QP state>"; 115 116 ib_state -= IB_QPS_RESET; 117 return state_str[ib_state]; 118} 119 120void c2_set_qp_state(struct c2_qp *qp, int c2_state) 121{ 122 int new_state = to_ib_state(c2_state); 123 124 pr_debug("%s: qp[%p] state modify %s --> %s\n", 125 __func__, 126 qp, 127 to_ib_state_str(qp->state), 128 to_ib_state_str(new_state)); 129 qp->state = new_state; 130} 131 132#define C2_QP_NO_ATTR_CHANGE 0xFFFFFFFF 133 134int c2_qp_modify(struct c2_dev *c2dev, struct c2_qp *qp, 135 struct ib_qp_attr *attr, int attr_mask) 136{ 137 struct c2wr_qp_modify_req wr; 138 struct c2wr_qp_modify_rep *reply; 139 struct c2_vq_req *vq_req; 140 unsigned long flags; 141 u8 next_state; 142 int err; 143 144 pr_debug("%s:%d qp=%p, %s --> %s\n", 145 __func__, __LINE__, 146 qp, 147 to_ib_state_str(qp->state), 148 to_ib_state_str(attr->qp_state)); 149 150 vq_req = vq_req_alloc(c2dev); 151 if (!vq_req) 152 return -ENOMEM; 153 154 c2_wr_set_id(&wr, CCWR_QP_MODIFY); 155 wr.hdr.context = (unsigned long) vq_req; 156 wr.rnic_handle = c2dev->adapter_handle; 157 wr.qp_handle = qp->adapter_handle; 158 wr.ord = cpu_to_be32(C2_QP_NO_ATTR_CHANGE); 159 wr.ird = cpu_to_be32(C2_QP_NO_ATTR_CHANGE); 160 wr.sq_depth = cpu_to_be32(C2_QP_NO_ATTR_CHANGE); 161 wr.rq_depth = cpu_to_be32(C2_QP_NO_ATTR_CHANGE); 162 163 if (attr_mask & IB_QP_STATE) { 164 /* Ensure the state is valid */ 165 if (attr->qp_state < 0 || attr->qp_state > IB_QPS_ERR) { 166 err = -EINVAL; 167 goto bail0; 168 } 169 170 wr.next_qp_state = cpu_to_be32(to_c2_state(attr->qp_state)); 171 172 if (attr->qp_state == IB_QPS_ERR) { 173 spin_lock_irqsave(&qp->lock, flags); 174 if (qp->cm_id && qp->state == IB_QPS_RTS) { 175 pr_debug("Generating CLOSE event for QP-->ERR, " 176 "qp=%p, cm_id=%p\n",qp,qp->cm_id); 177 /* Generate an CLOSE event */ 178 vq_req->cm_id = qp->cm_id; 179 vq_req->event = IW_CM_EVENT_CLOSE; 180 } 181 spin_unlock_irqrestore(&qp->lock, flags); 182 } 183 next_state = attr->qp_state; 184 185 } else if (attr_mask & IB_QP_CUR_STATE) { 186 187 if (attr->cur_qp_state != IB_QPS_RTR && 188 attr->cur_qp_state != IB_QPS_RTS && 189 attr->cur_qp_state != IB_QPS_SQD && 190 attr->cur_qp_state != IB_QPS_SQE) { 191 err = -EINVAL; 192 goto bail0; 193 } else 194 wr.next_qp_state = 195 cpu_to_be32(to_c2_state(attr->cur_qp_state)); 196 197 next_state = attr->cur_qp_state; 198 199 } else { 200 err = 0; 201 goto bail0; 202 } 203 204 /* reference the request struct */ 205 vq_req_get(c2dev, vq_req); 206 207 err = vq_send_wr(c2dev, (union c2wr *) & wr); 208 if (err) { 209 vq_req_put(c2dev, vq_req); 210 goto bail0; 211 } 212 213 err = vq_wait_for_reply(c2dev, vq_req); 214 if (err) 215 goto bail0; 216 217 reply = (struct c2wr_qp_modify_rep *) (unsigned long) vq_req->reply_msg; 218 if (!reply) { 219 err = -ENOMEM; 220 goto bail0; 221 } 222 223 err = c2_errno(reply); 224 if (!err) 225 qp->state = next_state; 226#ifdef DEBUG 227 else 228 pr_debug("%s: c2_errno=%d\n", __func__, err); 229#endif 230 /* 231 * If we're going to error and generating the event here, then 232 * we need to remove the reference because there will be no 233 * close event generated by the adapter 234 */ 235 spin_lock_irqsave(&qp->lock, flags); 236 if (vq_req->event==IW_CM_EVENT_CLOSE && qp->cm_id) { 237 qp->cm_id->rem_ref(qp->cm_id); 238 qp->cm_id = NULL; 239 } 240 spin_unlock_irqrestore(&qp->lock, flags); 241 242 vq_repbuf_free(c2dev, reply); 243 bail0: 244 vq_req_free(c2dev, vq_req); 245 246 pr_debug("%s:%d qp=%p, cur_state=%s\n", 247 __func__, __LINE__, 248 qp, 249 to_ib_state_str(qp->state)); 250 return err; 251} 252 253int c2_qp_set_read_limits(struct c2_dev *c2dev, struct c2_qp *qp, 254 int ord, int ird) 255{ 256 struct c2wr_qp_modify_req wr; 257 struct c2wr_qp_modify_rep *reply; 258 struct c2_vq_req *vq_req; 259 int err; 260 261 vq_req = vq_req_alloc(c2dev); 262 if (!vq_req) 263 return -ENOMEM; 264 265 c2_wr_set_id(&wr, CCWR_QP_MODIFY); 266 wr.hdr.context = (unsigned long) vq_req; 267 wr.rnic_handle = c2dev->adapter_handle; 268 wr.qp_handle = qp->adapter_handle; 269 wr.ord = cpu_to_be32(ord); 270 wr.ird = cpu_to_be32(ird); 271 wr.sq_depth = cpu_to_be32(C2_QP_NO_ATTR_CHANGE); 272 wr.rq_depth = cpu_to_be32(C2_QP_NO_ATTR_CHANGE); 273 wr.next_qp_state = cpu_to_be32(C2_QP_NO_ATTR_CHANGE); 274 275 /* reference the request struct */ 276 vq_req_get(c2dev, vq_req); 277 278 err = vq_send_wr(c2dev, (union c2wr *) & wr); 279 if (err) { 280 vq_req_put(c2dev, vq_req); 281 goto bail0; 282 } 283 284 err = vq_wait_for_reply(c2dev, vq_req); 285 if (err) 286 goto bail0; 287 288 reply = (struct c2wr_qp_modify_rep *) (unsigned long) 289 vq_req->reply_msg; 290 if (!reply) { 291 err = -ENOMEM; 292 goto bail0; 293 } 294 295 err = c2_errno(reply); 296 vq_repbuf_free(c2dev, reply); 297 bail0: 298 vq_req_free(c2dev, vq_req); 299 return err; 300} 301 302static int destroy_qp(struct c2_dev *c2dev, struct c2_qp *qp) 303{ 304 struct c2_vq_req *vq_req; 305 struct c2wr_qp_destroy_req wr; 306 struct c2wr_qp_destroy_rep *reply; 307 unsigned long flags; 308 int err; 309 310 /* 311 * Allocate a verb request message 312 */ 313 vq_req = vq_req_alloc(c2dev); 314 if (!vq_req) { 315 return -ENOMEM; 316 } 317 318 /* 319 * Initialize the WR 320 */ 321 c2_wr_set_id(&wr, CCWR_QP_DESTROY); 322 wr.hdr.context = (unsigned long) vq_req; 323 wr.rnic_handle = c2dev->adapter_handle; 324 wr.qp_handle = qp->adapter_handle; 325 326 /* 327 * reference the request struct. dereferenced in the int handler. 328 */ 329 vq_req_get(c2dev, vq_req); 330 331 spin_lock_irqsave(&qp->lock, flags); 332 if (qp->cm_id && qp->state == IB_QPS_RTS) { 333 pr_debug("destroy_qp: generating CLOSE event for QP-->ERR, " 334 "qp=%p, cm_id=%p\n",qp,qp->cm_id); 335 /* Generate an CLOSE event */ 336 vq_req->qp = qp; 337 vq_req->cm_id = qp->cm_id; 338 vq_req->event = IW_CM_EVENT_CLOSE; 339 } 340 spin_unlock_irqrestore(&qp->lock, flags); 341 342 /* 343 * Send WR to adapter 344 */ 345 err = vq_send_wr(c2dev, (union c2wr *) & wr); 346 if (err) { 347 vq_req_put(c2dev, vq_req); 348 goto bail0; 349 } 350 351 /* 352 * Wait for reply from adapter 353 */ 354 err = vq_wait_for_reply(c2dev, vq_req); 355 if (err) { 356 goto bail0; 357 } 358 359 /* 360 * Process reply 361 */ 362 reply = (struct c2wr_qp_destroy_rep *) (unsigned long) (vq_req->reply_msg); 363 if (!reply) { 364 err = -ENOMEM; 365 goto bail0; 366 } 367 368 spin_lock_irqsave(&qp->lock, flags); 369 if (qp->cm_id) { 370 qp->cm_id->rem_ref(qp->cm_id); 371 qp->cm_id = NULL; 372 } 373 spin_unlock_irqrestore(&qp->lock, flags); 374 375 vq_repbuf_free(c2dev, reply); 376 bail0: 377 vq_req_free(c2dev, vq_req); 378 return err; 379} 380 381static int c2_alloc_qpn(struct c2_dev *c2dev, struct c2_qp *qp) 382{ 383 int ret; 384 385 idr_preload(GFP_KERNEL); 386 spin_lock_irq(&c2dev->qp_table.lock); 387 388 ret = idr_alloc_cyclic(&c2dev->qp_table.idr, qp, 0, 0, GFP_NOWAIT); 389 if (ret >= 0) 390 qp->qpn = ret; 391 392 spin_unlock_irq(&c2dev->qp_table.lock); 393 idr_preload_end(); 394 return ret < 0 ? ret : 0; 395} 396 397static void c2_free_qpn(struct c2_dev *c2dev, int qpn) 398{ 399 spin_lock_irq(&c2dev->qp_table.lock); 400 idr_remove(&c2dev->qp_table.idr, qpn); 401 spin_unlock_irq(&c2dev->qp_table.lock); 402} 403 404struct c2_qp *c2_find_qpn(struct c2_dev *c2dev, int qpn) 405{ 406 unsigned long flags; 407 struct c2_qp *qp; 408 409 spin_lock_irqsave(&c2dev->qp_table.lock, flags); 410 qp = idr_find(&c2dev->qp_table.idr, qpn); 411 spin_unlock_irqrestore(&c2dev->qp_table.lock, flags); 412 return qp; 413} 414 415int c2_alloc_qp(struct c2_dev *c2dev, 416 struct c2_pd *pd, 417 struct ib_qp_init_attr *qp_attrs, struct c2_qp *qp) 418{ 419 struct c2wr_qp_create_req wr; 420 struct c2wr_qp_create_rep *reply; 421 struct c2_vq_req *vq_req; 422 struct c2_cq *send_cq = to_c2cq(qp_attrs->send_cq); 423 struct c2_cq *recv_cq = to_c2cq(qp_attrs->recv_cq); 424 unsigned long peer_pa; 425 u32 q_size, msg_size, mmap_size; 426 void __iomem *mmap; 427 int err; 428 429 err = c2_alloc_qpn(c2dev, qp); 430 if (err) 431 return err; 432 qp->ibqp.qp_num = qp->qpn; 433 qp->ibqp.qp_type = IB_QPT_RC; 434 435 /* Allocate the SQ and RQ shared pointers */ 436 qp->sq_mq.shared = c2_alloc_mqsp(c2dev, c2dev->kern_mqsp_pool, 437 &qp->sq_mq.shared_dma, GFP_KERNEL); 438 if (!qp->sq_mq.shared) { 439 err = -ENOMEM; 440 goto bail0; 441 } 442 443 qp->rq_mq.shared = c2_alloc_mqsp(c2dev, c2dev->kern_mqsp_pool, 444 &qp->rq_mq.shared_dma, GFP_KERNEL); 445 if (!qp->rq_mq.shared) { 446 err = -ENOMEM; 447 goto bail1; 448 } 449 450 /* Allocate the verbs request */ 451 vq_req = vq_req_alloc(c2dev); 452 if (vq_req == NULL) { 453 err = -ENOMEM; 454 goto bail2; 455 } 456 457 /* Initialize the work request */ 458 memset(&wr, 0, sizeof(wr)); 459 c2_wr_set_id(&wr, CCWR_QP_CREATE); 460 wr.hdr.context = (unsigned long) vq_req; 461 wr.rnic_handle = c2dev->adapter_handle; 462 wr.sq_cq_handle = send_cq->adapter_handle; 463 wr.rq_cq_handle = recv_cq->adapter_handle; 464 wr.sq_depth = cpu_to_be32(qp_attrs->cap.max_send_wr + 1); 465 wr.rq_depth = cpu_to_be32(qp_attrs->cap.max_recv_wr + 1); 466 wr.srq_handle = 0; 467 wr.flags = cpu_to_be32(QP_RDMA_READ | QP_RDMA_WRITE | QP_MW_BIND | 468 QP_ZERO_STAG | QP_RDMA_READ_RESPONSE); 469 wr.send_sgl_depth = cpu_to_be32(qp_attrs->cap.max_send_sge); 470 wr.recv_sgl_depth = cpu_to_be32(qp_attrs->cap.max_recv_sge); 471 wr.rdma_write_sgl_depth = cpu_to_be32(qp_attrs->cap.max_send_sge); 472 wr.shared_sq_ht = cpu_to_be64(qp->sq_mq.shared_dma); 473 wr.shared_rq_ht = cpu_to_be64(qp->rq_mq.shared_dma); 474 wr.ord = cpu_to_be32(C2_MAX_ORD_PER_QP); 475 wr.ird = cpu_to_be32(C2_MAX_IRD_PER_QP); 476 wr.pd_id = pd->pd_id; 477 wr.user_context = (unsigned long) qp; 478 479 vq_req_get(c2dev, vq_req); 480 481 /* Send the WR to the adapter */ 482 err = vq_send_wr(c2dev, (union c2wr *) & wr); 483 if (err) { 484 vq_req_put(c2dev, vq_req); 485 goto bail3; 486 } 487 488 /* Wait for the verb reply */ 489 err = vq_wait_for_reply(c2dev, vq_req); 490 if (err) { 491 goto bail3; 492 } 493 494 /* Process the reply */ 495 reply = (struct c2wr_qp_create_rep *) (unsigned long) (vq_req->reply_msg); 496 if (!reply) { 497 err = -ENOMEM; 498 goto bail3; 499 } 500 501 if ((err = c2_wr_get_result(reply)) != 0) { 502 goto bail4; 503 } 504 505 /* Fill in the kernel QP struct */ 506 atomic_set(&qp->refcount, 1); 507 qp->adapter_handle = reply->qp_handle; 508 qp->state = IB_QPS_RESET; 509 qp->send_sgl_depth = qp_attrs->cap.max_send_sge; 510 qp->rdma_write_sgl_depth = qp_attrs->cap.max_send_sge; 511 qp->recv_sgl_depth = qp_attrs->cap.max_recv_sge; 512 init_waitqueue_head(&qp->wait); 513 514 /* Initialize the SQ MQ */ 515 q_size = be32_to_cpu(reply->sq_depth); 516 msg_size = be32_to_cpu(reply->sq_msg_size); 517 peer_pa = c2dev->pa + be32_to_cpu(reply->sq_mq_start); 518 mmap_size = PAGE_ALIGN(sizeof(struct c2_mq_shared) + msg_size * q_size); 519 mmap = ioremap_nocache(peer_pa, mmap_size); 520 if (!mmap) { 521 err = -ENOMEM; 522 goto bail5; 523 } 524 525 c2_mq_req_init(&qp->sq_mq, 526 be32_to_cpu(reply->sq_mq_index), 527 q_size, 528 msg_size, 529 mmap + sizeof(struct c2_mq_shared), /* pool start */ 530 mmap, /* peer */ 531 C2_MQ_ADAPTER_TARGET); 532 533 /* Initialize the RQ mq */ 534 q_size = be32_to_cpu(reply->rq_depth); 535 msg_size = be32_to_cpu(reply->rq_msg_size); 536 peer_pa = c2dev->pa + be32_to_cpu(reply->rq_mq_start); 537 mmap_size = PAGE_ALIGN(sizeof(struct c2_mq_shared) + msg_size * q_size); 538 mmap = ioremap_nocache(peer_pa, mmap_size); 539 if (!mmap) { 540 err = -ENOMEM; 541 goto bail6; 542 } 543 544 c2_mq_req_init(&qp->rq_mq, 545 be32_to_cpu(reply->rq_mq_index), 546 q_size, 547 msg_size, 548 mmap + sizeof(struct c2_mq_shared), /* pool start */ 549 mmap, /* peer */ 550 C2_MQ_ADAPTER_TARGET); 551 552 vq_repbuf_free(c2dev, reply); 553 vq_req_free(c2dev, vq_req); 554 555 return 0; 556 557 bail6: 558 iounmap(qp->sq_mq.peer); 559 bail5: 560 destroy_qp(c2dev, qp); 561 bail4: 562 vq_repbuf_free(c2dev, reply); 563 bail3: 564 vq_req_free(c2dev, vq_req); 565 bail2: 566 c2_free_mqsp(qp->rq_mq.shared); 567 bail1: 568 c2_free_mqsp(qp->sq_mq.shared); 569 bail0: 570 c2_free_qpn(c2dev, qp->qpn); 571 return err; 572} 573 574static inline void c2_lock_cqs(struct c2_cq *send_cq, struct c2_cq *recv_cq) 575{ 576 if (send_cq == recv_cq) 577 spin_lock_irq(&send_cq->lock); 578 else if (send_cq > recv_cq) { 579 spin_lock_irq(&send_cq->lock); 580 spin_lock_nested(&recv_cq->lock, SINGLE_DEPTH_NESTING); 581 } else { 582 spin_lock_irq(&recv_cq->lock); 583 spin_lock_nested(&send_cq->lock, SINGLE_DEPTH_NESTING); 584 } 585} 586 587static inline void c2_unlock_cqs(struct c2_cq *send_cq, struct c2_cq *recv_cq) 588{ 589 if (send_cq == recv_cq) 590 spin_unlock_irq(&send_cq->lock); 591 else if (send_cq > recv_cq) { 592 spin_unlock(&recv_cq->lock); 593 spin_unlock_irq(&send_cq->lock); 594 } else { 595 spin_unlock(&send_cq->lock); 596 spin_unlock_irq(&recv_cq->lock); 597 } 598} 599 600void c2_free_qp(struct c2_dev *c2dev, struct c2_qp *qp) 601{ 602 struct c2_cq *send_cq; 603 struct c2_cq *recv_cq; 604 605 send_cq = to_c2cq(qp->ibqp.send_cq); 606 recv_cq = to_c2cq(qp->ibqp.recv_cq); 607 608 /* 609 * Lock CQs here, so that CQ polling code can do QP lookup 610 * without taking a lock. 611 */ 612 c2_lock_cqs(send_cq, recv_cq); 613 c2_free_qpn(c2dev, qp->qpn); 614 c2_unlock_cqs(send_cq, recv_cq); 615 616 /* 617 * Destroy qp in the rnic... 618 */ 619 destroy_qp(c2dev, qp); 620 621 /* 622 * Mark any unreaped CQEs as null and void. 623 */ 624 c2_cq_clean(c2dev, qp, send_cq->cqn); 625 if (send_cq != recv_cq) 626 c2_cq_clean(c2dev, qp, recv_cq->cqn); 627 /* 628 * Unmap the MQs and return the shared pointers 629 * to the message pool. 630 */ 631 iounmap(qp->sq_mq.peer); 632 iounmap(qp->rq_mq.peer); 633 c2_free_mqsp(qp->sq_mq.shared); 634 c2_free_mqsp(qp->rq_mq.shared); 635 636 atomic_dec(&qp->refcount); 637 wait_event(qp->wait, !atomic_read(&qp->refcount)); 638} 639 640/* 641 * Function: move_sgl 642 * 643 * Description: 644 * Move an SGL from the user's work request struct into a CCIL Work Request 645 * message, swapping to WR byte order and ensure the total length doesn't 646 * overflow. 647 * 648 * IN: 649 * dst - ptr to CCIL Work Request message SGL memory. 650 * src - ptr to the consumers SGL memory. 651 * 652 * OUT: none 653 * 654 * Return: 655 * CCIL status codes. 656 */ 657static int 658move_sgl(struct c2_data_addr * dst, struct ib_sge *src, int count, u32 * p_len, 659 u8 * actual_count) 660{ 661 u32 tot = 0; /* running total */ 662 u8 acount = 0; /* running total non-0 len sge's */ 663 664 while (count > 0) { 665 /* 666 * If the addition of this SGE causes the 667 * total SGL length to exceed 2^32-1, then 668 * fail-n-bail. 669 * 670 * If the current total plus the next element length 671 * wraps, then it will go negative and be less than the 672 * current total... 673 */ 674 if ((tot + src->length) < tot) { 675 return -EINVAL; 676 } 677 /* 678 * Bug: 1456 (as well as 1498 & 1643) 679 * Skip over any sge's supplied with len=0 680 */ 681 if (src->length) { 682 tot += src->length; 683 dst->stag = cpu_to_be32(src->lkey); 684 dst->to = cpu_to_be64(src->addr); 685 dst->length = cpu_to_be32(src->length); 686 dst++; 687 acount++; 688 } 689 src++; 690 count--; 691 } 692 693 if (acount == 0) { 694 /* 695 * Bug: 1476 (as well as 1498, 1456 and 1643) 696 * Setup the SGL in the WR to make it easier for the RNIC. 697 * This way, the FW doesn't have to deal with special cases. 698 * Setting length=0 should be sufficient. 699 */ 700 dst->stag = 0; 701 dst->to = 0; 702 dst->length = 0; 703 } 704 705 *p_len = tot; 706 *actual_count = acount; 707 return 0; 708} 709 710/* 711 * Function: c2_activity (private function) 712 * 713 * Description: 714 * Post an mq index to the host->adapter activity fifo. 715 * 716 * IN: 717 * c2dev - ptr to c2dev structure 718 * mq_index - mq index to post 719 * shared - value most recently written to shared 720 * 721 * OUT: 722 * 723 * Return: 724 * none 725 */ 726static inline void c2_activity(struct c2_dev *c2dev, u32 mq_index, u16 shared) 727{ 728 /* 729 * First read the register to see if the FIFO is full, and if so, 730 * spin until it's not. This isn't perfect -- there is no 731 * synchronization among the clients of the register, but in 732 * practice it prevents multiple CPU from hammering the bus 733 * with PCI RETRY. Note that when this does happen, the card 734 * cannot get on the bus and the card and system hang in a 735 * deadlock -- thus the need for this code. [TOT] 736 */ 737 while (readl(c2dev->regs + PCI_BAR0_ADAPTER_HINT) & 0x80000000) 738 udelay(10); 739 740 __raw_writel(C2_HINT_MAKE(mq_index, shared), 741 c2dev->regs + PCI_BAR0_ADAPTER_HINT); 742} 743 744/* 745 * Function: qp_wr_post 746 * 747 * Description: 748 * This in-line function allocates a MQ msg, then moves the host-copy of 749 * the completed WR into msg. Then it posts the message. 750 * 751 * IN: 752 * q - ptr to user MQ. 753 * wr - ptr to host-copy of the WR. 754 * qp - ptr to user qp 755 * size - Number of bytes to post. Assumed to be divisible by 4. 756 * 757 * OUT: none 758 * 759 * Return: 760 * CCIL status codes. 761 */ 762static int qp_wr_post(struct c2_mq *q, union c2wr * wr, struct c2_qp *qp, u32 size) 763{ 764 union c2wr *msg; 765 766 msg = c2_mq_alloc(q); 767 if (msg == NULL) { 768 return -EINVAL; 769 } 770#ifdef CCMSGMAGIC 771 ((c2wr_hdr_t *) wr)->magic = cpu_to_be32(CCWR_MAGIC); 772#endif 773 774 /* 775 * Since all header fields in the WR are the same as the 776 * CQE, set the following so the adapter need not. 777 */ 778 c2_wr_set_result(wr, CCERR_PENDING); 779 780 /* 781 * Copy the wr down to the adapter 782 */ 783 memcpy((void *) msg, (void *) wr, size); 784 785 c2_mq_produce(q); 786 return 0; 787} 788 789 790int c2_post_send(struct ib_qp *ibqp, struct ib_send_wr *ib_wr, 791 struct ib_send_wr **bad_wr) 792{ 793 struct c2_dev *c2dev = to_c2dev(ibqp->device); 794 struct c2_qp *qp = to_c2qp(ibqp); 795 union c2wr wr; 796 unsigned long lock_flags; 797 int err = 0; 798 799 u32 flags; 800 u32 tot_len; 801 u8 actual_sge_count; 802 u32 msg_size; 803 804 if (qp->state > IB_QPS_RTS) { 805 err = -EINVAL; 806 goto out; 807 } 808 809 while (ib_wr) { 810 811 flags = 0; 812 wr.sqwr.sq_hdr.user_hdr.hdr.context = ib_wr->wr_id; 813 if (ib_wr->send_flags & IB_SEND_SIGNALED) { 814 flags |= SQ_SIGNALED; 815 } 816 817 switch (ib_wr->opcode) { 818 case IB_WR_SEND: 819 case IB_WR_SEND_WITH_INV: 820 if (ib_wr->opcode == IB_WR_SEND) { 821 if (ib_wr->send_flags & IB_SEND_SOLICITED) 822 c2_wr_set_id(&wr, C2_WR_TYPE_SEND_SE); 823 else 824 c2_wr_set_id(&wr, C2_WR_TYPE_SEND); 825 wr.sqwr.send.remote_stag = 0; 826 } else { 827 if (ib_wr->send_flags & IB_SEND_SOLICITED) 828 c2_wr_set_id(&wr, C2_WR_TYPE_SEND_SE_INV); 829 else 830 c2_wr_set_id(&wr, C2_WR_TYPE_SEND_INV); 831 wr.sqwr.send.remote_stag = 832 cpu_to_be32(ib_wr->ex.invalidate_rkey); 833 } 834 835 msg_size = sizeof(struct c2wr_send_req) + 836 sizeof(struct c2_data_addr) * ib_wr->num_sge; 837 if (ib_wr->num_sge > qp->send_sgl_depth) { 838 err = -EINVAL; 839 break; 840 } 841 if (ib_wr->send_flags & IB_SEND_FENCE) { 842 flags |= SQ_READ_FENCE; 843 } 844 err = move_sgl((struct c2_data_addr *) & (wr.sqwr.send.data), 845 ib_wr->sg_list, 846 ib_wr->num_sge, 847 &tot_len, &actual_sge_count); 848 wr.sqwr.send.sge_len = cpu_to_be32(tot_len); 849 c2_wr_set_sge_count(&wr, actual_sge_count); 850 break; 851 case IB_WR_RDMA_WRITE: 852 c2_wr_set_id(&wr, C2_WR_TYPE_RDMA_WRITE); 853 msg_size = sizeof(struct c2wr_rdma_write_req) + 854 (sizeof(struct c2_data_addr) * ib_wr->num_sge); 855 if (ib_wr->num_sge > qp->rdma_write_sgl_depth) { 856 err = -EINVAL; 857 break; 858 } 859 if (ib_wr->send_flags & IB_SEND_FENCE) { 860 flags |= SQ_READ_FENCE; 861 } 862 wr.sqwr.rdma_write.remote_stag = 863 cpu_to_be32(ib_wr->wr.rdma.rkey); 864 wr.sqwr.rdma_write.remote_to = 865 cpu_to_be64(ib_wr->wr.rdma.remote_addr); 866 err = move_sgl((struct c2_data_addr *) 867 & (wr.sqwr.rdma_write.data), 868 ib_wr->sg_list, 869 ib_wr->num_sge, 870 &tot_len, &actual_sge_count); 871 wr.sqwr.rdma_write.sge_len = cpu_to_be32(tot_len); 872 c2_wr_set_sge_count(&wr, actual_sge_count); 873 break; 874 case IB_WR_RDMA_READ: 875 c2_wr_set_id(&wr, C2_WR_TYPE_RDMA_READ); 876 msg_size = sizeof(struct c2wr_rdma_read_req); 877 878 /* IWarp only suppots 1 sge for RDMA reads */ 879 if (ib_wr->num_sge > 1) { 880 err = -EINVAL; 881 break; 882 } 883 884 /* 885 * Move the local and remote stag/to/len into the WR. 886 */ 887 wr.sqwr.rdma_read.local_stag = 888 cpu_to_be32(ib_wr->sg_list->lkey); 889 wr.sqwr.rdma_read.local_to = 890 cpu_to_be64(ib_wr->sg_list->addr); 891 wr.sqwr.rdma_read.remote_stag = 892 cpu_to_be32(ib_wr->wr.rdma.rkey); 893 wr.sqwr.rdma_read.remote_to = 894 cpu_to_be64(ib_wr->wr.rdma.remote_addr); 895 wr.sqwr.rdma_read.length = 896 cpu_to_be32(ib_wr->sg_list->length); 897 break; 898 default: 899 /* error */ 900 msg_size = 0; 901 err = -EINVAL; 902 break; 903 } 904 905 /* 906 * If we had an error on the last wr build, then 907 * break out. Possible errors include bogus WR 908 * type, and a bogus SGL length... 909 */ 910 if (err) { 911 break; 912 } 913 914 /* 915 * Store flags 916 */ 917 c2_wr_set_flags(&wr, flags); 918 919 /* 920 * Post the puppy! 921 */ 922 spin_lock_irqsave(&qp->lock, lock_flags); 923 err = qp_wr_post(&qp->sq_mq, &wr, qp, msg_size); 924 if (err) { 925 spin_unlock_irqrestore(&qp->lock, lock_flags); 926 break; 927 } 928 929 /* 930 * Enqueue mq index to activity FIFO. 931 */ 932 c2_activity(c2dev, qp->sq_mq.index, qp->sq_mq.hint_count); 933 spin_unlock_irqrestore(&qp->lock, lock_flags); 934 935 ib_wr = ib_wr->next; 936 } 937 938out: 939 if (err) 940 *bad_wr = ib_wr; 941 return err; 942} 943 944int c2_post_receive(struct ib_qp *ibqp, struct ib_recv_wr *ib_wr, 945 struct ib_recv_wr **bad_wr) 946{ 947 struct c2_dev *c2dev = to_c2dev(ibqp->device); 948 struct c2_qp *qp = to_c2qp(ibqp); 949 union c2wr wr; 950 unsigned long lock_flags; 951 int err = 0; 952 953 if (qp->state > IB_QPS_RTS) { 954 err = -EINVAL; 955 goto out; 956 } 957 958 /* 959 * Try and post each work request 960 */ 961 while (ib_wr) { 962 u32 tot_len; 963 u8 actual_sge_count; 964 965 if (ib_wr->num_sge > qp->recv_sgl_depth) { 966 err = -EINVAL; 967 break; 968 } 969 970 /* 971 * Create local host-copy of the WR 972 */ 973 wr.rqwr.rq_hdr.user_hdr.hdr.context = ib_wr->wr_id; 974 c2_wr_set_id(&wr, CCWR_RECV); 975 c2_wr_set_flags(&wr, 0); 976 977 /* sge_count is limited to eight bits. */ 978 BUG_ON(ib_wr->num_sge >= 256); 979 err = move_sgl((struct c2_data_addr *) & (wr.rqwr.data), 980 ib_wr->sg_list, 981 ib_wr->num_sge, &tot_len, &actual_sge_count); 982 c2_wr_set_sge_count(&wr, actual_sge_count); 983 984 /* 985 * If we had an error on the last wr build, then 986 * break out. Possible errors include bogus WR 987 * type, and a bogus SGL length... 988 */ 989 if (err) { 990 break; 991 } 992 993 spin_lock_irqsave(&qp->lock, lock_flags); 994 err = qp_wr_post(&qp->rq_mq, &wr, qp, qp->rq_mq.msg_size); 995 if (err) { 996 spin_unlock_irqrestore(&qp->lock, lock_flags); 997 break; 998 } 999 1000 /* 1001 * Enqueue mq index to activity FIFO 1002 */ 1003 c2_activity(c2dev, qp->rq_mq.index, qp->rq_mq.hint_count); 1004 spin_unlock_irqrestore(&qp->lock, lock_flags); 1005 1006 ib_wr = ib_wr->next; 1007 } 1008 1009out: 1010 if (err) 1011 *bad_wr = ib_wr; 1012 return err; 1013} 1014 1015void c2_init_qp_table(struct c2_dev *c2dev) 1016{ 1017 spin_lock_init(&c2dev->qp_table.lock); 1018 idr_init(&c2dev->qp_table.idr); 1019} 1020 1021void c2_cleanup_qp_table(struct c2_dev *c2dev) 1022{ 1023 idr_destroy(&c2dev->qp_table.idr); 1024} 1025