Lines Matching refs:cq
42 static void mlx4_ib_cq_comp(struct mlx4_cq *cq) in mlx4_ib_cq_comp() argument
44 struct ib_cq *ibcq = &to_mibcq(cq)->ibcq; in mlx4_ib_cq_comp()
48 static void mlx4_ib_cq_event(struct mlx4_cq *cq, enum mlx4_event type) in mlx4_ib_cq_event() argument
55 "on CQ %06x\n", type, cq->cqn); in mlx4_ib_cq_event()
59 ibcq = &to_mibcq(cq)->ibcq; in mlx4_ib_cq_event()
63 event.element.cq = ibcq; in mlx4_ib_cq_event()
73 static void *get_cqe(struct mlx4_ib_cq *cq, int n) in get_cqe() argument
75 return get_cqe_from_buf(&cq->buf, n); in get_cqe()
78 static void *get_sw_cqe(struct mlx4_ib_cq *cq, int n) in get_sw_cqe() argument
80 struct mlx4_cqe *cqe = get_cqe(cq, n & cq->ibcq.cqe); in get_sw_cqe()
81 struct mlx4_cqe *tcqe = ((cq->buf.entry_size == 64) ? (cqe + 1) : cqe); in get_sw_cqe()
84 !!(n & (cq->ibcq.cqe + 1))) ? NULL : cqe; in get_sw_cqe()
87 static struct mlx4_cqe *next_cqe_sw(struct mlx4_ib_cq *cq) in next_cqe_sw() argument
89 return get_sw_cqe(cq, cq->mcq.cons_index); in next_cqe_sw()
92 int mlx4_ib_modify_cq(struct ib_cq *cq, u16 cq_count, u16 cq_period) in mlx4_ib_modify_cq() argument
94 struct mlx4_ib_cq *mcq = to_mcq(cq); in mlx4_ib_modify_cq()
95 struct mlx4_ib_dev *dev = to_mdev(cq->device); in mlx4_ib_modify_cq()
174 struct mlx4_ib_cq *cq; in mlx4_ib_create_cq() local
181 cq = kmalloc(sizeof *cq, GFP_KERNEL); in mlx4_ib_create_cq()
182 if (!cq) in mlx4_ib_create_cq()
186 cq->ibcq.cqe = entries - 1; in mlx4_ib_create_cq()
187 mutex_init(&cq->resize_mutex); in mlx4_ib_create_cq()
188 spin_lock_init(&cq->lock); in mlx4_ib_create_cq()
189 cq->resize_buf = NULL; in mlx4_ib_create_cq()
190 cq->resize_umem = NULL; in mlx4_ib_create_cq()
191 INIT_LIST_HEAD(&cq->send_qp_list); in mlx4_ib_create_cq()
192 INIT_LIST_HEAD(&cq->recv_qp_list); in mlx4_ib_create_cq()
202 err = mlx4_ib_get_cq_umem(dev, context, &cq->buf, &cq->umem, in mlx4_ib_create_cq()
208 &cq->db); in mlx4_ib_create_cq()
214 err = mlx4_db_alloc(dev->dev, &cq->db, 1, GFP_KERNEL); in mlx4_ib_create_cq()
218 cq->mcq.set_ci_db = cq->db.db; in mlx4_ib_create_cq()
219 cq->mcq.arm_db = cq->db.db + 1; in mlx4_ib_create_cq()
220 *cq->mcq.set_ci_db = 0; in mlx4_ib_create_cq()
221 *cq->mcq.arm_db = 0; in mlx4_ib_create_cq()
223 err = mlx4_ib_alloc_cq_buf(dev, &cq->buf, entries); in mlx4_ib_create_cq()
233 err = mlx4_cq_alloc(dev->dev, entries, &cq->buf.mtt, uar, in mlx4_ib_create_cq()
234 cq->db.dma, &cq->mcq, vector, 0, 0); in mlx4_ib_create_cq()
239 cq->mcq.tasklet_ctx.comp = mlx4_ib_cq_comp; in mlx4_ib_create_cq()
241 cq->mcq.comp = mlx4_ib_cq_comp; in mlx4_ib_create_cq()
242 cq->mcq.event = mlx4_ib_cq_event; in mlx4_ib_create_cq()
245 if (ib_copy_to_udata(udata, &cq->mcq.cqn, sizeof (__u32))) { in mlx4_ib_create_cq()
250 return &cq->ibcq; in mlx4_ib_create_cq()
254 mlx4_ib_db_unmap_user(to_mucontext(context), &cq->db); in mlx4_ib_create_cq()
257 mlx4_mtt_cleanup(dev->dev, &cq->buf.mtt); in mlx4_ib_create_cq()
260 ib_umem_release(cq->umem); in mlx4_ib_create_cq()
262 mlx4_ib_free_cq_buf(dev, &cq->buf, cq->ibcq.cqe); in mlx4_ib_create_cq()
266 mlx4_db_free(dev->dev, &cq->db); in mlx4_ib_create_cq()
269 kfree(cq); in mlx4_ib_create_cq()
274 static int mlx4_alloc_resize_buf(struct mlx4_ib_dev *dev, struct mlx4_ib_cq *cq, in mlx4_alloc_resize_buf() argument
279 if (cq->resize_buf) in mlx4_alloc_resize_buf()
282 cq->resize_buf = kmalloc(sizeof *cq->resize_buf, GFP_ATOMIC); in mlx4_alloc_resize_buf()
283 if (!cq->resize_buf) in mlx4_alloc_resize_buf()
286 err = mlx4_ib_alloc_cq_buf(dev, &cq->resize_buf->buf, entries); in mlx4_alloc_resize_buf()
288 kfree(cq->resize_buf); in mlx4_alloc_resize_buf()
289 cq->resize_buf = NULL; in mlx4_alloc_resize_buf()
293 cq->resize_buf->cqe = entries - 1; in mlx4_alloc_resize_buf()
298 static int mlx4_alloc_resize_umem(struct mlx4_ib_dev *dev, struct mlx4_ib_cq *cq, in mlx4_alloc_resize_umem() argument
304 if (cq->resize_umem) in mlx4_alloc_resize_umem()
310 cq->resize_buf = kmalloc(sizeof *cq->resize_buf, GFP_ATOMIC); in mlx4_alloc_resize_umem()
311 if (!cq->resize_buf) in mlx4_alloc_resize_umem()
314 err = mlx4_ib_get_cq_umem(dev, cq->umem->context, &cq->resize_buf->buf, in mlx4_alloc_resize_umem()
315 &cq->resize_umem, ucmd.buf_addr, entries); in mlx4_alloc_resize_umem()
317 kfree(cq->resize_buf); in mlx4_alloc_resize_umem()
318 cq->resize_buf = NULL; in mlx4_alloc_resize_umem()
322 cq->resize_buf->cqe = entries - 1; in mlx4_alloc_resize_umem()
327 static int mlx4_ib_get_outstanding_cqes(struct mlx4_ib_cq *cq) in mlx4_ib_get_outstanding_cqes() argument
331 i = cq->mcq.cons_index; in mlx4_ib_get_outstanding_cqes()
332 while (get_sw_cqe(cq, i)) in mlx4_ib_get_outstanding_cqes()
335 return i - cq->mcq.cons_index; in mlx4_ib_get_outstanding_cqes()
338 static void mlx4_ib_cq_resize_copy_cqes(struct mlx4_ib_cq *cq) in mlx4_ib_cq_resize_copy_cqes() argument
342 int cqe_size = cq->buf.entry_size; in mlx4_ib_cq_resize_copy_cqes()
345 i = cq->mcq.cons_index; in mlx4_ib_cq_resize_copy_cqes()
346 cqe = get_cqe(cq, i & cq->ibcq.cqe); in mlx4_ib_cq_resize_copy_cqes()
350 new_cqe = get_cqe_from_buf(&cq->resize_buf->buf, in mlx4_ib_cq_resize_copy_cqes()
351 (i + 1) & cq->resize_buf->cqe); in mlx4_ib_cq_resize_copy_cqes()
352 memcpy(new_cqe, get_cqe(cq, i & cq->ibcq.cqe), cqe_size); in mlx4_ib_cq_resize_copy_cqes()
356 (((i + 1) & (cq->resize_buf->cqe + 1)) ? MLX4_CQE_OWNER_MASK : 0); in mlx4_ib_cq_resize_copy_cqes()
357 cqe = get_cqe(cq, ++i & cq->ibcq.cqe); in mlx4_ib_cq_resize_copy_cqes()
360 ++cq->mcq.cons_index; in mlx4_ib_cq_resize_copy_cqes()
366 struct mlx4_ib_cq *cq = to_mcq(ibcq); in mlx4_ib_resize_cq() local
371 mutex_lock(&cq->resize_mutex); in mlx4_ib_resize_cq()
389 err = mlx4_alloc_resize_umem(dev, cq, entries, udata); in mlx4_ib_resize_cq()
394 outst_cqe = mlx4_ib_get_outstanding_cqes(cq); in mlx4_ib_resize_cq()
400 err = mlx4_alloc_resize_buf(dev, cq, entries); in mlx4_ib_resize_cq()
405 mtt = cq->buf.mtt; in mlx4_ib_resize_cq()
407 err = mlx4_cq_resize(dev->dev, &cq->mcq, entries, &cq->resize_buf->buf.mtt); in mlx4_ib_resize_cq()
413 cq->buf = cq->resize_buf->buf; in mlx4_ib_resize_cq()
414 cq->ibcq.cqe = cq->resize_buf->cqe; in mlx4_ib_resize_cq()
415 ib_umem_release(cq->umem); in mlx4_ib_resize_cq()
416 cq->umem = cq->resize_umem; in mlx4_ib_resize_cq()
418 kfree(cq->resize_buf); in mlx4_ib_resize_cq()
419 cq->resize_buf = NULL; in mlx4_ib_resize_cq()
420 cq->resize_umem = NULL; in mlx4_ib_resize_cq()
425 spin_lock_irq(&cq->lock); in mlx4_ib_resize_cq()
426 if (cq->resize_buf) { in mlx4_ib_resize_cq()
427 mlx4_ib_cq_resize_copy_cqes(cq); in mlx4_ib_resize_cq()
428 tmp_buf = cq->buf; in mlx4_ib_resize_cq()
429 tmp_cqe = cq->ibcq.cqe; in mlx4_ib_resize_cq()
430 cq->buf = cq->resize_buf->buf; in mlx4_ib_resize_cq()
431 cq->ibcq.cqe = cq->resize_buf->cqe; in mlx4_ib_resize_cq()
433 kfree(cq->resize_buf); in mlx4_ib_resize_cq()
434 cq->resize_buf = NULL; in mlx4_ib_resize_cq()
436 spin_unlock_irq(&cq->lock); in mlx4_ib_resize_cq()
445 mlx4_mtt_cleanup(dev->dev, &cq->resize_buf->buf.mtt); in mlx4_ib_resize_cq()
447 mlx4_ib_free_cq_buf(dev, &cq->resize_buf->buf, in mlx4_ib_resize_cq()
448 cq->resize_buf->cqe); in mlx4_ib_resize_cq()
450 kfree(cq->resize_buf); in mlx4_ib_resize_cq()
451 cq->resize_buf = NULL; in mlx4_ib_resize_cq()
453 if (cq->resize_umem) { in mlx4_ib_resize_cq()
454 ib_umem_release(cq->resize_umem); in mlx4_ib_resize_cq()
455 cq->resize_umem = NULL; in mlx4_ib_resize_cq()
459 mutex_unlock(&cq->resize_mutex); in mlx4_ib_resize_cq()
464 int mlx4_ib_destroy_cq(struct ib_cq *cq) in mlx4_ib_destroy_cq() argument
466 struct mlx4_ib_dev *dev = to_mdev(cq->device); in mlx4_ib_destroy_cq()
467 struct mlx4_ib_cq *mcq = to_mcq(cq); in mlx4_ib_destroy_cq()
472 if (cq->uobject) { in mlx4_ib_destroy_cq()
473 mlx4_ib_db_unmap_user(to_mucontext(cq->uobject->context), &mcq->db); in mlx4_ib_destroy_cq()
476 mlx4_ib_free_cq_buf(dev, &mcq->buf, cq->cqe); in mlx4_ib_destroy_cq()
570 static int use_tunnel_data(struct mlx4_ib_qp *qp, struct mlx4_ib_cq *cq, struct ib_wc *wc, in use_tunnel_data() argument
622 static void mlx4_ib_poll_sw_comp(struct mlx4_ib_cq *cq, int num_entries, in mlx4_ib_poll_sw_comp() argument
631 list_for_each_entry(qp, &cq->send_qp_list, cq_send_list) { in mlx4_ib_poll_sw_comp()
637 list_for_each_entry(qp, &cq->recv_qp_list, cq_recv_list) { in mlx4_ib_poll_sw_comp()
647 static int mlx4_ib_poll_one(struct mlx4_ib_cq *cq, in mlx4_ib_poll_one() argument
664 cqe = next_cqe_sw(cq); in mlx4_ib_poll_one()
668 if (cq->buf.entry_size == 64) in mlx4_ib_poll_one()
671 ++cq->mcq.cons_index; in mlx4_ib_poll_one()
691 if (cq->resize_buf) { in mlx4_ib_poll_one()
692 struct mlx4_ib_dev *dev = to_mdev(cq->ibcq.device); in mlx4_ib_poll_one()
694 mlx4_ib_free_cq_buf(dev, &cq->buf, cq->ibcq.cqe); in mlx4_ib_poll_one()
695 cq->buf = cq->resize_buf->buf; in mlx4_ib_poll_one()
696 cq->ibcq.cqe = cq->resize_buf->cqe; in mlx4_ib_poll_one()
698 kfree(cq->resize_buf); in mlx4_ib_poll_one()
699 cq->resize_buf = NULL; in mlx4_ib_poll_one()
712 mqp = __mlx4_qp_lookup(to_mdev(cq->ibcq.device)->dev, in mlx4_ib_poll_one()
716 cq->mcq.cqn, be32_to_cpu(cqe->vlan_my_qpn) & MLX4_CQE_QPN_MASK); in mlx4_ib_poll_one()
730 msrq = mlx4_srq_lookup(to_mdev(cq->ibcq.device)->dev, in mlx4_ib_poll_one()
734 cq->mcq.cqn, srq_num); in mlx4_ib_poll_one()
846 if (mlx4_is_mfunc(to_mdev(cq->ibcq.device)->dev)) { in mlx4_ib_poll_one()
850 return use_tunnel_data(*cur_qp, cq, wc, tail, in mlx4_ib_poll_one()
884 struct mlx4_ib_cq *cq = to_mcq(ibcq); in mlx4_ib_poll_cq() local
889 struct mlx4_ib_dev *mdev = to_mdev(cq->ibcq.device); in mlx4_ib_poll_cq()
891 spin_lock_irqsave(&cq->lock, flags); in mlx4_ib_poll_cq()
893 mlx4_ib_poll_sw_comp(cq, num_entries, wc, &npolled); in mlx4_ib_poll_cq()
898 err = mlx4_ib_poll_one(cq, &cur_qp, wc + npolled); in mlx4_ib_poll_cq()
903 mlx4_cq_set_ci(&cq->mcq); in mlx4_ib_poll_cq()
906 spin_unlock_irqrestore(&cq->lock, flags); in mlx4_ib_poll_cq()
925 void __mlx4_ib_cq_clean(struct mlx4_ib_cq *cq, u32 qpn, struct mlx4_ib_srq *srq) in __mlx4_ib_cq_clean() argument
931 int cqe_inc = cq->buf.entry_size == 64 ? 1 : 0; in __mlx4_ib_cq_clean()
940 for (prod_index = cq->mcq.cons_index; get_sw_cqe(cq, prod_index); ++prod_index) in __mlx4_ib_cq_clean()
941 if (prod_index == cq->mcq.cons_index + cq->ibcq.cqe) in __mlx4_ib_cq_clean()
948 while ((int) --prod_index - (int) cq->mcq.cons_index >= 0) { in __mlx4_ib_cq_clean()
949 cqe = get_cqe(cq, prod_index & cq->ibcq.cqe); in __mlx4_ib_cq_clean()
957 dest = get_cqe(cq, (prod_index + nfreed) & cq->ibcq.cqe); in __mlx4_ib_cq_clean()
968 cq->mcq.cons_index += nfreed; in __mlx4_ib_cq_clean()
974 mlx4_cq_set_ci(&cq->mcq); in __mlx4_ib_cq_clean()
978 void mlx4_ib_cq_clean(struct mlx4_ib_cq *cq, u32 qpn, struct mlx4_ib_srq *srq) in mlx4_ib_cq_clean() argument
980 spin_lock_irq(&cq->lock); in mlx4_ib_cq_clean()
981 __mlx4_ib_cq_clean(cq, qpn, srq); in mlx4_ib_cq_clean()
982 spin_unlock_irq(&cq->lock); in mlx4_ib_cq_clean()