This source file includes following definitions.
- hns_roce_qp_event
- hns_roce_ib_qp_event
- hns_roce_reserve_range_qp
- to_hns_roce_state
- hns_roce_gsi_qp_alloc
- hns_roce_qp_alloc
- hns_roce_qp_remove
- hns_roce_qp_free
- hns_roce_release_range_qp
- hns_roce_set_rq_size
- check_sq_size_with_integrity
- hns_roce_set_user_sq_size
- split_wqe_buf_region
- calc_wqe_bt_page_shift
- set_extend_sge_param
- hns_roce_set_kernel_sq_size
- hns_roce_qp_has_sq
- hns_roce_qp_has_rq
- alloc_rq_inline_buf
- free_rq_inline_buf
- hns_roce_create_qp_common
- hns_roce_create_qp
- to_hr_qp_type
- check_mtu_validate
- hns_roce_check_qp_attr
- hns_roce_modify_qp
- hns_roce_lock_cqs
- hns_roce_unlock_cqs
- get_wqe
- get_recv_wqe
- get_send_wqe
- get_send_extend_sge
- hns_roce_wq_overflow
- hns_roce_init_qp_table
- hns_roce_cleanup_qp_table
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34 #include <linux/pci.h>
35 #include <linux/platform_device.h>
36 #include <rdma/ib_addr.h>
37 #include <rdma/ib_umem.h>
38 #include <rdma/uverbs_ioctl.h>
39 #include "hns_roce_common.h"
40 #include "hns_roce_device.h"
41 #include "hns_roce_hem.h"
42 #include <rdma/hns-abi.h>
43
44 #define SQP_NUM (2 * HNS_ROCE_MAX_PORTS)
45
46 void hns_roce_qp_event(struct hns_roce_dev *hr_dev, u32 qpn, int event_type)
47 {
48 struct device *dev = hr_dev->dev;
49 struct hns_roce_qp *qp;
50
51 xa_lock(&hr_dev->qp_table_xa);
52 qp = __hns_roce_qp_lookup(hr_dev, qpn);
53 if (qp)
54 atomic_inc(&qp->refcount);
55 xa_unlock(&hr_dev->qp_table_xa);
56
57 if (!qp) {
58 dev_warn(dev, "Async event for bogus QP %08x\n", qpn);
59 return;
60 }
61
62 qp->event(qp, (enum hns_roce_event)event_type);
63
64 if (atomic_dec_and_test(&qp->refcount))
65 complete(&qp->free);
66 }
67
68 static void hns_roce_ib_qp_event(struct hns_roce_qp *hr_qp,
69 enum hns_roce_event type)
70 {
71 struct ib_event event;
72 struct ib_qp *ibqp = &hr_qp->ibqp;
73
74 if (ibqp->event_handler) {
75 event.device = ibqp->device;
76 event.element.qp = ibqp;
77 switch (type) {
78 case HNS_ROCE_EVENT_TYPE_PATH_MIG:
79 event.event = IB_EVENT_PATH_MIG;
80 break;
81 case HNS_ROCE_EVENT_TYPE_COMM_EST:
82 event.event = IB_EVENT_COMM_EST;
83 break;
84 case HNS_ROCE_EVENT_TYPE_SQ_DRAINED:
85 event.event = IB_EVENT_SQ_DRAINED;
86 break;
87 case HNS_ROCE_EVENT_TYPE_SRQ_LAST_WQE_REACH:
88 event.event = IB_EVENT_QP_LAST_WQE_REACHED;
89 break;
90 case HNS_ROCE_EVENT_TYPE_WQ_CATAS_ERROR:
91 event.event = IB_EVENT_QP_FATAL;
92 break;
93 case HNS_ROCE_EVENT_TYPE_PATH_MIG_FAILED:
94 event.event = IB_EVENT_PATH_MIG_ERR;
95 break;
96 case HNS_ROCE_EVENT_TYPE_INV_REQ_LOCAL_WQ_ERROR:
97 event.event = IB_EVENT_QP_REQ_ERR;
98 break;
99 case HNS_ROCE_EVENT_TYPE_LOCAL_WQ_ACCESS_ERROR:
100 event.event = IB_EVENT_QP_ACCESS_ERR;
101 break;
102 default:
103 dev_dbg(ibqp->device->dev.parent, "roce_ib: Unexpected event type %d on QP %06lx\n",
104 type, hr_qp->qpn);
105 return;
106 }
107 ibqp->event_handler(&event, ibqp->qp_context);
108 }
109 }
110
111 static int hns_roce_reserve_range_qp(struct hns_roce_dev *hr_dev, int cnt,
112 int align, unsigned long *base)
113 {
114 struct hns_roce_qp_table *qp_table = &hr_dev->qp_table;
115
116 return hns_roce_bitmap_alloc_range(&qp_table->bitmap, cnt, align,
117 base) ?
118 -ENOMEM :
119 0;
120 }
121
122 enum hns_roce_qp_state to_hns_roce_state(enum ib_qp_state state)
123 {
124 switch (state) {
125 case IB_QPS_RESET:
126 return HNS_ROCE_QP_STATE_RST;
127 case IB_QPS_INIT:
128 return HNS_ROCE_QP_STATE_INIT;
129 case IB_QPS_RTR:
130 return HNS_ROCE_QP_STATE_RTR;
131 case IB_QPS_RTS:
132 return HNS_ROCE_QP_STATE_RTS;
133 case IB_QPS_SQD:
134 return HNS_ROCE_QP_STATE_SQD;
135 case IB_QPS_ERR:
136 return HNS_ROCE_QP_STATE_ERR;
137 default:
138 return HNS_ROCE_QP_NUM_STATE;
139 }
140 }
141
142 static int hns_roce_gsi_qp_alloc(struct hns_roce_dev *hr_dev, unsigned long qpn,
143 struct hns_roce_qp *hr_qp)
144 {
145 struct xarray *xa = &hr_dev->qp_table_xa;
146 int ret;
147
148 if (!qpn)
149 return -EINVAL;
150
151 hr_qp->qpn = qpn;
152 atomic_set(&hr_qp->refcount, 1);
153 init_completion(&hr_qp->free);
154
155 ret = xa_err(xa_store_irq(xa, hr_qp->qpn & (hr_dev->caps.num_qps - 1),
156 hr_qp, GFP_KERNEL));
157 if (ret)
158 dev_err(hr_dev->dev, "QPC xa_store failed\n");
159
160 return ret;
161 }
162
163 static int hns_roce_qp_alloc(struct hns_roce_dev *hr_dev, unsigned long qpn,
164 struct hns_roce_qp *hr_qp)
165 {
166 struct hns_roce_qp_table *qp_table = &hr_dev->qp_table;
167 struct device *dev = hr_dev->dev;
168 int ret;
169
170 if (!qpn)
171 return -EINVAL;
172
173 hr_qp->qpn = qpn;
174
175
176 ret = hns_roce_table_get(hr_dev, &qp_table->qp_table, hr_qp->qpn);
177 if (ret) {
178 dev_err(dev, "QPC table get failed\n");
179 goto err_out;
180 }
181
182
183 ret = hns_roce_table_get(hr_dev, &qp_table->irrl_table, hr_qp->qpn);
184 if (ret) {
185 dev_err(dev, "IRRL table get failed\n");
186 goto err_put_qp;
187 }
188
189 if (hr_dev->caps.trrl_entry_sz) {
190
191 ret = hns_roce_table_get(hr_dev, &qp_table->trrl_table,
192 hr_qp->qpn);
193 if (ret) {
194 dev_err(dev, "TRRL table get failed\n");
195 goto err_put_irrl;
196 }
197 }
198
199 if (hr_dev->caps.sccc_entry_sz) {
200
201 ret = hns_roce_table_get(hr_dev, &qp_table->sccc_table,
202 hr_qp->qpn);
203 if (ret) {
204 dev_err(dev, "SCC CTX table get failed\n");
205 goto err_put_trrl;
206 }
207 }
208
209 ret = hns_roce_gsi_qp_alloc(hr_dev, qpn, hr_qp);
210 if (ret)
211 goto err_put_sccc;
212
213 return 0;
214
215 err_put_sccc:
216 if (hr_dev->caps.sccc_entry_sz)
217 hns_roce_table_put(hr_dev, &qp_table->sccc_table,
218 hr_qp->qpn);
219
220 err_put_trrl:
221 if (hr_dev->caps.trrl_entry_sz)
222 hns_roce_table_put(hr_dev, &qp_table->trrl_table, hr_qp->qpn);
223
224 err_put_irrl:
225 hns_roce_table_put(hr_dev, &qp_table->irrl_table, hr_qp->qpn);
226
227 err_put_qp:
228 hns_roce_table_put(hr_dev, &qp_table->qp_table, hr_qp->qpn);
229
230 err_out:
231 return ret;
232 }
233
234 void hns_roce_qp_remove(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp)
235 {
236 struct xarray *xa = &hr_dev->qp_table_xa;
237 unsigned long flags;
238
239 xa_lock_irqsave(xa, flags);
240 __xa_erase(xa, hr_qp->qpn & (hr_dev->caps.num_qps - 1));
241 xa_unlock_irqrestore(xa, flags);
242 }
243
244 void hns_roce_qp_free(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp)
245 {
246 struct hns_roce_qp_table *qp_table = &hr_dev->qp_table;
247
248 if (atomic_dec_and_test(&hr_qp->refcount))
249 complete(&hr_qp->free);
250 wait_for_completion(&hr_qp->free);
251
252 if ((hr_qp->ibqp.qp_type) != IB_QPT_GSI) {
253 if (hr_dev->caps.trrl_entry_sz)
254 hns_roce_table_put(hr_dev, &qp_table->trrl_table,
255 hr_qp->qpn);
256 hns_roce_table_put(hr_dev, &qp_table->irrl_table, hr_qp->qpn);
257 }
258 }
259
260 void hns_roce_release_range_qp(struct hns_roce_dev *hr_dev, int base_qpn,
261 int cnt)
262 {
263 struct hns_roce_qp_table *qp_table = &hr_dev->qp_table;
264
265 if (base_qpn < hr_dev->caps.reserved_qps)
266 return;
267
268 hns_roce_bitmap_free_range(&qp_table->bitmap, base_qpn, cnt, BITMAP_RR);
269 }
270
271 static int hns_roce_set_rq_size(struct hns_roce_dev *hr_dev,
272 struct ib_qp_cap *cap, bool is_user, int has_rq,
273 struct hns_roce_qp *hr_qp)
274 {
275 struct device *dev = hr_dev->dev;
276 u32 max_cnt;
277
278
279 if (cap->max_recv_wr > hr_dev->caps.max_wqes ||
280 cap->max_recv_sge > hr_dev->caps.max_rq_sg) {
281 dev_err(dev, "RQ WR or sge error!max_recv_wr=%d max_recv_sge=%d\n",
282 cap->max_recv_wr, cap->max_recv_sge);
283 return -EINVAL;
284 }
285
286
287 if (!has_rq) {
288 hr_qp->rq.wqe_cnt = 0;
289 hr_qp->rq.max_gs = 0;
290 cap->max_recv_wr = 0;
291 cap->max_recv_sge = 0;
292 } else {
293 if (is_user && (!cap->max_recv_wr || !cap->max_recv_sge)) {
294 dev_err(dev, "user space no need config max_recv_wr max_recv_sge\n");
295 return -EINVAL;
296 }
297
298 if (hr_dev->caps.min_wqes)
299 max_cnt = max(cap->max_recv_wr, hr_dev->caps.min_wqes);
300 else
301 max_cnt = cap->max_recv_wr;
302
303 hr_qp->rq.wqe_cnt = roundup_pow_of_two(max_cnt);
304
305 if ((u32)hr_qp->rq.wqe_cnt > hr_dev->caps.max_wqes) {
306 dev_err(dev, "while setting rq size, rq.wqe_cnt too large\n");
307 return -EINVAL;
308 }
309
310 max_cnt = max(1U, cap->max_recv_sge);
311 hr_qp->rq.max_gs = roundup_pow_of_two(max_cnt);
312 if (hr_dev->caps.max_rq_sg <= 2)
313 hr_qp->rq.wqe_shift =
314 ilog2(hr_dev->caps.max_rq_desc_sz);
315 else
316 hr_qp->rq.wqe_shift =
317 ilog2(hr_dev->caps.max_rq_desc_sz
318 * hr_qp->rq.max_gs);
319 }
320
321 cap->max_recv_wr = hr_qp->rq.max_post = hr_qp->rq.wqe_cnt;
322 cap->max_recv_sge = hr_qp->rq.max_gs;
323
324 return 0;
325 }
326
327 static int check_sq_size_with_integrity(struct hns_roce_dev *hr_dev,
328 struct ib_qp_cap *cap,
329 struct hns_roce_ib_create_qp *ucmd)
330 {
331 u32 roundup_sq_stride = roundup_pow_of_two(hr_dev->caps.max_sq_desc_sz);
332 u8 max_sq_stride = ilog2(roundup_sq_stride);
333
334
335 if (ucmd->log_sq_stride > max_sq_stride ||
336 ucmd->log_sq_stride < HNS_ROCE_IB_MIN_SQ_STRIDE) {
337 ibdev_err(&hr_dev->ib_dev, "check SQ size error!\n");
338 return -EINVAL;
339 }
340
341 if (cap->max_send_sge > hr_dev->caps.max_sq_sg) {
342 ibdev_err(&hr_dev->ib_dev, "SQ sge error! max_send_sge=%d\n",
343 cap->max_send_sge);
344 return -EINVAL;
345 }
346
347 return 0;
348 }
349
350 static int hns_roce_set_user_sq_size(struct hns_roce_dev *hr_dev,
351 struct ib_qp_cap *cap,
352 struct hns_roce_qp *hr_qp,
353 struct hns_roce_ib_create_qp *ucmd)
354 {
355 u32 ex_sge_num;
356 u32 page_size;
357 u32 max_cnt;
358 int ret;
359
360 if (check_shl_overflow(1, ucmd->log_sq_bb_count, &hr_qp->sq.wqe_cnt) ||
361 hr_qp->sq.wqe_cnt > hr_dev->caps.max_wqes)
362 return -EINVAL;
363
364 ret = check_sq_size_with_integrity(hr_dev, cap, ucmd);
365 if (ret) {
366 ibdev_err(&hr_dev->ib_dev, "Sanity check sq size failed\n");
367 return ret;
368 }
369
370 hr_qp->sq.wqe_shift = ucmd->log_sq_stride;
371
372 max_cnt = max(1U, cap->max_send_sge);
373 if (hr_dev->caps.max_sq_sg <= 2)
374 hr_qp->sq.max_gs = roundup_pow_of_two(max_cnt);
375 else
376 hr_qp->sq.max_gs = max_cnt;
377
378 if (hr_qp->sq.max_gs > 2)
379 hr_qp->sge.sge_cnt = roundup_pow_of_two(hr_qp->sq.wqe_cnt *
380 (hr_qp->sq.max_gs - 2));
381
382 if ((hr_qp->sq.max_gs > 2) && (hr_dev->pci_dev->revision == 0x20)) {
383 if (hr_qp->sge.sge_cnt > hr_dev->caps.max_extend_sg) {
384 dev_err(hr_dev->dev,
385 "The extended sge cnt error! sge_cnt=%d\n",
386 hr_qp->sge.sge_cnt);
387 return -EINVAL;
388 }
389 }
390
391 hr_qp->sge.sge_shift = 4;
392 ex_sge_num = hr_qp->sge.sge_cnt;
393
394
395 if (hr_dev->caps.max_sq_sg <= 2) {
396 hr_qp->buff_size = HNS_ROCE_ALOGN_UP((hr_qp->rq.wqe_cnt <<
397 hr_qp->rq.wqe_shift), PAGE_SIZE) +
398 HNS_ROCE_ALOGN_UP((hr_qp->sq.wqe_cnt <<
399 hr_qp->sq.wqe_shift), PAGE_SIZE);
400
401 hr_qp->sq.offset = 0;
402 hr_qp->rq.offset = HNS_ROCE_ALOGN_UP((hr_qp->sq.wqe_cnt <<
403 hr_qp->sq.wqe_shift), PAGE_SIZE);
404 } else {
405 page_size = 1 << (hr_dev->caps.mtt_buf_pg_sz + PAGE_SHIFT);
406 hr_qp->sge.sge_cnt = ex_sge_num ?
407 max(page_size / (1 << hr_qp->sge.sge_shift), ex_sge_num) : 0;
408 hr_qp->buff_size = HNS_ROCE_ALOGN_UP((hr_qp->rq.wqe_cnt <<
409 hr_qp->rq.wqe_shift), page_size) +
410 HNS_ROCE_ALOGN_UP((hr_qp->sge.sge_cnt <<
411 hr_qp->sge.sge_shift), page_size) +
412 HNS_ROCE_ALOGN_UP((hr_qp->sq.wqe_cnt <<
413 hr_qp->sq.wqe_shift), page_size);
414
415 hr_qp->sq.offset = 0;
416 if (ex_sge_num) {
417 hr_qp->sge.offset = HNS_ROCE_ALOGN_UP(
418 (hr_qp->sq.wqe_cnt <<
419 hr_qp->sq.wqe_shift),
420 page_size);
421 hr_qp->rq.offset = hr_qp->sge.offset +
422 HNS_ROCE_ALOGN_UP((hr_qp->sge.sge_cnt <<
423 hr_qp->sge.sge_shift),
424 page_size);
425 } else {
426 hr_qp->rq.offset = HNS_ROCE_ALOGN_UP(
427 (hr_qp->sq.wqe_cnt <<
428 hr_qp->sq.wqe_shift),
429 page_size);
430 }
431 }
432
433 return 0;
434 }
435
436 static int split_wqe_buf_region(struct hns_roce_dev *hr_dev,
437 struct hns_roce_qp *hr_qp,
438 struct hns_roce_buf_region *regions,
439 int region_max, int page_shift)
440 {
441 int page_size = 1 << page_shift;
442 bool is_extend_sge;
443 int region_cnt = 0;
444 int buf_size;
445 int buf_cnt;
446
447 if (hr_qp->buff_size < 1 || region_max < 1)
448 return region_cnt;
449
450 if (hr_qp->sge.sge_cnt > 0)
451 is_extend_sge = true;
452 else
453 is_extend_sge = false;
454
455
456 if (is_extend_sge)
457 buf_size = hr_qp->sge.offset - hr_qp->sq.offset;
458 else
459 buf_size = hr_qp->rq.offset - hr_qp->sq.offset;
460
461 if (buf_size > 0 && region_cnt < region_max) {
462 buf_cnt = DIV_ROUND_UP(buf_size, page_size);
463 hns_roce_init_buf_region(®ions[region_cnt],
464 hr_dev->caps.wqe_sq_hop_num,
465 hr_qp->sq.offset / page_size,
466 buf_cnt);
467 region_cnt++;
468 }
469
470
471 if (is_extend_sge) {
472 buf_size = hr_qp->rq.offset - hr_qp->sge.offset;
473 if (buf_size > 0 && region_cnt < region_max) {
474 buf_cnt = DIV_ROUND_UP(buf_size, page_size);
475 hns_roce_init_buf_region(®ions[region_cnt],
476 hr_dev->caps.wqe_sge_hop_num,
477 hr_qp->sge.offset / page_size,
478 buf_cnt);
479 region_cnt++;
480 }
481 }
482
483
484 buf_size = hr_qp->buff_size - hr_qp->rq.offset;
485 if (buf_size > 0) {
486 buf_cnt = DIV_ROUND_UP(buf_size, page_size);
487 hns_roce_init_buf_region(®ions[region_cnt],
488 hr_dev->caps.wqe_rq_hop_num,
489 hr_qp->rq.offset / page_size,
490 buf_cnt);
491 region_cnt++;
492 }
493
494 return region_cnt;
495 }
496
497 static int calc_wqe_bt_page_shift(struct hns_roce_dev *hr_dev,
498 struct hns_roce_buf_region *regions,
499 int region_cnt)
500 {
501 int bt_pg_shift;
502 int ba_num;
503 int ret;
504
505 bt_pg_shift = PAGE_SHIFT + hr_dev->caps.mtt_ba_pg_sz;
506
507
508 do {
509 ba_num = (1 << bt_pg_shift) / BA_BYTE_LEN;
510 ret = hns_roce_hem_list_calc_root_ba(regions, region_cnt,
511 ba_num);
512 if (ret <= ba_num)
513 break;
514
515 bt_pg_shift++;
516 } while (ret > ba_num);
517
518 return bt_pg_shift - PAGE_SHIFT;
519 }
520
521 static int set_extend_sge_param(struct hns_roce_dev *hr_dev,
522 struct hns_roce_qp *hr_qp)
523 {
524 struct device *dev = hr_dev->dev;
525
526 if (hr_qp->sq.max_gs > 2) {
527 hr_qp->sge.sge_cnt = roundup_pow_of_two(hr_qp->sq.wqe_cnt *
528 (hr_qp->sq.max_gs - 2));
529 hr_qp->sge.sge_shift = 4;
530 }
531
532
533 if (hr_dev->caps.max_sq_sg > 2 && hr_qp->ibqp.qp_type == IB_QPT_GSI) {
534 hr_qp->sge.sge_cnt = roundup_pow_of_two(hr_qp->sq.wqe_cnt *
535 hr_qp->sq.max_gs);
536 hr_qp->sge.sge_shift = 4;
537 }
538
539 if ((hr_qp->sq.max_gs > 2) && hr_dev->pci_dev->revision == 0x20) {
540 if (hr_qp->sge.sge_cnt > hr_dev->caps.max_extend_sg) {
541 dev_err(dev, "The extended sge cnt error! sge_cnt=%d\n",
542 hr_qp->sge.sge_cnt);
543 return -EINVAL;
544 }
545 }
546
547 return 0;
548 }
549
550 static int hns_roce_set_kernel_sq_size(struct hns_roce_dev *hr_dev,
551 struct ib_qp_cap *cap,
552 struct hns_roce_qp *hr_qp)
553 {
554 struct device *dev = hr_dev->dev;
555 u32 page_size;
556 u32 max_cnt;
557 int size;
558 int ret;
559
560 if (cap->max_send_wr > hr_dev->caps.max_wqes ||
561 cap->max_send_sge > hr_dev->caps.max_sq_sg ||
562 cap->max_inline_data > hr_dev->caps.max_sq_inline) {
563 dev_err(dev, "SQ WR or sge or inline data error!\n");
564 return -EINVAL;
565 }
566
567 hr_qp->sq.wqe_shift = ilog2(hr_dev->caps.max_sq_desc_sz);
568
569 if (hr_dev->caps.min_wqes)
570 max_cnt = max(cap->max_send_wr, hr_dev->caps.min_wqes);
571 else
572 max_cnt = cap->max_send_wr;
573
574 hr_qp->sq.wqe_cnt = roundup_pow_of_two(max_cnt);
575 if ((u32)hr_qp->sq.wqe_cnt > hr_dev->caps.max_wqes) {
576 dev_err(dev, "while setting kernel sq size, sq.wqe_cnt too large\n");
577 return -EINVAL;
578 }
579
580
581 max_cnt = max(1U, cap->max_send_sge);
582 if (hr_dev->caps.max_sq_sg <= 2)
583 hr_qp->sq.max_gs = roundup_pow_of_two(max_cnt);
584 else
585 hr_qp->sq.max_gs = max_cnt;
586
587 ret = set_extend_sge_param(hr_dev, hr_qp);
588 if (ret) {
589 dev_err(dev, "set extend sge parameters fail\n");
590 return ret;
591 }
592
593
594 page_size = 1 << (hr_dev->caps.mtt_buf_pg_sz + PAGE_SHIFT);
595 hr_qp->sq.offset = 0;
596 size = HNS_ROCE_ALOGN_UP(hr_qp->sq.wqe_cnt << hr_qp->sq.wqe_shift,
597 page_size);
598
599 if (hr_dev->caps.max_sq_sg > 2 && hr_qp->sge.sge_cnt) {
600 hr_qp->sge.sge_cnt = max(page_size/(1 << hr_qp->sge.sge_shift),
601 (u32)hr_qp->sge.sge_cnt);
602 hr_qp->sge.offset = size;
603 size += HNS_ROCE_ALOGN_UP(hr_qp->sge.sge_cnt <<
604 hr_qp->sge.sge_shift, page_size);
605 }
606
607 hr_qp->rq.offset = size;
608 size += HNS_ROCE_ALOGN_UP((hr_qp->rq.wqe_cnt << hr_qp->rq.wqe_shift),
609 page_size);
610 hr_qp->buff_size = size;
611
612
613 cap->max_send_wr = hr_qp->sq.max_post = hr_qp->sq.wqe_cnt;
614 cap->max_send_sge = hr_qp->sq.max_gs;
615
616
617 cap->max_inline_data = 0;
618
619 return 0;
620 }
621
622 static int hns_roce_qp_has_sq(struct ib_qp_init_attr *attr)
623 {
624 if (attr->qp_type == IB_QPT_XRC_TGT || !attr->cap.max_send_wr)
625 return 0;
626
627 return 1;
628 }
629
630 static int hns_roce_qp_has_rq(struct ib_qp_init_attr *attr)
631 {
632 if (attr->qp_type == IB_QPT_XRC_INI ||
633 attr->qp_type == IB_QPT_XRC_TGT || attr->srq ||
634 !attr->cap.max_recv_wr)
635 return 0;
636
637 return 1;
638 }
639
640 static int alloc_rq_inline_buf(struct hns_roce_qp *hr_qp,
641 struct ib_qp_init_attr *init_attr)
642 {
643 u32 max_recv_sge = init_attr->cap.max_recv_sge;
644 struct hns_roce_rinl_wqe *wqe_list;
645 u32 wqe_cnt = hr_qp->rq.wqe_cnt;
646 int i;
647
648
649 wqe_list = kcalloc(wqe_cnt, sizeof(struct hns_roce_rinl_wqe),
650 GFP_KERNEL);
651
652 if (!wqe_list)
653 goto err;
654
655
656 wqe_list[0].sg_list = kcalloc(wqe_cnt, (max_recv_sge *
657 sizeof(struct hns_roce_rinl_sge)),
658 GFP_KERNEL);
659 if (!wqe_list[0].sg_list)
660 goto err_wqe_list;
661
662
663 for (i = 1; i < wqe_cnt; i++)
664 wqe_list[i].sg_list = &wqe_list[0].sg_list[i * max_recv_sge];
665
666 hr_qp->rq_inl_buf.wqe_list = wqe_list;
667 hr_qp->rq_inl_buf.wqe_cnt = wqe_cnt;
668
669 return 0;
670
671 err_wqe_list:
672 kfree(wqe_list);
673
674 err:
675 return -ENOMEM;
676 }
677
678 static void free_rq_inline_buf(struct hns_roce_qp *hr_qp)
679 {
680 kfree(hr_qp->rq_inl_buf.wqe_list[0].sg_list);
681 kfree(hr_qp->rq_inl_buf.wqe_list);
682 }
683
684 static int hns_roce_create_qp_common(struct hns_roce_dev *hr_dev,
685 struct ib_pd *ib_pd,
686 struct ib_qp_init_attr *init_attr,
687 struct ib_udata *udata, unsigned long sqpn,
688 struct hns_roce_qp *hr_qp)
689 {
690 dma_addr_t *buf_list[ARRAY_SIZE(hr_qp->regions)] = { NULL };
691 struct device *dev = hr_dev->dev;
692 struct hns_roce_ib_create_qp ucmd;
693 struct hns_roce_ib_create_qp_resp resp = {};
694 struct hns_roce_ucontext *uctx = rdma_udata_to_drv_context(
695 udata, struct hns_roce_ucontext, ibucontext);
696 struct hns_roce_buf_region *r;
697 unsigned long qpn = 0;
698 u32 page_shift;
699 int buf_count;
700 int ret;
701 int i;
702
703 mutex_init(&hr_qp->mutex);
704 spin_lock_init(&hr_qp->sq.lock);
705 spin_lock_init(&hr_qp->rq.lock);
706
707 hr_qp->state = IB_QPS_RESET;
708
709 hr_qp->ibqp.qp_type = init_attr->qp_type;
710
711 if (init_attr->sq_sig_type == IB_SIGNAL_ALL_WR)
712 hr_qp->sq_signal_bits = IB_SIGNAL_ALL_WR;
713 else
714 hr_qp->sq_signal_bits = IB_SIGNAL_REQ_WR;
715
716 ret = hns_roce_set_rq_size(hr_dev, &init_attr->cap, udata,
717 hns_roce_qp_has_rq(init_attr), hr_qp);
718 if (ret) {
719 dev_err(dev, "hns_roce_set_rq_size failed\n");
720 goto err_out;
721 }
722
723 if ((hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RQ_INLINE) &&
724 hns_roce_qp_has_rq(init_attr)) {
725 ret = alloc_rq_inline_buf(hr_qp, init_attr);
726 if (ret) {
727 dev_err(dev, "allocate receive inline buffer failed\n");
728 goto err_out;
729 }
730 }
731
732 page_shift = PAGE_SHIFT + hr_dev->caps.mtt_buf_pg_sz;
733 if (udata) {
734 if (ib_copy_from_udata(&ucmd, udata, sizeof(ucmd))) {
735 dev_err(dev, "ib_copy_from_udata error for create qp\n");
736 ret = -EFAULT;
737 goto err_alloc_rq_inline_buf;
738 }
739
740 ret = hns_roce_set_user_sq_size(hr_dev, &init_attr->cap, hr_qp,
741 &ucmd);
742 if (ret) {
743 dev_err(dev, "hns_roce_set_user_sq_size error for create qp\n");
744 goto err_alloc_rq_inline_buf;
745 }
746
747 hr_qp->umem = ib_umem_get(udata, ucmd.buf_addr,
748 hr_qp->buff_size, 0, 0);
749 if (IS_ERR(hr_qp->umem)) {
750 dev_err(dev, "ib_umem_get error for create qp\n");
751 ret = PTR_ERR(hr_qp->umem);
752 goto err_alloc_rq_inline_buf;
753 }
754 hr_qp->region_cnt = split_wqe_buf_region(hr_dev, hr_qp,
755 hr_qp->regions, ARRAY_SIZE(hr_qp->regions),
756 page_shift);
757 ret = hns_roce_alloc_buf_list(hr_qp->regions, buf_list,
758 hr_qp->region_cnt);
759 if (ret) {
760 dev_err(dev, "alloc buf_list error for create qp\n");
761 goto err_alloc_list;
762 }
763
764 for (i = 0; i < hr_qp->region_cnt; i++) {
765 r = &hr_qp->regions[i];
766 buf_count = hns_roce_get_umem_bufs(hr_dev,
767 buf_list[i], r->count, r->offset,
768 hr_qp->umem, page_shift);
769 if (buf_count != r->count) {
770 dev_err(dev,
771 "get umem buf err, expect %d,ret %d.\n",
772 r->count, buf_count);
773 ret = -ENOBUFS;
774 goto err_get_bufs;
775 }
776 }
777
778 if ((hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_SQ_RECORD_DB) &&
779 (udata->inlen >= sizeof(ucmd)) &&
780 (udata->outlen >= sizeof(resp)) &&
781 hns_roce_qp_has_sq(init_attr)) {
782 ret = hns_roce_db_map_user(uctx, udata, ucmd.sdb_addr,
783 &hr_qp->sdb);
784 if (ret) {
785 dev_err(dev, "sq record doorbell map failed!\n");
786 goto err_get_bufs;
787 }
788
789
790 resp.cap_flags |= HNS_ROCE_SUPPORT_SQ_RECORD_DB;
791 hr_qp->sdb_en = 1;
792 }
793
794 if ((hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RECORD_DB) &&
795 (udata->outlen >= sizeof(resp)) &&
796 hns_roce_qp_has_rq(init_attr)) {
797 ret = hns_roce_db_map_user(uctx, udata, ucmd.db_addr,
798 &hr_qp->rdb);
799 if (ret) {
800 dev_err(dev, "rq record doorbell map failed!\n");
801 goto err_sq_dbmap;
802 }
803
804
805 resp.cap_flags |= HNS_ROCE_SUPPORT_RQ_RECORD_DB;
806 hr_qp->rdb_en = 1;
807 }
808 } else {
809 if (init_attr->create_flags &
810 IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK) {
811 dev_err(dev, "init_attr->create_flags error!\n");
812 ret = -EINVAL;
813 goto err_alloc_rq_inline_buf;
814 }
815
816 if (init_attr->create_flags & IB_QP_CREATE_IPOIB_UD_LSO) {
817 dev_err(dev, "init_attr->create_flags error!\n");
818 ret = -EINVAL;
819 goto err_alloc_rq_inline_buf;
820 }
821
822
823 ret = hns_roce_set_kernel_sq_size(hr_dev, &init_attr->cap,
824 hr_qp);
825 if (ret) {
826 dev_err(dev, "hns_roce_set_kernel_sq_size error!\n");
827 goto err_alloc_rq_inline_buf;
828 }
829
830
831 hr_qp->sq.db_reg_l = hr_dev->reg_base + hr_dev->sdb_offset +
832 DB_REG_OFFSET * hr_dev->priv_uar.index;
833 hr_qp->rq.db_reg_l = hr_dev->reg_base + hr_dev->odb_offset +
834 DB_REG_OFFSET * hr_dev->priv_uar.index;
835
836 if ((hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RECORD_DB) &&
837 hns_roce_qp_has_rq(init_attr)) {
838 ret = hns_roce_alloc_db(hr_dev, &hr_qp->rdb, 0);
839 if (ret) {
840 dev_err(dev, "rq record doorbell alloc failed!\n");
841 goto err_alloc_rq_inline_buf;
842 }
843 *hr_qp->rdb.db_record = 0;
844 hr_qp->rdb_en = 1;
845 }
846
847
848 if (hns_roce_buf_alloc(hr_dev, hr_qp->buff_size,
849 (1 << page_shift) * 2,
850 &hr_qp->hr_buf, page_shift)) {
851 dev_err(dev, "hns_roce_buf_alloc error!\n");
852 ret = -ENOMEM;
853 goto err_db;
854 }
855 hr_qp->region_cnt = split_wqe_buf_region(hr_dev, hr_qp,
856 hr_qp->regions, ARRAY_SIZE(hr_qp->regions),
857 page_shift);
858 ret = hns_roce_alloc_buf_list(hr_qp->regions, buf_list,
859 hr_qp->region_cnt);
860 if (ret) {
861 dev_err(dev, "alloc buf_list error for create qp!\n");
862 goto err_alloc_list;
863 }
864
865 for (i = 0; i < hr_qp->region_cnt; i++) {
866 r = &hr_qp->regions[i];
867 buf_count = hns_roce_get_kmem_bufs(hr_dev,
868 buf_list[i], r->count, r->offset,
869 &hr_qp->hr_buf);
870 if (buf_count != r->count) {
871 dev_err(dev,
872 "get kmem buf err, expect %d,ret %d.\n",
873 r->count, buf_count);
874 ret = -ENOBUFS;
875 goto err_get_bufs;
876 }
877 }
878
879 hr_qp->sq.wrid = kcalloc(hr_qp->sq.wqe_cnt, sizeof(u64),
880 GFP_KERNEL);
881 if (ZERO_OR_NULL_PTR(hr_qp->sq.wrid)) {
882 ret = -ENOMEM;
883 goto err_get_bufs;
884 }
885
886 if (hr_qp->rq.wqe_cnt) {
887 hr_qp->rq.wrid = kcalloc(hr_qp->rq.wqe_cnt, sizeof(u64),
888 GFP_KERNEL);
889 if (ZERO_OR_NULL_PTR(hr_qp->rq.wrid)) {
890 ret = -ENOMEM;
891 goto err_sq_wrid;
892 }
893 }
894 }
895
896 if (sqpn) {
897 qpn = sqpn;
898 } else {
899
900 ret = hns_roce_reserve_range_qp(hr_dev, 1, 1, &qpn);
901 if (ret) {
902 dev_err(dev, "hns_roce_reserve_range_qp alloc qpn error\n");
903 goto err_wrid;
904 }
905 }
906
907 hr_qp->wqe_bt_pg_shift = calc_wqe_bt_page_shift(hr_dev, hr_qp->regions,
908 hr_qp->region_cnt);
909 hns_roce_mtr_init(&hr_qp->mtr, PAGE_SHIFT + hr_qp->wqe_bt_pg_shift,
910 page_shift);
911 ret = hns_roce_mtr_attach(hr_dev, &hr_qp->mtr, buf_list,
912 hr_qp->regions, hr_qp->region_cnt);
913 if (ret) {
914 dev_err(dev, "mtr attach error for create qp\n");
915 goto err_mtr;
916 }
917
918 if (init_attr->qp_type == IB_QPT_GSI &&
919 hr_dev->hw_rev == HNS_ROCE_HW_VER1) {
920
921 ret = hns_roce_gsi_qp_alloc(hr_dev, qpn, hr_qp);
922 if (ret) {
923 dev_err(dev, "hns_roce_qp_alloc failed!\n");
924 goto err_qpn;
925 }
926 } else {
927 ret = hns_roce_qp_alloc(hr_dev, qpn, hr_qp);
928 if (ret) {
929 dev_err(dev, "hns_roce_qp_alloc failed!\n");
930 goto err_qpn;
931 }
932 }
933
934 if (sqpn)
935 hr_qp->doorbell_qpn = 1;
936 else
937 hr_qp->doorbell_qpn = (u32)hr_qp->qpn;
938
939 if (udata) {
940 ret = ib_copy_to_udata(udata, &resp,
941 min(udata->outlen, sizeof(resp)));
942 if (ret)
943 goto err_qp;
944 }
945
946 if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_QP_FLOW_CTRL) {
947 ret = hr_dev->hw->qp_flow_control_init(hr_dev, hr_qp);
948 if (ret)
949 goto err_qp;
950 }
951
952 hr_qp->event = hns_roce_ib_qp_event;
953 hns_roce_free_buf_list(buf_list, hr_qp->region_cnt);
954
955 return 0;
956
957 err_qp:
958 if (init_attr->qp_type == IB_QPT_GSI &&
959 hr_dev->hw_rev == HNS_ROCE_HW_VER1)
960 hns_roce_qp_remove(hr_dev, hr_qp);
961 else
962 hns_roce_qp_free(hr_dev, hr_qp);
963
964 err_qpn:
965 if (!sqpn)
966 hns_roce_release_range_qp(hr_dev, qpn, 1);
967
968 err_mtr:
969 hns_roce_mtr_cleanup(hr_dev, &hr_qp->mtr);
970
971 err_wrid:
972 if (udata) {
973 if ((hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RECORD_DB) &&
974 (udata->outlen >= sizeof(resp)) &&
975 hns_roce_qp_has_rq(init_attr))
976 hns_roce_db_unmap_user(uctx, &hr_qp->rdb);
977 } else {
978 if (hr_qp->rq.wqe_cnt)
979 kfree(hr_qp->rq.wrid);
980 }
981
982 err_sq_dbmap:
983 if (udata)
984 if ((hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_SQ_RECORD_DB) &&
985 (udata->inlen >= sizeof(ucmd)) &&
986 (udata->outlen >= sizeof(resp)) &&
987 hns_roce_qp_has_sq(init_attr))
988 hns_roce_db_unmap_user(uctx, &hr_qp->sdb);
989
990 err_sq_wrid:
991 if (!udata)
992 kfree(hr_qp->sq.wrid);
993
994 err_get_bufs:
995 hns_roce_free_buf_list(buf_list, hr_qp->region_cnt);
996
997 err_alloc_list:
998 if (!hr_qp->umem)
999 hns_roce_buf_free(hr_dev, hr_qp->buff_size, &hr_qp->hr_buf);
1000 ib_umem_release(hr_qp->umem);
1001
1002 err_db:
1003 if (!udata && hns_roce_qp_has_rq(init_attr) &&
1004 (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RECORD_DB))
1005 hns_roce_free_db(hr_dev, &hr_qp->rdb);
1006
1007 err_alloc_rq_inline_buf:
1008 if ((hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RQ_INLINE) &&
1009 hns_roce_qp_has_rq(init_attr))
1010 free_rq_inline_buf(hr_qp);
1011
1012 err_out:
1013 return ret;
1014 }
1015
1016 struct ib_qp *hns_roce_create_qp(struct ib_pd *pd,
1017 struct ib_qp_init_attr *init_attr,
1018 struct ib_udata *udata)
1019 {
1020 struct hns_roce_dev *hr_dev = to_hr_dev(pd->device);
1021 struct ib_device *ibdev = &hr_dev->ib_dev;
1022 struct hns_roce_sqp *hr_sqp;
1023 struct hns_roce_qp *hr_qp;
1024 int ret;
1025
1026 switch (init_attr->qp_type) {
1027 case IB_QPT_RC: {
1028 hr_qp = kzalloc(sizeof(*hr_qp), GFP_KERNEL);
1029 if (!hr_qp)
1030 return ERR_PTR(-ENOMEM);
1031
1032 ret = hns_roce_create_qp_common(hr_dev, pd, init_attr, udata, 0,
1033 hr_qp);
1034 if (ret) {
1035 ibdev_err(ibdev, "Create RC QP 0x%06lx failed(%d)\n",
1036 hr_qp->qpn, ret);
1037 kfree(hr_qp);
1038 return ERR_PTR(ret);
1039 }
1040
1041 hr_qp->ibqp.qp_num = hr_qp->qpn;
1042
1043 break;
1044 }
1045 case IB_QPT_GSI: {
1046
1047 if (udata) {
1048 ibdev_err(ibdev, "not support usr space GSI\n");
1049 return ERR_PTR(-EINVAL);
1050 }
1051
1052 hr_sqp = kzalloc(sizeof(*hr_sqp), GFP_KERNEL);
1053 if (!hr_sqp)
1054 return ERR_PTR(-ENOMEM);
1055
1056 hr_qp = &hr_sqp->hr_qp;
1057 hr_qp->port = init_attr->port_num - 1;
1058 hr_qp->phy_port = hr_dev->iboe.phy_port[hr_qp->port];
1059
1060
1061 if (hr_dev->caps.max_sq_sg <= 2)
1062 hr_qp->ibqp.qp_num = HNS_ROCE_MAX_PORTS +
1063 hr_dev->iboe.phy_port[hr_qp->port];
1064 else
1065 hr_qp->ibqp.qp_num = 1;
1066
1067 ret = hns_roce_create_qp_common(hr_dev, pd, init_attr, udata,
1068 hr_qp->ibqp.qp_num, hr_qp);
1069 if (ret) {
1070 ibdev_err(ibdev, "Create GSI QP failed!\n");
1071 kfree(hr_sqp);
1072 return ERR_PTR(ret);
1073 }
1074
1075 break;
1076 }
1077 default:{
1078 ibdev_err(ibdev, "not support QP type %d\n",
1079 init_attr->qp_type);
1080 return ERR_PTR(-EINVAL);
1081 }
1082 }
1083
1084 return &hr_qp->ibqp;
1085 }
1086
1087 int to_hr_qp_type(int qp_type)
1088 {
1089 int transport_type;
1090
1091 if (qp_type == IB_QPT_RC)
1092 transport_type = SERV_TYPE_RC;
1093 else if (qp_type == IB_QPT_UC)
1094 transport_type = SERV_TYPE_UC;
1095 else if (qp_type == IB_QPT_UD)
1096 transport_type = SERV_TYPE_UD;
1097 else if (qp_type == IB_QPT_GSI)
1098 transport_type = SERV_TYPE_UD;
1099 else
1100 transport_type = -1;
1101
1102 return transport_type;
1103 }
1104
1105 static int check_mtu_validate(struct hns_roce_dev *hr_dev,
1106 struct hns_roce_qp *hr_qp,
1107 struct ib_qp_attr *attr, int attr_mask)
1108 {
1109 enum ib_mtu active_mtu;
1110 int p;
1111
1112 p = attr_mask & IB_QP_PORT ? (attr->port_num - 1) : hr_qp->port;
1113 active_mtu = iboe_get_mtu(hr_dev->iboe.netdevs[p]->mtu);
1114
1115 if ((hr_dev->caps.max_mtu >= IB_MTU_2048 &&
1116 attr->path_mtu > hr_dev->caps.max_mtu) ||
1117 attr->path_mtu < IB_MTU_256 || attr->path_mtu > active_mtu) {
1118 ibdev_err(&hr_dev->ib_dev,
1119 "attr path_mtu(%d)invalid while modify qp",
1120 attr->path_mtu);
1121 return -EINVAL;
1122 }
1123
1124 return 0;
1125 }
1126
1127 static int hns_roce_check_qp_attr(struct ib_qp *ibqp, struct ib_qp_attr *attr,
1128 int attr_mask)
1129 {
1130 struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
1131 struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
1132 int p;
1133
1134 if ((attr_mask & IB_QP_PORT) &&
1135 (attr->port_num == 0 || attr->port_num > hr_dev->caps.num_ports)) {
1136 ibdev_err(&hr_dev->ib_dev,
1137 "attr port_num invalid.attr->port_num=%d\n",
1138 attr->port_num);
1139 return -EINVAL;
1140 }
1141
1142 if (attr_mask & IB_QP_PKEY_INDEX) {
1143 p = attr_mask & IB_QP_PORT ? (attr->port_num - 1) : hr_qp->port;
1144 if (attr->pkey_index >= hr_dev->caps.pkey_table_len[p]) {
1145 ibdev_err(&hr_dev->ib_dev,
1146 "attr pkey_index invalid.attr->pkey_index=%d\n",
1147 attr->pkey_index);
1148 return -EINVAL;
1149 }
1150 }
1151
1152 if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC &&
1153 attr->max_rd_atomic > hr_dev->caps.max_qp_init_rdma) {
1154 ibdev_err(&hr_dev->ib_dev,
1155 "attr max_rd_atomic invalid.attr->max_rd_atomic=%d\n",
1156 attr->max_rd_atomic);
1157 return -EINVAL;
1158 }
1159
1160 if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC &&
1161 attr->max_dest_rd_atomic > hr_dev->caps.max_qp_dest_rdma) {
1162 ibdev_err(&hr_dev->ib_dev,
1163 "attr max_dest_rd_atomic invalid.attr->max_dest_rd_atomic=%d\n",
1164 attr->max_dest_rd_atomic);
1165 return -EINVAL;
1166 }
1167
1168 if (attr_mask & IB_QP_PATH_MTU)
1169 return check_mtu_validate(hr_dev, hr_qp, attr, attr_mask);
1170
1171 return 0;
1172 }
1173
1174 int hns_roce_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
1175 int attr_mask, struct ib_udata *udata)
1176 {
1177 struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
1178 struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
1179 enum ib_qp_state cur_state, new_state;
1180 int ret = -EINVAL;
1181
1182 mutex_lock(&hr_qp->mutex);
1183
1184 cur_state = attr_mask & IB_QP_CUR_STATE ?
1185 attr->cur_qp_state : (enum ib_qp_state)hr_qp->state;
1186 new_state = attr_mask & IB_QP_STATE ? attr->qp_state : cur_state;
1187
1188 if (ibqp->uobject &&
1189 (attr_mask & IB_QP_STATE) && new_state == IB_QPS_ERR) {
1190 if (hr_qp->sdb_en == 1) {
1191 hr_qp->sq.head = *(int *)(hr_qp->sdb.virt_addr);
1192
1193 if (hr_qp->rdb_en == 1)
1194 hr_qp->rq.head = *(int *)(hr_qp->rdb.virt_addr);
1195 } else {
1196 ibdev_warn(&hr_dev->ib_dev,
1197 "flush cqe is not supported in userspace!\n");
1198 goto out;
1199 }
1200 }
1201
1202 if (!ib_modify_qp_is_ok(cur_state, new_state, ibqp->qp_type,
1203 attr_mask)) {
1204 ibdev_err(&hr_dev->ib_dev, "ib_modify_qp_is_ok failed\n");
1205 goto out;
1206 }
1207
1208 ret = hns_roce_check_qp_attr(ibqp, attr, attr_mask);
1209 if (ret)
1210 goto out;
1211
1212 if (cur_state == new_state && cur_state == IB_QPS_RESET) {
1213 if (hr_dev->caps.min_wqes) {
1214 ret = -EPERM;
1215 ibdev_err(&hr_dev->ib_dev,
1216 "cur_state=%d new_state=%d\n", cur_state,
1217 new_state);
1218 } else {
1219 ret = 0;
1220 }
1221
1222 goto out;
1223 }
1224
1225 ret = hr_dev->hw->modify_qp(ibqp, attr, attr_mask, cur_state,
1226 new_state);
1227
1228 out:
1229 mutex_unlock(&hr_qp->mutex);
1230
1231 return ret;
1232 }
1233
1234 void hns_roce_lock_cqs(struct hns_roce_cq *send_cq, struct hns_roce_cq *recv_cq)
1235 __acquires(&send_cq->lock) __acquires(&recv_cq->lock)
1236 {
1237 if (send_cq == recv_cq) {
1238 spin_lock_irq(&send_cq->lock);
1239 __acquire(&recv_cq->lock);
1240 } else if (send_cq->cqn < recv_cq->cqn) {
1241 spin_lock_irq(&send_cq->lock);
1242 spin_lock_nested(&recv_cq->lock, SINGLE_DEPTH_NESTING);
1243 } else {
1244 spin_lock_irq(&recv_cq->lock);
1245 spin_lock_nested(&send_cq->lock, SINGLE_DEPTH_NESTING);
1246 }
1247 }
1248
1249 void hns_roce_unlock_cqs(struct hns_roce_cq *send_cq,
1250 struct hns_roce_cq *recv_cq) __releases(&send_cq->lock)
1251 __releases(&recv_cq->lock)
1252 {
1253 if (send_cq == recv_cq) {
1254 __release(&recv_cq->lock);
1255 spin_unlock_irq(&send_cq->lock);
1256 } else if (send_cq->cqn < recv_cq->cqn) {
1257 spin_unlock(&recv_cq->lock);
1258 spin_unlock_irq(&send_cq->lock);
1259 } else {
1260 spin_unlock(&send_cq->lock);
1261 spin_unlock_irq(&recv_cq->lock);
1262 }
1263 }
1264
1265 static void *get_wqe(struct hns_roce_qp *hr_qp, int offset)
1266 {
1267
1268 return hns_roce_buf_offset(&hr_qp->hr_buf, offset);
1269 }
1270
1271 void *get_recv_wqe(struct hns_roce_qp *hr_qp, int n)
1272 {
1273 return get_wqe(hr_qp, hr_qp->rq.offset + (n << hr_qp->rq.wqe_shift));
1274 }
1275
1276 void *get_send_wqe(struct hns_roce_qp *hr_qp, int n)
1277 {
1278 return get_wqe(hr_qp, hr_qp->sq.offset + (n << hr_qp->sq.wqe_shift));
1279 }
1280
1281 void *get_send_extend_sge(struct hns_roce_qp *hr_qp, int n)
1282 {
1283 return hns_roce_buf_offset(&hr_qp->hr_buf, hr_qp->sge.offset +
1284 (n << hr_qp->sge.sge_shift));
1285 }
1286
1287 bool hns_roce_wq_overflow(struct hns_roce_wq *hr_wq, int nreq,
1288 struct ib_cq *ib_cq)
1289 {
1290 struct hns_roce_cq *hr_cq;
1291 u32 cur;
1292
1293 cur = hr_wq->head - hr_wq->tail;
1294 if (likely(cur + nreq < hr_wq->max_post))
1295 return false;
1296
1297 hr_cq = to_hr_cq(ib_cq);
1298 spin_lock(&hr_cq->lock);
1299 cur = hr_wq->head - hr_wq->tail;
1300 spin_unlock(&hr_cq->lock);
1301
1302 return cur + nreq >= hr_wq->max_post;
1303 }
1304
1305 int hns_roce_init_qp_table(struct hns_roce_dev *hr_dev)
1306 {
1307 struct hns_roce_qp_table *qp_table = &hr_dev->qp_table;
1308 int reserved_from_top = 0;
1309 int reserved_from_bot;
1310 int ret;
1311
1312 mutex_init(&qp_table->scc_mutex);
1313 xa_init(&hr_dev->qp_table_xa);
1314
1315 reserved_from_bot = hr_dev->caps.reserved_qps;
1316
1317 ret = hns_roce_bitmap_init(&qp_table->bitmap, hr_dev->caps.num_qps,
1318 hr_dev->caps.num_qps - 1, reserved_from_bot,
1319 reserved_from_top);
1320 if (ret) {
1321 dev_err(hr_dev->dev, "qp bitmap init failed!error=%d\n",
1322 ret);
1323 return ret;
1324 }
1325
1326 return 0;
1327 }
1328
1329 void hns_roce_cleanup_qp_table(struct hns_roce_dev *hr_dev)
1330 {
1331 hns_roce_bitmap_cleanup(&hr_dev->qp_table.bitmap);
1332 }