This source file includes following definitions.
- hw_index_to_key
- key_to_hw_index
- hns_roce_sw2hw_mpt
- hns_roce_hw2sw_mpt
- hns_roce_buddy_alloc
- hns_roce_buddy_free
- hns_roce_buddy_init
- hns_roce_buddy_cleanup
- hns_roce_alloc_mtt_range
- hns_roce_mtt_init
- hns_roce_mtt_cleanup
- hns_roce_loop_free
- pbl_1hop_alloc
- pbl_2hop_alloc
- pbl_3hop_alloc
- hns_roce_mhop_alloc
- hns_roce_mr_alloc
- hns_roce_mhop_free
- hns_roce_mr_free
- hns_roce_mr_enable
- hns_roce_write_mtt_chunk
- hns_roce_write_mtt
- hns_roce_buf_write_mtt
- hns_roce_init_mr_table
- hns_roce_cleanup_mr_table
- hns_roce_get_dma_mr
- hns_roce_ib_umem_write_mtt
- hns_roce_ib_umem_write_mr
- hns_roce_reg_user_mr
- rereg_mr_trans
- hns_roce_rereg_user_mr
- hns_roce_dereg_mr
- hns_roce_alloc_mr
- hns_roce_set_page
- hns_roce_map_mr_sg
- hns_roce_mw_free
- hns_roce_mw_enable
- hns_roce_alloc_mw
- hns_roce_dealloc_mw
- hns_roce_mtr_init
- hns_roce_mtr_cleanup
- hns_roce_write_mtr
- hns_roce_mtr_attach
- hns_roce_mtr_find
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34 #include <linux/platform_device.h>
35 #include <linux/vmalloc.h>
36 #include <rdma/ib_umem.h>
37 #include "hns_roce_device.h"
38 #include "hns_roce_cmd.h"
39 #include "hns_roce_hem.h"
40
41 static u32 hw_index_to_key(unsigned long ind)
42 {
43 return (u32)(ind >> 24) | (ind << 8);
44 }
45
46 unsigned long key_to_hw_index(u32 key)
47 {
48 return (key << 24) | (key >> 8);
49 }
50
51 static int hns_roce_sw2hw_mpt(struct hns_roce_dev *hr_dev,
52 struct hns_roce_cmd_mailbox *mailbox,
53 unsigned long mpt_index)
54 {
55 return hns_roce_cmd_mbox(hr_dev, mailbox->dma, 0, mpt_index, 0,
56 HNS_ROCE_CMD_SW2HW_MPT,
57 HNS_ROCE_CMD_TIMEOUT_MSECS);
58 }
59
60 int hns_roce_hw2sw_mpt(struct hns_roce_dev *hr_dev,
61 struct hns_roce_cmd_mailbox *mailbox,
62 unsigned long mpt_index)
63 {
64 return hns_roce_cmd_mbox(hr_dev, 0, mailbox ? mailbox->dma : 0,
65 mpt_index, !mailbox, HNS_ROCE_CMD_HW2SW_MPT,
66 HNS_ROCE_CMD_TIMEOUT_MSECS);
67 }
68
69 static int hns_roce_buddy_alloc(struct hns_roce_buddy *buddy, int order,
70 unsigned long *seg)
71 {
72 int o;
73 u32 m;
74
75 spin_lock(&buddy->lock);
76
77 for (o = order; o <= buddy->max_order; ++o) {
78 if (buddy->num_free[o]) {
79 m = 1 << (buddy->max_order - o);
80 *seg = find_first_bit(buddy->bits[o], m);
81 if (*seg < m)
82 goto found;
83 }
84 }
85 spin_unlock(&buddy->lock);
86 return -1;
87
88 found:
89 clear_bit(*seg, buddy->bits[o]);
90 --buddy->num_free[o];
91
92 while (o > order) {
93 --o;
94 *seg <<= 1;
95 set_bit(*seg ^ 1, buddy->bits[o]);
96 ++buddy->num_free[o];
97 }
98
99 spin_unlock(&buddy->lock);
100
101 *seg <<= order;
102 return 0;
103 }
104
105 static void hns_roce_buddy_free(struct hns_roce_buddy *buddy, unsigned long seg,
106 int order)
107 {
108 seg >>= order;
109
110 spin_lock(&buddy->lock);
111
112 while (test_bit(seg ^ 1, buddy->bits[order])) {
113 clear_bit(seg ^ 1, buddy->bits[order]);
114 --buddy->num_free[order];
115 seg >>= 1;
116 ++order;
117 }
118
119 set_bit(seg, buddy->bits[order]);
120 ++buddy->num_free[order];
121
122 spin_unlock(&buddy->lock);
123 }
124
125 static int hns_roce_buddy_init(struct hns_roce_buddy *buddy, int max_order)
126 {
127 int i, s;
128
129 buddy->max_order = max_order;
130 spin_lock_init(&buddy->lock);
131 buddy->bits = kcalloc(buddy->max_order + 1,
132 sizeof(*buddy->bits),
133 GFP_KERNEL);
134 buddy->num_free = kcalloc(buddy->max_order + 1,
135 sizeof(*buddy->num_free),
136 GFP_KERNEL);
137 if (!buddy->bits || !buddy->num_free)
138 goto err_out;
139
140 for (i = 0; i <= buddy->max_order; ++i) {
141 s = BITS_TO_LONGS(1 << (buddy->max_order - i));
142 buddy->bits[i] = kcalloc(s, sizeof(long), GFP_KERNEL |
143 __GFP_NOWARN);
144 if (!buddy->bits[i]) {
145 buddy->bits[i] = vzalloc(array_size(s, sizeof(long)));
146 if (!buddy->bits[i])
147 goto err_out_free;
148 }
149 }
150
151 set_bit(0, buddy->bits[buddy->max_order]);
152 buddy->num_free[buddy->max_order] = 1;
153
154 return 0;
155
156 err_out_free:
157 for (i = 0; i <= buddy->max_order; ++i)
158 kvfree(buddy->bits[i]);
159
160 err_out:
161 kfree(buddy->bits);
162 kfree(buddy->num_free);
163 return -ENOMEM;
164 }
165
166 static void hns_roce_buddy_cleanup(struct hns_roce_buddy *buddy)
167 {
168 int i;
169
170 for (i = 0; i <= buddy->max_order; ++i)
171 kvfree(buddy->bits[i]);
172
173 kfree(buddy->bits);
174 kfree(buddy->num_free);
175 }
176
177 static int hns_roce_alloc_mtt_range(struct hns_roce_dev *hr_dev, int order,
178 unsigned long *seg, u32 mtt_type)
179 {
180 struct hns_roce_mr_table *mr_table = &hr_dev->mr_table;
181 struct hns_roce_hem_table *table;
182 struct hns_roce_buddy *buddy;
183 int ret;
184
185 switch (mtt_type) {
186 case MTT_TYPE_WQE:
187 buddy = &mr_table->mtt_buddy;
188 table = &mr_table->mtt_table;
189 break;
190 case MTT_TYPE_CQE:
191 buddy = &mr_table->mtt_cqe_buddy;
192 table = &mr_table->mtt_cqe_table;
193 break;
194 case MTT_TYPE_SRQWQE:
195 buddy = &mr_table->mtt_srqwqe_buddy;
196 table = &mr_table->mtt_srqwqe_table;
197 break;
198 case MTT_TYPE_IDX:
199 buddy = &mr_table->mtt_idx_buddy;
200 table = &mr_table->mtt_idx_table;
201 break;
202 default:
203 dev_err(hr_dev->dev, "Unsupport MTT table type: %d\n",
204 mtt_type);
205 return -EINVAL;
206 }
207
208 ret = hns_roce_buddy_alloc(buddy, order, seg);
209 if (ret == -1)
210 return -1;
211
212 if (hns_roce_table_get_range(hr_dev, table, *seg,
213 *seg + (1 << order) - 1)) {
214 hns_roce_buddy_free(buddy, *seg, order);
215 return -1;
216 }
217
218 return 0;
219 }
220
221 int hns_roce_mtt_init(struct hns_roce_dev *hr_dev, int npages, int page_shift,
222 struct hns_roce_mtt *mtt)
223 {
224 int ret;
225 int i;
226
227
228 if (!npages) {
229 mtt->order = -1;
230 mtt->page_shift = HNS_ROCE_HEM_PAGE_SHIFT;
231 return 0;
232 }
233
234
235 mtt->page_shift = page_shift;
236
237
238 for (mtt->order = 0, i = HNS_ROCE_MTT_ENTRY_PER_SEG; i < npages;
239 i <<= 1)
240 ++mtt->order;
241
242
243 ret = hns_roce_alloc_mtt_range(hr_dev, mtt->order, &mtt->first_seg,
244 mtt->mtt_type);
245 if (ret == -1)
246 return -ENOMEM;
247
248 return 0;
249 }
250
251 void hns_roce_mtt_cleanup(struct hns_roce_dev *hr_dev, struct hns_roce_mtt *mtt)
252 {
253 struct hns_roce_mr_table *mr_table = &hr_dev->mr_table;
254
255 if (mtt->order < 0)
256 return;
257
258 switch (mtt->mtt_type) {
259 case MTT_TYPE_WQE:
260 hns_roce_buddy_free(&mr_table->mtt_buddy, mtt->first_seg,
261 mtt->order);
262 hns_roce_table_put_range(hr_dev, &mr_table->mtt_table,
263 mtt->first_seg,
264 mtt->first_seg + (1 << mtt->order) - 1);
265 break;
266 case MTT_TYPE_CQE:
267 hns_roce_buddy_free(&mr_table->mtt_cqe_buddy, mtt->first_seg,
268 mtt->order);
269 hns_roce_table_put_range(hr_dev, &mr_table->mtt_cqe_table,
270 mtt->first_seg,
271 mtt->first_seg + (1 << mtt->order) - 1);
272 break;
273 case MTT_TYPE_SRQWQE:
274 hns_roce_buddy_free(&mr_table->mtt_srqwqe_buddy, mtt->first_seg,
275 mtt->order);
276 hns_roce_table_put_range(hr_dev, &mr_table->mtt_srqwqe_table,
277 mtt->first_seg,
278 mtt->first_seg + (1 << mtt->order) - 1);
279 break;
280 case MTT_TYPE_IDX:
281 hns_roce_buddy_free(&mr_table->mtt_idx_buddy, mtt->first_seg,
282 mtt->order);
283 hns_roce_table_put_range(hr_dev, &mr_table->mtt_idx_table,
284 mtt->first_seg,
285 mtt->first_seg + (1 << mtt->order) - 1);
286 break;
287 default:
288 dev_err(hr_dev->dev,
289 "Unsupport mtt type %d, clean mtt failed\n",
290 mtt->mtt_type);
291 break;
292 }
293 }
294
295 static void hns_roce_loop_free(struct hns_roce_dev *hr_dev,
296 struct hns_roce_mr *mr, int err_loop_index,
297 int loop_i, int loop_j)
298 {
299 struct device *dev = hr_dev->dev;
300 u32 mhop_num;
301 u32 pbl_bt_sz;
302 u64 bt_idx;
303 int i, j;
304
305 pbl_bt_sz = 1 << (hr_dev->caps.pbl_ba_pg_sz + PAGE_SHIFT);
306 mhop_num = hr_dev->caps.pbl_hop_num;
307
308 i = loop_i;
309 if (mhop_num == 3 && err_loop_index == 2) {
310 for (; i >= 0; i--) {
311 dma_free_coherent(dev, pbl_bt_sz, mr->pbl_bt_l1[i],
312 mr->pbl_l1_dma_addr[i]);
313
314 for (j = 0; j < pbl_bt_sz / BA_BYTE_LEN; j++) {
315 if (i == loop_i && j >= loop_j)
316 break;
317
318 bt_idx = i * pbl_bt_sz / BA_BYTE_LEN + j;
319 dma_free_coherent(dev, pbl_bt_sz,
320 mr->pbl_bt_l2[bt_idx],
321 mr->pbl_l2_dma_addr[bt_idx]);
322 }
323 }
324 } else if (mhop_num == 3 && err_loop_index == 1) {
325 for (i -= 1; i >= 0; i--) {
326 dma_free_coherent(dev, pbl_bt_sz, mr->pbl_bt_l1[i],
327 mr->pbl_l1_dma_addr[i]);
328
329 for (j = 0; j < pbl_bt_sz / BA_BYTE_LEN; j++) {
330 bt_idx = i * pbl_bt_sz / BA_BYTE_LEN + j;
331 dma_free_coherent(dev, pbl_bt_sz,
332 mr->pbl_bt_l2[bt_idx],
333 mr->pbl_l2_dma_addr[bt_idx]);
334 }
335 }
336 } else if (mhop_num == 2 && err_loop_index == 1) {
337 for (i -= 1; i >= 0; i--)
338 dma_free_coherent(dev, pbl_bt_sz, mr->pbl_bt_l1[i],
339 mr->pbl_l1_dma_addr[i]);
340 } else {
341 dev_warn(dev, "not support: mhop_num=%d, err_loop_index=%d.",
342 mhop_num, err_loop_index);
343 return;
344 }
345
346 dma_free_coherent(dev, pbl_bt_sz, mr->pbl_bt_l0, mr->pbl_l0_dma_addr);
347 mr->pbl_bt_l0 = NULL;
348 mr->pbl_l0_dma_addr = 0;
349 }
350 static int pbl_1hop_alloc(struct hns_roce_dev *hr_dev, int npages,
351 struct hns_roce_mr *mr, u32 pbl_bt_sz)
352 {
353 struct device *dev = hr_dev->dev;
354
355 if (npages > pbl_bt_sz / 8) {
356 dev_err(dev, "npages %d is larger than buf_pg_sz!",
357 npages);
358 return -EINVAL;
359 }
360 mr->pbl_buf = dma_alloc_coherent(dev, npages * 8,
361 &(mr->pbl_dma_addr),
362 GFP_KERNEL);
363 if (!mr->pbl_buf)
364 return -ENOMEM;
365
366 mr->pbl_size = npages;
367 mr->pbl_ba = mr->pbl_dma_addr;
368 mr->pbl_hop_num = 1;
369 mr->pbl_ba_pg_sz = hr_dev->caps.pbl_ba_pg_sz;
370 mr->pbl_buf_pg_sz = hr_dev->caps.pbl_buf_pg_sz;
371 return 0;
372
373 }
374
375
376 static int pbl_2hop_alloc(struct hns_roce_dev *hr_dev, int npages,
377 struct hns_roce_mr *mr, u32 pbl_bt_sz)
378 {
379 struct device *dev = hr_dev->dev;
380 int npages_allocated;
381 u64 pbl_last_bt_num;
382 u64 pbl_bt_cnt = 0;
383 u64 size;
384 int i;
385
386 pbl_last_bt_num = (npages + pbl_bt_sz / 8 - 1) / (pbl_bt_sz / 8);
387
388
389 for (i = 0; i < pbl_bt_sz / 8; i++) {
390 if (pbl_bt_cnt + 1 < pbl_last_bt_num) {
391 size = pbl_bt_sz;
392 } else {
393 npages_allocated = i * (pbl_bt_sz / 8);
394 size = (npages - npages_allocated) * 8;
395 }
396 mr->pbl_bt_l1[i] = dma_alloc_coherent(dev, size,
397 &(mr->pbl_l1_dma_addr[i]),
398 GFP_KERNEL);
399 if (!mr->pbl_bt_l1[i]) {
400 hns_roce_loop_free(hr_dev, mr, 1, i, 0);
401 return -ENOMEM;
402 }
403
404 *(mr->pbl_bt_l0 + i) = mr->pbl_l1_dma_addr[i];
405
406 pbl_bt_cnt++;
407 if (pbl_bt_cnt >= pbl_last_bt_num)
408 break;
409 }
410
411 mr->l0_chunk_last_num = i + 1;
412
413 return 0;
414 }
415
416 static int pbl_3hop_alloc(struct hns_roce_dev *hr_dev, int npages,
417 struct hns_roce_mr *mr, u32 pbl_bt_sz)
418 {
419 struct device *dev = hr_dev->dev;
420 int mr_alloc_done = 0;
421 int npages_allocated;
422 u64 pbl_last_bt_num;
423 u64 pbl_bt_cnt = 0;
424 u64 bt_idx;
425 u64 size;
426 int i;
427 int j = 0;
428
429 pbl_last_bt_num = (npages + pbl_bt_sz / 8 - 1) / (pbl_bt_sz / 8);
430
431 mr->pbl_l2_dma_addr = kcalloc(pbl_last_bt_num,
432 sizeof(*mr->pbl_l2_dma_addr),
433 GFP_KERNEL);
434 if (!mr->pbl_l2_dma_addr)
435 return -ENOMEM;
436
437 mr->pbl_bt_l2 = kcalloc(pbl_last_bt_num,
438 sizeof(*mr->pbl_bt_l2),
439 GFP_KERNEL);
440 if (!mr->pbl_bt_l2)
441 goto err_kcalloc_bt_l2;
442
443
444 for (i = 0; i < pbl_bt_sz / 8; i++) {
445 mr->pbl_bt_l1[i] = dma_alloc_coherent(dev, pbl_bt_sz,
446 &(mr->pbl_l1_dma_addr[i]),
447 GFP_KERNEL);
448 if (!mr->pbl_bt_l1[i]) {
449 hns_roce_loop_free(hr_dev, mr, 1, i, 0);
450 goto err_dma_alloc_l0;
451 }
452
453 *(mr->pbl_bt_l0 + i) = mr->pbl_l1_dma_addr[i];
454
455 for (j = 0; j < pbl_bt_sz / 8; j++) {
456 bt_idx = i * pbl_bt_sz / 8 + j;
457
458 if (pbl_bt_cnt + 1 < pbl_last_bt_num) {
459 size = pbl_bt_sz;
460 } else {
461 npages_allocated = bt_idx *
462 (pbl_bt_sz / 8);
463 size = (npages - npages_allocated) * 8;
464 }
465 mr->pbl_bt_l2[bt_idx] = dma_alloc_coherent(
466 dev, size,
467 &(mr->pbl_l2_dma_addr[bt_idx]),
468 GFP_KERNEL);
469 if (!mr->pbl_bt_l2[bt_idx]) {
470 hns_roce_loop_free(hr_dev, mr, 2, i, j);
471 goto err_dma_alloc_l0;
472 }
473
474 *(mr->pbl_bt_l1[i] + j) =
475 mr->pbl_l2_dma_addr[bt_idx];
476
477 pbl_bt_cnt++;
478 if (pbl_bt_cnt >= pbl_last_bt_num) {
479 mr_alloc_done = 1;
480 break;
481 }
482 }
483
484 if (mr_alloc_done)
485 break;
486 }
487
488 mr->l0_chunk_last_num = i + 1;
489 mr->l1_chunk_last_num = j + 1;
490
491
492 return 0;
493
494 err_dma_alloc_l0:
495 kfree(mr->pbl_bt_l2);
496 mr->pbl_bt_l2 = NULL;
497
498 err_kcalloc_bt_l2:
499 kfree(mr->pbl_l2_dma_addr);
500 mr->pbl_l2_dma_addr = NULL;
501
502 return -ENOMEM;
503 }
504
505
506
507 static int hns_roce_mhop_alloc(struct hns_roce_dev *hr_dev, int npages,
508 struct hns_roce_mr *mr)
509 {
510 struct device *dev = hr_dev->dev;
511 u32 pbl_bt_sz;
512 u32 mhop_num;
513
514 mhop_num = (mr->type == MR_TYPE_FRMR ? 1 : hr_dev->caps.pbl_hop_num);
515 pbl_bt_sz = 1 << (hr_dev->caps.pbl_ba_pg_sz + PAGE_SHIFT);
516
517 if (mhop_num == HNS_ROCE_HOP_NUM_0)
518 return 0;
519
520 if (mhop_num == 1)
521 return pbl_1hop_alloc(hr_dev, npages, mr, pbl_bt_sz);
522
523 mr->pbl_l1_dma_addr = kcalloc(pbl_bt_sz / 8,
524 sizeof(*mr->pbl_l1_dma_addr),
525 GFP_KERNEL);
526 if (!mr->pbl_l1_dma_addr)
527 return -ENOMEM;
528
529 mr->pbl_bt_l1 = kcalloc(pbl_bt_sz / 8, sizeof(*mr->pbl_bt_l1),
530 GFP_KERNEL);
531 if (!mr->pbl_bt_l1)
532 goto err_kcalloc_bt_l1;
533
534
535 mr->pbl_bt_l0 = dma_alloc_coherent(dev, pbl_bt_sz,
536 &(mr->pbl_l0_dma_addr),
537 GFP_KERNEL);
538 if (!mr->pbl_bt_l0)
539 goto err_kcalloc_l2_dma;
540
541 if (mhop_num == 2) {
542 if (pbl_2hop_alloc(hr_dev, npages, mr, pbl_bt_sz))
543 goto err_kcalloc_l2_dma;
544 }
545
546 if (mhop_num == 3) {
547 if (pbl_3hop_alloc(hr_dev, npages, mr, pbl_bt_sz))
548 goto err_kcalloc_l2_dma;
549 }
550
551
552 mr->pbl_size = npages;
553 mr->pbl_ba = mr->pbl_l0_dma_addr;
554 mr->pbl_hop_num = hr_dev->caps.pbl_hop_num;
555 mr->pbl_ba_pg_sz = hr_dev->caps.pbl_ba_pg_sz;
556 mr->pbl_buf_pg_sz = hr_dev->caps.pbl_buf_pg_sz;
557
558 return 0;
559
560 err_kcalloc_l2_dma:
561 kfree(mr->pbl_bt_l1);
562 mr->pbl_bt_l1 = NULL;
563
564 err_kcalloc_bt_l1:
565 kfree(mr->pbl_l1_dma_addr);
566 mr->pbl_l1_dma_addr = NULL;
567
568 return -ENOMEM;
569 }
570
571 static int hns_roce_mr_alloc(struct hns_roce_dev *hr_dev, u32 pd, u64 iova,
572 u64 size, u32 access, int npages,
573 struct hns_roce_mr *mr)
574 {
575 struct device *dev = hr_dev->dev;
576 unsigned long index = 0;
577 int ret;
578
579
580 ret = hns_roce_bitmap_alloc(&hr_dev->mr_table.mtpt_bitmap, &index);
581 if (ret == -1)
582 return -ENOMEM;
583
584 mr->iova = iova;
585 mr->size = size;
586 mr->pd = pd;
587 mr->access = access;
588 mr->enabled = 0;
589 mr->key = hw_index_to_key(index);
590
591 if (size == ~0ull) {
592 mr->pbl_buf = NULL;
593 mr->pbl_dma_addr = 0;
594
595 mr->pbl_bt_l2 = NULL;
596 mr->pbl_bt_l1 = NULL;
597 mr->pbl_bt_l0 = NULL;
598 mr->pbl_l2_dma_addr = NULL;
599 mr->pbl_l1_dma_addr = NULL;
600 mr->pbl_l0_dma_addr = 0;
601 } else {
602 if (!hr_dev->caps.pbl_hop_num) {
603 mr->pbl_buf = dma_alloc_coherent(dev,
604 npages * BA_BYTE_LEN,
605 &(mr->pbl_dma_addr),
606 GFP_KERNEL);
607 if (!mr->pbl_buf)
608 return -ENOMEM;
609 } else {
610 ret = hns_roce_mhop_alloc(hr_dev, npages, mr);
611 }
612 }
613
614 return ret;
615 }
616
617 static void hns_roce_mhop_free(struct hns_roce_dev *hr_dev,
618 struct hns_roce_mr *mr)
619 {
620 struct device *dev = hr_dev->dev;
621 int npages_allocated;
622 int npages;
623 int i, j;
624 u32 pbl_bt_sz;
625 u32 mhop_num;
626 u64 bt_idx;
627
628 npages = mr->pbl_size;
629 pbl_bt_sz = 1 << (hr_dev->caps.pbl_ba_pg_sz + PAGE_SHIFT);
630 mhop_num = (mr->type == MR_TYPE_FRMR) ? 1 : hr_dev->caps.pbl_hop_num;
631
632 if (mhop_num == HNS_ROCE_HOP_NUM_0)
633 return;
634
635 if (mhop_num == 1) {
636 dma_free_coherent(dev, (unsigned int)(npages * BA_BYTE_LEN),
637 mr->pbl_buf, mr->pbl_dma_addr);
638 return;
639 }
640
641 dma_free_coherent(dev, pbl_bt_sz, mr->pbl_bt_l0,
642 mr->pbl_l0_dma_addr);
643
644 if (mhop_num == 2) {
645 for (i = 0; i < mr->l0_chunk_last_num; i++) {
646 if (i == mr->l0_chunk_last_num - 1) {
647 npages_allocated =
648 i * (pbl_bt_sz / BA_BYTE_LEN);
649
650 dma_free_coherent(dev,
651 (npages - npages_allocated) * BA_BYTE_LEN,
652 mr->pbl_bt_l1[i],
653 mr->pbl_l1_dma_addr[i]);
654
655 break;
656 }
657
658 dma_free_coherent(dev, pbl_bt_sz, mr->pbl_bt_l1[i],
659 mr->pbl_l1_dma_addr[i]);
660 }
661 } else if (mhop_num == 3) {
662 for (i = 0; i < mr->l0_chunk_last_num; i++) {
663 dma_free_coherent(dev, pbl_bt_sz, mr->pbl_bt_l1[i],
664 mr->pbl_l1_dma_addr[i]);
665
666 for (j = 0; j < pbl_bt_sz / BA_BYTE_LEN; j++) {
667 bt_idx = i * (pbl_bt_sz / BA_BYTE_LEN) + j;
668
669 if ((i == mr->l0_chunk_last_num - 1)
670 && j == mr->l1_chunk_last_num - 1) {
671 npages_allocated = bt_idx *
672 (pbl_bt_sz / BA_BYTE_LEN);
673
674 dma_free_coherent(dev,
675 (npages - npages_allocated) *
676 BA_BYTE_LEN,
677 mr->pbl_bt_l2[bt_idx],
678 mr->pbl_l2_dma_addr[bt_idx]);
679
680 break;
681 }
682
683 dma_free_coherent(dev, pbl_bt_sz,
684 mr->pbl_bt_l2[bt_idx],
685 mr->pbl_l2_dma_addr[bt_idx]);
686 }
687 }
688 }
689
690 kfree(mr->pbl_bt_l1);
691 kfree(mr->pbl_l1_dma_addr);
692 mr->pbl_bt_l1 = NULL;
693 mr->pbl_l1_dma_addr = NULL;
694 if (mhop_num == 3) {
695 kfree(mr->pbl_bt_l2);
696 kfree(mr->pbl_l2_dma_addr);
697 mr->pbl_bt_l2 = NULL;
698 mr->pbl_l2_dma_addr = NULL;
699 }
700 }
701
702 static void hns_roce_mr_free(struct hns_roce_dev *hr_dev,
703 struct hns_roce_mr *mr)
704 {
705 struct device *dev = hr_dev->dev;
706 int npages = 0;
707 int ret;
708
709 if (mr->enabled) {
710 ret = hns_roce_hw2sw_mpt(hr_dev, NULL, key_to_hw_index(mr->key)
711 & (hr_dev->caps.num_mtpts - 1));
712 if (ret)
713 dev_warn(dev, "HW2SW_MPT failed (%d)\n", ret);
714 }
715
716 if (mr->size != ~0ULL) {
717 if (mr->type == MR_TYPE_MR)
718 npages = ib_umem_page_count(mr->umem);
719
720 if (!hr_dev->caps.pbl_hop_num)
721 dma_free_coherent(dev,
722 (unsigned int)(npages * BA_BYTE_LEN),
723 mr->pbl_buf, mr->pbl_dma_addr);
724 else
725 hns_roce_mhop_free(hr_dev, mr);
726 }
727
728 if (mr->enabled)
729 hns_roce_table_put(hr_dev, &hr_dev->mr_table.mtpt_table,
730 key_to_hw_index(mr->key));
731
732 hns_roce_bitmap_free(&hr_dev->mr_table.mtpt_bitmap,
733 key_to_hw_index(mr->key), BITMAP_NO_RR);
734 }
735
736 static int hns_roce_mr_enable(struct hns_roce_dev *hr_dev,
737 struct hns_roce_mr *mr)
738 {
739 int ret;
740 unsigned long mtpt_idx = key_to_hw_index(mr->key);
741 struct device *dev = hr_dev->dev;
742 struct hns_roce_cmd_mailbox *mailbox;
743 struct hns_roce_mr_table *mr_table = &hr_dev->mr_table;
744
745
746 ret = hns_roce_table_get(hr_dev, &mr_table->mtpt_table, mtpt_idx);
747 if (ret)
748 return ret;
749
750
751 mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
752 if (IS_ERR(mailbox)) {
753 ret = PTR_ERR(mailbox);
754 goto err_table;
755 }
756
757 if (mr->type != MR_TYPE_FRMR)
758 ret = hr_dev->hw->write_mtpt(mailbox->buf, mr, mtpt_idx);
759 else
760 ret = hr_dev->hw->frmr_write_mtpt(mailbox->buf, mr);
761 if (ret) {
762 dev_err(dev, "Write mtpt fail!\n");
763 goto err_page;
764 }
765
766 ret = hns_roce_sw2hw_mpt(hr_dev, mailbox,
767 mtpt_idx & (hr_dev->caps.num_mtpts - 1));
768 if (ret) {
769 dev_err(dev, "SW2HW_MPT failed (%d)\n", ret);
770 goto err_page;
771 }
772
773 mr->enabled = 1;
774 hns_roce_free_cmd_mailbox(hr_dev, mailbox);
775
776 return 0;
777
778 err_page:
779 hns_roce_free_cmd_mailbox(hr_dev, mailbox);
780
781 err_table:
782 hns_roce_table_put(hr_dev, &mr_table->mtpt_table, mtpt_idx);
783 return ret;
784 }
785
786 static int hns_roce_write_mtt_chunk(struct hns_roce_dev *hr_dev,
787 struct hns_roce_mtt *mtt, u32 start_index,
788 u32 npages, u64 *page_list)
789 {
790 struct hns_roce_hem_table *table;
791 dma_addr_t dma_handle;
792 __le64 *mtts;
793 u32 bt_page_size;
794 u32 i;
795
796 switch (mtt->mtt_type) {
797 case MTT_TYPE_WQE:
798 table = &hr_dev->mr_table.mtt_table;
799 bt_page_size = 1 << (hr_dev->caps.mtt_ba_pg_sz + PAGE_SHIFT);
800 break;
801 case MTT_TYPE_CQE:
802 table = &hr_dev->mr_table.mtt_cqe_table;
803 bt_page_size = 1 << (hr_dev->caps.cqe_ba_pg_sz + PAGE_SHIFT);
804 break;
805 case MTT_TYPE_SRQWQE:
806 table = &hr_dev->mr_table.mtt_srqwqe_table;
807 bt_page_size = 1 << (hr_dev->caps.srqwqe_ba_pg_sz + PAGE_SHIFT);
808 break;
809 case MTT_TYPE_IDX:
810 table = &hr_dev->mr_table.mtt_idx_table;
811 bt_page_size = 1 << (hr_dev->caps.idx_ba_pg_sz + PAGE_SHIFT);
812 break;
813 default:
814 return -EINVAL;
815 }
816
817
818 if (start_index / (bt_page_size / sizeof(u64)) !=
819 (start_index + npages - 1) / (bt_page_size / sizeof(u64)))
820 return -EINVAL;
821
822 if (start_index & (HNS_ROCE_MTT_ENTRY_PER_SEG - 1))
823 return -EINVAL;
824
825 mtts = hns_roce_table_find(hr_dev, table,
826 mtt->first_seg +
827 start_index / HNS_ROCE_MTT_ENTRY_PER_SEG,
828 &dma_handle);
829 if (!mtts)
830 return -ENOMEM;
831
832
833 for (i = 0; i < npages; ++i) {
834 if (!hr_dev->caps.mtt_hop_num)
835 mtts[i] = cpu_to_le64(page_list[i] >> PAGE_ADDR_SHIFT);
836 else
837 mtts[i] = cpu_to_le64(page_list[i]);
838 }
839
840 return 0;
841 }
842
843 static int hns_roce_write_mtt(struct hns_roce_dev *hr_dev,
844 struct hns_roce_mtt *mtt, u32 start_index,
845 u32 npages, u64 *page_list)
846 {
847 int chunk;
848 int ret;
849 u32 bt_page_size;
850
851 if (mtt->order < 0)
852 return -EINVAL;
853
854 switch (mtt->mtt_type) {
855 case MTT_TYPE_WQE:
856 bt_page_size = 1 << (hr_dev->caps.mtt_ba_pg_sz + PAGE_SHIFT);
857 break;
858 case MTT_TYPE_CQE:
859 bt_page_size = 1 << (hr_dev->caps.cqe_ba_pg_sz + PAGE_SHIFT);
860 break;
861 case MTT_TYPE_SRQWQE:
862 bt_page_size = 1 << (hr_dev->caps.srqwqe_ba_pg_sz + PAGE_SHIFT);
863 break;
864 case MTT_TYPE_IDX:
865 bt_page_size = 1 << (hr_dev->caps.idx_ba_pg_sz + PAGE_SHIFT);
866 break;
867 default:
868 dev_err(hr_dev->dev,
869 "Unsupport mtt type %d, write mtt failed\n",
870 mtt->mtt_type);
871 return -EINVAL;
872 }
873
874 while (npages > 0) {
875 chunk = min_t(int, bt_page_size / sizeof(u64), npages);
876
877 ret = hns_roce_write_mtt_chunk(hr_dev, mtt, start_index, chunk,
878 page_list);
879 if (ret)
880 return ret;
881
882 npages -= chunk;
883 start_index += chunk;
884 page_list += chunk;
885 }
886
887 return 0;
888 }
889
890 int hns_roce_buf_write_mtt(struct hns_roce_dev *hr_dev,
891 struct hns_roce_mtt *mtt, struct hns_roce_buf *buf)
892 {
893 u64 *page_list;
894 int ret;
895 u32 i;
896
897 page_list = kmalloc_array(buf->npages, sizeof(*page_list), GFP_KERNEL);
898 if (!page_list)
899 return -ENOMEM;
900
901 for (i = 0; i < buf->npages; ++i) {
902 if (buf->nbufs == 1)
903 page_list[i] = buf->direct.map + (i << buf->page_shift);
904 else
905 page_list[i] = buf->page_list[i].map;
906
907 }
908 ret = hns_roce_write_mtt(hr_dev, mtt, 0, buf->npages, page_list);
909
910 kfree(page_list);
911
912 return ret;
913 }
914
915 int hns_roce_init_mr_table(struct hns_roce_dev *hr_dev)
916 {
917 struct hns_roce_mr_table *mr_table = &hr_dev->mr_table;
918 int ret;
919
920 ret = hns_roce_bitmap_init(&mr_table->mtpt_bitmap,
921 hr_dev->caps.num_mtpts,
922 hr_dev->caps.num_mtpts - 1,
923 hr_dev->caps.reserved_mrws, 0);
924 if (ret)
925 return ret;
926
927 ret = hns_roce_buddy_init(&mr_table->mtt_buddy,
928 ilog2(hr_dev->caps.num_mtt_segs));
929 if (ret)
930 goto err_buddy;
931
932 if (hns_roce_check_whether_mhop(hr_dev, HEM_TYPE_CQE)) {
933 ret = hns_roce_buddy_init(&mr_table->mtt_cqe_buddy,
934 ilog2(hr_dev->caps.num_cqe_segs));
935 if (ret)
936 goto err_buddy_cqe;
937 }
938
939 if (hr_dev->caps.num_srqwqe_segs) {
940 ret = hns_roce_buddy_init(&mr_table->mtt_srqwqe_buddy,
941 ilog2(hr_dev->caps.num_srqwqe_segs));
942 if (ret)
943 goto err_buddy_srqwqe;
944 }
945
946 if (hr_dev->caps.num_idx_segs) {
947 ret = hns_roce_buddy_init(&mr_table->mtt_idx_buddy,
948 ilog2(hr_dev->caps.num_idx_segs));
949 if (ret)
950 goto err_buddy_idx;
951 }
952
953 return 0;
954
955 err_buddy_idx:
956 if (hr_dev->caps.num_srqwqe_segs)
957 hns_roce_buddy_cleanup(&mr_table->mtt_srqwqe_buddy);
958
959 err_buddy_srqwqe:
960 if (hns_roce_check_whether_mhop(hr_dev, HEM_TYPE_CQE))
961 hns_roce_buddy_cleanup(&mr_table->mtt_cqe_buddy);
962
963 err_buddy_cqe:
964 hns_roce_buddy_cleanup(&mr_table->mtt_buddy);
965
966 err_buddy:
967 hns_roce_bitmap_cleanup(&mr_table->mtpt_bitmap);
968 return ret;
969 }
970
971 void hns_roce_cleanup_mr_table(struct hns_roce_dev *hr_dev)
972 {
973 struct hns_roce_mr_table *mr_table = &hr_dev->mr_table;
974
975 if (hr_dev->caps.num_idx_segs)
976 hns_roce_buddy_cleanup(&mr_table->mtt_idx_buddy);
977 if (hr_dev->caps.num_srqwqe_segs)
978 hns_roce_buddy_cleanup(&mr_table->mtt_srqwqe_buddy);
979 hns_roce_buddy_cleanup(&mr_table->mtt_buddy);
980 if (hns_roce_check_whether_mhop(hr_dev, HEM_TYPE_CQE))
981 hns_roce_buddy_cleanup(&mr_table->mtt_cqe_buddy);
982 hns_roce_bitmap_cleanup(&mr_table->mtpt_bitmap);
983 }
984
985 struct ib_mr *hns_roce_get_dma_mr(struct ib_pd *pd, int acc)
986 {
987 struct hns_roce_mr *mr;
988 int ret;
989
990 mr = kmalloc(sizeof(*mr), GFP_KERNEL);
991 if (mr == NULL)
992 return ERR_PTR(-ENOMEM);
993
994 mr->type = MR_TYPE_DMA;
995
996
997 ret = hns_roce_mr_alloc(to_hr_dev(pd->device), to_hr_pd(pd)->pdn, 0,
998 ~0ULL, acc, 0, mr);
999 if (ret)
1000 goto err_free;
1001
1002 ret = hns_roce_mr_enable(to_hr_dev(pd->device), mr);
1003 if (ret)
1004 goto err_mr;
1005
1006 mr->ibmr.rkey = mr->ibmr.lkey = mr->key;
1007 mr->umem = NULL;
1008
1009 return &mr->ibmr;
1010
1011 err_mr:
1012 hns_roce_mr_free(to_hr_dev(pd->device), mr);
1013
1014 err_free:
1015 kfree(mr);
1016 return ERR_PTR(ret);
1017 }
1018
1019 int hns_roce_ib_umem_write_mtt(struct hns_roce_dev *hr_dev,
1020 struct hns_roce_mtt *mtt, struct ib_umem *umem)
1021 {
1022 struct device *dev = hr_dev->dev;
1023 struct sg_dma_page_iter sg_iter;
1024 unsigned int order;
1025 int npage = 0;
1026 int ret = 0;
1027 int i;
1028 u64 page_addr;
1029 u64 *pages;
1030 u32 bt_page_size;
1031 u32 n;
1032
1033 switch (mtt->mtt_type) {
1034 case MTT_TYPE_WQE:
1035 order = hr_dev->caps.mtt_ba_pg_sz;
1036 break;
1037 case MTT_TYPE_CQE:
1038 order = hr_dev->caps.cqe_ba_pg_sz;
1039 break;
1040 case MTT_TYPE_SRQWQE:
1041 order = hr_dev->caps.srqwqe_ba_pg_sz;
1042 break;
1043 case MTT_TYPE_IDX:
1044 order = hr_dev->caps.idx_ba_pg_sz;
1045 break;
1046 default:
1047 dev_err(dev, "Unsupport mtt type %d, write mtt failed\n",
1048 mtt->mtt_type);
1049 return -EINVAL;
1050 }
1051
1052 bt_page_size = 1 << (order + PAGE_SHIFT);
1053
1054 pages = (u64 *) __get_free_pages(GFP_KERNEL, order);
1055 if (!pages)
1056 return -ENOMEM;
1057
1058 i = n = 0;
1059
1060 for_each_sg_dma_page(umem->sg_head.sgl, &sg_iter, umem->nmap, 0) {
1061 page_addr = sg_page_iter_dma_address(&sg_iter);
1062 if (!(npage % (1 << (mtt->page_shift - PAGE_SHIFT)))) {
1063 if (page_addr & ((1 << mtt->page_shift) - 1)) {
1064 dev_err(dev,
1065 "page_addr is not page_shift %d alignment!\n",
1066 mtt->page_shift);
1067 ret = -EINVAL;
1068 goto out;
1069 }
1070 pages[i++] = page_addr;
1071 }
1072 npage++;
1073 if (i == bt_page_size / sizeof(u64)) {
1074 ret = hns_roce_write_mtt(hr_dev, mtt, n, i, pages);
1075 if (ret)
1076 goto out;
1077 n += i;
1078 i = 0;
1079 }
1080 }
1081
1082 if (i)
1083 ret = hns_roce_write_mtt(hr_dev, mtt, n, i, pages);
1084
1085 out:
1086 free_pages((unsigned long) pages, order);
1087 return ret;
1088 }
1089
1090 static int hns_roce_ib_umem_write_mr(struct hns_roce_dev *hr_dev,
1091 struct hns_roce_mr *mr,
1092 struct ib_umem *umem)
1093 {
1094 struct sg_dma_page_iter sg_iter;
1095 int i = 0, j = 0;
1096 u64 page_addr;
1097 u32 pbl_bt_sz;
1098
1099 if (hr_dev->caps.pbl_hop_num == HNS_ROCE_HOP_NUM_0)
1100 return 0;
1101
1102 pbl_bt_sz = 1 << (hr_dev->caps.pbl_ba_pg_sz + PAGE_SHIFT);
1103 for_each_sg_dma_page(umem->sg_head.sgl, &sg_iter, umem->nmap, 0) {
1104 page_addr = sg_page_iter_dma_address(&sg_iter);
1105 if (!hr_dev->caps.pbl_hop_num) {
1106
1107 mr->pbl_buf[i++] = page_addr >> 12;
1108 } else if (hr_dev->caps.pbl_hop_num == 1) {
1109 mr->pbl_buf[i++] = page_addr;
1110 } else {
1111 if (hr_dev->caps.pbl_hop_num == 2)
1112 mr->pbl_bt_l1[i][j] = page_addr;
1113 else if (hr_dev->caps.pbl_hop_num == 3)
1114 mr->pbl_bt_l2[i][j] = page_addr;
1115
1116 j++;
1117 if (j >= (pbl_bt_sz / BA_BYTE_LEN)) {
1118 i++;
1119 j = 0;
1120 }
1121 }
1122 }
1123
1124
1125 mb();
1126
1127 return 0;
1128 }
1129
1130 struct ib_mr *hns_roce_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
1131 u64 virt_addr, int access_flags,
1132 struct ib_udata *udata)
1133 {
1134 struct hns_roce_dev *hr_dev = to_hr_dev(pd->device);
1135 struct device *dev = hr_dev->dev;
1136 struct hns_roce_mr *mr;
1137 int bt_size;
1138 int ret;
1139 int n;
1140 int i;
1141
1142 mr = kmalloc(sizeof(*mr), GFP_KERNEL);
1143 if (!mr)
1144 return ERR_PTR(-ENOMEM);
1145
1146 mr->umem = ib_umem_get(udata, start, length, access_flags, 0);
1147 if (IS_ERR(mr->umem)) {
1148 ret = PTR_ERR(mr->umem);
1149 goto err_free;
1150 }
1151
1152 n = ib_umem_page_count(mr->umem);
1153
1154 if (!hr_dev->caps.pbl_hop_num) {
1155 if (n > HNS_ROCE_MAX_MTPT_PBL_NUM) {
1156 dev_err(dev,
1157 " MR len %lld err. MR is limited to 4G at most!\n",
1158 length);
1159 ret = -EINVAL;
1160 goto err_umem;
1161 }
1162 } else {
1163 u64 pbl_size = 1;
1164
1165 bt_size = (1 << (hr_dev->caps.pbl_ba_pg_sz + PAGE_SHIFT)) /
1166 BA_BYTE_LEN;
1167 for (i = 0; i < hr_dev->caps.pbl_hop_num; i++)
1168 pbl_size *= bt_size;
1169 if (n > pbl_size) {
1170 dev_err(dev,
1171 " MR len %lld err. MR page num is limited to %lld!\n",
1172 length, pbl_size);
1173 ret = -EINVAL;
1174 goto err_umem;
1175 }
1176 }
1177
1178 mr->type = MR_TYPE_MR;
1179
1180 ret = hns_roce_mr_alloc(hr_dev, to_hr_pd(pd)->pdn, virt_addr, length,
1181 access_flags, n, mr);
1182 if (ret)
1183 goto err_umem;
1184
1185 ret = hns_roce_ib_umem_write_mr(hr_dev, mr, mr->umem);
1186 if (ret)
1187 goto err_mr;
1188
1189 ret = hns_roce_mr_enable(hr_dev, mr);
1190 if (ret)
1191 goto err_mr;
1192
1193 mr->ibmr.rkey = mr->ibmr.lkey = mr->key;
1194
1195 return &mr->ibmr;
1196
1197 err_mr:
1198 hns_roce_mr_free(hr_dev, mr);
1199
1200 err_umem:
1201 ib_umem_release(mr->umem);
1202
1203 err_free:
1204 kfree(mr);
1205 return ERR_PTR(ret);
1206 }
1207
1208 static int rereg_mr_trans(struct ib_mr *ibmr, int flags,
1209 u64 start, u64 length,
1210 u64 virt_addr, int mr_access_flags,
1211 struct hns_roce_cmd_mailbox *mailbox,
1212 u32 pdn, struct ib_udata *udata)
1213 {
1214 struct hns_roce_dev *hr_dev = to_hr_dev(ibmr->device);
1215 struct hns_roce_mr *mr = to_hr_mr(ibmr);
1216 struct device *dev = hr_dev->dev;
1217 int npages;
1218 int ret;
1219
1220 if (mr->size != ~0ULL) {
1221 npages = ib_umem_page_count(mr->umem);
1222
1223 if (hr_dev->caps.pbl_hop_num)
1224 hns_roce_mhop_free(hr_dev, mr);
1225 else
1226 dma_free_coherent(dev, npages * 8,
1227 mr->pbl_buf, mr->pbl_dma_addr);
1228 }
1229 ib_umem_release(mr->umem);
1230
1231 mr->umem = ib_umem_get(udata, start, length, mr_access_flags, 0);
1232 if (IS_ERR(mr->umem)) {
1233 ret = PTR_ERR(mr->umem);
1234 mr->umem = NULL;
1235 return -ENOMEM;
1236 }
1237 npages = ib_umem_page_count(mr->umem);
1238
1239 if (hr_dev->caps.pbl_hop_num) {
1240 ret = hns_roce_mhop_alloc(hr_dev, npages, mr);
1241 if (ret)
1242 goto release_umem;
1243 } else {
1244 mr->pbl_buf = dma_alloc_coherent(dev, npages * 8,
1245 &(mr->pbl_dma_addr),
1246 GFP_KERNEL);
1247 if (!mr->pbl_buf) {
1248 ret = -ENOMEM;
1249 goto release_umem;
1250 }
1251 }
1252
1253 ret = hr_dev->hw->rereg_write_mtpt(hr_dev, mr, flags, pdn,
1254 mr_access_flags, virt_addr,
1255 length, mailbox->buf);
1256 if (ret)
1257 goto release_umem;
1258
1259
1260 ret = hns_roce_ib_umem_write_mr(hr_dev, mr, mr->umem);
1261 if (ret) {
1262 if (mr->size != ~0ULL) {
1263 npages = ib_umem_page_count(mr->umem);
1264
1265 if (hr_dev->caps.pbl_hop_num)
1266 hns_roce_mhop_free(hr_dev, mr);
1267 else
1268 dma_free_coherent(dev, npages * 8,
1269 mr->pbl_buf,
1270 mr->pbl_dma_addr);
1271 }
1272
1273 goto release_umem;
1274 }
1275
1276 return 0;
1277
1278 release_umem:
1279 ib_umem_release(mr->umem);
1280 return ret;
1281
1282 }
1283
1284
1285 int hns_roce_rereg_user_mr(struct ib_mr *ibmr, int flags, u64 start, u64 length,
1286 u64 virt_addr, int mr_access_flags, struct ib_pd *pd,
1287 struct ib_udata *udata)
1288 {
1289 struct hns_roce_dev *hr_dev = to_hr_dev(ibmr->device);
1290 struct hns_roce_mr *mr = to_hr_mr(ibmr);
1291 struct hns_roce_cmd_mailbox *mailbox;
1292 struct device *dev = hr_dev->dev;
1293 unsigned long mtpt_idx;
1294 u32 pdn = 0;
1295 int ret;
1296
1297 if (!mr->enabled)
1298 return -EINVAL;
1299
1300 mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
1301 if (IS_ERR(mailbox))
1302 return PTR_ERR(mailbox);
1303
1304 mtpt_idx = key_to_hw_index(mr->key) & (hr_dev->caps.num_mtpts - 1);
1305 ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox->dma, mtpt_idx, 0,
1306 HNS_ROCE_CMD_QUERY_MPT,
1307 HNS_ROCE_CMD_TIMEOUT_MSECS);
1308 if (ret)
1309 goto free_cmd_mbox;
1310
1311 ret = hns_roce_hw2sw_mpt(hr_dev, NULL, mtpt_idx);
1312 if (ret)
1313 dev_warn(dev, "HW2SW_MPT failed (%d)\n", ret);
1314
1315 mr->enabled = 0;
1316
1317 if (flags & IB_MR_REREG_PD)
1318 pdn = to_hr_pd(pd)->pdn;
1319
1320 if (flags & IB_MR_REREG_TRANS) {
1321 ret = rereg_mr_trans(ibmr, flags,
1322 start, length,
1323 virt_addr, mr_access_flags,
1324 mailbox, pdn, udata);
1325 if (ret)
1326 goto free_cmd_mbox;
1327 } else {
1328 ret = hr_dev->hw->rereg_write_mtpt(hr_dev, mr, flags, pdn,
1329 mr_access_flags, virt_addr,
1330 length, mailbox->buf);
1331 if (ret)
1332 goto free_cmd_mbox;
1333 }
1334
1335 ret = hns_roce_sw2hw_mpt(hr_dev, mailbox, mtpt_idx);
1336 if (ret) {
1337 dev_err(dev, "SW2HW_MPT failed (%d)\n", ret);
1338 ib_umem_release(mr->umem);
1339 goto free_cmd_mbox;
1340 }
1341
1342 mr->enabled = 1;
1343 if (flags & IB_MR_REREG_ACCESS)
1344 mr->access = mr_access_flags;
1345
1346 hns_roce_free_cmd_mailbox(hr_dev, mailbox);
1347
1348 return 0;
1349
1350 free_cmd_mbox:
1351 hns_roce_free_cmd_mailbox(hr_dev, mailbox);
1352
1353 return ret;
1354 }
1355
1356 int hns_roce_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata)
1357 {
1358 struct hns_roce_dev *hr_dev = to_hr_dev(ibmr->device);
1359 struct hns_roce_mr *mr = to_hr_mr(ibmr);
1360 int ret = 0;
1361
1362 if (hr_dev->hw->dereg_mr) {
1363 ret = hr_dev->hw->dereg_mr(hr_dev, mr, udata);
1364 } else {
1365 hns_roce_mr_free(hr_dev, mr);
1366
1367 ib_umem_release(mr->umem);
1368 kfree(mr);
1369 }
1370
1371 return ret;
1372 }
1373
1374 struct ib_mr *hns_roce_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type,
1375 u32 max_num_sg, struct ib_udata *udata)
1376 {
1377 struct hns_roce_dev *hr_dev = to_hr_dev(pd->device);
1378 struct device *dev = hr_dev->dev;
1379 struct hns_roce_mr *mr;
1380 u64 length;
1381 u32 page_size;
1382 int ret;
1383
1384 page_size = 1 << (hr_dev->caps.pbl_buf_pg_sz + PAGE_SHIFT);
1385 length = max_num_sg * page_size;
1386
1387 if (mr_type != IB_MR_TYPE_MEM_REG)
1388 return ERR_PTR(-EINVAL);
1389
1390 if (max_num_sg > HNS_ROCE_FRMR_MAX_PA) {
1391 dev_err(dev, "max_num_sg larger than %d\n",
1392 HNS_ROCE_FRMR_MAX_PA);
1393 return ERR_PTR(-EINVAL);
1394 }
1395
1396 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
1397 if (!mr)
1398 return ERR_PTR(-ENOMEM);
1399
1400 mr->type = MR_TYPE_FRMR;
1401
1402
1403 ret = hns_roce_mr_alloc(hr_dev, to_hr_pd(pd)->pdn, 0, length,
1404 0, max_num_sg, mr);
1405 if (ret)
1406 goto err_free;
1407
1408 ret = hns_roce_mr_enable(hr_dev, mr);
1409 if (ret)
1410 goto err_mr;
1411
1412 mr->ibmr.rkey = mr->ibmr.lkey = mr->key;
1413 mr->umem = NULL;
1414
1415 return &mr->ibmr;
1416
1417 err_mr:
1418 hns_roce_mr_free(to_hr_dev(pd->device), mr);
1419
1420 err_free:
1421 kfree(mr);
1422 return ERR_PTR(ret);
1423 }
1424
1425 static int hns_roce_set_page(struct ib_mr *ibmr, u64 addr)
1426 {
1427 struct hns_roce_mr *mr = to_hr_mr(ibmr);
1428
1429 mr->pbl_buf[mr->npages++] = addr;
1430
1431 return 0;
1432 }
1433
1434 int hns_roce_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents,
1435 unsigned int *sg_offset)
1436 {
1437 struct hns_roce_mr *mr = to_hr_mr(ibmr);
1438
1439 mr->npages = 0;
1440
1441 return ib_sg_to_pages(ibmr, sg, sg_nents, sg_offset, hns_roce_set_page);
1442 }
1443
1444 static void hns_roce_mw_free(struct hns_roce_dev *hr_dev,
1445 struct hns_roce_mw *mw)
1446 {
1447 struct device *dev = hr_dev->dev;
1448 int ret;
1449
1450 if (mw->enabled) {
1451 ret = hns_roce_hw2sw_mpt(hr_dev, NULL, key_to_hw_index(mw->rkey)
1452 & (hr_dev->caps.num_mtpts - 1));
1453 if (ret)
1454 dev_warn(dev, "MW HW2SW_MPT failed (%d)\n", ret);
1455
1456 hns_roce_table_put(hr_dev, &hr_dev->mr_table.mtpt_table,
1457 key_to_hw_index(mw->rkey));
1458 }
1459
1460 hns_roce_bitmap_free(&hr_dev->mr_table.mtpt_bitmap,
1461 key_to_hw_index(mw->rkey), BITMAP_NO_RR);
1462 }
1463
1464 static int hns_roce_mw_enable(struct hns_roce_dev *hr_dev,
1465 struct hns_roce_mw *mw)
1466 {
1467 struct hns_roce_mr_table *mr_table = &hr_dev->mr_table;
1468 struct hns_roce_cmd_mailbox *mailbox;
1469 struct device *dev = hr_dev->dev;
1470 unsigned long mtpt_idx = key_to_hw_index(mw->rkey);
1471 int ret;
1472
1473
1474 ret = hns_roce_table_get(hr_dev, &mr_table->mtpt_table, mtpt_idx);
1475 if (ret)
1476 return ret;
1477
1478 mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
1479 if (IS_ERR(mailbox)) {
1480 ret = PTR_ERR(mailbox);
1481 goto err_table;
1482 }
1483
1484 ret = hr_dev->hw->mw_write_mtpt(mailbox->buf, mw);
1485 if (ret) {
1486 dev_err(dev, "MW write mtpt fail!\n");
1487 goto err_page;
1488 }
1489
1490 ret = hns_roce_sw2hw_mpt(hr_dev, mailbox,
1491 mtpt_idx & (hr_dev->caps.num_mtpts - 1));
1492 if (ret) {
1493 dev_err(dev, "MW sw2hw_mpt failed (%d)\n", ret);
1494 goto err_page;
1495 }
1496
1497 mw->enabled = 1;
1498
1499 hns_roce_free_cmd_mailbox(hr_dev, mailbox);
1500
1501 return 0;
1502
1503 err_page:
1504 hns_roce_free_cmd_mailbox(hr_dev, mailbox);
1505
1506 err_table:
1507 hns_roce_table_put(hr_dev, &mr_table->mtpt_table, mtpt_idx);
1508
1509 return ret;
1510 }
1511
1512 struct ib_mw *hns_roce_alloc_mw(struct ib_pd *ib_pd, enum ib_mw_type type,
1513 struct ib_udata *udata)
1514 {
1515 struct hns_roce_dev *hr_dev = to_hr_dev(ib_pd->device);
1516 struct hns_roce_mw *mw;
1517 unsigned long index = 0;
1518 int ret;
1519
1520 mw = kmalloc(sizeof(*mw), GFP_KERNEL);
1521 if (!mw)
1522 return ERR_PTR(-ENOMEM);
1523
1524
1525 ret = hns_roce_bitmap_alloc(&hr_dev->mr_table.mtpt_bitmap, &index);
1526 if (ret)
1527 goto err_bitmap;
1528
1529 mw->rkey = hw_index_to_key(index);
1530
1531 mw->ibmw.rkey = mw->rkey;
1532 mw->ibmw.type = type;
1533 mw->pdn = to_hr_pd(ib_pd)->pdn;
1534 mw->pbl_hop_num = hr_dev->caps.pbl_hop_num;
1535 mw->pbl_ba_pg_sz = hr_dev->caps.pbl_ba_pg_sz;
1536 mw->pbl_buf_pg_sz = hr_dev->caps.pbl_buf_pg_sz;
1537
1538 ret = hns_roce_mw_enable(hr_dev, mw);
1539 if (ret)
1540 goto err_mw;
1541
1542 return &mw->ibmw;
1543
1544 err_mw:
1545 hns_roce_mw_free(hr_dev, mw);
1546
1547 err_bitmap:
1548 kfree(mw);
1549
1550 return ERR_PTR(ret);
1551 }
1552
1553 int hns_roce_dealloc_mw(struct ib_mw *ibmw)
1554 {
1555 struct hns_roce_dev *hr_dev = to_hr_dev(ibmw->device);
1556 struct hns_roce_mw *mw = to_hr_mw(ibmw);
1557
1558 hns_roce_mw_free(hr_dev, mw);
1559 kfree(mw);
1560
1561 return 0;
1562 }
1563
1564 void hns_roce_mtr_init(struct hns_roce_mtr *mtr, int bt_pg_shift,
1565 int buf_pg_shift)
1566 {
1567 hns_roce_hem_list_init(&mtr->hem_list, bt_pg_shift);
1568 mtr->buf_pg_shift = buf_pg_shift;
1569 }
1570
1571 void hns_roce_mtr_cleanup(struct hns_roce_dev *hr_dev,
1572 struct hns_roce_mtr *mtr)
1573 {
1574 hns_roce_hem_list_release(hr_dev, &mtr->hem_list);
1575 }
1576
1577 static int hns_roce_write_mtr(struct hns_roce_dev *hr_dev,
1578 struct hns_roce_mtr *mtr, dma_addr_t *bufs,
1579 struct hns_roce_buf_region *r)
1580 {
1581 int offset;
1582 int count;
1583 int npage;
1584 u64 *mtts;
1585 int end;
1586 int i;
1587
1588 offset = r->offset;
1589 end = offset + r->count;
1590 npage = 0;
1591 while (offset < end) {
1592 mtts = hns_roce_hem_list_find_mtt(hr_dev, &mtr->hem_list,
1593 offset, &count, NULL);
1594 if (!mtts)
1595 return -ENOBUFS;
1596
1597
1598 for (i = 0; i < count; i++) {
1599 if (hr_dev->hw_rev == HNS_ROCE_HW_VER1)
1600 mtts[i] = bufs[npage] >> PAGE_ADDR_SHIFT;
1601 else
1602 mtts[i] = bufs[npage];
1603
1604 npage++;
1605 }
1606 offset += count;
1607 }
1608
1609 return 0;
1610 }
1611
1612 int hns_roce_mtr_attach(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr,
1613 dma_addr_t **bufs, struct hns_roce_buf_region *regions,
1614 int region_cnt)
1615 {
1616 struct hns_roce_buf_region *r;
1617 int ret;
1618 int i;
1619
1620 ret = hns_roce_hem_list_request(hr_dev, &mtr->hem_list, regions,
1621 region_cnt);
1622 if (ret)
1623 return ret;
1624
1625 for (i = 0; i < region_cnt; i++) {
1626 r = ®ions[i];
1627 ret = hns_roce_write_mtr(hr_dev, mtr, bufs[i], r);
1628 if (ret) {
1629 dev_err(hr_dev->dev,
1630 "write mtr[%d/%d] err %d,offset=%d.\n",
1631 i, region_cnt, ret, r->offset);
1632 goto err_write;
1633 }
1634 }
1635
1636 return 0;
1637
1638 err_write:
1639 hns_roce_hem_list_release(hr_dev, &mtr->hem_list);
1640
1641 return ret;
1642 }
1643
1644 int hns_roce_mtr_find(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr,
1645 int offset, u64 *mtt_buf, int mtt_max, u64 *base_addr)
1646 {
1647 u64 *mtts = mtt_buf;
1648 int mtt_count;
1649 int total = 0;
1650 u64 *addr;
1651 int npage;
1652 int left;
1653
1654 if (mtts == NULL || mtt_max < 1)
1655 goto done;
1656
1657 left = mtt_max;
1658 while (left > 0) {
1659 mtt_count = 0;
1660 addr = hns_roce_hem_list_find_mtt(hr_dev, &mtr->hem_list,
1661 offset + total,
1662 &mtt_count, NULL);
1663 if (!addr || !mtt_count)
1664 goto done;
1665
1666 npage = min(mtt_count, left);
1667 memcpy(&mtts[total], addr, BA_BYTE_LEN * npage);
1668 left -= npage;
1669 total += npage;
1670 }
1671
1672 done:
1673 if (base_addr)
1674 *base_addr = mtr->hem_list.root_ba;
1675
1676 return total;
1677 }