This source file includes following definitions.
- hns_roce_check_whether_mhop
- hns_roce_check_hem_null
- hns_roce_check_bt_null
- hns_roce_get_bt_num
- get_hem_table_config
- hns_roce_calc_hem_mhop
- hns_roce_alloc_hem
- hns_roce_free_hem
- hns_roce_set_hem
- hns_roce_table_mhop_get
- hns_roce_table_get
- hns_roce_table_mhop_put
- hns_roce_table_put
- hns_roce_table_find
- hns_roce_table_get_range
- hns_roce_table_put_range
- hns_roce_init_hem_table
- hns_roce_cleanup_mhop_hem_table
- hns_roce_cleanup_hem_table
- hns_roce_cleanup_hem
- hem_list_alloc_item
- hem_list_free_item
- hem_list_free_all
- hem_list_link_bt
- hem_list_assign_bt
- hem_list_page_is_in_range
- hem_list_search_item
- hem_list_is_bottom_bt
- hem_list_calc_ba_range
- hns_roce_hem_list_calc_root_ba
- hem_list_alloc_mid_bt
- hem_list_alloc_root_bt
- hns_roce_hem_list_request
- hns_roce_hem_list_release
- hns_roce_hem_list_init
- hns_roce_hem_list_find_mtt
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34 #include <linux/platform_device.h>
35 #include "hns_roce_device.h"
36 #include "hns_roce_hem.h"
37 #include "hns_roce_common.h"
38
39 #define DMA_ADDR_T_SHIFT 12
40 #define BT_BA_SHIFT 32
41
42 bool hns_roce_check_whether_mhop(struct hns_roce_dev *hr_dev, u32 type)
43 {
44 int hop_num = 0;
45
46 switch (type) {
47 case HEM_TYPE_QPC:
48 hop_num = hr_dev->caps.qpc_hop_num;
49 break;
50 case HEM_TYPE_MTPT:
51 hop_num = hr_dev->caps.mpt_hop_num;
52 break;
53 case HEM_TYPE_CQC:
54 hop_num = hr_dev->caps.cqc_hop_num;
55 break;
56 case HEM_TYPE_SRQC:
57 hop_num = hr_dev->caps.srqc_hop_num;
58 break;
59 case HEM_TYPE_SCCC:
60 hop_num = hr_dev->caps.sccc_hop_num;
61 break;
62 case HEM_TYPE_QPC_TIMER:
63 hop_num = hr_dev->caps.qpc_timer_hop_num;
64 break;
65 case HEM_TYPE_CQC_TIMER:
66 hop_num = hr_dev->caps.cqc_timer_hop_num;
67 break;
68 case HEM_TYPE_CQE:
69 hop_num = hr_dev->caps.cqe_hop_num;
70 break;
71 case HEM_TYPE_MTT:
72 hop_num = hr_dev->caps.mtt_hop_num;
73 break;
74 case HEM_TYPE_SRQWQE:
75 hop_num = hr_dev->caps.srqwqe_hop_num;
76 break;
77 case HEM_TYPE_IDX:
78 hop_num = hr_dev->caps.idx_hop_num;
79 break;
80 default:
81 return false;
82 }
83
84 return hop_num ? true : false;
85 }
86
87 static bool hns_roce_check_hem_null(struct hns_roce_hem **hem, u64 start_idx,
88 u32 bt_chunk_num, u64 hem_max_num)
89 {
90 u64 check_max_num = start_idx + bt_chunk_num;
91 u64 i;
92
93 for (i = start_idx; (i < check_max_num) && (i < hem_max_num); i++)
94 if (hem[i])
95 return false;
96
97 return true;
98 }
99
100 static bool hns_roce_check_bt_null(u64 **bt, u64 start_idx, u32 bt_chunk_num)
101 {
102 int i;
103
104 for (i = 0; i < bt_chunk_num; i++)
105 if (bt[start_idx + i])
106 return false;
107
108 return true;
109 }
110
111 static int hns_roce_get_bt_num(u32 table_type, u32 hop_num)
112 {
113 if (check_whether_bt_num_3(table_type, hop_num))
114 return 3;
115 else if (check_whether_bt_num_2(table_type, hop_num))
116 return 2;
117 else if (check_whether_bt_num_1(table_type, hop_num))
118 return 1;
119 else
120 return 0;
121 }
122
123 static int get_hem_table_config(struct hns_roce_dev *hr_dev,
124 struct hns_roce_hem_mhop *mhop,
125 u32 type)
126 {
127 struct device *dev = hr_dev->dev;
128
129 switch (type) {
130 case HEM_TYPE_QPC:
131 mhop->buf_chunk_size = 1 << (hr_dev->caps.qpc_buf_pg_sz
132 + PAGE_SHIFT);
133 mhop->bt_chunk_size = 1 << (hr_dev->caps.qpc_ba_pg_sz
134 + PAGE_SHIFT);
135 mhop->ba_l0_num = hr_dev->caps.qpc_bt_num;
136 mhop->hop_num = hr_dev->caps.qpc_hop_num;
137 break;
138 case HEM_TYPE_MTPT:
139 mhop->buf_chunk_size = 1 << (hr_dev->caps.mpt_buf_pg_sz
140 + PAGE_SHIFT);
141 mhop->bt_chunk_size = 1 << (hr_dev->caps.mpt_ba_pg_sz
142 + PAGE_SHIFT);
143 mhop->ba_l0_num = hr_dev->caps.mpt_bt_num;
144 mhop->hop_num = hr_dev->caps.mpt_hop_num;
145 break;
146 case HEM_TYPE_CQC:
147 mhop->buf_chunk_size = 1 << (hr_dev->caps.cqc_buf_pg_sz
148 + PAGE_SHIFT);
149 mhop->bt_chunk_size = 1 << (hr_dev->caps.cqc_ba_pg_sz
150 + PAGE_SHIFT);
151 mhop->ba_l0_num = hr_dev->caps.cqc_bt_num;
152 mhop->hop_num = hr_dev->caps.cqc_hop_num;
153 break;
154 case HEM_TYPE_SCCC:
155 mhop->buf_chunk_size = 1 << (hr_dev->caps.sccc_buf_pg_sz
156 + PAGE_SHIFT);
157 mhop->bt_chunk_size = 1 << (hr_dev->caps.sccc_ba_pg_sz
158 + PAGE_SHIFT);
159 mhop->ba_l0_num = hr_dev->caps.sccc_bt_num;
160 mhop->hop_num = hr_dev->caps.sccc_hop_num;
161 break;
162 case HEM_TYPE_QPC_TIMER:
163 mhop->buf_chunk_size = 1 << (hr_dev->caps.qpc_timer_buf_pg_sz
164 + PAGE_SHIFT);
165 mhop->bt_chunk_size = 1 << (hr_dev->caps.qpc_timer_ba_pg_sz
166 + PAGE_SHIFT);
167 mhop->ba_l0_num = hr_dev->caps.qpc_timer_bt_num;
168 mhop->hop_num = hr_dev->caps.qpc_timer_hop_num;
169 break;
170 case HEM_TYPE_CQC_TIMER:
171 mhop->buf_chunk_size = 1 << (hr_dev->caps.cqc_timer_buf_pg_sz
172 + PAGE_SHIFT);
173 mhop->bt_chunk_size = 1 << (hr_dev->caps.cqc_timer_ba_pg_sz
174 + PAGE_SHIFT);
175 mhop->ba_l0_num = hr_dev->caps.cqc_timer_bt_num;
176 mhop->hop_num = hr_dev->caps.cqc_timer_hop_num;
177 break;
178 case HEM_TYPE_SRQC:
179 mhop->buf_chunk_size = 1 << (hr_dev->caps.srqc_buf_pg_sz
180 + PAGE_SHIFT);
181 mhop->bt_chunk_size = 1 << (hr_dev->caps.srqc_ba_pg_sz
182 + PAGE_SHIFT);
183 mhop->ba_l0_num = hr_dev->caps.srqc_bt_num;
184 mhop->hop_num = hr_dev->caps.srqc_hop_num;
185 break;
186 case HEM_TYPE_MTT:
187 mhop->buf_chunk_size = 1 << (hr_dev->caps.mtt_buf_pg_sz
188 + PAGE_SHIFT);
189 mhop->bt_chunk_size = 1 << (hr_dev->caps.mtt_ba_pg_sz
190 + PAGE_SHIFT);
191 mhop->ba_l0_num = mhop->bt_chunk_size / BA_BYTE_LEN;
192 mhop->hop_num = hr_dev->caps.mtt_hop_num;
193 break;
194 case HEM_TYPE_CQE:
195 mhop->buf_chunk_size = 1 << (hr_dev->caps.cqe_buf_pg_sz
196 + PAGE_SHIFT);
197 mhop->bt_chunk_size = 1 << (hr_dev->caps.cqe_ba_pg_sz
198 + PAGE_SHIFT);
199 mhop->ba_l0_num = mhop->bt_chunk_size / BA_BYTE_LEN;
200 mhop->hop_num = hr_dev->caps.cqe_hop_num;
201 break;
202 case HEM_TYPE_SRQWQE:
203 mhop->buf_chunk_size = 1 << (hr_dev->caps.srqwqe_buf_pg_sz
204 + PAGE_SHIFT);
205 mhop->bt_chunk_size = 1 << (hr_dev->caps.srqwqe_ba_pg_sz
206 + PAGE_SHIFT);
207 mhop->ba_l0_num = mhop->bt_chunk_size / BA_BYTE_LEN;
208 mhop->hop_num = hr_dev->caps.srqwqe_hop_num;
209 break;
210 case HEM_TYPE_IDX:
211 mhop->buf_chunk_size = 1 << (hr_dev->caps.idx_buf_pg_sz
212 + PAGE_SHIFT);
213 mhop->bt_chunk_size = 1 << (hr_dev->caps.idx_ba_pg_sz
214 + PAGE_SHIFT);
215 mhop->ba_l0_num = mhop->bt_chunk_size / BA_BYTE_LEN;
216 mhop->hop_num = hr_dev->caps.idx_hop_num;
217 break;
218 default:
219 dev_err(dev, "Table %d not support multi-hop addressing!\n",
220 type);
221 return -EINVAL;
222 }
223
224 return 0;
225 }
226
227 int hns_roce_calc_hem_mhop(struct hns_roce_dev *hr_dev,
228 struct hns_roce_hem_table *table, unsigned long *obj,
229 struct hns_roce_hem_mhop *mhop)
230 {
231 struct device *dev = hr_dev->dev;
232 u32 chunk_ba_num;
233 u32 table_idx;
234 u32 bt_num;
235 u32 chunk_size;
236
237 if (get_hem_table_config(hr_dev, mhop, table->type))
238 return -EINVAL;
239
240 if (!obj)
241 return 0;
242
243
244
245
246
247 bt_num = hns_roce_get_bt_num(table->type, mhop->hop_num);
248 chunk_ba_num = mhop->bt_chunk_size / BA_BYTE_LEN;
249 chunk_size = table->type < HEM_TYPE_MTT ? mhop->buf_chunk_size :
250 mhop->bt_chunk_size;
251 table_idx = (*obj & (table->num_obj - 1)) /
252 (chunk_size / table->obj_size);
253 switch (bt_num) {
254 case 3:
255 mhop->l2_idx = table_idx & (chunk_ba_num - 1);
256 mhop->l1_idx = table_idx / chunk_ba_num & (chunk_ba_num - 1);
257 mhop->l0_idx = (table_idx / chunk_ba_num) / chunk_ba_num;
258 break;
259 case 2:
260 mhop->l1_idx = table_idx & (chunk_ba_num - 1);
261 mhop->l0_idx = table_idx / chunk_ba_num;
262 break;
263 case 1:
264 mhop->l0_idx = table_idx;
265 break;
266 default:
267 dev_err(dev, "Table %d not support hop_num = %d!\n",
268 table->type, mhop->hop_num);
269 return -EINVAL;
270 }
271 if (mhop->l0_idx >= mhop->ba_l0_num)
272 mhop->l0_idx %= mhop->ba_l0_num;
273
274 return 0;
275 }
276
277 static struct hns_roce_hem *hns_roce_alloc_hem(struct hns_roce_dev *hr_dev,
278 int npages,
279 unsigned long hem_alloc_size,
280 gfp_t gfp_mask)
281 {
282 struct hns_roce_hem_chunk *chunk = NULL;
283 struct hns_roce_hem *hem;
284 struct scatterlist *mem;
285 int order;
286 void *buf;
287
288 WARN_ON(gfp_mask & __GFP_HIGHMEM);
289
290 hem = kmalloc(sizeof(*hem),
291 gfp_mask & ~(__GFP_HIGHMEM | __GFP_NOWARN));
292 if (!hem)
293 return NULL;
294
295 hem->refcount = 0;
296 INIT_LIST_HEAD(&hem->chunk_list);
297
298 order = get_order(hem_alloc_size);
299
300 while (npages > 0) {
301 if (!chunk) {
302 chunk = kmalloc(sizeof(*chunk),
303 gfp_mask & ~(__GFP_HIGHMEM | __GFP_NOWARN));
304 if (!chunk)
305 goto fail;
306
307 sg_init_table(chunk->mem, HNS_ROCE_HEM_CHUNK_LEN);
308 chunk->npages = 0;
309 chunk->nsg = 0;
310 memset(chunk->buf, 0, sizeof(chunk->buf));
311 list_add_tail(&chunk->list, &hem->chunk_list);
312 }
313
314 while (1 << order > npages)
315 --order;
316
317
318
319
320
321 mem = &chunk->mem[chunk->npages];
322 buf = dma_alloc_coherent(hr_dev->dev, PAGE_SIZE << order,
323 &sg_dma_address(mem), gfp_mask);
324 if (!buf)
325 goto fail;
326
327 chunk->buf[chunk->npages] = buf;
328 sg_dma_len(mem) = PAGE_SIZE << order;
329
330 ++chunk->npages;
331 ++chunk->nsg;
332 npages -= 1 << order;
333 }
334
335 return hem;
336
337 fail:
338 hns_roce_free_hem(hr_dev, hem);
339 return NULL;
340 }
341
342 void hns_roce_free_hem(struct hns_roce_dev *hr_dev, struct hns_roce_hem *hem)
343 {
344 struct hns_roce_hem_chunk *chunk, *tmp;
345 int i;
346
347 if (!hem)
348 return;
349
350 list_for_each_entry_safe(chunk, tmp, &hem->chunk_list, list) {
351 for (i = 0; i < chunk->npages; ++i)
352 dma_free_coherent(hr_dev->dev,
353 sg_dma_len(&chunk->mem[i]),
354 chunk->buf[i],
355 sg_dma_address(&chunk->mem[i]));
356 kfree(chunk);
357 }
358
359 kfree(hem);
360 }
361
362 static int hns_roce_set_hem(struct hns_roce_dev *hr_dev,
363 struct hns_roce_hem_table *table, unsigned long obj)
364 {
365 spinlock_t *lock = &hr_dev->bt_cmd_lock;
366 struct device *dev = hr_dev->dev;
367 long end;
368 unsigned long flags;
369 struct hns_roce_hem_iter iter;
370 void __iomem *bt_cmd;
371 __le32 bt_cmd_val[2];
372 __le32 bt_cmd_h = 0;
373 __le32 bt_cmd_l = 0;
374 u64 bt_ba = 0;
375 int ret = 0;
376
377
378 unsigned long i = (obj & (table->num_obj - 1)) /
379 (table->table_chunk_size / table->obj_size);
380
381 switch (table->type) {
382 case HEM_TYPE_QPC:
383 case HEM_TYPE_MTPT:
384 case HEM_TYPE_CQC:
385 case HEM_TYPE_SRQC:
386 roce_set_field(bt_cmd_h, ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_M,
387 ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_S, table->type);
388 break;
389 default:
390 return ret;
391 }
392
393 roce_set_field(bt_cmd_h, ROCEE_BT_CMD_H_ROCEE_BT_CMD_IN_MDF_M,
394 ROCEE_BT_CMD_H_ROCEE_BT_CMD_IN_MDF_S, obj);
395 roce_set_bit(bt_cmd_h, ROCEE_BT_CMD_H_ROCEE_BT_CMD_S, 0);
396 roce_set_bit(bt_cmd_h, ROCEE_BT_CMD_H_ROCEE_BT_CMD_HW_SYNS_S, 1);
397
398
399 for (hns_roce_hem_first(table->hem[i], &iter);
400 !hns_roce_hem_last(&iter); hns_roce_hem_next(&iter)) {
401 bt_ba = hns_roce_hem_addr(&iter) >> DMA_ADDR_T_SHIFT;
402
403 spin_lock_irqsave(lock, flags);
404
405 bt_cmd = hr_dev->reg_base + ROCEE_BT_CMD_H_REG;
406
407 end = HW_SYNC_TIMEOUT_MSECS;
408 while (end > 0) {
409 if (!(readl(bt_cmd) >> BT_CMD_SYNC_SHIFT))
410 break;
411
412 mdelay(HW_SYNC_SLEEP_TIME_INTERVAL);
413 end -= HW_SYNC_SLEEP_TIME_INTERVAL;
414 }
415
416 if (end <= 0) {
417 dev_err(dev, "Write bt_cmd err,hw_sync is not zero.\n");
418 spin_unlock_irqrestore(lock, flags);
419 return -EBUSY;
420 }
421
422 bt_cmd_l = cpu_to_le32(bt_ba);
423 roce_set_field(bt_cmd_h, ROCEE_BT_CMD_H_ROCEE_BT_CMD_BA_H_M,
424 ROCEE_BT_CMD_H_ROCEE_BT_CMD_BA_H_S,
425 bt_ba >> BT_BA_SHIFT);
426
427 bt_cmd_val[0] = bt_cmd_l;
428 bt_cmd_val[1] = bt_cmd_h;
429 hns_roce_write64_k(bt_cmd_val,
430 hr_dev->reg_base + ROCEE_BT_CMD_L_REG);
431 spin_unlock_irqrestore(lock, flags);
432 }
433
434 return ret;
435 }
436
437 static int hns_roce_table_mhop_get(struct hns_roce_dev *hr_dev,
438 struct hns_roce_hem_table *table,
439 unsigned long obj)
440 {
441 struct device *dev = hr_dev->dev;
442 struct hns_roce_hem_mhop mhop;
443 struct hns_roce_hem_iter iter;
444 u32 buf_chunk_size;
445 u32 bt_chunk_size;
446 u32 chunk_ba_num;
447 u32 hop_num;
448 u32 size;
449 u32 bt_num;
450 u64 hem_idx;
451 u64 bt_l1_idx = 0;
452 u64 bt_l0_idx = 0;
453 u64 bt_ba;
454 unsigned long mhop_obj = obj;
455 int bt_l1_allocated = 0;
456 int bt_l0_allocated = 0;
457 int step_idx;
458 int ret;
459
460 ret = hns_roce_calc_hem_mhop(hr_dev, table, &mhop_obj, &mhop);
461 if (ret)
462 return ret;
463
464 buf_chunk_size = mhop.buf_chunk_size;
465 bt_chunk_size = mhop.bt_chunk_size;
466 hop_num = mhop.hop_num;
467 chunk_ba_num = bt_chunk_size / BA_BYTE_LEN;
468
469 bt_num = hns_roce_get_bt_num(table->type, hop_num);
470 switch (bt_num) {
471 case 3:
472 hem_idx = mhop.l0_idx * chunk_ba_num * chunk_ba_num +
473 mhop.l1_idx * chunk_ba_num + mhop.l2_idx;
474 bt_l1_idx = mhop.l0_idx * chunk_ba_num + mhop.l1_idx;
475 bt_l0_idx = mhop.l0_idx;
476 break;
477 case 2:
478 hem_idx = mhop.l0_idx * chunk_ba_num + mhop.l1_idx;
479 bt_l0_idx = mhop.l0_idx;
480 break;
481 case 1:
482 hem_idx = mhop.l0_idx;
483 break;
484 default:
485 dev_err(dev, "Table %d not support hop_num = %d!\n",
486 table->type, hop_num);
487 return -EINVAL;
488 }
489
490 if (unlikely(hem_idx >= table->num_hem)) {
491 dev_err(dev, "Table %d exceed hem limt idx = %llu,max = %lu!\n",
492 table->type, hem_idx, table->num_hem);
493 return -EINVAL;
494 }
495
496 mutex_lock(&table->mutex);
497
498 if (table->hem[hem_idx]) {
499 ++table->hem[hem_idx]->refcount;
500 goto out;
501 }
502
503
504 if ((check_whether_bt_num_3(table->type, hop_num) ||
505 check_whether_bt_num_2(table->type, hop_num)) &&
506 !table->bt_l0[bt_l0_idx]) {
507 table->bt_l0[bt_l0_idx] = dma_alloc_coherent(dev, bt_chunk_size,
508 &(table->bt_l0_dma_addr[bt_l0_idx]),
509 GFP_KERNEL);
510 if (!table->bt_l0[bt_l0_idx]) {
511 ret = -ENOMEM;
512 goto out;
513 }
514 bt_l0_allocated = 1;
515
516
517 if (table->type < HEM_TYPE_MTT) {
518 step_idx = 0;
519 if (hr_dev->hw->set_hem(hr_dev, table, obj, step_idx)) {
520 ret = -ENODEV;
521 dev_err(dev, "set HEM base address to HW failed!\n");
522 goto err_dma_alloc_l1;
523 }
524 }
525 }
526
527
528 if (check_whether_bt_num_3(table->type, hop_num) &&
529 !table->bt_l1[bt_l1_idx]) {
530 table->bt_l1[bt_l1_idx] = dma_alloc_coherent(dev, bt_chunk_size,
531 &(table->bt_l1_dma_addr[bt_l1_idx]),
532 GFP_KERNEL);
533 if (!table->bt_l1[bt_l1_idx]) {
534 ret = -ENOMEM;
535 goto err_dma_alloc_l1;
536 }
537 bt_l1_allocated = 1;
538 *(table->bt_l0[bt_l0_idx] + mhop.l1_idx) =
539 table->bt_l1_dma_addr[bt_l1_idx];
540
541
542 step_idx = 1;
543 if (hr_dev->hw->set_hem(hr_dev, table, obj, step_idx)) {
544 ret = -ENODEV;
545 dev_err(dev, "set HEM base address to HW failed!\n");
546 goto err_alloc_hem_buf;
547 }
548 }
549
550
551
552
553
554 size = table->type < HEM_TYPE_MTT ? buf_chunk_size : bt_chunk_size;
555 table->hem[hem_idx] = hns_roce_alloc_hem(hr_dev,
556 size >> PAGE_SHIFT,
557 size,
558 (table->lowmem ? GFP_KERNEL :
559 GFP_HIGHUSER) | __GFP_NOWARN);
560 if (!table->hem[hem_idx]) {
561 ret = -ENOMEM;
562 goto err_alloc_hem_buf;
563 }
564
565 hns_roce_hem_first(table->hem[hem_idx], &iter);
566 bt_ba = hns_roce_hem_addr(&iter);
567
568 if (table->type < HEM_TYPE_MTT) {
569 if (hop_num == 2) {
570 *(table->bt_l1[bt_l1_idx] + mhop.l2_idx) = bt_ba;
571 step_idx = 2;
572 } else if (hop_num == 1) {
573 *(table->bt_l0[bt_l0_idx] + mhop.l1_idx) = bt_ba;
574 step_idx = 1;
575 } else if (hop_num == HNS_ROCE_HOP_NUM_0) {
576 step_idx = 0;
577 } else {
578 ret = -EINVAL;
579 goto err_dma_alloc_l1;
580 }
581
582
583 if (hr_dev->hw->set_hem(hr_dev, table, obj, step_idx)) {
584 ret = -ENODEV;
585 dev_err(dev, "set HEM base address to HW failed!\n");
586 goto err_alloc_hem_buf;
587 }
588 } else if (hop_num == 2) {
589 *(table->bt_l0[bt_l0_idx] + mhop.l1_idx) = bt_ba;
590 }
591
592 ++table->hem[hem_idx]->refcount;
593 goto out;
594
595 err_alloc_hem_buf:
596 if (bt_l1_allocated) {
597 dma_free_coherent(dev, bt_chunk_size, table->bt_l1[bt_l1_idx],
598 table->bt_l1_dma_addr[bt_l1_idx]);
599 table->bt_l1[bt_l1_idx] = NULL;
600 }
601
602 err_dma_alloc_l1:
603 if (bt_l0_allocated) {
604 dma_free_coherent(dev, bt_chunk_size, table->bt_l0[bt_l0_idx],
605 table->bt_l0_dma_addr[bt_l0_idx]);
606 table->bt_l0[bt_l0_idx] = NULL;
607 }
608
609 out:
610 mutex_unlock(&table->mutex);
611 return ret;
612 }
613
614 int hns_roce_table_get(struct hns_roce_dev *hr_dev,
615 struct hns_roce_hem_table *table, unsigned long obj)
616 {
617 struct device *dev = hr_dev->dev;
618 int ret = 0;
619 unsigned long i;
620
621 if (hns_roce_check_whether_mhop(hr_dev, table->type))
622 return hns_roce_table_mhop_get(hr_dev, table, obj);
623
624 i = (obj & (table->num_obj - 1)) / (table->table_chunk_size /
625 table->obj_size);
626
627 mutex_lock(&table->mutex);
628
629 if (table->hem[i]) {
630 ++table->hem[i]->refcount;
631 goto out;
632 }
633
634 table->hem[i] = hns_roce_alloc_hem(hr_dev,
635 table->table_chunk_size >> PAGE_SHIFT,
636 table->table_chunk_size,
637 (table->lowmem ? GFP_KERNEL :
638 GFP_HIGHUSER) | __GFP_NOWARN);
639 if (!table->hem[i]) {
640 ret = -ENOMEM;
641 goto out;
642 }
643
644
645 if (hns_roce_set_hem(hr_dev, table, obj)) {
646 hns_roce_free_hem(hr_dev, table->hem[i]);
647 table->hem[i] = NULL;
648 ret = -ENODEV;
649 dev_err(dev, "set HEM base address to HW failed.\n");
650 goto out;
651 }
652
653 ++table->hem[i]->refcount;
654 out:
655 mutex_unlock(&table->mutex);
656 return ret;
657 }
658
659 static void hns_roce_table_mhop_put(struct hns_roce_dev *hr_dev,
660 struct hns_roce_hem_table *table,
661 unsigned long obj,
662 int check_refcount)
663 {
664 struct device *dev = hr_dev->dev;
665 struct hns_roce_hem_mhop mhop;
666 unsigned long mhop_obj = obj;
667 u32 bt_chunk_size;
668 u32 chunk_ba_num;
669 u32 hop_num;
670 u32 start_idx;
671 u32 bt_num;
672 u64 hem_idx;
673 u64 bt_l1_idx = 0;
674 int ret;
675
676 ret = hns_roce_calc_hem_mhop(hr_dev, table, &mhop_obj, &mhop);
677 if (ret)
678 return;
679
680 bt_chunk_size = mhop.bt_chunk_size;
681 hop_num = mhop.hop_num;
682 chunk_ba_num = bt_chunk_size / BA_BYTE_LEN;
683
684 bt_num = hns_roce_get_bt_num(table->type, hop_num);
685 switch (bt_num) {
686 case 3:
687 hem_idx = mhop.l0_idx * chunk_ba_num * chunk_ba_num +
688 mhop.l1_idx * chunk_ba_num + mhop.l2_idx;
689 bt_l1_idx = mhop.l0_idx * chunk_ba_num + mhop.l1_idx;
690 break;
691 case 2:
692 hem_idx = mhop.l0_idx * chunk_ba_num + mhop.l1_idx;
693 break;
694 case 1:
695 hem_idx = mhop.l0_idx;
696 break;
697 default:
698 dev_err(dev, "Table %d not support hop_num = %d!\n",
699 table->type, hop_num);
700 return;
701 }
702
703 mutex_lock(&table->mutex);
704
705 if (check_refcount && (--table->hem[hem_idx]->refcount > 0)) {
706 mutex_unlock(&table->mutex);
707 return;
708 }
709
710 if (table->type < HEM_TYPE_MTT && hop_num == 1) {
711 if (hr_dev->hw->clear_hem(hr_dev, table, obj, 1))
712 dev_warn(dev, "Clear HEM base address failed.\n");
713 } else if (table->type < HEM_TYPE_MTT && hop_num == 2) {
714 if (hr_dev->hw->clear_hem(hr_dev, table, obj, 2))
715 dev_warn(dev, "Clear HEM base address failed.\n");
716 } else if (table->type < HEM_TYPE_MTT &&
717 hop_num == HNS_ROCE_HOP_NUM_0) {
718 if (hr_dev->hw->clear_hem(hr_dev, table, obj, 0))
719 dev_warn(dev, "Clear HEM base address failed.\n");
720 }
721
722
723
724
725
726 hns_roce_free_hem(hr_dev, table->hem[hem_idx]);
727 table->hem[hem_idx] = NULL;
728
729 if (check_whether_bt_num_2(table->type, hop_num)) {
730 start_idx = mhop.l0_idx * chunk_ba_num;
731 if (hns_roce_check_hem_null(table->hem, start_idx,
732 chunk_ba_num, table->num_hem)) {
733 if (table->type < HEM_TYPE_MTT &&
734 hr_dev->hw->clear_hem(hr_dev, table, obj, 0))
735 dev_warn(dev, "Clear HEM base address failed.\n");
736
737 dma_free_coherent(dev, bt_chunk_size,
738 table->bt_l0[mhop.l0_idx],
739 table->bt_l0_dma_addr[mhop.l0_idx]);
740 table->bt_l0[mhop.l0_idx] = NULL;
741 }
742 } else if (check_whether_bt_num_3(table->type, hop_num)) {
743 start_idx = mhop.l0_idx * chunk_ba_num * chunk_ba_num +
744 mhop.l1_idx * chunk_ba_num;
745 if (hns_roce_check_hem_null(table->hem, start_idx,
746 chunk_ba_num, table->num_hem)) {
747 if (hr_dev->hw->clear_hem(hr_dev, table, obj, 1))
748 dev_warn(dev, "Clear HEM base address failed.\n");
749
750 dma_free_coherent(dev, bt_chunk_size,
751 table->bt_l1[bt_l1_idx],
752 table->bt_l1_dma_addr[bt_l1_idx]);
753 table->bt_l1[bt_l1_idx] = NULL;
754
755 start_idx = mhop.l0_idx * chunk_ba_num;
756 if (hns_roce_check_bt_null(table->bt_l1, start_idx,
757 chunk_ba_num)) {
758 if (hr_dev->hw->clear_hem(hr_dev, table, obj,
759 0))
760 dev_warn(dev, "Clear HEM base address failed.\n");
761
762 dma_free_coherent(dev, bt_chunk_size,
763 table->bt_l0[mhop.l0_idx],
764 table->bt_l0_dma_addr[mhop.l0_idx]);
765 table->bt_l0[mhop.l0_idx] = NULL;
766 }
767 }
768 }
769
770 mutex_unlock(&table->mutex);
771 }
772
773 void hns_roce_table_put(struct hns_roce_dev *hr_dev,
774 struct hns_roce_hem_table *table, unsigned long obj)
775 {
776 struct device *dev = hr_dev->dev;
777 unsigned long i;
778
779 if (hns_roce_check_whether_mhop(hr_dev, table->type)) {
780 hns_roce_table_mhop_put(hr_dev, table, obj, 1);
781 return;
782 }
783
784 i = (obj & (table->num_obj - 1)) /
785 (table->table_chunk_size / table->obj_size);
786
787 mutex_lock(&table->mutex);
788
789 if (--table->hem[i]->refcount == 0) {
790
791 if (hr_dev->hw->clear_hem(hr_dev, table, obj, 0))
792 dev_warn(dev, "Clear HEM base address failed.\n");
793
794 hns_roce_free_hem(hr_dev, table->hem[i]);
795 table->hem[i] = NULL;
796 }
797
798 mutex_unlock(&table->mutex);
799 }
800
801 void *hns_roce_table_find(struct hns_roce_dev *hr_dev,
802 struct hns_roce_hem_table *table,
803 unsigned long obj, dma_addr_t *dma_handle)
804 {
805 struct hns_roce_hem_chunk *chunk;
806 struct hns_roce_hem_mhop mhop;
807 struct hns_roce_hem *hem;
808 void *addr = NULL;
809 unsigned long mhop_obj = obj;
810 unsigned long obj_per_chunk;
811 unsigned long idx_offset;
812 int offset, dma_offset;
813 int length;
814 int i, j;
815 u32 hem_idx = 0;
816
817 if (!table->lowmem)
818 return NULL;
819
820 mutex_lock(&table->mutex);
821
822 if (!hns_roce_check_whether_mhop(hr_dev, table->type)) {
823 obj_per_chunk = table->table_chunk_size / table->obj_size;
824 hem = table->hem[(obj & (table->num_obj - 1)) / obj_per_chunk];
825 idx_offset = (obj & (table->num_obj - 1)) % obj_per_chunk;
826 dma_offset = offset = idx_offset * table->obj_size;
827 } else {
828 u32 seg_size = 64;
829
830 if (hns_roce_calc_hem_mhop(hr_dev, table, &mhop_obj, &mhop))
831 goto out;
832
833 i = mhop.l0_idx;
834 j = mhop.l1_idx;
835 if (mhop.hop_num == 2)
836 hem_idx = i * (mhop.bt_chunk_size / BA_BYTE_LEN) + j;
837 else if (mhop.hop_num == 1 ||
838 mhop.hop_num == HNS_ROCE_HOP_NUM_0)
839 hem_idx = i;
840
841 hem = table->hem[hem_idx];
842 dma_offset = offset = (obj & (table->num_obj - 1)) * seg_size %
843 mhop.bt_chunk_size;
844 if (mhop.hop_num == 2)
845 dma_offset = offset = 0;
846 }
847
848 if (!hem)
849 goto out;
850
851 list_for_each_entry(chunk, &hem->chunk_list, list) {
852 for (i = 0; i < chunk->npages; ++i) {
853 length = sg_dma_len(&chunk->mem[i]);
854 if (dma_handle && dma_offset >= 0) {
855 if (length > (u32)dma_offset)
856 *dma_handle = sg_dma_address(
857 &chunk->mem[i]) + dma_offset;
858 dma_offset -= length;
859 }
860
861 if (length > (u32)offset) {
862 addr = chunk->buf[i] + offset;
863 goto out;
864 }
865 offset -= length;
866 }
867 }
868
869 out:
870 mutex_unlock(&table->mutex);
871 return addr;
872 }
873
874 int hns_roce_table_get_range(struct hns_roce_dev *hr_dev,
875 struct hns_roce_hem_table *table,
876 unsigned long start, unsigned long end)
877 {
878 struct hns_roce_hem_mhop mhop;
879 unsigned long inc = table->table_chunk_size / table->obj_size;
880 unsigned long i = 0;
881 int ret;
882
883 if (hns_roce_check_whether_mhop(hr_dev, table->type)) {
884 ret = hns_roce_calc_hem_mhop(hr_dev, table, NULL, &mhop);
885 if (ret)
886 goto fail;
887 inc = mhop.bt_chunk_size / table->obj_size;
888 }
889
890
891 for (i = start; i <= end; i += inc) {
892 ret = hns_roce_table_get(hr_dev, table, i);
893 if (ret)
894 goto fail;
895 }
896
897 return 0;
898
899 fail:
900 while (i > start) {
901 i -= inc;
902 hns_roce_table_put(hr_dev, table, i);
903 }
904 return ret;
905 }
906
907 void hns_roce_table_put_range(struct hns_roce_dev *hr_dev,
908 struct hns_roce_hem_table *table,
909 unsigned long start, unsigned long end)
910 {
911 struct hns_roce_hem_mhop mhop;
912 unsigned long inc = table->table_chunk_size / table->obj_size;
913 unsigned long i;
914
915 if (hns_roce_check_whether_mhop(hr_dev, table->type)) {
916 if (hns_roce_calc_hem_mhop(hr_dev, table, NULL, &mhop))
917 return;
918 inc = mhop.bt_chunk_size / table->obj_size;
919 }
920
921 for (i = start; i <= end; i += inc)
922 hns_roce_table_put(hr_dev, table, i);
923 }
924
925 int hns_roce_init_hem_table(struct hns_roce_dev *hr_dev,
926 struct hns_roce_hem_table *table, u32 type,
927 unsigned long obj_size, unsigned long nobj,
928 int use_lowmem)
929 {
930 unsigned long obj_per_chunk;
931 unsigned long num_hem;
932
933 if (!hns_roce_check_whether_mhop(hr_dev, type)) {
934 table->table_chunk_size = hr_dev->caps.chunk_sz;
935 obj_per_chunk = table->table_chunk_size / obj_size;
936 num_hem = (nobj + obj_per_chunk - 1) / obj_per_chunk;
937
938 table->hem = kcalloc(num_hem, sizeof(*table->hem), GFP_KERNEL);
939 if (!table->hem)
940 return -ENOMEM;
941 } else {
942 struct hns_roce_hem_mhop mhop = {};
943 unsigned long buf_chunk_size;
944 unsigned long bt_chunk_size;
945 unsigned long bt_chunk_num;
946 unsigned long num_bt_l0 = 0;
947 u32 hop_num;
948
949 if (get_hem_table_config(hr_dev, &mhop, type))
950 return -EINVAL;
951
952 buf_chunk_size = mhop.buf_chunk_size;
953 bt_chunk_size = mhop.bt_chunk_size;
954 num_bt_l0 = mhop.ba_l0_num;
955 hop_num = mhop.hop_num;
956
957 obj_per_chunk = buf_chunk_size / obj_size;
958 num_hem = (nobj + obj_per_chunk - 1) / obj_per_chunk;
959 bt_chunk_num = bt_chunk_size / BA_BYTE_LEN;
960 if (type >= HEM_TYPE_MTT)
961 num_bt_l0 = bt_chunk_num;
962
963 table->hem = kcalloc(num_hem, sizeof(*table->hem),
964 GFP_KERNEL);
965 if (!table->hem)
966 goto err_kcalloc_hem_buf;
967
968 if (check_whether_bt_num_3(type, hop_num)) {
969 unsigned long num_bt_l1;
970
971 num_bt_l1 = (num_hem + bt_chunk_num - 1) /
972 bt_chunk_num;
973 table->bt_l1 = kcalloc(num_bt_l1,
974 sizeof(*table->bt_l1),
975 GFP_KERNEL);
976 if (!table->bt_l1)
977 goto err_kcalloc_bt_l1;
978
979 table->bt_l1_dma_addr = kcalloc(num_bt_l1,
980 sizeof(*table->bt_l1_dma_addr),
981 GFP_KERNEL);
982
983 if (!table->bt_l1_dma_addr)
984 goto err_kcalloc_l1_dma;
985 }
986
987 if (check_whether_bt_num_2(type, hop_num) ||
988 check_whether_bt_num_3(type, hop_num)) {
989 table->bt_l0 = kcalloc(num_bt_l0, sizeof(*table->bt_l0),
990 GFP_KERNEL);
991 if (!table->bt_l0)
992 goto err_kcalloc_bt_l0;
993
994 table->bt_l0_dma_addr = kcalloc(num_bt_l0,
995 sizeof(*table->bt_l0_dma_addr),
996 GFP_KERNEL);
997 if (!table->bt_l0_dma_addr)
998 goto err_kcalloc_l0_dma;
999 }
1000 }
1001
1002 table->type = type;
1003 table->num_hem = num_hem;
1004 table->num_obj = nobj;
1005 table->obj_size = obj_size;
1006 table->lowmem = use_lowmem;
1007 mutex_init(&table->mutex);
1008
1009 return 0;
1010
1011 err_kcalloc_l0_dma:
1012 kfree(table->bt_l0);
1013 table->bt_l0 = NULL;
1014
1015 err_kcalloc_bt_l0:
1016 kfree(table->bt_l1_dma_addr);
1017 table->bt_l1_dma_addr = NULL;
1018
1019 err_kcalloc_l1_dma:
1020 kfree(table->bt_l1);
1021 table->bt_l1 = NULL;
1022
1023 err_kcalloc_bt_l1:
1024 kfree(table->hem);
1025 table->hem = NULL;
1026
1027 err_kcalloc_hem_buf:
1028 return -ENOMEM;
1029 }
1030
1031 static void hns_roce_cleanup_mhop_hem_table(struct hns_roce_dev *hr_dev,
1032 struct hns_roce_hem_table *table)
1033 {
1034 struct hns_roce_hem_mhop mhop;
1035 u32 buf_chunk_size;
1036 int i;
1037 u64 obj;
1038
1039 if (hns_roce_calc_hem_mhop(hr_dev, table, NULL, &mhop))
1040 return;
1041 buf_chunk_size = table->type < HEM_TYPE_MTT ? mhop.buf_chunk_size :
1042 mhop.bt_chunk_size;
1043
1044 for (i = 0; i < table->num_hem; ++i) {
1045 obj = i * buf_chunk_size / table->obj_size;
1046 if (table->hem[i])
1047 hns_roce_table_mhop_put(hr_dev, table, obj, 0);
1048 }
1049
1050 kfree(table->hem);
1051 table->hem = NULL;
1052 kfree(table->bt_l1);
1053 table->bt_l1 = NULL;
1054 kfree(table->bt_l1_dma_addr);
1055 table->bt_l1_dma_addr = NULL;
1056 kfree(table->bt_l0);
1057 table->bt_l0 = NULL;
1058 kfree(table->bt_l0_dma_addr);
1059 table->bt_l0_dma_addr = NULL;
1060 }
1061
1062 void hns_roce_cleanup_hem_table(struct hns_roce_dev *hr_dev,
1063 struct hns_roce_hem_table *table)
1064 {
1065 struct device *dev = hr_dev->dev;
1066 unsigned long i;
1067
1068 if (hns_roce_check_whether_mhop(hr_dev, table->type)) {
1069 hns_roce_cleanup_mhop_hem_table(hr_dev, table);
1070 return;
1071 }
1072
1073 for (i = 0; i < table->num_hem; ++i)
1074 if (table->hem[i]) {
1075 if (hr_dev->hw->clear_hem(hr_dev, table,
1076 i * table->table_chunk_size / table->obj_size, 0))
1077 dev_err(dev, "Clear HEM base address failed.\n");
1078
1079 hns_roce_free_hem(hr_dev, table->hem[i]);
1080 }
1081
1082 kfree(table->hem);
1083 }
1084
1085 void hns_roce_cleanup_hem(struct hns_roce_dev *hr_dev)
1086 {
1087 if ((hr_dev->caps.num_idx_segs))
1088 hns_roce_cleanup_hem_table(hr_dev,
1089 &hr_dev->mr_table.mtt_idx_table);
1090 if (hr_dev->caps.num_srqwqe_segs)
1091 hns_roce_cleanup_hem_table(hr_dev,
1092 &hr_dev->mr_table.mtt_srqwqe_table);
1093 if (hr_dev->caps.srqc_entry_sz)
1094 hns_roce_cleanup_hem_table(hr_dev,
1095 &hr_dev->srq_table.table);
1096 hns_roce_cleanup_hem_table(hr_dev, &hr_dev->cq_table.table);
1097 if (hr_dev->caps.qpc_timer_entry_sz)
1098 hns_roce_cleanup_hem_table(hr_dev,
1099 &hr_dev->qpc_timer_table);
1100 if (hr_dev->caps.cqc_timer_entry_sz)
1101 hns_roce_cleanup_hem_table(hr_dev,
1102 &hr_dev->cqc_timer_table);
1103 if (hr_dev->caps.sccc_entry_sz)
1104 hns_roce_cleanup_hem_table(hr_dev,
1105 &hr_dev->qp_table.sccc_table);
1106 if (hr_dev->caps.trrl_entry_sz)
1107 hns_roce_cleanup_hem_table(hr_dev,
1108 &hr_dev->qp_table.trrl_table);
1109 hns_roce_cleanup_hem_table(hr_dev, &hr_dev->qp_table.irrl_table);
1110 hns_roce_cleanup_hem_table(hr_dev, &hr_dev->qp_table.qp_table);
1111 hns_roce_cleanup_hem_table(hr_dev, &hr_dev->mr_table.mtpt_table);
1112 if (hns_roce_check_whether_mhop(hr_dev, HEM_TYPE_CQE))
1113 hns_roce_cleanup_hem_table(hr_dev,
1114 &hr_dev->mr_table.mtt_cqe_table);
1115 hns_roce_cleanup_hem_table(hr_dev, &hr_dev->mr_table.mtt_table);
1116 }
1117
1118 struct roce_hem_item {
1119 struct list_head list;
1120 struct list_head sibling;
1121 void *addr;
1122 dma_addr_t dma_addr;
1123 size_t count;
1124 int start;
1125 int end;
1126 };
1127
1128 static struct roce_hem_item *hem_list_alloc_item(struct hns_roce_dev *hr_dev,
1129 int start, int end,
1130 int count, bool exist_bt,
1131 int bt_level)
1132 {
1133 struct roce_hem_item *hem;
1134
1135 hem = kzalloc(sizeof(*hem), GFP_KERNEL);
1136 if (!hem)
1137 return NULL;
1138
1139 if (exist_bt) {
1140 hem->addr = dma_alloc_coherent(hr_dev->dev,
1141 count * BA_BYTE_LEN,
1142 &hem->dma_addr, GFP_KERNEL);
1143 if (!hem->addr) {
1144 kfree(hem);
1145 return NULL;
1146 }
1147 }
1148
1149 hem->count = count;
1150 hem->start = start;
1151 hem->end = end;
1152 INIT_LIST_HEAD(&hem->list);
1153 INIT_LIST_HEAD(&hem->sibling);
1154
1155 return hem;
1156 }
1157
1158 static void hem_list_free_item(struct hns_roce_dev *hr_dev,
1159 struct roce_hem_item *hem, bool exist_bt)
1160 {
1161 if (exist_bt)
1162 dma_free_coherent(hr_dev->dev, hem->count * BA_BYTE_LEN,
1163 hem->addr, hem->dma_addr);
1164 kfree(hem);
1165 }
1166
1167 static void hem_list_free_all(struct hns_roce_dev *hr_dev,
1168 struct list_head *head, bool exist_bt)
1169 {
1170 struct roce_hem_item *hem, *temp_hem;
1171
1172 list_for_each_entry_safe(hem, temp_hem, head, list) {
1173 list_del(&hem->list);
1174 hem_list_free_item(hr_dev, hem, exist_bt);
1175 }
1176 }
1177
1178 static void hem_list_link_bt(struct hns_roce_dev *hr_dev, void *base_addr,
1179 u64 table_addr)
1180 {
1181 *(u64 *)(base_addr) = table_addr;
1182 }
1183
1184
1185 static void hem_list_assign_bt(struct hns_roce_dev *hr_dev,
1186 struct roce_hem_item *hem, void *cpu_addr,
1187 u64 phy_addr)
1188 {
1189 hem->addr = cpu_addr;
1190 hem->dma_addr = (dma_addr_t)phy_addr;
1191 }
1192
1193 static inline bool hem_list_page_is_in_range(struct roce_hem_item *hem,
1194 int offset)
1195 {
1196 return (hem->start <= offset && offset <= hem->end);
1197 }
1198
1199 static struct roce_hem_item *hem_list_search_item(struct list_head *ba_list,
1200 int page_offset)
1201 {
1202 struct roce_hem_item *hem, *temp_hem;
1203 struct roce_hem_item *found = NULL;
1204
1205 list_for_each_entry_safe(hem, temp_hem, ba_list, list) {
1206 if (hem_list_page_is_in_range(hem, page_offset)) {
1207 found = hem;
1208 break;
1209 }
1210 }
1211
1212 return found;
1213 }
1214
1215 static bool hem_list_is_bottom_bt(int hopnum, int bt_level)
1216 {
1217
1218
1219
1220
1221
1222
1223
1224 return bt_level >= (hopnum ? hopnum - 1 : hopnum);
1225 }
1226
1227
1228
1229
1230
1231
1232
1233 static u32 hem_list_calc_ba_range(int hopnum, int bt_level, int unit)
1234 {
1235 u32 step;
1236 int max;
1237 int i;
1238
1239 if (hopnum <= bt_level)
1240 return 0;
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252 step = 1;
1253 max = hopnum - bt_level;
1254 for (i = 0; i < max; i++)
1255 step = step * unit;
1256
1257 return step;
1258 }
1259
1260
1261
1262
1263
1264
1265
1266 int hns_roce_hem_list_calc_root_ba(const struct hns_roce_buf_region *regions,
1267 int region_cnt, int unit)
1268 {
1269 struct hns_roce_buf_region *r;
1270 int total = 0;
1271 int step;
1272 int i;
1273
1274 for (i = 0; i < region_cnt; i++) {
1275 r = (struct hns_roce_buf_region *)®ions[i];
1276 if (r->hopnum > 1) {
1277 step = hem_list_calc_ba_range(r->hopnum, 1, unit);
1278 if (step > 0)
1279 total += (r->count + step - 1) / step;
1280 } else {
1281 total += r->count;
1282 }
1283 }
1284
1285 return total;
1286 }
1287
1288 static int hem_list_alloc_mid_bt(struct hns_roce_dev *hr_dev,
1289 const struct hns_roce_buf_region *r, int unit,
1290 int offset, struct list_head *mid_bt,
1291 struct list_head *btm_bt)
1292 {
1293 struct roce_hem_item *hem_ptrs[HNS_ROCE_MAX_BT_LEVEL] = { NULL };
1294 struct list_head temp_list[HNS_ROCE_MAX_BT_LEVEL];
1295 struct roce_hem_item *cur, *pre;
1296 const int hopnum = r->hopnum;
1297 int start_aligned;
1298 int distance;
1299 int ret = 0;
1300 int max_ofs;
1301 int level;
1302 u32 step;
1303 int end;
1304
1305 if (hopnum <= 1)
1306 return 0;
1307
1308 if (hopnum > HNS_ROCE_MAX_BT_LEVEL) {
1309 dev_err(hr_dev->dev, "invalid hopnum %d!\n", hopnum);
1310 return -EINVAL;
1311 }
1312
1313 if (offset < r->offset) {
1314 dev_err(hr_dev->dev, "invalid offset %d,min %d!\n",
1315 offset, r->offset);
1316 return -EINVAL;
1317 }
1318
1319 distance = offset - r->offset;
1320 max_ofs = r->offset + r->count - 1;
1321 for (level = 0; level < hopnum; level++)
1322 INIT_LIST_HEAD(&temp_list[level]);
1323
1324
1325 for (level = 1; level < hopnum; level++) {
1326 cur = hem_list_search_item(&mid_bt[level], offset);
1327 if (cur) {
1328 hem_ptrs[level] = cur;
1329 continue;
1330 }
1331
1332 step = hem_list_calc_ba_range(hopnum, level, unit);
1333 if (step < 1) {
1334 ret = -EINVAL;
1335 goto err_exit;
1336 }
1337
1338 start_aligned = (distance / step) * step + r->offset;
1339 end = min_t(int, start_aligned + step - 1, max_ofs);
1340 cur = hem_list_alloc_item(hr_dev, start_aligned, end, unit,
1341 true, level);
1342 if (!cur) {
1343 ret = -ENOMEM;
1344 goto err_exit;
1345 }
1346 hem_ptrs[level] = cur;
1347 list_add(&cur->list, &temp_list[level]);
1348 if (hem_list_is_bottom_bt(hopnum, level))
1349 list_add(&cur->sibling, &temp_list[0]);
1350
1351
1352 if (level > 1) {
1353 pre = hem_ptrs[level - 1];
1354 step = (cur->start - pre->start) / step * BA_BYTE_LEN;
1355 hem_list_link_bt(hr_dev, pre->addr + step,
1356 cur->dma_addr);
1357 }
1358 }
1359
1360 list_splice(&temp_list[0], btm_bt);
1361 for (level = 1; level < hopnum; level++)
1362 list_splice(&temp_list[level], &mid_bt[level]);
1363
1364 return 0;
1365
1366 err_exit:
1367 for (level = 1; level < hopnum; level++)
1368 hem_list_free_all(hr_dev, &temp_list[level], true);
1369
1370 return ret;
1371 }
1372
1373 static int hem_list_alloc_root_bt(struct hns_roce_dev *hr_dev,
1374 struct hns_roce_hem_list *hem_list, int unit,
1375 const struct hns_roce_buf_region *regions,
1376 int region_cnt)
1377 {
1378 struct roce_hem_item *hem, *temp_hem, *root_hem;
1379 struct list_head temp_list[HNS_ROCE_MAX_BT_REGION];
1380 const struct hns_roce_buf_region *r;
1381 struct list_head temp_root;
1382 struct list_head temp_btm;
1383 void *cpu_base;
1384 u64 phy_base;
1385 int ret = 0;
1386 int offset;
1387 int total;
1388 int step;
1389 int i;
1390
1391 r = ®ions[0];
1392 root_hem = hem_list_search_item(&hem_list->root_bt, r->offset);
1393 if (root_hem)
1394 return 0;
1395
1396 INIT_LIST_HEAD(&temp_root);
1397 total = r->offset;
1398
1399 r = ®ions[region_cnt - 1];
1400 root_hem = hem_list_alloc_item(hr_dev, total, r->offset + r->count - 1,
1401 unit, true, 0);
1402 if (!root_hem)
1403 return -ENOMEM;
1404 list_add(&root_hem->list, &temp_root);
1405
1406 hem_list->root_ba = root_hem->dma_addr;
1407
1408 INIT_LIST_HEAD(&temp_btm);
1409 for (i = 0; i < region_cnt; i++)
1410 INIT_LIST_HEAD(&temp_list[i]);
1411
1412 total = 0;
1413 for (i = 0; i < region_cnt && total < unit; i++) {
1414 r = ®ions[i];
1415 if (!r->count)
1416 continue;
1417
1418
1419 cpu_base = root_hem->addr + total * BA_BYTE_LEN;
1420 phy_base = root_hem->dma_addr + total * BA_BYTE_LEN;
1421
1422
1423
1424
1425 if (hem_list_is_bottom_bt(r->hopnum, 0)) {
1426 hem = hem_list_alloc_item(hr_dev, r->offset,
1427 r->offset + r->count - 1,
1428 r->count, false, 0);
1429 if (!hem) {
1430 ret = -ENOMEM;
1431 goto err_exit;
1432 }
1433 hem_list_assign_bt(hr_dev, hem, cpu_base, phy_base);
1434 list_add(&hem->list, &temp_list[i]);
1435 list_add(&hem->sibling, &temp_btm);
1436 total += r->count;
1437 } else {
1438 step = hem_list_calc_ba_range(r->hopnum, 1, unit);
1439 if (step < 1) {
1440 ret = -EINVAL;
1441 goto err_exit;
1442 }
1443
1444 list_for_each_entry_safe(hem, temp_hem,
1445 &hem_list->mid_bt[i][1], list) {
1446 offset = hem->start / step * BA_BYTE_LEN;
1447 hem_list_link_bt(hr_dev, cpu_base + offset,
1448 hem->dma_addr);
1449 total++;
1450 }
1451 }
1452 }
1453
1454 list_splice(&temp_btm, &hem_list->btm_bt);
1455 list_splice(&temp_root, &hem_list->root_bt);
1456 for (i = 0; i < region_cnt; i++)
1457 list_splice(&temp_list[i], &hem_list->mid_bt[i][0]);
1458
1459 return 0;
1460
1461 err_exit:
1462 for (i = 0; i < region_cnt; i++)
1463 hem_list_free_all(hr_dev, &temp_list[i], false);
1464
1465 hem_list_free_all(hr_dev, &temp_root, true);
1466
1467 return ret;
1468 }
1469
1470
1471 int hns_roce_hem_list_request(struct hns_roce_dev *hr_dev,
1472 struct hns_roce_hem_list *hem_list,
1473 const struct hns_roce_buf_region *regions,
1474 int region_cnt)
1475 {
1476 const struct hns_roce_buf_region *r;
1477 int ofs, end;
1478 int ret = 0;
1479 int unit;
1480 int i;
1481
1482 if (region_cnt > HNS_ROCE_MAX_BT_REGION) {
1483 dev_err(hr_dev->dev, "invalid region region_cnt %d!\n",
1484 region_cnt);
1485 return -EINVAL;
1486 }
1487
1488 unit = (1 << hem_list->bt_pg_shift) / BA_BYTE_LEN;
1489 for (i = 0; i < region_cnt; i++) {
1490 r = ®ions[i];
1491 if (!r->count)
1492 continue;
1493
1494 end = r->offset + r->count;
1495 for (ofs = r->offset; ofs < end; ofs += unit) {
1496 ret = hem_list_alloc_mid_bt(hr_dev, r, unit, ofs,
1497 hem_list->mid_bt[i],
1498 &hem_list->btm_bt);
1499 if (ret) {
1500 dev_err(hr_dev->dev,
1501 "alloc hem trunk fail ret=%d!\n", ret);
1502 goto err_alloc;
1503 }
1504 }
1505 }
1506
1507 ret = hem_list_alloc_root_bt(hr_dev, hem_list, unit, regions,
1508 region_cnt);
1509 if (ret)
1510 dev_err(hr_dev->dev, "alloc hem root fail ret=%d!\n", ret);
1511 else
1512 return 0;
1513
1514 err_alloc:
1515 hns_roce_hem_list_release(hr_dev, hem_list);
1516
1517 return ret;
1518 }
1519
1520 void hns_roce_hem_list_release(struct hns_roce_dev *hr_dev,
1521 struct hns_roce_hem_list *hem_list)
1522 {
1523 int i, j;
1524
1525 for (i = 0; i < HNS_ROCE_MAX_BT_REGION; i++)
1526 for (j = 0; j < HNS_ROCE_MAX_BT_LEVEL; j++)
1527 hem_list_free_all(hr_dev, &hem_list->mid_bt[i][j],
1528 j != 0);
1529
1530 hem_list_free_all(hr_dev, &hem_list->root_bt, true);
1531 INIT_LIST_HEAD(&hem_list->btm_bt);
1532 hem_list->root_ba = 0;
1533 }
1534
1535 void hns_roce_hem_list_init(struct hns_roce_hem_list *hem_list,
1536 int bt_page_order)
1537 {
1538 int i, j;
1539
1540 INIT_LIST_HEAD(&hem_list->root_bt);
1541 INIT_LIST_HEAD(&hem_list->btm_bt);
1542 for (i = 0; i < HNS_ROCE_MAX_BT_REGION; i++)
1543 for (j = 0; j < HNS_ROCE_MAX_BT_LEVEL; j++)
1544 INIT_LIST_HEAD(&hem_list->mid_bt[i][j]);
1545
1546 hem_list->bt_pg_shift = bt_page_order;
1547 }
1548
1549 void *hns_roce_hem_list_find_mtt(struct hns_roce_dev *hr_dev,
1550 struct hns_roce_hem_list *hem_list,
1551 int offset, int *mtt_cnt, u64 *phy_addr)
1552 {
1553 struct list_head *head = &hem_list->btm_bt;
1554 struct roce_hem_item *hem, *temp_hem;
1555 void *cpu_base = NULL;
1556 u64 phy_base = 0;
1557 int nr = 0;
1558
1559 list_for_each_entry_safe(hem, temp_hem, head, sibling) {
1560 if (hem_list_page_is_in_range(hem, offset)) {
1561 nr = offset - hem->start;
1562 cpu_base = hem->addr + nr * BA_BYTE_LEN;
1563 phy_base = hem->dma_addr + nr * BA_BYTE_LEN;
1564 nr = hem->end + 1 - offset;
1565 break;
1566 }
1567 }
1568
1569 if (mtt_cnt)
1570 *mtt_cnt = nr;
1571
1572 if (phy_addr)
1573 *phy_addr = phy_base;
1574
1575 return cpu_base;
1576 }