This source file includes following definitions.
- mlx5_core_alloc_transport_domain
- mlx5_core_dealloc_transport_domain
- mlx5_core_create_rq
- mlx5_core_modify_rq
- mlx5_core_destroy_rq
- mlx5_core_query_rq
- mlx5_core_create_sq
- mlx5_core_modify_sq
- mlx5_core_destroy_sq
- mlx5_core_query_sq
- mlx5_core_query_sq_state
- mlx5_core_create_tir_out
- mlx5_core_create_tir
- mlx5_core_modify_tir
- mlx5_core_destroy_tir
- mlx5_core_create_tis
- mlx5_core_modify_tis
- mlx5_core_destroy_tis
- mlx5_core_create_rqt
- mlx5_core_modify_rqt
- mlx5_core_destroy_rqt
- mlx5_hairpin_create_rq
- mlx5_hairpin_create_sq
- mlx5_hairpin_create_queues
- mlx5_hairpin_destroy_queues
- mlx5_hairpin_modify_rq
- mlx5_hairpin_modify_sq
- mlx5_hairpin_pair_queues
- mlx5_hairpin_unpair_queues
- mlx5_core_hairpin_create
- mlx5_core_hairpin_destroy
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33 #include <linux/mlx5/driver.h>
34 #include "mlx5_core.h"
35 #include <linux/mlx5/transobj.h>
36
37 int mlx5_core_alloc_transport_domain(struct mlx5_core_dev *dev, u32 *tdn)
38 {
39 u32 in[MLX5_ST_SZ_DW(alloc_transport_domain_in)] = {0};
40 u32 out[MLX5_ST_SZ_DW(alloc_transport_domain_out)] = {0};
41 int err;
42
43 MLX5_SET(alloc_transport_domain_in, in, opcode,
44 MLX5_CMD_OP_ALLOC_TRANSPORT_DOMAIN);
45
46 err = mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
47 if (!err)
48 *tdn = MLX5_GET(alloc_transport_domain_out, out,
49 transport_domain);
50
51 return err;
52 }
53 EXPORT_SYMBOL(mlx5_core_alloc_transport_domain);
54
55 void mlx5_core_dealloc_transport_domain(struct mlx5_core_dev *dev, u32 tdn)
56 {
57 u32 in[MLX5_ST_SZ_DW(dealloc_transport_domain_in)] = {0};
58 u32 out[MLX5_ST_SZ_DW(dealloc_transport_domain_out)] = {0};
59
60 MLX5_SET(dealloc_transport_domain_in, in, opcode,
61 MLX5_CMD_OP_DEALLOC_TRANSPORT_DOMAIN);
62 MLX5_SET(dealloc_transport_domain_in, in, transport_domain, tdn);
63 mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
64 }
65 EXPORT_SYMBOL(mlx5_core_dealloc_transport_domain);
66
67 int mlx5_core_create_rq(struct mlx5_core_dev *dev, u32 *in, int inlen, u32 *rqn)
68 {
69 u32 out[MLX5_ST_SZ_DW(create_rq_out)] = {0};
70 int err;
71
72 MLX5_SET(create_rq_in, in, opcode, MLX5_CMD_OP_CREATE_RQ);
73 err = mlx5_cmd_exec(dev, in, inlen, out, sizeof(out));
74 if (!err)
75 *rqn = MLX5_GET(create_rq_out, out, rqn);
76
77 return err;
78 }
79 EXPORT_SYMBOL(mlx5_core_create_rq);
80
81 int mlx5_core_modify_rq(struct mlx5_core_dev *dev, u32 rqn, u32 *in, int inlen)
82 {
83 u32 out[MLX5_ST_SZ_DW(modify_rq_out)];
84
85 MLX5_SET(modify_rq_in, in, rqn, rqn);
86 MLX5_SET(modify_rq_in, in, opcode, MLX5_CMD_OP_MODIFY_RQ);
87
88 memset(out, 0, sizeof(out));
89 return mlx5_cmd_exec(dev, in, inlen, out, sizeof(out));
90 }
91 EXPORT_SYMBOL(mlx5_core_modify_rq);
92
93 void mlx5_core_destroy_rq(struct mlx5_core_dev *dev, u32 rqn)
94 {
95 u32 in[MLX5_ST_SZ_DW(destroy_rq_in)] = {0};
96 u32 out[MLX5_ST_SZ_DW(destroy_rq_out)] = {0};
97
98 MLX5_SET(destroy_rq_in, in, opcode, MLX5_CMD_OP_DESTROY_RQ);
99 MLX5_SET(destroy_rq_in, in, rqn, rqn);
100 mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
101 }
102 EXPORT_SYMBOL(mlx5_core_destroy_rq);
103
104 int mlx5_core_query_rq(struct mlx5_core_dev *dev, u32 rqn, u32 *out)
105 {
106 u32 in[MLX5_ST_SZ_DW(query_rq_in)] = {0};
107 int outlen = MLX5_ST_SZ_BYTES(query_rq_out);
108
109 MLX5_SET(query_rq_in, in, opcode, MLX5_CMD_OP_QUERY_RQ);
110 MLX5_SET(query_rq_in, in, rqn, rqn);
111
112 return mlx5_cmd_exec(dev, in, sizeof(in), out, outlen);
113 }
114 EXPORT_SYMBOL(mlx5_core_query_rq);
115
116 int mlx5_core_create_sq(struct mlx5_core_dev *dev, u32 *in, int inlen, u32 *sqn)
117 {
118 u32 out[MLX5_ST_SZ_DW(create_sq_out)] = {0};
119 int err;
120
121 MLX5_SET(create_sq_in, in, opcode, MLX5_CMD_OP_CREATE_SQ);
122 err = mlx5_cmd_exec(dev, in, inlen, out, sizeof(out));
123 if (!err)
124 *sqn = MLX5_GET(create_sq_out, out, sqn);
125
126 return err;
127 }
128
129 int mlx5_core_modify_sq(struct mlx5_core_dev *dev, u32 sqn, u32 *in, int inlen)
130 {
131 u32 out[MLX5_ST_SZ_DW(modify_sq_out)] = {0};
132
133 MLX5_SET(modify_sq_in, in, sqn, sqn);
134 MLX5_SET(modify_sq_in, in, opcode, MLX5_CMD_OP_MODIFY_SQ);
135 return mlx5_cmd_exec(dev, in, inlen, out, sizeof(out));
136 }
137 EXPORT_SYMBOL(mlx5_core_modify_sq);
138
139 void mlx5_core_destroy_sq(struct mlx5_core_dev *dev, u32 sqn)
140 {
141 u32 in[MLX5_ST_SZ_DW(destroy_sq_in)] = {0};
142 u32 out[MLX5_ST_SZ_DW(destroy_sq_out)] = {0};
143
144 MLX5_SET(destroy_sq_in, in, opcode, MLX5_CMD_OP_DESTROY_SQ);
145 MLX5_SET(destroy_sq_in, in, sqn, sqn);
146 mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
147 }
148
149 int mlx5_core_query_sq(struct mlx5_core_dev *dev, u32 sqn, u32 *out)
150 {
151 u32 in[MLX5_ST_SZ_DW(query_sq_in)] = {0};
152 int outlen = MLX5_ST_SZ_BYTES(query_sq_out);
153
154 MLX5_SET(query_sq_in, in, opcode, MLX5_CMD_OP_QUERY_SQ);
155 MLX5_SET(query_sq_in, in, sqn, sqn);
156 return mlx5_cmd_exec(dev, in, sizeof(in), out, outlen);
157 }
158 EXPORT_SYMBOL(mlx5_core_query_sq);
159
160 int mlx5_core_query_sq_state(struct mlx5_core_dev *dev, u32 sqn, u8 *state)
161 {
162 void *out;
163 void *sqc;
164 int inlen;
165 int err;
166
167 inlen = MLX5_ST_SZ_BYTES(query_sq_out);
168 out = kvzalloc(inlen, GFP_KERNEL);
169 if (!out)
170 return -ENOMEM;
171
172 err = mlx5_core_query_sq(dev, sqn, out);
173 if (err)
174 goto out;
175
176 sqc = MLX5_ADDR_OF(query_sq_out, out, sq_context);
177 *state = MLX5_GET(sqc, sqc, state);
178
179 out:
180 kvfree(out);
181 return err;
182 }
183 EXPORT_SYMBOL_GPL(mlx5_core_query_sq_state);
184
185 int mlx5_core_create_tir_out(struct mlx5_core_dev *dev,
186 u32 *in, int inlen,
187 u32 *out, int outlen)
188 {
189 MLX5_SET(create_tir_in, in, opcode, MLX5_CMD_OP_CREATE_TIR);
190
191 return mlx5_cmd_exec(dev, in, inlen, out, outlen);
192 }
193 EXPORT_SYMBOL(mlx5_core_create_tir_out);
194
195 int mlx5_core_create_tir(struct mlx5_core_dev *dev, u32 *in, int inlen,
196 u32 *tirn)
197 {
198 u32 out[MLX5_ST_SZ_DW(create_tir_out)] = {};
199 int err;
200
201 err = mlx5_core_create_tir_out(dev, in, inlen,
202 out, sizeof(out));
203 if (!err)
204 *tirn = MLX5_GET(create_tir_out, out, tirn);
205
206 return err;
207 }
208 EXPORT_SYMBOL(mlx5_core_create_tir);
209
210 int mlx5_core_modify_tir(struct mlx5_core_dev *dev, u32 tirn, u32 *in,
211 int inlen)
212 {
213 u32 out[MLX5_ST_SZ_DW(modify_tir_out)] = {0};
214
215 MLX5_SET(modify_tir_in, in, tirn, tirn);
216 MLX5_SET(modify_tir_in, in, opcode, MLX5_CMD_OP_MODIFY_TIR);
217 return mlx5_cmd_exec(dev, in, inlen, out, sizeof(out));
218 }
219
220 void mlx5_core_destroy_tir(struct mlx5_core_dev *dev, u32 tirn)
221 {
222 u32 in[MLX5_ST_SZ_DW(destroy_tir_in)] = {0};
223 u32 out[MLX5_ST_SZ_DW(destroy_tir_out)] = {0};
224
225 MLX5_SET(destroy_tir_in, in, opcode, MLX5_CMD_OP_DESTROY_TIR);
226 MLX5_SET(destroy_tir_in, in, tirn, tirn);
227 mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
228 }
229 EXPORT_SYMBOL(mlx5_core_destroy_tir);
230
231 int mlx5_core_create_tis(struct mlx5_core_dev *dev, u32 *in, int inlen,
232 u32 *tisn)
233 {
234 u32 out[MLX5_ST_SZ_DW(create_tis_out)] = {0};
235 int err;
236
237 MLX5_SET(create_tis_in, in, opcode, MLX5_CMD_OP_CREATE_TIS);
238 err = mlx5_cmd_exec(dev, in, inlen, out, sizeof(out));
239 if (!err)
240 *tisn = MLX5_GET(create_tis_out, out, tisn);
241
242 return err;
243 }
244 EXPORT_SYMBOL(mlx5_core_create_tis);
245
246 int mlx5_core_modify_tis(struct mlx5_core_dev *dev, u32 tisn, u32 *in,
247 int inlen)
248 {
249 u32 out[MLX5_ST_SZ_DW(modify_tis_out)] = {0};
250
251 MLX5_SET(modify_tis_in, in, tisn, tisn);
252 MLX5_SET(modify_tis_in, in, opcode, MLX5_CMD_OP_MODIFY_TIS);
253
254 return mlx5_cmd_exec(dev, in, inlen, out, sizeof(out));
255 }
256 EXPORT_SYMBOL(mlx5_core_modify_tis);
257
258 void mlx5_core_destroy_tis(struct mlx5_core_dev *dev, u32 tisn)
259 {
260 u32 in[MLX5_ST_SZ_DW(destroy_tis_in)] = {0};
261 u32 out[MLX5_ST_SZ_DW(destroy_tis_out)] = {0};
262
263 MLX5_SET(destroy_tis_in, in, opcode, MLX5_CMD_OP_DESTROY_TIS);
264 MLX5_SET(destroy_tis_in, in, tisn, tisn);
265 mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
266 }
267 EXPORT_SYMBOL(mlx5_core_destroy_tis);
268
269 int mlx5_core_create_rqt(struct mlx5_core_dev *dev, u32 *in, int inlen,
270 u32 *rqtn)
271 {
272 u32 out[MLX5_ST_SZ_DW(create_rqt_out)] = {0};
273 int err;
274
275 MLX5_SET(create_rqt_in, in, opcode, MLX5_CMD_OP_CREATE_RQT);
276 err = mlx5_cmd_exec(dev, in, inlen, out, sizeof(out));
277 if (!err)
278 *rqtn = MLX5_GET(create_rqt_out, out, rqtn);
279
280 return err;
281 }
282 EXPORT_SYMBOL(mlx5_core_create_rqt);
283
284 int mlx5_core_modify_rqt(struct mlx5_core_dev *dev, u32 rqtn, u32 *in,
285 int inlen)
286 {
287 u32 out[MLX5_ST_SZ_DW(modify_rqt_out)] = {0};
288
289 MLX5_SET(modify_rqt_in, in, rqtn, rqtn);
290 MLX5_SET(modify_rqt_in, in, opcode, MLX5_CMD_OP_MODIFY_RQT);
291 return mlx5_cmd_exec(dev, in, inlen, out, sizeof(out));
292 }
293
294 void mlx5_core_destroy_rqt(struct mlx5_core_dev *dev, u32 rqtn)
295 {
296 u32 in[MLX5_ST_SZ_DW(destroy_rqt_in)] = {0};
297 u32 out[MLX5_ST_SZ_DW(destroy_rqt_out)] = {0};
298
299 MLX5_SET(destroy_rqt_in, in, opcode, MLX5_CMD_OP_DESTROY_RQT);
300 MLX5_SET(destroy_rqt_in, in, rqtn, rqtn);
301 mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
302 }
303 EXPORT_SYMBOL(mlx5_core_destroy_rqt);
304
305 static int mlx5_hairpin_create_rq(struct mlx5_core_dev *mdev,
306 struct mlx5_hairpin_params *params, u32 *rqn)
307 {
308 u32 in[MLX5_ST_SZ_DW(create_rq_in)] = {0};
309 void *rqc, *wq;
310
311 rqc = MLX5_ADDR_OF(create_rq_in, in, ctx);
312 wq = MLX5_ADDR_OF(rqc, rqc, wq);
313
314 MLX5_SET(rqc, rqc, hairpin, 1);
315 MLX5_SET(rqc, rqc, state, MLX5_RQC_STATE_RST);
316 MLX5_SET(rqc, rqc, counter_set_id, params->q_counter);
317
318 MLX5_SET(wq, wq, log_hairpin_data_sz, params->log_data_size);
319 MLX5_SET(wq, wq, log_hairpin_num_packets, params->log_num_packets);
320
321 return mlx5_core_create_rq(mdev, in, MLX5_ST_SZ_BYTES(create_rq_in), rqn);
322 }
323
324 static int mlx5_hairpin_create_sq(struct mlx5_core_dev *mdev,
325 struct mlx5_hairpin_params *params, u32 *sqn)
326 {
327 u32 in[MLX5_ST_SZ_DW(create_sq_in)] = {0};
328 void *sqc, *wq;
329
330 sqc = MLX5_ADDR_OF(create_sq_in, in, ctx);
331 wq = MLX5_ADDR_OF(sqc, sqc, wq);
332
333 MLX5_SET(sqc, sqc, hairpin, 1);
334 MLX5_SET(sqc, sqc, state, MLX5_SQC_STATE_RST);
335
336 MLX5_SET(wq, wq, log_hairpin_data_sz, params->log_data_size);
337 MLX5_SET(wq, wq, log_hairpin_num_packets, params->log_num_packets);
338
339 return mlx5_core_create_sq(mdev, in, MLX5_ST_SZ_BYTES(create_sq_in), sqn);
340 }
341
342 static int mlx5_hairpin_create_queues(struct mlx5_hairpin *hp,
343 struct mlx5_hairpin_params *params)
344 {
345 int i, j, err;
346
347 for (i = 0; i < hp->num_channels; i++) {
348 err = mlx5_hairpin_create_rq(hp->func_mdev, params, &hp->rqn[i]);
349 if (err)
350 goto out_err_rq;
351 }
352
353 for (i = 0; i < hp->num_channels; i++) {
354 err = mlx5_hairpin_create_sq(hp->peer_mdev, params, &hp->sqn[i]);
355 if (err)
356 goto out_err_sq;
357 }
358
359 return 0;
360
361 out_err_sq:
362 for (j = 0; j < i; j++)
363 mlx5_core_destroy_sq(hp->peer_mdev, hp->sqn[j]);
364 i = hp->num_channels;
365 out_err_rq:
366 for (j = 0; j < i; j++)
367 mlx5_core_destroy_rq(hp->func_mdev, hp->rqn[j]);
368 return err;
369 }
370
371 static void mlx5_hairpin_destroy_queues(struct mlx5_hairpin *hp)
372 {
373 int i;
374
375 for (i = 0; i < hp->num_channels; i++) {
376 mlx5_core_destroy_rq(hp->func_mdev, hp->rqn[i]);
377 if (!hp->peer_gone)
378 mlx5_core_destroy_sq(hp->peer_mdev, hp->sqn[i]);
379 }
380 }
381
382 static int mlx5_hairpin_modify_rq(struct mlx5_core_dev *func_mdev, u32 rqn,
383 int curr_state, int next_state,
384 u16 peer_vhca, u32 peer_sq)
385 {
386 u32 in[MLX5_ST_SZ_DW(modify_rq_in)] = {0};
387 void *rqc;
388
389 rqc = MLX5_ADDR_OF(modify_rq_in, in, ctx);
390
391 if (next_state == MLX5_RQC_STATE_RDY) {
392 MLX5_SET(rqc, rqc, hairpin_peer_sq, peer_sq);
393 MLX5_SET(rqc, rqc, hairpin_peer_vhca, peer_vhca);
394 }
395
396 MLX5_SET(modify_rq_in, in, rq_state, curr_state);
397 MLX5_SET(rqc, rqc, state, next_state);
398
399 return mlx5_core_modify_rq(func_mdev, rqn,
400 in, MLX5_ST_SZ_BYTES(modify_rq_in));
401 }
402
403 static int mlx5_hairpin_modify_sq(struct mlx5_core_dev *peer_mdev, u32 sqn,
404 int curr_state, int next_state,
405 u16 peer_vhca, u32 peer_rq)
406 {
407 u32 in[MLX5_ST_SZ_DW(modify_sq_in)] = {0};
408 void *sqc;
409
410 sqc = MLX5_ADDR_OF(modify_sq_in, in, ctx);
411
412 if (next_state == MLX5_SQC_STATE_RDY) {
413 MLX5_SET(sqc, sqc, hairpin_peer_rq, peer_rq);
414 MLX5_SET(sqc, sqc, hairpin_peer_vhca, peer_vhca);
415 }
416
417 MLX5_SET(modify_sq_in, in, sq_state, curr_state);
418 MLX5_SET(sqc, sqc, state, next_state);
419
420 return mlx5_core_modify_sq(peer_mdev, sqn,
421 in, MLX5_ST_SZ_BYTES(modify_sq_in));
422 }
423
424 static int mlx5_hairpin_pair_queues(struct mlx5_hairpin *hp)
425 {
426 int i, j, err;
427
428
429 for (i = 0; i < hp->num_channels; i++) {
430 err = mlx5_hairpin_modify_sq(hp->peer_mdev, hp->sqn[i],
431 MLX5_SQC_STATE_RST, MLX5_SQC_STATE_RDY,
432 MLX5_CAP_GEN(hp->func_mdev, vhca_id), hp->rqn[i]);
433 if (err)
434 goto err_modify_sq;
435 }
436
437
438 for (i = 0; i < hp->num_channels; i++) {
439 err = mlx5_hairpin_modify_rq(hp->func_mdev, hp->rqn[i],
440 MLX5_RQC_STATE_RST, MLX5_RQC_STATE_RDY,
441 MLX5_CAP_GEN(hp->peer_mdev, vhca_id), hp->sqn[i]);
442 if (err)
443 goto err_modify_rq;
444 }
445
446 return 0;
447
448 err_modify_rq:
449 for (j = 0; j < i; j++)
450 mlx5_hairpin_modify_rq(hp->func_mdev, hp->rqn[j], MLX5_RQC_STATE_RDY,
451 MLX5_RQC_STATE_RST, 0, 0);
452 i = hp->num_channels;
453 err_modify_sq:
454 for (j = 0; j < i; j++)
455 mlx5_hairpin_modify_sq(hp->peer_mdev, hp->sqn[j], MLX5_SQC_STATE_RDY,
456 MLX5_SQC_STATE_RST, 0, 0);
457 return err;
458 }
459
460 static void mlx5_hairpin_unpair_queues(struct mlx5_hairpin *hp)
461 {
462 int i;
463
464
465 for (i = 0; i < hp->num_channels; i++)
466 mlx5_hairpin_modify_rq(hp->func_mdev, hp->rqn[i], MLX5_RQC_STATE_RDY,
467 MLX5_RQC_STATE_RST, 0, 0);
468
469
470 if (hp->peer_gone)
471 return;
472 for (i = 0; i < hp->num_channels; i++)
473 mlx5_hairpin_modify_sq(hp->peer_mdev, hp->sqn[i], MLX5_SQC_STATE_RDY,
474 MLX5_SQC_STATE_RST, 0, 0);
475 }
476
477 struct mlx5_hairpin *
478 mlx5_core_hairpin_create(struct mlx5_core_dev *func_mdev,
479 struct mlx5_core_dev *peer_mdev,
480 struct mlx5_hairpin_params *params)
481 {
482 struct mlx5_hairpin *hp;
483 int size, err;
484
485 size = sizeof(*hp) + params->num_channels * 2 * sizeof(u32);
486 hp = kzalloc(size, GFP_KERNEL);
487 if (!hp)
488 return ERR_PTR(-ENOMEM);
489
490 hp->func_mdev = func_mdev;
491 hp->peer_mdev = peer_mdev;
492 hp->num_channels = params->num_channels;
493
494 hp->rqn = (void *)hp + sizeof(*hp);
495 hp->sqn = hp->rqn + params->num_channels;
496
497
498 err = mlx5_hairpin_create_queues(hp, params);
499 if (err)
500 goto err_create_queues;
501
502 err = mlx5_hairpin_pair_queues(hp);
503 if (err)
504 goto err_pair_queues;
505
506 return hp;
507
508 err_pair_queues:
509 mlx5_hairpin_destroy_queues(hp);
510 err_create_queues:
511 kfree(hp);
512 return ERR_PTR(err);
513 }
514
515 void mlx5_core_hairpin_destroy(struct mlx5_hairpin *hp)
516 {
517 mlx5_hairpin_unpair_queues(hp);
518 mlx5_hairpin_destroy_queues(hp);
519 kfree(hp);
520 }