This source file includes following definitions.
- mlx5_fpga_is_ipsec_device
- mlx5_fpga_ipsec_send_complete
- syndrome_to_errno
- mlx5_fpga_ipsec_recv
- mlx5_fpga_ipsec_cmd_exec
- mlx5_fpga_ipsec_cmd_wait
- is_v2_sadb_supported
- mlx5_fpga_ipsec_update_hw_sa
- mlx5_fpga_ipsec_device_caps
- mlx5_fpga_ipsec_counters_count
- mlx5_fpga_ipsec_counters_read
- mlx5_fpga_ipsec_set_caps
- mlx5_fpga_ipsec_enable_supported_caps
- mlx5_fpga_ipsec_build_hw_xfrm
- mlx5_fpga_ipsec_build_hw_sa
- is_full_mask
- validate_fpga_full_mask
- mlx5_is_fpga_ipsec_rule
- mlx5_is_fpga_egress_ipsec_rule
- mlx5_fpga_ipsec_create_sa_ctx
- mlx5_fpga_ipsec_fs_create_sa_ctx
- mlx5_fpga_ipsec_release_sa_ctx
- mlx5_fpga_ipsec_delete_sa_ctx
- _rule_search
- rule_search
- _rule_insert
- rule_insert
- _rule_delete
- rule_delete
- restore_spec_mailbox
- modify_spec_mailbox
- egress_to_fs_ft
- fpga_ipsec_fs_create_flow_group
- fpga_ipsec_fs_create_fte
- fpga_ipsec_fs_update_fte
- fpga_ipsec_fs_delete_fte
- mlx5_fpga_ipsec_fs_create_flow_group_egress
- mlx5_fpga_ipsec_fs_create_fte_egress
- mlx5_fpga_ipsec_fs_update_fte_egress
- mlx5_fpga_ipsec_fs_delete_fte_egress
- mlx5_fpga_ipsec_fs_create_flow_group_ingress
- mlx5_fpga_ipsec_fs_create_fte_ingress
- mlx5_fpga_ipsec_fs_update_fte_ingress
- mlx5_fpga_ipsec_fs_delete_fte_ingress
- mlx5_fs_cmd_get_default_ipsec_fpga_cmds
- mlx5_fpga_ipsec_init
- destroy_rules_rb
- mlx5_fpga_ipsec_cleanup
- mlx5_fpga_ipsec_build_fs_cmds
- mlx5_fpga_esp_validate_xfrm_attrs
- mlx5_fpga_esp_create_xfrm
- mlx5_fpga_esp_destroy_xfrm
- mlx5_fpga_esp_modify_xfrm
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34 #include <linux/rhashtable.h>
35 #include <linux/mlx5/driver.h>
36 #include <linux/mlx5/fs_helpers.h>
37 #include <linux/mlx5/fs.h>
38 #include <linux/rbtree.h>
39
40 #include "mlx5_core.h"
41 #include "fs_cmd.h"
42 #include "fpga/ipsec.h"
43 #include "fpga/sdk.h"
44 #include "fpga/core.h"
45
46 enum mlx5_fpga_ipsec_cmd_status {
47 MLX5_FPGA_IPSEC_CMD_PENDING,
48 MLX5_FPGA_IPSEC_CMD_SEND_FAIL,
49 MLX5_FPGA_IPSEC_CMD_COMPLETE,
50 };
51
52 struct mlx5_fpga_ipsec_cmd_context {
53 struct mlx5_fpga_dma_buf buf;
54 enum mlx5_fpga_ipsec_cmd_status status;
55 struct mlx5_ifc_fpga_ipsec_cmd_resp resp;
56 int status_code;
57 struct completion complete;
58 struct mlx5_fpga_device *dev;
59 struct list_head list;
60 u8 command[0];
61 };
62
63 struct mlx5_fpga_esp_xfrm;
64
65 struct mlx5_fpga_ipsec_sa_ctx {
66 struct rhash_head hash;
67 struct mlx5_ifc_fpga_ipsec_sa hw_sa;
68 struct mlx5_core_dev *dev;
69 struct mlx5_fpga_esp_xfrm *fpga_xfrm;
70 };
71
72 struct mlx5_fpga_esp_xfrm {
73 unsigned int num_rules;
74 struct mlx5_fpga_ipsec_sa_ctx *sa_ctx;
75 struct mutex lock;
76 struct mlx5_accel_esp_xfrm accel_xfrm;
77 };
78
79 struct mlx5_fpga_ipsec_rule {
80 struct rb_node node;
81 struct fs_fte *fte;
82 struct mlx5_fpga_ipsec_sa_ctx *ctx;
83 };
84
85 static const struct rhashtable_params rhash_sa = {
86
87
88
89
90 .key_len = FIELD_SIZEOF(struct mlx5_fpga_ipsec_sa_ctx, hw_sa) -
91 FIELD_SIZEOF(struct mlx5_ifc_fpga_ipsec_sa_v1, cmd),
92 .key_offset = offsetof(struct mlx5_fpga_ipsec_sa_ctx, hw_sa) +
93 FIELD_SIZEOF(struct mlx5_ifc_fpga_ipsec_sa_v1, cmd),
94 .head_offset = offsetof(struct mlx5_fpga_ipsec_sa_ctx, hash),
95 .automatic_shrinking = true,
96 .min_size = 1,
97 };
98
99 struct mlx5_fpga_ipsec {
100 struct mlx5_fpga_device *fdev;
101 struct list_head pending_cmds;
102 spinlock_t pending_cmds_lock;
103 u32 caps[MLX5_ST_SZ_DW(ipsec_extended_cap)];
104 struct mlx5_fpga_conn *conn;
105
106 struct notifier_block fs_notifier_ingress_bypass;
107 struct notifier_block fs_notifier_egress;
108
109
110
111
112
113
114 struct rhashtable sa_hash;
115 struct mutex sa_hash_lock;
116
117
118
119
120 struct rb_root rules_rb;
121 struct mutex rules_rb_lock;
122 };
123
124 static bool mlx5_fpga_is_ipsec_device(struct mlx5_core_dev *mdev)
125 {
126 if (!mdev->fpga || !MLX5_CAP_GEN(mdev, fpga))
127 return false;
128
129 if (MLX5_CAP_FPGA(mdev, ieee_vendor_id) !=
130 MLX5_FPGA_CAP_SANDBOX_VENDOR_ID_MLNX)
131 return false;
132
133 if (MLX5_CAP_FPGA(mdev, sandbox_product_id) !=
134 MLX5_FPGA_CAP_SANDBOX_PRODUCT_ID_IPSEC)
135 return false;
136
137 return true;
138 }
139
140 static void mlx5_fpga_ipsec_send_complete(struct mlx5_fpga_conn *conn,
141 struct mlx5_fpga_device *fdev,
142 struct mlx5_fpga_dma_buf *buf,
143 u8 status)
144 {
145 struct mlx5_fpga_ipsec_cmd_context *context;
146
147 if (status) {
148 context = container_of(buf, struct mlx5_fpga_ipsec_cmd_context,
149 buf);
150 mlx5_fpga_warn(fdev, "IPSec command send failed with status %u\n",
151 status);
152 context->status = MLX5_FPGA_IPSEC_CMD_SEND_FAIL;
153 complete(&context->complete);
154 }
155 }
156
157 static inline
158 int syndrome_to_errno(enum mlx5_ifc_fpga_ipsec_response_syndrome syndrome)
159 {
160 switch (syndrome) {
161 case MLX5_FPGA_IPSEC_RESPONSE_SUCCESS:
162 return 0;
163 case MLX5_FPGA_IPSEC_RESPONSE_SADB_ISSUE:
164 return -EEXIST;
165 case MLX5_FPGA_IPSEC_RESPONSE_ILLEGAL_REQUEST:
166 return -EINVAL;
167 case MLX5_FPGA_IPSEC_RESPONSE_WRITE_RESPONSE_ISSUE:
168 return -EIO;
169 }
170 return -EIO;
171 }
172
173 static void mlx5_fpga_ipsec_recv(void *cb_arg, struct mlx5_fpga_dma_buf *buf)
174 {
175 struct mlx5_ifc_fpga_ipsec_cmd_resp *resp = buf->sg[0].data;
176 struct mlx5_fpga_ipsec_cmd_context *context;
177 enum mlx5_ifc_fpga_ipsec_response_syndrome syndrome;
178 struct mlx5_fpga_device *fdev = cb_arg;
179 unsigned long flags;
180
181 if (buf->sg[0].size < sizeof(*resp)) {
182 mlx5_fpga_warn(fdev, "Short receive from FPGA IPSec: %u < %zu bytes\n",
183 buf->sg[0].size, sizeof(*resp));
184 return;
185 }
186
187 mlx5_fpga_dbg(fdev, "mlx5_ipsec recv_cb syndrome %08x\n",
188 ntohl(resp->syndrome));
189
190 spin_lock_irqsave(&fdev->ipsec->pending_cmds_lock, flags);
191 context = list_first_entry_or_null(&fdev->ipsec->pending_cmds,
192 struct mlx5_fpga_ipsec_cmd_context,
193 list);
194 if (context)
195 list_del(&context->list);
196 spin_unlock_irqrestore(&fdev->ipsec->pending_cmds_lock, flags);
197
198 if (!context) {
199 mlx5_fpga_warn(fdev, "Received IPSec offload response without pending command request\n");
200 return;
201 }
202 mlx5_fpga_dbg(fdev, "Handling response for %p\n", context);
203
204 syndrome = ntohl(resp->syndrome);
205 context->status_code = syndrome_to_errno(syndrome);
206 context->status = MLX5_FPGA_IPSEC_CMD_COMPLETE;
207 memcpy(&context->resp, resp, sizeof(*resp));
208
209 if (context->status_code)
210 mlx5_fpga_warn(fdev, "IPSec command failed with syndrome %08x\n",
211 syndrome);
212
213 complete(&context->complete);
214 }
215
216 static void *mlx5_fpga_ipsec_cmd_exec(struct mlx5_core_dev *mdev,
217 const void *cmd, int cmd_size)
218 {
219 struct mlx5_fpga_ipsec_cmd_context *context;
220 struct mlx5_fpga_device *fdev = mdev->fpga;
221 unsigned long flags;
222 int res;
223
224 if (!fdev || !fdev->ipsec)
225 return ERR_PTR(-EOPNOTSUPP);
226
227 if (cmd_size & 3)
228 return ERR_PTR(-EINVAL);
229
230 context = kzalloc(sizeof(*context) + cmd_size, GFP_ATOMIC);
231 if (!context)
232 return ERR_PTR(-ENOMEM);
233
234 context->status = MLX5_FPGA_IPSEC_CMD_PENDING;
235 context->dev = fdev;
236 context->buf.complete = mlx5_fpga_ipsec_send_complete;
237 init_completion(&context->complete);
238 memcpy(&context->command, cmd, cmd_size);
239 context->buf.sg[0].size = cmd_size;
240 context->buf.sg[0].data = &context->command;
241
242 spin_lock_irqsave(&fdev->ipsec->pending_cmds_lock, flags);
243 res = mlx5_fpga_sbu_conn_sendmsg(fdev->ipsec->conn, &context->buf);
244 if (!res)
245 list_add_tail(&context->list, &fdev->ipsec->pending_cmds);
246 spin_unlock_irqrestore(&fdev->ipsec->pending_cmds_lock, flags);
247
248 if (res) {
249 mlx5_fpga_warn(fdev, "Failed to send IPSec command: %d\n", res);
250 kfree(context);
251 return ERR_PTR(res);
252 }
253
254
255 return context;
256 }
257
258 static int mlx5_fpga_ipsec_cmd_wait(void *ctx)
259 {
260 struct mlx5_fpga_ipsec_cmd_context *context = ctx;
261 unsigned long timeout =
262 msecs_to_jiffies(MLX5_FPGA_CMD_TIMEOUT_MSEC);
263 int res;
264
265 res = wait_for_completion_timeout(&context->complete, timeout);
266 if (!res) {
267 mlx5_fpga_warn(context->dev, "Failure waiting for IPSec command response\n");
268 return -ETIMEDOUT;
269 }
270
271 if (context->status == MLX5_FPGA_IPSEC_CMD_COMPLETE)
272 res = context->status_code;
273 else
274 res = -EIO;
275
276 return res;
277 }
278
279 static inline bool is_v2_sadb_supported(struct mlx5_fpga_ipsec *fipsec)
280 {
281 if (MLX5_GET(ipsec_extended_cap, fipsec->caps, v2_command))
282 return true;
283 return false;
284 }
285
286 static int mlx5_fpga_ipsec_update_hw_sa(struct mlx5_fpga_device *fdev,
287 struct mlx5_ifc_fpga_ipsec_sa *hw_sa,
288 int opcode)
289 {
290 struct mlx5_core_dev *dev = fdev->mdev;
291 struct mlx5_ifc_fpga_ipsec_sa *sa;
292 struct mlx5_fpga_ipsec_cmd_context *cmd_context;
293 size_t sa_cmd_size;
294 int err;
295
296 hw_sa->ipsec_sa_v1.cmd = htonl(opcode);
297 if (is_v2_sadb_supported(fdev->ipsec))
298 sa_cmd_size = sizeof(*hw_sa);
299 else
300 sa_cmd_size = sizeof(hw_sa->ipsec_sa_v1);
301
302 cmd_context = (struct mlx5_fpga_ipsec_cmd_context *)
303 mlx5_fpga_ipsec_cmd_exec(dev, hw_sa, sa_cmd_size);
304 if (IS_ERR(cmd_context))
305 return PTR_ERR(cmd_context);
306
307 err = mlx5_fpga_ipsec_cmd_wait(cmd_context);
308 if (err)
309 goto out;
310
311 sa = (struct mlx5_ifc_fpga_ipsec_sa *)&cmd_context->command;
312 if (sa->ipsec_sa_v1.sw_sa_handle != cmd_context->resp.sw_sa_handle) {
313 mlx5_fpga_err(fdev, "mismatch SA handle. cmd 0x%08x vs resp 0x%08x\n",
314 ntohl(sa->ipsec_sa_v1.sw_sa_handle),
315 ntohl(cmd_context->resp.sw_sa_handle));
316 err = -EIO;
317 }
318
319 out:
320 kfree(cmd_context);
321 return err;
322 }
323
324 u32 mlx5_fpga_ipsec_device_caps(struct mlx5_core_dev *mdev)
325 {
326 struct mlx5_fpga_device *fdev = mdev->fpga;
327 u32 ret = 0;
328
329 if (mlx5_fpga_is_ipsec_device(mdev)) {
330 ret |= MLX5_ACCEL_IPSEC_CAP_DEVICE;
331 ret |= MLX5_ACCEL_IPSEC_CAP_REQUIRED_METADATA;
332 } else {
333 return ret;
334 }
335
336 if (!fdev->ipsec)
337 return ret;
338
339 if (MLX5_GET(ipsec_extended_cap, fdev->ipsec->caps, esp))
340 ret |= MLX5_ACCEL_IPSEC_CAP_ESP;
341
342 if (MLX5_GET(ipsec_extended_cap, fdev->ipsec->caps, ipv6))
343 ret |= MLX5_ACCEL_IPSEC_CAP_IPV6;
344
345 if (MLX5_GET(ipsec_extended_cap, fdev->ipsec->caps, lso))
346 ret |= MLX5_ACCEL_IPSEC_CAP_LSO;
347
348 if (MLX5_GET(ipsec_extended_cap, fdev->ipsec->caps, rx_no_trailer))
349 ret |= MLX5_ACCEL_IPSEC_CAP_RX_NO_TRAILER;
350
351 if (MLX5_GET(ipsec_extended_cap, fdev->ipsec->caps, esn)) {
352 ret |= MLX5_ACCEL_IPSEC_CAP_ESN;
353 ret |= MLX5_ACCEL_IPSEC_CAP_TX_IV_IS_ESN;
354 }
355
356 return ret;
357 }
358
359 unsigned int mlx5_fpga_ipsec_counters_count(struct mlx5_core_dev *mdev)
360 {
361 struct mlx5_fpga_device *fdev = mdev->fpga;
362
363 if (!fdev || !fdev->ipsec)
364 return 0;
365
366 return MLX5_GET(ipsec_extended_cap, fdev->ipsec->caps,
367 number_of_ipsec_counters);
368 }
369
370 int mlx5_fpga_ipsec_counters_read(struct mlx5_core_dev *mdev, u64 *counters,
371 unsigned int counters_count)
372 {
373 struct mlx5_fpga_device *fdev = mdev->fpga;
374 unsigned int i;
375 __be32 *data;
376 u32 count;
377 u64 addr;
378 int ret;
379
380 if (!fdev || !fdev->ipsec)
381 return 0;
382
383 addr = (u64)MLX5_GET(ipsec_extended_cap, fdev->ipsec->caps,
384 ipsec_counters_addr_low) +
385 ((u64)MLX5_GET(ipsec_extended_cap, fdev->ipsec->caps,
386 ipsec_counters_addr_high) << 32);
387
388 count = mlx5_fpga_ipsec_counters_count(mdev);
389
390 data = kzalloc(array3_size(sizeof(*data), count, 2), GFP_KERNEL);
391 if (!data) {
392 ret = -ENOMEM;
393 goto out;
394 }
395
396 ret = mlx5_fpga_mem_read(fdev, count * sizeof(u64), addr, data,
397 MLX5_FPGA_ACCESS_TYPE_DONTCARE);
398 if (ret < 0) {
399 mlx5_fpga_err(fdev, "Failed to read IPSec counters from HW: %d\n",
400 ret);
401 goto out;
402 }
403 ret = 0;
404
405 if (count > counters_count)
406 count = counters_count;
407
408
409 for (i = 0; i < count; i++)
410 counters[i] = (u64)ntohl(data[i * 2]) |
411 ((u64)ntohl(data[i * 2 + 1]) << 32);
412
413 out:
414 kfree(data);
415 return ret;
416 }
417
418 static int mlx5_fpga_ipsec_set_caps(struct mlx5_core_dev *mdev, u32 flags)
419 {
420 struct mlx5_fpga_ipsec_cmd_context *context;
421 struct mlx5_ifc_fpga_ipsec_cmd_cap cmd = {0};
422 int err;
423
424 cmd.cmd = htonl(MLX5_FPGA_IPSEC_CMD_OP_SET_CAP);
425 cmd.flags = htonl(flags);
426 context = mlx5_fpga_ipsec_cmd_exec(mdev, &cmd, sizeof(cmd));
427 if (IS_ERR(context))
428 return PTR_ERR(context);
429
430 err = mlx5_fpga_ipsec_cmd_wait(context);
431 if (err)
432 goto out;
433
434 if ((context->resp.flags & cmd.flags) != cmd.flags) {
435 mlx5_fpga_err(context->dev, "Failed to set capabilities. cmd 0x%08x vs resp 0x%08x\n",
436 cmd.flags,
437 context->resp.flags);
438 err = -EIO;
439 }
440
441 out:
442 kfree(context);
443 return err;
444 }
445
446 static int mlx5_fpga_ipsec_enable_supported_caps(struct mlx5_core_dev *mdev)
447 {
448 u32 dev_caps = mlx5_fpga_ipsec_device_caps(mdev);
449 u32 flags = 0;
450
451 if (dev_caps & MLX5_ACCEL_IPSEC_CAP_RX_NO_TRAILER)
452 flags |= MLX5_FPGA_IPSEC_CAP_NO_TRAILER;
453
454 return mlx5_fpga_ipsec_set_caps(mdev, flags);
455 }
456
457 static void
458 mlx5_fpga_ipsec_build_hw_xfrm(struct mlx5_core_dev *mdev,
459 const struct mlx5_accel_esp_xfrm_attrs *xfrm_attrs,
460 struct mlx5_ifc_fpga_ipsec_sa *hw_sa)
461 {
462 const struct aes_gcm_keymat *aes_gcm = &xfrm_attrs->keymat.aes_gcm;
463
464
465 memcpy(&hw_sa->ipsec_sa_v1.key_enc, aes_gcm->aes_key,
466 aes_gcm->key_len / 8);
467
468 if (aes_gcm->key_len == 128)
469 memcpy(&hw_sa->ipsec_sa_v1.key_enc[16],
470 aes_gcm->aes_key, aes_gcm->key_len / 8);
471
472
473 memcpy(&hw_sa->ipsec_sa_v1.gcm.salt_iv, &aes_gcm->seq_iv,
474 sizeof(aes_gcm->seq_iv));
475 memcpy(&hw_sa->ipsec_sa_v1.gcm.salt, &aes_gcm->salt,
476 sizeof(aes_gcm->salt));
477
478
479 if (xfrm_attrs->flags & MLX5_ACCEL_ESP_FLAGS_ESN_TRIGGERED) {
480 hw_sa->ipsec_sa_v1.flags |= MLX5_FPGA_IPSEC_SA_ESN_EN;
481 hw_sa->ipsec_sa_v1.flags |=
482 (xfrm_attrs->flags &
483 MLX5_ACCEL_ESP_FLAGS_ESN_STATE_OVERLAP) ?
484 MLX5_FPGA_IPSEC_SA_ESN_OVERLAP : 0;
485 hw_sa->esn = htonl(xfrm_attrs->esn);
486 } else {
487 hw_sa->ipsec_sa_v1.flags &= ~MLX5_FPGA_IPSEC_SA_ESN_EN;
488 hw_sa->ipsec_sa_v1.flags &=
489 ~(xfrm_attrs->flags &
490 MLX5_ACCEL_ESP_FLAGS_ESN_STATE_OVERLAP) ?
491 MLX5_FPGA_IPSEC_SA_ESN_OVERLAP : 0;
492 hw_sa->esn = 0;
493 }
494
495
496 hw_sa->ipsec_sa_v1.sw_sa_handle = htonl(xfrm_attrs->sa_handle);
497
498
499 switch (aes_gcm->key_len) {
500 case 128:
501 hw_sa->ipsec_sa_v1.enc_mode =
502 MLX5_FPGA_IPSEC_SA_ENC_MODE_AES_GCM_128_AUTH_128;
503 break;
504 case 256:
505 hw_sa->ipsec_sa_v1.enc_mode =
506 MLX5_FPGA_IPSEC_SA_ENC_MODE_AES_GCM_256_AUTH_128;
507 break;
508 }
509
510
511 hw_sa->ipsec_sa_v1.flags |= MLX5_FPGA_IPSEC_SA_SA_VALID |
512 MLX5_FPGA_IPSEC_SA_SPI_EN |
513 MLX5_FPGA_IPSEC_SA_IP_ESP;
514
515 if (xfrm_attrs->action & MLX5_ACCEL_ESP_ACTION_ENCRYPT)
516 hw_sa->ipsec_sa_v1.flags |= MLX5_FPGA_IPSEC_SA_DIR_SX;
517 else
518 hw_sa->ipsec_sa_v1.flags &= ~MLX5_FPGA_IPSEC_SA_DIR_SX;
519 }
520
521 static void
522 mlx5_fpga_ipsec_build_hw_sa(struct mlx5_core_dev *mdev,
523 struct mlx5_accel_esp_xfrm_attrs *xfrm_attrs,
524 const __be32 saddr[4],
525 const __be32 daddr[4],
526 const __be32 spi, bool is_ipv6,
527 struct mlx5_ifc_fpga_ipsec_sa *hw_sa)
528 {
529 mlx5_fpga_ipsec_build_hw_xfrm(mdev, xfrm_attrs, hw_sa);
530
531
532 memcpy(hw_sa->ipsec_sa_v1.sip, saddr, sizeof(hw_sa->ipsec_sa_v1.sip));
533 memcpy(hw_sa->ipsec_sa_v1.dip, daddr, sizeof(hw_sa->ipsec_sa_v1.dip));
534
535
536 hw_sa->ipsec_sa_v1.spi = spi;
537
538
539 if (is_ipv6)
540 hw_sa->ipsec_sa_v1.flags |= MLX5_FPGA_IPSEC_SA_IPV6;
541 }
542
543 static bool is_full_mask(const void *p, size_t len)
544 {
545 WARN_ON(len % 4);
546
547 return !memchr_inv(p, 0xff, len);
548 }
549
550 static bool validate_fpga_full_mask(struct mlx5_core_dev *dev,
551 const u32 *match_c,
552 const u32 *match_v)
553 {
554 const void *misc_params_c = MLX5_ADDR_OF(fte_match_param,
555 match_c,
556 misc_parameters);
557 const void *headers_c = MLX5_ADDR_OF(fte_match_param,
558 match_c,
559 outer_headers);
560 const void *headers_v = MLX5_ADDR_OF(fte_match_param,
561 match_v,
562 outer_headers);
563
564 if (mlx5_fs_is_outer_ipv4_flow(dev, headers_c, headers_v)) {
565 const void *s_ipv4_c = MLX5_ADDR_OF(fte_match_set_lyr_2_4,
566 headers_c,
567 src_ipv4_src_ipv6.ipv4_layout.ipv4);
568 const void *d_ipv4_c = MLX5_ADDR_OF(fte_match_set_lyr_2_4,
569 headers_c,
570 dst_ipv4_dst_ipv6.ipv4_layout.ipv4);
571
572 if (!is_full_mask(s_ipv4_c, MLX5_FLD_SZ_BYTES(ipv4_layout,
573 ipv4)) ||
574 !is_full_mask(d_ipv4_c, MLX5_FLD_SZ_BYTES(ipv4_layout,
575 ipv4)))
576 return false;
577 } else {
578 const void *s_ipv6_c = MLX5_ADDR_OF(fte_match_set_lyr_2_4,
579 headers_c,
580 src_ipv4_src_ipv6.ipv6_layout.ipv6);
581 const void *d_ipv6_c = MLX5_ADDR_OF(fte_match_set_lyr_2_4,
582 headers_c,
583 dst_ipv4_dst_ipv6.ipv6_layout.ipv6);
584
585 if (!is_full_mask(s_ipv6_c, MLX5_FLD_SZ_BYTES(ipv6_layout,
586 ipv6)) ||
587 !is_full_mask(d_ipv6_c, MLX5_FLD_SZ_BYTES(ipv6_layout,
588 ipv6)))
589 return false;
590 }
591
592 if (!is_full_mask(MLX5_ADDR_OF(fte_match_set_misc, misc_params_c,
593 outer_esp_spi),
594 MLX5_FLD_SZ_BYTES(fte_match_set_misc, outer_esp_spi)))
595 return false;
596
597 return true;
598 }
599
600 static bool mlx5_is_fpga_ipsec_rule(struct mlx5_core_dev *dev,
601 u8 match_criteria_enable,
602 const u32 *match_c,
603 const u32 *match_v)
604 {
605 u32 ipsec_dev_caps = mlx5_accel_ipsec_device_caps(dev);
606 bool ipv6_flow;
607
608 ipv6_flow = mlx5_fs_is_outer_ipv6_flow(dev, match_c, match_v);
609
610 if (!(match_criteria_enable & MLX5_MATCH_OUTER_HEADERS) ||
611 mlx5_fs_is_outer_udp_flow(match_c, match_v) ||
612 mlx5_fs_is_outer_tcp_flow(match_c, match_v) ||
613 mlx5_fs_is_vxlan_flow(match_c) ||
614 !(mlx5_fs_is_outer_ipv4_flow(dev, match_c, match_v) ||
615 ipv6_flow))
616 return false;
617
618 if (!(ipsec_dev_caps & MLX5_ACCEL_IPSEC_CAP_DEVICE))
619 return false;
620
621 if (!(ipsec_dev_caps & MLX5_ACCEL_IPSEC_CAP_ESP) &&
622 mlx5_fs_is_outer_ipsec_flow(match_c))
623 return false;
624
625 if (!(ipsec_dev_caps & MLX5_ACCEL_IPSEC_CAP_IPV6) &&
626 ipv6_flow)
627 return false;
628
629 if (!validate_fpga_full_mask(dev, match_c, match_v))
630 return false;
631
632 return true;
633 }
634
635 static bool mlx5_is_fpga_egress_ipsec_rule(struct mlx5_core_dev *dev,
636 u8 match_criteria_enable,
637 const u32 *match_c,
638 const u32 *match_v,
639 struct mlx5_flow_act *flow_act,
640 struct mlx5_flow_context *flow_context)
641 {
642 const void *outer_c = MLX5_ADDR_OF(fte_match_param, match_c,
643 outer_headers);
644 bool is_dmac = MLX5_GET(fte_match_set_lyr_2_4, outer_c, dmac_47_16) ||
645 MLX5_GET(fte_match_set_lyr_2_4, outer_c, dmac_15_0);
646 bool is_smac = MLX5_GET(fte_match_set_lyr_2_4, outer_c, smac_47_16) ||
647 MLX5_GET(fte_match_set_lyr_2_4, outer_c, smac_15_0);
648 int ret;
649
650 ret = mlx5_is_fpga_ipsec_rule(dev, match_criteria_enable, match_c,
651 match_v);
652 if (!ret)
653 return ret;
654
655 if (is_dmac || is_smac ||
656 (match_criteria_enable &
657 ~(MLX5_MATCH_OUTER_HEADERS | MLX5_MATCH_MISC_PARAMETERS)) ||
658 (flow_act->action & ~(MLX5_FLOW_CONTEXT_ACTION_ENCRYPT | MLX5_FLOW_CONTEXT_ACTION_ALLOW)) ||
659 (flow_context->flags & FLOW_CONTEXT_HAS_TAG))
660 return false;
661
662 return true;
663 }
664
665 void *mlx5_fpga_ipsec_create_sa_ctx(struct mlx5_core_dev *mdev,
666 struct mlx5_accel_esp_xfrm *accel_xfrm,
667 const __be32 saddr[4],
668 const __be32 daddr[4],
669 const __be32 spi, bool is_ipv6)
670 {
671 struct mlx5_fpga_ipsec_sa_ctx *sa_ctx;
672 struct mlx5_fpga_esp_xfrm *fpga_xfrm =
673 container_of(accel_xfrm, typeof(*fpga_xfrm),
674 accel_xfrm);
675 struct mlx5_fpga_device *fdev = mdev->fpga;
676 struct mlx5_fpga_ipsec *fipsec = fdev->ipsec;
677 int opcode, err;
678 void *context;
679
680
681 sa_ctx = kzalloc(sizeof(*sa_ctx), GFP_KERNEL);
682 if (!sa_ctx)
683 return ERR_PTR(-ENOMEM);
684
685 sa_ctx->dev = mdev;
686
687
688 mlx5_fpga_ipsec_build_hw_sa(mdev, &accel_xfrm->attrs,
689 saddr, daddr, spi, is_ipv6,
690 &sa_ctx->hw_sa);
691
692 mutex_lock(&fpga_xfrm->lock);
693
694 if (fpga_xfrm->sa_ctx) {
695
696 if (memcmp(&sa_ctx->hw_sa, &fpga_xfrm->sa_ctx->hw_sa,
697 sizeof(sa_ctx->hw_sa))) {
698 context = ERR_PTR(-EINVAL);
699 goto exists;
700 }
701
702 ++fpga_xfrm->num_rules;
703 context = fpga_xfrm->sa_ctx;
704 goto exists;
705 }
706
707
708 mutex_lock(&fipsec->sa_hash_lock);
709
710 err = rhashtable_lookup_insert_fast(&fipsec->sa_hash, &sa_ctx->hash,
711 rhash_sa);
712 if (err) {
713
714
715
716
717 context = ERR_PTR(-EEXIST);
718 goto unlock_hash;
719 }
720
721
722 opcode = is_v2_sadb_supported(fdev->ipsec) ?
723 MLX5_FPGA_IPSEC_CMD_OP_ADD_SA_V2 :
724 MLX5_FPGA_IPSEC_CMD_OP_ADD_SA;
725 err = mlx5_fpga_ipsec_update_hw_sa(fdev, &sa_ctx->hw_sa, opcode);
726 sa_ctx->hw_sa.ipsec_sa_v1.cmd = 0;
727 if (err) {
728 context = ERR_PTR(err);
729 goto delete_hash;
730 }
731
732 mutex_unlock(&fipsec->sa_hash_lock);
733
734 ++fpga_xfrm->num_rules;
735 fpga_xfrm->sa_ctx = sa_ctx;
736 sa_ctx->fpga_xfrm = fpga_xfrm;
737
738 mutex_unlock(&fpga_xfrm->lock);
739
740 return sa_ctx;
741
742 delete_hash:
743 WARN_ON(rhashtable_remove_fast(&fipsec->sa_hash, &sa_ctx->hash,
744 rhash_sa));
745 unlock_hash:
746 mutex_unlock(&fipsec->sa_hash_lock);
747
748 exists:
749 mutex_unlock(&fpga_xfrm->lock);
750 kfree(sa_ctx);
751 return context;
752 }
753
754 static void *
755 mlx5_fpga_ipsec_fs_create_sa_ctx(struct mlx5_core_dev *mdev,
756 struct fs_fte *fte,
757 bool is_egress)
758 {
759 struct mlx5_accel_esp_xfrm *accel_xfrm;
760 __be32 saddr[4], daddr[4], spi;
761 struct mlx5_flow_group *fg;
762 bool is_ipv6 = false;
763
764 fs_get_obj(fg, fte->node.parent);
765
766 if (is_egress &&
767 !mlx5_is_fpga_egress_ipsec_rule(mdev,
768 fg->mask.match_criteria_enable,
769 fg->mask.match_criteria,
770 fte->val,
771 &fte->action,
772 &fte->flow_context))
773 return ERR_PTR(-EINVAL);
774 else if (!mlx5_is_fpga_ipsec_rule(mdev,
775 fg->mask.match_criteria_enable,
776 fg->mask.match_criteria,
777 fte->val))
778 return ERR_PTR(-EINVAL);
779
780
781 accel_xfrm =
782 (struct mlx5_accel_esp_xfrm *)fte->action.esp_id;
783
784
785 if (mlx5_fs_is_outer_ipv4_flow(mdev, fg->mask.match_criteria,
786 fte->val)) {
787 memcpy(&saddr[3],
788 MLX5_ADDR_OF(fte_match_set_lyr_2_4,
789 fte->val,
790 src_ipv4_src_ipv6.ipv4_layout.ipv4),
791 sizeof(saddr[3]));
792 memcpy(&daddr[3],
793 MLX5_ADDR_OF(fte_match_set_lyr_2_4,
794 fte->val,
795 dst_ipv4_dst_ipv6.ipv4_layout.ipv4),
796 sizeof(daddr[3]));
797 } else {
798 memcpy(saddr,
799 MLX5_ADDR_OF(fte_match_param,
800 fte->val,
801 outer_headers.src_ipv4_src_ipv6.ipv6_layout.ipv6),
802 sizeof(saddr));
803 memcpy(daddr,
804 MLX5_ADDR_OF(fte_match_param,
805 fte->val,
806 outer_headers.dst_ipv4_dst_ipv6.ipv6_layout.ipv6),
807 sizeof(daddr));
808 is_ipv6 = true;
809 }
810
811
812 spi = MLX5_GET_BE(typeof(spi),
813 fte_match_param, fte->val,
814 misc_parameters.outer_esp_spi);
815
816
817 return mlx5_fpga_ipsec_create_sa_ctx(mdev, accel_xfrm,
818 saddr, daddr,
819 spi, is_ipv6);
820 }
821
822 static void
823 mlx5_fpga_ipsec_release_sa_ctx(struct mlx5_fpga_ipsec_sa_ctx *sa_ctx)
824 {
825 struct mlx5_fpga_device *fdev = sa_ctx->dev->fpga;
826 struct mlx5_fpga_ipsec *fipsec = fdev->ipsec;
827 int opcode = is_v2_sadb_supported(fdev->ipsec) ?
828 MLX5_FPGA_IPSEC_CMD_OP_DEL_SA_V2 :
829 MLX5_FPGA_IPSEC_CMD_OP_DEL_SA;
830 int err;
831
832 err = mlx5_fpga_ipsec_update_hw_sa(fdev, &sa_ctx->hw_sa, opcode);
833 sa_ctx->hw_sa.ipsec_sa_v1.cmd = 0;
834 if (err) {
835 WARN_ON(err);
836 return;
837 }
838
839 mutex_lock(&fipsec->sa_hash_lock);
840 WARN_ON(rhashtable_remove_fast(&fipsec->sa_hash, &sa_ctx->hash,
841 rhash_sa));
842 mutex_unlock(&fipsec->sa_hash_lock);
843 }
844
845 void mlx5_fpga_ipsec_delete_sa_ctx(void *context)
846 {
847 struct mlx5_fpga_esp_xfrm *fpga_xfrm =
848 ((struct mlx5_fpga_ipsec_sa_ctx *)context)->fpga_xfrm;
849
850 mutex_lock(&fpga_xfrm->lock);
851 if (!--fpga_xfrm->num_rules) {
852 mlx5_fpga_ipsec_release_sa_ctx(fpga_xfrm->sa_ctx);
853 kfree(fpga_xfrm->sa_ctx);
854 fpga_xfrm->sa_ctx = NULL;
855 }
856 mutex_unlock(&fpga_xfrm->lock);
857 }
858
859 static inline struct mlx5_fpga_ipsec_rule *
860 _rule_search(struct rb_root *root, struct fs_fte *fte)
861 {
862 struct rb_node *node = root->rb_node;
863
864 while (node) {
865 struct mlx5_fpga_ipsec_rule *rule =
866 container_of(node, struct mlx5_fpga_ipsec_rule,
867 node);
868
869 if (rule->fte < fte)
870 node = node->rb_left;
871 else if (rule->fte > fte)
872 node = node->rb_right;
873 else
874 return rule;
875 }
876 return NULL;
877 }
878
879 static struct mlx5_fpga_ipsec_rule *
880 rule_search(struct mlx5_fpga_ipsec *ipsec_dev, struct fs_fte *fte)
881 {
882 struct mlx5_fpga_ipsec_rule *rule;
883
884 mutex_lock(&ipsec_dev->rules_rb_lock);
885 rule = _rule_search(&ipsec_dev->rules_rb, fte);
886 mutex_unlock(&ipsec_dev->rules_rb_lock);
887
888 return rule;
889 }
890
891 static inline int _rule_insert(struct rb_root *root,
892 struct mlx5_fpga_ipsec_rule *rule)
893 {
894 struct rb_node **new = &root->rb_node, *parent = NULL;
895
896
897 while (*new) {
898 struct mlx5_fpga_ipsec_rule *this =
899 container_of(*new, struct mlx5_fpga_ipsec_rule,
900 node);
901
902 parent = *new;
903 if (rule->fte < this->fte)
904 new = &((*new)->rb_left);
905 else if (rule->fte > this->fte)
906 new = &((*new)->rb_right);
907 else
908 return -EEXIST;
909 }
910
911
912 rb_link_node(&rule->node, parent, new);
913 rb_insert_color(&rule->node, root);
914
915 return 0;
916 }
917
918 static int rule_insert(struct mlx5_fpga_ipsec *ipsec_dev,
919 struct mlx5_fpga_ipsec_rule *rule)
920 {
921 int ret;
922
923 mutex_lock(&ipsec_dev->rules_rb_lock);
924 ret = _rule_insert(&ipsec_dev->rules_rb, rule);
925 mutex_unlock(&ipsec_dev->rules_rb_lock);
926
927 return ret;
928 }
929
930 static inline void _rule_delete(struct mlx5_fpga_ipsec *ipsec_dev,
931 struct mlx5_fpga_ipsec_rule *rule)
932 {
933 struct rb_root *root = &ipsec_dev->rules_rb;
934
935 mutex_lock(&ipsec_dev->rules_rb_lock);
936 rb_erase(&rule->node, root);
937 mutex_unlock(&ipsec_dev->rules_rb_lock);
938 }
939
940 static void rule_delete(struct mlx5_fpga_ipsec *ipsec_dev,
941 struct mlx5_fpga_ipsec_rule *rule)
942 {
943 _rule_delete(ipsec_dev, rule);
944 kfree(rule);
945 }
946
947 struct mailbox_mod {
948 uintptr_t saved_esp_id;
949 u32 saved_action;
950 u32 saved_outer_esp_spi_value;
951 };
952
953 static void restore_spec_mailbox(struct fs_fte *fte,
954 struct mailbox_mod *mbox_mod)
955 {
956 char *misc_params_v = MLX5_ADDR_OF(fte_match_param,
957 fte->val,
958 misc_parameters);
959
960 MLX5_SET(fte_match_set_misc, misc_params_v, outer_esp_spi,
961 mbox_mod->saved_outer_esp_spi_value);
962 fte->action.action |= mbox_mod->saved_action;
963 fte->action.esp_id = (uintptr_t)mbox_mod->saved_esp_id;
964 }
965
966 static void modify_spec_mailbox(struct mlx5_core_dev *mdev,
967 struct fs_fte *fte,
968 struct mailbox_mod *mbox_mod)
969 {
970 char *misc_params_v = MLX5_ADDR_OF(fte_match_param,
971 fte->val,
972 misc_parameters);
973
974 mbox_mod->saved_esp_id = fte->action.esp_id;
975 mbox_mod->saved_action = fte->action.action &
976 (MLX5_FLOW_CONTEXT_ACTION_ENCRYPT |
977 MLX5_FLOW_CONTEXT_ACTION_DECRYPT);
978 mbox_mod->saved_outer_esp_spi_value =
979 MLX5_GET(fte_match_set_misc, misc_params_v,
980 outer_esp_spi);
981
982 fte->action.esp_id = 0;
983 fte->action.action &= ~(MLX5_FLOW_CONTEXT_ACTION_ENCRYPT |
984 MLX5_FLOW_CONTEXT_ACTION_DECRYPT);
985 if (!MLX5_CAP_FLOWTABLE(mdev,
986 flow_table_properties_nic_receive.ft_field_support.outer_esp_spi))
987 MLX5_SET(fte_match_set_misc, misc_params_v, outer_esp_spi, 0);
988 }
989
990 static enum fs_flow_table_type egress_to_fs_ft(bool egress)
991 {
992 return egress ? FS_FT_NIC_TX : FS_FT_NIC_RX;
993 }
994
995 static int fpga_ipsec_fs_create_flow_group(struct mlx5_flow_root_namespace *ns,
996 struct mlx5_flow_table *ft,
997 u32 *in,
998 struct mlx5_flow_group *fg,
999 bool is_egress)
1000 {
1001 int (*create_flow_group)(struct mlx5_flow_root_namespace *ns,
1002 struct mlx5_flow_table *ft, u32 *in,
1003 struct mlx5_flow_group *fg) =
1004 mlx5_fs_cmd_get_default(egress_to_fs_ft(is_egress))->create_flow_group;
1005 char *misc_params_c = MLX5_ADDR_OF(create_flow_group_in, in,
1006 match_criteria.misc_parameters);
1007 struct mlx5_core_dev *dev = ns->dev;
1008 u32 saved_outer_esp_spi_mask;
1009 u8 match_criteria_enable;
1010 int ret;
1011
1012 if (MLX5_CAP_FLOWTABLE(dev,
1013 flow_table_properties_nic_receive.ft_field_support.outer_esp_spi))
1014 return create_flow_group(ns, ft, in, fg);
1015
1016 match_criteria_enable =
1017 MLX5_GET(create_flow_group_in, in, match_criteria_enable);
1018 saved_outer_esp_spi_mask =
1019 MLX5_GET(fte_match_set_misc, misc_params_c, outer_esp_spi);
1020 if (!match_criteria_enable || !saved_outer_esp_spi_mask)
1021 return create_flow_group(ns, ft, in, fg);
1022
1023 MLX5_SET(fte_match_set_misc, misc_params_c, outer_esp_spi, 0);
1024
1025 if (!(*misc_params_c) &&
1026 !memcmp(misc_params_c, misc_params_c + 1, MLX5_ST_SZ_BYTES(fte_match_set_misc) - 1))
1027 MLX5_SET(create_flow_group_in, in, match_criteria_enable,
1028 match_criteria_enable & ~MLX5_MATCH_MISC_PARAMETERS);
1029
1030 ret = create_flow_group(ns, ft, in, fg);
1031
1032 MLX5_SET(fte_match_set_misc, misc_params_c, outer_esp_spi, saved_outer_esp_spi_mask);
1033 MLX5_SET(create_flow_group_in, in, match_criteria_enable, match_criteria_enable);
1034
1035 return ret;
1036 }
1037
1038 static int fpga_ipsec_fs_create_fte(struct mlx5_flow_root_namespace *ns,
1039 struct mlx5_flow_table *ft,
1040 struct mlx5_flow_group *fg,
1041 struct fs_fte *fte,
1042 bool is_egress)
1043 {
1044 int (*create_fte)(struct mlx5_flow_root_namespace *ns,
1045 struct mlx5_flow_table *ft,
1046 struct mlx5_flow_group *fg,
1047 struct fs_fte *fte) =
1048 mlx5_fs_cmd_get_default(egress_to_fs_ft(is_egress))->create_fte;
1049 struct mlx5_core_dev *dev = ns->dev;
1050 struct mlx5_fpga_device *fdev = dev->fpga;
1051 struct mlx5_fpga_ipsec *fipsec = fdev->ipsec;
1052 struct mlx5_fpga_ipsec_rule *rule;
1053 bool is_esp = fte->action.esp_id;
1054 struct mailbox_mod mbox_mod;
1055 int ret;
1056
1057 if (!is_esp ||
1058 !(fte->action.action &
1059 (MLX5_FLOW_CONTEXT_ACTION_ENCRYPT |
1060 MLX5_FLOW_CONTEXT_ACTION_DECRYPT)))
1061 return create_fte(ns, ft, fg, fte);
1062
1063 rule = kzalloc(sizeof(*rule), GFP_KERNEL);
1064 if (!rule)
1065 return -ENOMEM;
1066
1067 rule->ctx = mlx5_fpga_ipsec_fs_create_sa_ctx(dev, fte, is_egress);
1068 if (IS_ERR(rule->ctx)) {
1069 int err = PTR_ERR(rule->ctx);
1070 kfree(rule);
1071 return err;
1072 }
1073
1074 rule->fte = fte;
1075 WARN_ON(rule_insert(fipsec, rule));
1076
1077 modify_spec_mailbox(dev, fte, &mbox_mod);
1078 ret = create_fte(ns, ft, fg, fte);
1079 restore_spec_mailbox(fte, &mbox_mod);
1080 if (ret) {
1081 _rule_delete(fipsec, rule);
1082 mlx5_fpga_ipsec_delete_sa_ctx(rule->ctx);
1083 kfree(rule);
1084 }
1085
1086 return ret;
1087 }
1088
1089 static int fpga_ipsec_fs_update_fte(struct mlx5_flow_root_namespace *ns,
1090 struct mlx5_flow_table *ft,
1091 struct mlx5_flow_group *fg,
1092 int modify_mask,
1093 struct fs_fte *fte,
1094 bool is_egress)
1095 {
1096 int (*update_fte)(struct mlx5_flow_root_namespace *ns,
1097 struct mlx5_flow_table *ft,
1098 struct mlx5_flow_group *fg,
1099 int modify_mask,
1100 struct fs_fte *fte) =
1101 mlx5_fs_cmd_get_default(egress_to_fs_ft(is_egress))->update_fte;
1102 struct mlx5_core_dev *dev = ns->dev;
1103 bool is_esp = fte->action.esp_id;
1104 struct mailbox_mod mbox_mod;
1105 int ret;
1106
1107 if (!is_esp ||
1108 !(fte->action.action &
1109 (MLX5_FLOW_CONTEXT_ACTION_ENCRYPT |
1110 MLX5_FLOW_CONTEXT_ACTION_DECRYPT)))
1111 return update_fte(ns, ft, fg, modify_mask, fte);
1112
1113 modify_spec_mailbox(dev, fte, &mbox_mod);
1114 ret = update_fte(ns, ft, fg, modify_mask, fte);
1115 restore_spec_mailbox(fte, &mbox_mod);
1116
1117 return ret;
1118 }
1119
1120 static int fpga_ipsec_fs_delete_fte(struct mlx5_flow_root_namespace *ns,
1121 struct mlx5_flow_table *ft,
1122 struct fs_fte *fte,
1123 bool is_egress)
1124 {
1125 int (*delete_fte)(struct mlx5_flow_root_namespace *ns,
1126 struct mlx5_flow_table *ft,
1127 struct fs_fte *fte) =
1128 mlx5_fs_cmd_get_default(egress_to_fs_ft(is_egress))->delete_fte;
1129 struct mlx5_core_dev *dev = ns->dev;
1130 struct mlx5_fpga_device *fdev = dev->fpga;
1131 struct mlx5_fpga_ipsec *fipsec = fdev->ipsec;
1132 struct mlx5_fpga_ipsec_rule *rule;
1133 bool is_esp = fte->action.esp_id;
1134 struct mailbox_mod mbox_mod;
1135 int ret;
1136
1137 if (!is_esp ||
1138 !(fte->action.action &
1139 (MLX5_FLOW_CONTEXT_ACTION_ENCRYPT |
1140 MLX5_FLOW_CONTEXT_ACTION_DECRYPT)))
1141 return delete_fte(ns, ft, fte);
1142
1143 rule = rule_search(fipsec, fte);
1144 if (!rule)
1145 return -ENOENT;
1146
1147 mlx5_fpga_ipsec_delete_sa_ctx(rule->ctx);
1148 rule_delete(fipsec, rule);
1149
1150 modify_spec_mailbox(dev, fte, &mbox_mod);
1151 ret = delete_fte(ns, ft, fte);
1152 restore_spec_mailbox(fte, &mbox_mod);
1153
1154 return ret;
1155 }
1156
1157 static int
1158 mlx5_fpga_ipsec_fs_create_flow_group_egress(struct mlx5_flow_root_namespace *ns,
1159 struct mlx5_flow_table *ft,
1160 u32 *in,
1161 struct mlx5_flow_group *fg)
1162 {
1163 return fpga_ipsec_fs_create_flow_group(ns, ft, in, fg, true);
1164 }
1165
1166 static int
1167 mlx5_fpga_ipsec_fs_create_fte_egress(struct mlx5_flow_root_namespace *ns,
1168 struct mlx5_flow_table *ft,
1169 struct mlx5_flow_group *fg,
1170 struct fs_fte *fte)
1171 {
1172 return fpga_ipsec_fs_create_fte(ns, ft, fg, fte, true);
1173 }
1174
1175 static int
1176 mlx5_fpga_ipsec_fs_update_fte_egress(struct mlx5_flow_root_namespace *ns,
1177 struct mlx5_flow_table *ft,
1178 struct mlx5_flow_group *fg,
1179 int modify_mask,
1180 struct fs_fte *fte)
1181 {
1182 return fpga_ipsec_fs_update_fte(ns, ft, fg, modify_mask, fte,
1183 true);
1184 }
1185
1186 static int
1187 mlx5_fpga_ipsec_fs_delete_fte_egress(struct mlx5_flow_root_namespace *ns,
1188 struct mlx5_flow_table *ft,
1189 struct fs_fte *fte)
1190 {
1191 return fpga_ipsec_fs_delete_fte(ns, ft, fte, true);
1192 }
1193
1194 static int
1195 mlx5_fpga_ipsec_fs_create_flow_group_ingress(struct mlx5_flow_root_namespace *ns,
1196 struct mlx5_flow_table *ft,
1197 u32 *in,
1198 struct mlx5_flow_group *fg)
1199 {
1200 return fpga_ipsec_fs_create_flow_group(ns, ft, in, fg, false);
1201 }
1202
1203 static int
1204 mlx5_fpga_ipsec_fs_create_fte_ingress(struct mlx5_flow_root_namespace *ns,
1205 struct mlx5_flow_table *ft,
1206 struct mlx5_flow_group *fg,
1207 struct fs_fte *fte)
1208 {
1209 return fpga_ipsec_fs_create_fte(ns, ft, fg, fte, false);
1210 }
1211
1212 static int
1213 mlx5_fpga_ipsec_fs_update_fte_ingress(struct mlx5_flow_root_namespace *ns,
1214 struct mlx5_flow_table *ft,
1215 struct mlx5_flow_group *fg,
1216 int modify_mask,
1217 struct fs_fte *fte)
1218 {
1219 return fpga_ipsec_fs_update_fte(ns, ft, fg, modify_mask, fte,
1220 false);
1221 }
1222
1223 static int
1224 mlx5_fpga_ipsec_fs_delete_fte_ingress(struct mlx5_flow_root_namespace *ns,
1225 struct mlx5_flow_table *ft,
1226 struct fs_fte *fte)
1227 {
1228 return fpga_ipsec_fs_delete_fte(ns, ft, fte, false);
1229 }
1230
1231 static struct mlx5_flow_cmds fpga_ipsec_ingress;
1232 static struct mlx5_flow_cmds fpga_ipsec_egress;
1233
1234 const struct mlx5_flow_cmds *mlx5_fs_cmd_get_default_ipsec_fpga_cmds(enum fs_flow_table_type type)
1235 {
1236 switch (type) {
1237 case FS_FT_NIC_RX:
1238 return &fpga_ipsec_ingress;
1239 case FS_FT_NIC_TX:
1240 return &fpga_ipsec_egress;
1241 default:
1242 WARN_ON(true);
1243 return NULL;
1244 }
1245 }
1246
1247 int mlx5_fpga_ipsec_init(struct mlx5_core_dev *mdev)
1248 {
1249 struct mlx5_fpga_conn_attr init_attr = {0};
1250 struct mlx5_fpga_device *fdev = mdev->fpga;
1251 struct mlx5_fpga_conn *conn;
1252 int err;
1253
1254 if (!mlx5_fpga_is_ipsec_device(mdev))
1255 return 0;
1256
1257 fdev->ipsec = kzalloc(sizeof(*fdev->ipsec), GFP_KERNEL);
1258 if (!fdev->ipsec)
1259 return -ENOMEM;
1260
1261 fdev->ipsec->fdev = fdev;
1262
1263 err = mlx5_fpga_get_sbu_caps(fdev, sizeof(fdev->ipsec->caps),
1264 fdev->ipsec->caps);
1265 if (err) {
1266 mlx5_fpga_err(fdev, "Failed to retrieve IPSec extended capabilities: %d\n",
1267 err);
1268 goto error;
1269 }
1270
1271 INIT_LIST_HEAD(&fdev->ipsec->pending_cmds);
1272 spin_lock_init(&fdev->ipsec->pending_cmds_lock);
1273
1274 init_attr.rx_size = SBU_QP_QUEUE_SIZE;
1275 init_attr.tx_size = SBU_QP_QUEUE_SIZE;
1276 init_attr.recv_cb = mlx5_fpga_ipsec_recv;
1277 init_attr.cb_arg = fdev;
1278 conn = mlx5_fpga_sbu_conn_create(fdev, &init_attr);
1279 if (IS_ERR(conn)) {
1280 err = PTR_ERR(conn);
1281 mlx5_fpga_err(fdev, "Error creating IPSec command connection %d\n",
1282 err);
1283 goto error;
1284 }
1285 fdev->ipsec->conn = conn;
1286
1287 err = rhashtable_init(&fdev->ipsec->sa_hash, &rhash_sa);
1288 if (err)
1289 goto err_destroy_conn;
1290 mutex_init(&fdev->ipsec->sa_hash_lock);
1291
1292 fdev->ipsec->rules_rb = RB_ROOT;
1293 mutex_init(&fdev->ipsec->rules_rb_lock);
1294
1295 err = mlx5_fpga_ipsec_enable_supported_caps(mdev);
1296 if (err) {
1297 mlx5_fpga_err(fdev, "Failed to enable IPSec extended capabilities: %d\n",
1298 err);
1299 goto err_destroy_hash;
1300 }
1301
1302 return 0;
1303
1304 err_destroy_hash:
1305 rhashtable_destroy(&fdev->ipsec->sa_hash);
1306
1307 err_destroy_conn:
1308 mlx5_fpga_sbu_conn_destroy(conn);
1309
1310 error:
1311 kfree(fdev->ipsec);
1312 fdev->ipsec = NULL;
1313 return err;
1314 }
1315
1316 static void destroy_rules_rb(struct rb_root *root)
1317 {
1318 struct mlx5_fpga_ipsec_rule *r, *tmp;
1319
1320 rbtree_postorder_for_each_entry_safe(r, tmp, root, node) {
1321 rb_erase(&r->node, root);
1322 mlx5_fpga_ipsec_delete_sa_ctx(r->ctx);
1323 kfree(r);
1324 }
1325 }
1326
1327 void mlx5_fpga_ipsec_cleanup(struct mlx5_core_dev *mdev)
1328 {
1329 struct mlx5_fpga_device *fdev = mdev->fpga;
1330
1331 if (!mlx5_fpga_is_ipsec_device(mdev))
1332 return;
1333
1334 destroy_rules_rb(&fdev->ipsec->rules_rb);
1335 rhashtable_destroy(&fdev->ipsec->sa_hash);
1336
1337 mlx5_fpga_sbu_conn_destroy(fdev->ipsec->conn);
1338 kfree(fdev->ipsec);
1339 fdev->ipsec = NULL;
1340 }
1341
1342 void mlx5_fpga_ipsec_build_fs_cmds(void)
1343 {
1344
1345 fpga_ipsec_ingress.create_flow_table =
1346 mlx5_fs_cmd_get_default(egress_to_fs_ft(false))->create_flow_table;
1347 fpga_ipsec_ingress.destroy_flow_table =
1348 mlx5_fs_cmd_get_default(egress_to_fs_ft(false))->destroy_flow_table;
1349 fpga_ipsec_ingress.modify_flow_table =
1350 mlx5_fs_cmd_get_default(egress_to_fs_ft(false))->modify_flow_table;
1351 fpga_ipsec_ingress.create_flow_group =
1352 mlx5_fpga_ipsec_fs_create_flow_group_ingress;
1353 fpga_ipsec_ingress.destroy_flow_group =
1354 mlx5_fs_cmd_get_default(egress_to_fs_ft(false))->destroy_flow_group;
1355 fpga_ipsec_ingress.create_fte =
1356 mlx5_fpga_ipsec_fs_create_fte_ingress;
1357 fpga_ipsec_ingress.update_fte =
1358 mlx5_fpga_ipsec_fs_update_fte_ingress;
1359 fpga_ipsec_ingress.delete_fte =
1360 mlx5_fpga_ipsec_fs_delete_fte_ingress;
1361 fpga_ipsec_ingress.update_root_ft =
1362 mlx5_fs_cmd_get_default(egress_to_fs_ft(false))->update_root_ft;
1363
1364
1365 fpga_ipsec_egress.create_flow_table =
1366 mlx5_fs_cmd_get_default(egress_to_fs_ft(true))->create_flow_table;
1367 fpga_ipsec_egress.destroy_flow_table =
1368 mlx5_fs_cmd_get_default(egress_to_fs_ft(true))->destroy_flow_table;
1369 fpga_ipsec_egress.modify_flow_table =
1370 mlx5_fs_cmd_get_default(egress_to_fs_ft(true))->modify_flow_table;
1371 fpga_ipsec_egress.create_flow_group =
1372 mlx5_fpga_ipsec_fs_create_flow_group_egress;
1373 fpga_ipsec_egress.destroy_flow_group =
1374 mlx5_fs_cmd_get_default(egress_to_fs_ft(true))->destroy_flow_group;
1375 fpga_ipsec_egress.create_fte =
1376 mlx5_fpga_ipsec_fs_create_fte_egress;
1377 fpga_ipsec_egress.update_fte =
1378 mlx5_fpga_ipsec_fs_update_fte_egress;
1379 fpga_ipsec_egress.delete_fte =
1380 mlx5_fpga_ipsec_fs_delete_fte_egress;
1381 fpga_ipsec_egress.update_root_ft =
1382 mlx5_fs_cmd_get_default(egress_to_fs_ft(true))->update_root_ft;
1383 }
1384
1385 static int
1386 mlx5_fpga_esp_validate_xfrm_attrs(struct mlx5_core_dev *mdev,
1387 const struct mlx5_accel_esp_xfrm_attrs *attrs)
1388 {
1389 if (attrs->tfc_pad) {
1390 mlx5_core_err(mdev, "Cannot offload xfrm states with tfc padding\n");
1391 return -EOPNOTSUPP;
1392 }
1393
1394 if (attrs->replay_type != MLX5_ACCEL_ESP_REPLAY_NONE) {
1395 mlx5_core_err(mdev, "Cannot offload xfrm states with anti replay\n");
1396 return -EOPNOTSUPP;
1397 }
1398
1399 if (attrs->keymat_type != MLX5_ACCEL_ESP_KEYMAT_AES_GCM) {
1400 mlx5_core_err(mdev, "Only aes gcm keymat is supported\n");
1401 return -EOPNOTSUPP;
1402 }
1403
1404 if (attrs->keymat.aes_gcm.iv_algo !=
1405 MLX5_ACCEL_ESP_AES_GCM_IV_ALGO_SEQ) {
1406 mlx5_core_err(mdev, "Only iv sequence algo is supported\n");
1407 return -EOPNOTSUPP;
1408 }
1409
1410 if (attrs->keymat.aes_gcm.icv_len != 128) {
1411 mlx5_core_err(mdev, "Cannot offload xfrm states with AEAD ICV length other than 128bit\n");
1412 return -EOPNOTSUPP;
1413 }
1414
1415 if (attrs->keymat.aes_gcm.key_len != 128 &&
1416 attrs->keymat.aes_gcm.key_len != 256) {
1417 mlx5_core_err(mdev, "Cannot offload xfrm states with AEAD key length other than 128/256 bit\n");
1418 return -EOPNOTSUPP;
1419 }
1420
1421 if ((attrs->flags & MLX5_ACCEL_ESP_FLAGS_ESN_TRIGGERED) &&
1422 (!MLX5_GET(ipsec_extended_cap, mdev->fpga->ipsec->caps,
1423 v2_command))) {
1424 mlx5_core_err(mdev, "Cannot offload xfrm states with AEAD key length other than 128/256 bit\n");
1425 return -EOPNOTSUPP;
1426 }
1427
1428 return 0;
1429 }
1430
1431 struct mlx5_accel_esp_xfrm *
1432 mlx5_fpga_esp_create_xfrm(struct mlx5_core_dev *mdev,
1433 const struct mlx5_accel_esp_xfrm_attrs *attrs,
1434 u32 flags)
1435 {
1436 struct mlx5_fpga_esp_xfrm *fpga_xfrm;
1437
1438 if (!(flags & MLX5_ACCEL_XFRM_FLAG_REQUIRE_METADATA)) {
1439 mlx5_core_warn(mdev, "Tried to create an esp action without metadata\n");
1440 return ERR_PTR(-EINVAL);
1441 }
1442
1443 if (mlx5_fpga_esp_validate_xfrm_attrs(mdev, attrs)) {
1444 mlx5_core_warn(mdev, "Tried to create an esp with unsupported attrs\n");
1445 return ERR_PTR(-EOPNOTSUPP);
1446 }
1447
1448 fpga_xfrm = kzalloc(sizeof(*fpga_xfrm), GFP_KERNEL);
1449 if (!fpga_xfrm)
1450 return ERR_PTR(-ENOMEM);
1451
1452 mutex_init(&fpga_xfrm->lock);
1453 memcpy(&fpga_xfrm->accel_xfrm.attrs, attrs,
1454 sizeof(fpga_xfrm->accel_xfrm.attrs));
1455
1456 return &fpga_xfrm->accel_xfrm;
1457 }
1458
1459 void mlx5_fpga_esp_destroy_xfrm(struct mlx5_accel_esp_xfrm *xfrm)
1460 {
1461 struct mlx5_fpga_esp_xfrm *fpga_xfrm =
1462 container_of(xfrm, struct mlx5_fpga_esp_xfrm,
1463 accel_xfrm);
1464
1465 kfree(fpga_xfrm);
1466 }
1467
1468 int mlx5_fpga_esp_modify_xfrm(struct mlx5_accel_esp_xfrm *xfrm,
1469 const struct mlx5_accel_esp_xfrm_attrs *attrs)
1470 {
1471 struct mlx5_core_dev *mdev = xfrm->mdev;
1472 struct mlx5_fpga_device *fdev = mdev->fpga;
1473 struct mlx5_fpga_ipsec *fipsec = fdev->ipsec;
1474 struct mlx5_fpga_esp_xfrm *fpga_xfrm;
1475 struct mlx5_ifc_fpga_ipsec_sa org_hw_sa;
1476
1477 int err = 0;
1478
1479 if (!memcmp(&xfrm->attrs, attrs, sizeof(xfrm->attrs)))
1480 return 0;
1481
1482 if (mlx5_fpga_esp_validate_xfrm_attrs(mdev, attrs)) {
1483 mlx5_core_warn(mdev, "Tried to create an esp with unsupported attrs\n");
1484 return -EOPNOTSUPP;
1485 }
1486
1487 if (is_v2_sadb_supported(fipsec)) {
1488 mlx5_core_warn(mdev, "Modify esp is not supported\n");
1489 return -EOPNOTSUPP;
1490 }
1491
1492 fpga_xfrm = container_of(xfrm, struct mlx5_fpga_esp_xfrm, accel_xfrm);
1493
1494 mutex_lock(&fpga_xfrm->lock);
1495
1496 if (!fpga_xfrm->sa_ctx)
1497
1498 goto change_sw_xfrm_attrs;
1499
1500
1501 memcpy(&org_hw_sa, &fpga_xfrm->sa_ctx->hw_sa, sizeof(org_hw_sa));
1502 mutex_lock(&fipsec->sa_hash_lock);
1503
1504 WARN_ON(rhashtable_remove_fast(&fipsec->sa_hash,
1505 &fpga_xfrm->sa_ctx->hash, rhash_sa));
1506
1507 mlx5_fpga_ipsec_build_hw_xfrm(xfrm->mdev, attrs,
1508 &fpga_xfrm->sa_ctx->hw_sa);
1509
1510 err = rhashtable_insert_fast(&fipsec->sa_hash,
1511 &fpga_xfrm->sa_ctx->hash, rhash_sa);
1512 if (err)
1513 goto rollback_sa;
1514
1515
1516 err = mlx5_fpga_ipsec_update_hw_sa(fdev, &fpga_xfrm->sa_ctx->hw_sa,
1517 MLX5_FPGA_IPSEC_CMD_OP_MOD_SA_V2);
1518 fpga_xfrm->sa_ctx->hw_sa.ipsec_sa_v1.cmd = 0;
1519 if (err)
1520 WARN_ON(rhashtable_remove_fast(&fipsec->sa_hash,
1521 &fpga_xfrm->sa_ctx->hash,
1522 rhash_sa));
1523 rollback_sa:
1524 if (err) {
1525
1526 memcpy(&fpga_xfrm->sa_ctx->hw_sa, &org_hw_sa,
1527 sizeof(org_hw_sa));
1528 WARN_ON(rhashtable_insert_fast(&fipsec->sa_hash,
1529 &fpga_xfrm->sa_ctx->hash,
1530 rhash_sa));
1531 }
1532 mutex_unlock(&fipsec->sa_hash_lock);
1533
1534 change_sw_xfrm_attrs:
1535 if (!err)
1536 memcpy(&xfrm->attrs, attrs, sizeof(xfrm->attrs));
1537 mutex_unlock(&fpga_xfrm->lock);
1538 return err;
1539 }