root/drivers/net/ethernet/mellanox/mlx5/core/fpga/tls.c

DEFINITIONS

1. mlx5_fpga_tls_put_command_ctx
2. mlx5_fpga_tls_cmd_complete
3. mlx5_fpga_cmd_send_complete
4. mlx5_fpga_tls_cmd_send
5. mlx5_fpga_tls_alloc_swid
6. mlx5_fpga_tls_release_swid
7. mlx_tls_kfree_complete
8. mlx5_fpga_tls_teardown_completion
9. mlx5_fpga_tls_flow_to_cmd
10. mlx5_fpga_tls_resync_rx
11. mlx5_fpga_tls_send_teardown_cmd
12. mlx5_fpga_tls_del_flow
13. mlx5_fpga_tls_setup_completion
14. mlx5_fpga_tls_setup_stream_cmd
15. mlx5_fpga_tls_hw_qp_recv_cb
16. mlx5_fpga_is_tls_device
17. mlx5_fpga_tls_get_caps
18. mlx5_fpga_tls_init
19. mlx5_fpga_tls_cleanup
20. mlx5_fpga_tls_set_aes_gcm128_ctx
21. mlx5_fpga_tls_set_key_material
22. _mlx5_fpga_tls_add_flow
23. mlx5_fpga_tls_add_flow

   1 /*
   2  * Copyright (c) 2018 Mellanox Technologies. All rights reserved.
   3  *
   4  * This software is available to you under a choice of one of two
   5  * licenses.  You may choose to be licensed under the terms of the GNU
   6  * General Public License (GPL) Version 2, available from the file
   7  * COPYING in the main directory of this source tree, or the
   8  * OpenIB.org BSD license below:
   9  *
  10  *     Redistribution and use in source and binary forms, with or
  11  *     without modification, are permitted provided that the following
  12  *     conditions are met:
  13  *
  14  *      - Redistributions of source code must retain the above
  15  *        copyright notice, this list of conditions and the following
  16  *        disclaimer.
  17  *
  18  *      - Redistributions in binary form must reproduce the above
  19  *        copyright notice, this list of conditions and the following
  20  *        disclaimer in the documentation and/or other materials
  21  *        provided with the distribution.
  22  *
  23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30  * SOFTWARE.
  31  *
  32  */
  33 
  34 #include <linux/mlx5/device.h>
  35 #include "fpga/tls.h"
  36 #include "fpga/cmd.h"
  37 #include "fpga/sdk.h"
  38 #include "fpga/core.h"
  39 #include "accel/tls.h"
  40 
  41 struct mlx5_fpga_tls_command_context;
  42 
  43 typedef void (*mlx5_fpga_tls_command_complete)
  44         (struct mlx5_fpga_conn *conn, struct mlx5_fpga_device *fdev,
  45          struct mlx5_fpga_tls_command_context *ctx,
  46          struct mlx5_fpga_dma_buf *resp);
  47 
  48 struct mlx5_fpga_tls_command_context {
  49         struct list_head list;
  50         /* There is no guarantee on the order between the TX completion
  51          * and the command response.
  52          * The TX completion is going to touch cmd->buf even in
  53          * the case of successful transmission.
  54          * So instead of requiring separate allocations for cmd
  55          * and cmd->buf we've decided to use a reference counter
  56          */
  57         refcount_t ref;
  58         struct mlx5_fpga_dma_buf buf;
  59         mlx5_fpga_tls_command_complete complete;
  60 };
  61 
  62 static void
  63 mlx5_fpga_tls_put_command_ctx(struct mlx5_fpga_tls_command_context *ctx)
  64 {
  65         if (refcount_dec_and_test(&ctx->ref))
  66                 kfree(ctx);
  67 }
  68 
  69 static void mlx5_fpga_tls_cmd_complete(struct mlx5_fpga_device *fdev,
  70                                        struct mlx5_fpga_dma_buf *resp)
  71 {
  72         struct mlx5_fpga_conn *conn = fdev->tls->conn;
  73         struct mlx5_fpga_tls_command_context *ctx;
  74         struct mlx5_fpga_tls *tls = fdev->tls;
  75         unsigned long flags;
  76 
  77         spin_lock_irqsave(&tls->pending_cmds_lock, flags);
  78         ctx = list_first_entry(&tls->pending_cmds,
  79                                struct mlx5_fpga_tls_command_context, list);
  80         list_del(&ctx->list);
  81         spin_unlock_irqrestore(&tls->pending_cmds_lock, flags);
  82         ctx->complete(conn, fdev, ctx, resp);
  83 }
  84 
  85 static void mlx5_fpga_cmd_send_complete(struct mlx5_fpga_conn *conn,
  86                                         struct mlx5_fpga_device *fdev,
  87                                         struct mlx5_fpga_dma_buf *buf,
  88                                         u8 status)
  89 {
  90         struct mlx5_fpga_tls_command_context *ctx =
  91             container_of(buf, struct mlx5_fpga_tls_command_context, buf);
  92 
  93         mlx5_fpga_tls_put_command_ctx(ctx);
  94 
  95         if (unlikely(status))
  96                 mlx5_fpga_tls_cmd_complete(fdev, NULL);
  97 }
  98 
  99 static void mlx5_fpga_tls_cmd_send(struct mlx5_fpga_device *fdev,
 100                                    struct mlx5_fpga_tls_command_context *cmd,
 101                                    mlx5_fpga_tls_command_complete complete)
 102 {
 103         struct mlx5_fpga_tls *tls = fdev->tls;
 104         unsigned long flags;
 105         int ret;
 106 
 107         refcount_set(&cmd->ref, 2);
 108         cmd->complete = complete;
 109         cmd->buf.complete = mlx5_fpga_cmd_send_complete;
 110 
 111         spin_lock_irqsave(&tls->pending_cmds_lock, flags);
 112         /* mlx5_fpga_sbu_conn_sendmsg is called under pending_cmds_lock
 113          * to make sure commands are inserted to the tls->pending_cmds list
 114          * and the command QP in the same order.
 115          */
 116         ret = mlx5_fpga_sbu_conn_sendmsg(tls->conn, &cmd->buf);
 117         if (likely(!ret))
 118                 list_add_tail(&cmd->list, &tls->pending_cmds);
 119         else
 120                 complete(tls->conn, fdev, cmd, NULL);
 121         spin_unlock_irqrestore(&tls->pending_cmds_lock, flags);
 122 }
 123 
 124 /* Start of context identifiers range (inclusive) */
 125 #define SWID_START      0
 126 /* End of context identifiers range (exclusive) */
 127 #define SWID_END        BIT(24)
 128 
 129 static int mlx5_fpga_tls_alloc_swid(struct idr *idr, spinlock_t *idr_spinlock,
 130                                     void *ptr)
 131 {
 132         unsigned long flags;
 133         int ret;
 134 
 135         /* TLS metadata format is 1 byte for syndrome followed
 136          * by 3 bytes of swid (software ID)
 137          * swid must not exceed 3 bytes.
 138          * See tls_rxtx.c:insert_pet() for details
 139          */
 140         BUILD_BUG_ON((SWID_END - 1) & 0xFF000000);
 141 
 142         idr_preload(GFP_KERNEL);
 143         spin_lock_irqsave(idr_spinlock, flags);
 144         ret = idr_alloc(idr, ptr, SWID_START, SWID_END, GFP_ATOMIC);
 145         spin_unlock_irqrestore(idr_spinlock, flags);
 146         idr_preload_end();
 147 
 148         return ret;
 149 }
 150 
 151 static void *mlx5_fpga_tls_release_swid(struct idr *idr,
 152                                         spinlock_t *idr_spinlock, u32 swid)
 153 {
 154         unsigned long flags;
 155         void *ptr;
 156 
 157         spin_lock_irqsave(idr_spinlock, flags);
 158         ptr = idr_remove(idr, swid);
 159         spin_unlock_irqrestore(idr_spinlock, flags);
 160         return ptr;
 161 }
 162 
 163 static void mlx_tls_kfree_complete(struct mlx5_fpga_conn *conn,
 164                                    struct mlx5_fpga_device *fdev,
 165                                    struct mlx5_fpga_dma_buf *buf, u8 status)
 166 {
 167         kfree(buf);
 168 }
 169 
 170 static void
 171 mlx5_fpga_tls_teardown_completion(struct mlx5_fpga_conn *conn,
 172                                   struct mlx5_fpga_device *fdev,
 173                                   struct mlx5_fpga_tls_command_context *cmd,
 174                                   struct mlx5_fpga_dma_buf *resp)
 175 {
 176         if (resp) {
 177                 u32 syndrome = MLX5_GET(tls_resp, resp->sg[0].data, syndrome);
 178 
 179                 if (syndrome)
 180                         mlx5_fpga_err(fdev,
 181                                       "Teardown stream failed with syndrome = %d",
 182                                       syndrome);
 183         }
 184         mlx5_fpga_tls_put_command_ctx(cmd);
 185 }
 186 
 187 static void mlx5_fpga_tls_flow_to_cmd(void *flow, void *cmd)
 188 {
 189         memcpy(MLX5_ADDR_OF(tls_cmd, cmd, src_port), flow,
 190                MLX5_BYTE_OFF(tls_flow, ipv6));
 191 
 192         MLX5_SET(tls_cmd, cmd, ipv6, MLX5_GET(tls_flow, flow, ipv6));
 193         MLX5_SET(tls_cmd, cmd, direction_sx,
 194                  MLX5_GET(tls_flow, flow, direction_sx));
 195 }
 196 
 197 int mlx5_fpga_tls_resync_rx(struct mlx5_core_dev *mdev, u32 handle, u32 seq,
 198                             u64 rcd_sn)
 199 {
 200         struct mlx5_fpga_dma_buf *buf;
 201         int size = sizeof(*buf) + MLX5_TLS_COMMAND_SIZE;
 202         void *flow;
 203         void *cmd;
 204         int ret;
 205 
 206         buf = kzalloc(size, GFP_ATOMIC);
 207         if (!buf)
 208                 return -ENOMEM;
 209 
 210         cmd = (buf + 1);
 211 
 212         rcu_read_lock();
 213         flow = idr_find(&mdev->fpga->tls->rx_idr, ntohl(handle));
 214         if (unlikely(!flow)) {
 215                 rcu_read_unlock();
 216                 WARN_ONCE(1, "Received NULL pointer for handle\n");
 217                 kfree(buf);
 218                 return -EINVAL;
 219         }
 220         mlx5_fpga_tls_flow_to_cmd(flow, cmd);
 221         rcu_read_unlock();
 222 
 223         MLX5_SET(tls_cmd, cmd, swid, ntohl(handle));
 224         MLX5_SET64(tls_cmd, cmd, tls_rcd_sn, be64_to_cpu(rcd_sn));
 225         MLX5_SET(tls_cmd, cmd, tcp_sn, seq);
 226         MLX5_SET(tls_cmd, cmd, command_type, CMD_RESYNC_RX);
 227 
 228         buf->sg[0].data = cmd;
 229         buf->sg[0].size = MLX5_TLS_COMMAND_SIZE;
 230         buf->complete = mlx_tls_kfree_complete;
 231 
 232         ret = mlx5_fpga_sbu_conn_sendmsg(mdev->fpga->tls->conn, buf);
 233         if (ret < 0)
 234                 kfree(buf);
 235 
 236         return ret;
 237 }
 238 
 239 static void mlx5_fpga_tls_send_teardown_cmd(struct mlx5_core_dev *mdev,
 240                                             void *flow, u32 swid, gfp_t flags)
 241 {
 242         struct mlx5_fpga_tls_command_context *ctx;
 243         struct mlx5_fpga_dma_buf *buf;
 244         void *cmd;
 245 
 246         ctx = kzalloc(sizeof(*ctx) + MLX5_TLS_COMMAND_SIZE, flags);
 247         if (!ctx)
 248                 return;
 249 
 250         buf = &ctx->buf;
 251         cmd = (ctx + 1);
 252         MLX5_SET(tls_cmd, cmd, command_type, CMD_TEARDOWN_STREAM);
 253         MLX5_SET(tls_cmd, cmd, swid, swid);
 254 
 255         mlx5_fpga_tls_flow_to_cmd(flow, cmd);
 256         kfree(flow);
 257 
 258         buf->sg[0].data = cmd;
 259         buf->sg[0].size = MLX5_TLS_COMMAND_SIZE;
 260 
 261         mlx5_fpga_tls_cmd_send(mdev->fpga, ctx,
 262                                mlx5_fpga_tls_teardown_completion);
 263 }
 264 
 265 void mlx5_fpga_tls_del_flow(struct mlx5_core_dev *mdev, u32 swid,
 266                             gfp_t flags, bool direction_sx)
 267 {
 268         struct mlx5_fpga_tls *tls = mdev->fpga->tls;
 269         void *flow;
 270 
 271         if (direction_sx)
 272                 flow = mlx5_fpga_tls_release_swid(&tls->tx_idr,
 273                                                   &tls->tx_idr_spinlock,
 274                                                   swid);
 275         else
 276                 flow = mlx5_fpga_tls_release_swid(&tls->rx_idr,
 277                                                   &tls->rx_idr_spinlock,
 278                                                   swid);
 279 
 280         if (!flow) {
 281                 mlx5_fpga_err(mdev->fpga, "No flow information for swid %u\n",
 282                               swid);
 283                 return;
 284         }
 285 
 286         synchronize_rcu(); /* before kfree(flow) */
 287         mlx5_fpga_tls_send_teardown_cmd(mdev, flow, swid, flags);
 288 }
 289 
 290 enum mlx5_fpga_setup_stream_status {
 291         MLX5_FPGA_CMD_PENDING,
 292         MLX5_FPGA_CMD_SEND_FAILED,
 293         MLX5_FPGA_CMD_RESPONSE_RECEIVED,
 294         MLX5_FPGA_CMD_ABANDONED,
 295 };
 296 
 297 struct mlx5_setup_stream_context {
 298         struct mlx5_fpga_tls_command_context cmd;
 299         atomic_t status;
 300         u32 syndrome;
 301         struct completion comp;
 302 };
 303 
 304 static void
 305 mlx5_fpga_tls_setup_completion(struct mlx5_fpga_conn *conn,
 306                                struct mlx5_fpga_device *fdev,
 307                                struct mlx5_fpga_tls_command_context *cmd,
 308                                struct mlx5_fpga_dma_buf *resp)
 309 {
 310         struct mlx5_setup_stream_context *ctx =
 311             container_of(cmd, struct mlx5_setup_stream_context, cmd);
 312         int status = MLX5_FPGA_CMD_SEND_FAILED;
 313         void *tls_cmd = ctx + 1;
 314 
 315         /* If we failed to send to command resp == NULL */
 316         if (resp) {
 317                 ctx->syndrome = MLX5_GET(tls_resp, resp->sg[0].data, syndrome);
 318                 status = MLX5_FPGA_CMD_RESPONSE_RECEIVED;
 319         }
 320 
 321         status = atomic_xchg_release(&ctx->status, status);
 322         if (likely(status != MLX5_FPGA_CMD_ABANDONED)) {
 323                 complete(&ctx->comp);
 324                 return;
 325         }
 326 
 327         mlx5_fpga_err(fdev, "Command was abandoned, syndrome = %u\n",
 328                       ctx->syndrome);
 329 
 330         if (!ctx->syndrome) {
 331                 /* The process was killed while waiting for the context to be
 332                  * added, and the add completed successfully.
 333                  * We need to destroy the HW context, and we can't can't reuse
 334                  * the command context because we might not have received
 335                  * the tx completion yet.
 336                  */
 337                 mlx5_fpga_tls_del_flow(fdev->mdev,
 338                                        MLX5_GET(tls_cmd, tls_cmd, swid),
 339                                        GFP_ATOMIC,
 340                                        MLX5_GET(tls_cmd, tls_cmd,
 341                                                 direction_sx));
 342         }
 343 
 344         mlx5_fpga_tls_put_command_ctx(cmd);
 345 }
 346 
 347 static int mlx5_fpga_tls_setup_stream_cmd(struct mlx5_core_dev *mdev,
 348                                           struct mlx5_setup_stream_context *ctx)
 349 {
 350         struct mlx5_fpga_dma_buf *buf;
 351         void *cmd = ctx + 1;
 352         int status, ret = 0;
 353 
 354         buf = &ctx->cmd.buf;
 355         buf->sg[0].data = cmd;
 356         buf->sg[0].size = MLX5_TLS_COMMAND_SIZE;
 357         MLX5_SET(tls_cmd, cmd, command_type, CMD_SETUP_STREAM);
 358 
 359         init_completion(&ctx->comp);
 360         atomic_set(&ctx->status, MLX5_FPGA_CMD_PENDING);
 361         ctx->syndrome = -1;
 362 
 363         mlx5_fpga_tls_cmd_send(mdev->fpga, &ctx->cmd,
 364                                mlx5_fpga_tls_setup_completion);
 365         wait_for_completion_killable(&ctx->comp);
 366 
 367         status = atomic_xchg_acquire(&ctx->status, MLX5_FPGA_CMD_ABANDONED);
 368         if (unlikely(status == MLX5_FPGA_CMD_PENDING))
 369         /* ctx is going to be released in mlx5_fpga_tls_setup_completion */
 370                 return -EINTR;
 371 
 372         if (unlikely(ctx->syndrome))
 373                 ret = -ENOMEM;
 374 
 375         mlx5_fpga_tls_put_command_ctx(&ctx->cmd);
 376         return ret;
 377 }
 378 
 379 static void mlx5_fpga_tls_hw_qp_recv_cb(void *cb_arg,
 380                                         struct mlx5_fpga_dma_buf *buf)
 381 {
 382         struct mlx5_fpga_device *fdev = (struct mlx5_fpga_device *)cb_arg;
 383 
 384         mlx5_fpga_tls_cmd_complete(fdev, buf);
 385 }
 386 
 387 bool mlx5_fpga_is_tls_device(struct mlx5_core_dev *mdev)
 388 {
 389         if (!mdev->fpga || !MLX5_CAP_GEN(mdev, fpga))
 390                 return false;
 391 
 392         if (MLX5_CAP_FPGA(mdev, ieee_vendor_id) !=
 393             MLX5_FPGA_CAP_SANDBOX_VENDOR_ID_MLNX)
 394                 return false;
 395 
 396         if (MLX5_CAP_FPGA(mdev, sandbox_product_id) !=
 397             MLX5_FPGA_CAP_SANDBOX_PRODUCT_ID_TLS)
 398                 return false;
 399 
 400         if (MLX5_CAP_FPGA(mdev, sandbox_product_version) != 0)
 401                 return false;
 402 
 403         return true;
 404 }
 405 
 406 static int mlx5_fpga_tls_get_caps(struct mlx5_fpga_device *fdev,
 407                                   u32 *p_caps)
 408 {
 409         int err, cap_size = MLX5_ST_SZ_BYTES(tls_extended_cap);
 410         u32 caps = 0;
 411         void *buf;
 412 
 413         buf = kzalloc(cap_size, GFP_KERNEL);
 414         if (!buf)
 415                 return -ENOMEM;
 416 
 417         err = mlx5_fpga_get_sbu_caps(fdev, cap_size, buf);
 418         if (err)
 419                 goto out;
 420 
 421         if (MLX5_GET(tls_extended_cap, buf, tx))
 422                 caps |= MLX5_ACCEL_TLS_TX;
 423         if (MLX5_GET(tls_extended_cap, buf, rx))
 424                 caps |= MLX5_ACCEL_TLS_RX;
 425         if (MLX5_GET(tls_extended_cap, buf, tls_v12))
 426                 caps |= MLX5_ACCEL_TLS_V12;
 427         if (MLX5_GET(tls_extended_cap, buf, tls_v13))
 428                 caps |= MLX5_ACCEL_TLS_V13;
 429         if (MLX5_GET(tls_extended_cap, buf, lro))
 430                 caps |= MLX5_ACCEL_TLS_LRO;
 431         if (MLX5_GET(tls_extended_cap, buf, ipv6))
 432                 caps |= MLX5_ACCEL_TLS_IPV6;
 433 
 434         if (MLX5_GET(tls_extended_cap, buf, aes_gcm_128))
 435                 caps |= MLX5_ACCEL_TLS_AES_GCM128;
 436         if (MLX5_GET(tls_extended_cap, buf, aes_gcm_256))
 437                 caps |= MLX5_ACCEL_TLS_AES_GCM256;
 438 
 439         *p_caps = caps;
 440         err = 0;
 441 out:
 442         kfree(buf);
 443         return err;
 444 }
 445 
 446 int mlx5_fpga_tls_init(struct mlx5_core_dev *mdev)
 447 {
 448         struct mlx5_fpga_device *fdev = mdev->fpga;
 449         struct mlx5_fpga_conn_attr init_attr = {0};
 450         struct mlx5_fpga_conn *conn;
 451         struct mlx5_fpga_tls *tls;
 452         int err = 0;
 453 
 454         if (!mlx5_fpga_is_tls_device(mdev) || !fdev)
 455                 return 0;
 456 
 457         tls = kzalloc(sizeof(*tls), GFP_KERNEL);
 458         if (!tls)
 459                 return -ENOMEM;
 460 
 461         err = mlx5_fpga_tls_get_caps(fdev, &tls->caps);
 462         if (err)
 463                 goto error;
 464 
 465         if (!(tls->caps & (MLX5_ACCEL_TLS_V12 | MLX5_ACCEL_TLS_AES_GCM128))) {
 466                 err = -ENOTSUPP;
 467                 goto error;
 468         }
 469 
 470         init_attr.rx_size = SBU_QP_QUEUE_SIZE;
 471         init_attr.tx_size = SBU_QP_QUEUE_SIZE;
 472         init_attr.recv_cb = mlx5_fpga_tls_hw_qp_recv_cb;
 473         init_attr.cb_arg = fdev;
 474         conn = mlx5_fpga_sbu_conn_create(fdev, &init_attr);
 475         if (IS_ERR(conn)) {
 476                 err = PTR_ERR(conn);
 477                 mlx5_fpga_err(fdev, "Error creating TLS command connection %d\n",
 478                               err);
 479                 goto error;
 480         }
 481 
 482         tls->conn = conn;
 483         spin_lock_init(&tls->pending_cmds_lock);
 484         INIT_LIST_HEAD(&tls->pending_cmds);
 485 
 486         idr_init(&tls->tx_idr);
 487         idr_init(&tls->rx_idr);
 488         spin_lock_init(&tls->tx_idr_spinlock);
 489         spin_lock_init(&tls->rx_idr_spinlock);
 490         fdev->tls = tls;
 491         return 0;
 492 
 493 error:
 494         kfree(tls);
 495         return err;
 496 }
 497 
 498 void mlx5_fpga_tls_cleanup(struct mlx5_core_dev *mdev)
 499 {
 500         struct mlx5_fpga_device *fdev = mdev->fpga;
 501 
 502         if (!fdev || !fdev->tls)
 503                 return;
 504 
 505         mlx5_fpga_sbu_conn_destroy(fdev->tls->conn);
 506         kfree(fdev->tls);
 507         fdev->tls = NULL;
 508 }
 509 
 510 static void mlx5_fpga_tls_set_aes_gcm128_ctx(void *cmd,
 511                                              struct tls_crypto_info *info,
 512                                              __be64 *rcd_sn)
 513 {
 514         struct tls12_crypto_info_aes_gcm_128 *crypto_info =
 515             (struct tls12_crypto_info_aes_gcm_128 *)info;
 516 
 517         memcpy(MLX5_ADDR_OF(tls_cmd, cmd, tls_rcd_sn), crypto_info->rec_seq,
 518                TLS_CIPHER_AES_GCM_128_REC_SEQ_SIZE);
 519 
 520         memcpy(MLX5_ADDR_OF(tls_cmd, cmd, tls_implicit_iv),
 521                crypto_info->salt, TLS_CIPHER_AES_GCM_128_SALT_SIZE);
 522         memcpy(MLX5_ADDR_OF(tls_cmd, cmd, encryption_key),
 523                crypto_info->key, TLS_CIPHER_AES_GCM_128_KEY_SIZE);
 524 
 525         /* in AES-GCM 128 we need to write the key twice */
 526         memcpy(MLX5_ADDR_OF(tls_cmd, cmd, encryption_key) +
 527                    TLS_CIPHER_AES_GCM_128_KEY_SIZE,
 528                crypto_info->key, TLS_CIPHER_AES_GCM_128_KEY_SIZE);
 529 
 530         MLX5_SET(tls_cmd, cmd, alg, MLX5_TLS_ALG_AES_GCM_128);
 531 }
 532 
 533 static int mlx5_fpga_tls_set_key_material(void *cmd, u32 caps,
 534                                           struct tls_crypto_info *crypto_info)
 535 {
 536         __be64 rcd_sn;
 537 
 538         switch (crypto_info->cipher_type) {
 539         case TLS_CIPHER_AES_GCM_128:
 540                 if (!(caps & MLX5_ACCEL_TLS_AES_GCM128))
 541                         return -EINVAL;
 542                 mlx5_fpga_tls_set_aes_gcm128_ctx(cmd, crypto_info, &rcd_sn);
 543                 break;
 544         default:
 545                 return -EINVAL;
 546         }
 547 
 548         return 0;
 549 }
 550 
 551 static int _mlx5_fpga_tls_add_flow(struct mlx5_core_dev *mdev, void *flow,
 552                                    struct tls_crypto_info *crypto_info,
 553                                    u32 swid, u32 tcp_sn)
 554 {
 555         u32 caps = mlx5_fpga_tls_device_caps(mdev);
 556         struct mlx5_setup_stream_context *ctx;
 557         int ret = -ENOMEM;
 558         size_t cmd_size;
 559         void *cmd;
 560 
 561         cmd_size = MLX5_TLS_COMMAND_SIZE + sizeof(*ctx);
 562         ctx = kzalloc(cmd_size, GFP_KERNEL);
 563         if (!ctx)
 564                 goto out;
 565 
 566         cmd = ctx + 1;
 567         ret = mlx5_fpga_tls_set_key_material(cmd, caps, crypto_info);
 568         if (ret)
 569                 goto free_ctx;
 570 
 571         mlx5_fpga_tls_flow_to_cmd(flow, cmd);
 572 
 573         MLX5_SET(tls_cmd, cmd, swid, swid);
 574         MLX5_SET(tls_cmd, cmd, tcp_sn, tcp_sn);
 575 
 576         return mlx5_fpga_tls_setup_stream_cmd(mdev, ctx);
 577 
 578 free_ctx:
 579         kfree(ctx);
 580 out:
 581         return ret;
 582 }
 583 
 584 int mlx5_fpga_tls_add_flow(struct mlx5_core_dev *mdev, void *flow,
 585                            struct tls_crypto_info *crypto_info,
 586                            u32 start_offload_tcp_sn, u32 *p_swid,
 587                            bool direction_sx)
 588 {
 589         struct mlx5_fpga_tls *tls = mdev->fpga->tls;
 590         int ret = -ENOMEM;
 591         u32 swid;
 592 
 593         if (direction_sx)
 594                 ret = mlx5_fpga_tls_alloc_swid(&tls->tx_idr,
 595                                                &tls->tx_idr_spinlock, flow);
 596         else
 597                 ret = mlx5_fpga_tls_alloc_swid(&tls->rx_idr,
 598                                                &tls->rx_idr_spinlock, flow);
 599 
 600         if (ret < 0)
 601                 return ret;
 602 
 603         swid = ret;
 604         MLX5_SET(tls_flow, flow, direction_sx, direction_sx ? 1 : 0);
 605 
 606         ret = _mlx5_fpga_tls_add_flow(mdev, flow, crypto_info, swid,
 607                                       start_offload_tcp_sn);
 608         if (ret && ret != -EINTR)
 609                 goto free_swid;
 610 
 611         *p_swid = swid;
 612         return 0;
 613 free_swid:
 614         if (direction_sx)
 615                 mlx5_fpga_tls_release_swid(&tls->tx_idr,
 616                                            &tls->tx_idr_spinlock, swid);
 617         else
 618                 mlx5_fpga_tls_release_swid(&tls->rx_idr,
 619                                            &tls->rx_idr_spinlock, swid);
 620 
 621         return ret;
 622 }