root/drivers/net/ethernet/netronome/nfp/crypto/tls.c

/* [<][>][^][v][top][bottom][index][help] */

DEFINITIONS

This source file includes following definitions.
  1. nfp_net_crypto_set_op
  2. __nfp_net_tls_conn_cnt_changed
  3. nfp_net_tls_conn_cnt_changed
  4. nfp_net_tls_conn_add
  5. nfp_net_tls_conn_remove
  6. nfp_net_tls_alloc_simple
  7. nfp_net_tls_communicate_simple
  8. nfp_net_tls_del_fw
  9. nfp_net_tls_set_ipver_vlan
  10. nfp_net_tls_assign_conn_id
  11. nfp_net_tls_set_ipv4
  12. nfp_net_tls_set_ipv6
  13. nfp_net_tls_set_l4
  14. nfp_tls_1_2_dir_to_opcode
  15. nfp_net_cipher_supported
  16. nfp_net_tls_add
  17. nfp_net_tls_del
  18. nfp_net_tls_resync
  19. nfp_net_tls_reset
  20. nfp_net_tls_init

   1 // SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
   2 /* Copyright (C) 2019 Netronome Systems, Inc. */
   3 
   4 #include <linux/bitfield.h>
   5 #include <linux/ipv6.h>
   6 #include <linux/skbuff.h>
   7 #include <linux/string.h>
   8 #include <net/tls.h>
   9 
  10 #include "../ccm.h"
  11 #include "../nfp_net.h"
  12 #include "crypto.h"
  13 #include "fw.h"
  14 
  15 #define NFP_NET_TLS_CCM_MBOX_OPS_MASK           \
  16         (BIT(NFP_CCM_TYPE_CRYPTO_RESET) |       \
  17          BIT(NFP_CCM_TYPE_CRYPTO_ADD) |         \
  18          BIT(NFP_CCM_TYPE_CRYPTO_DEL) |         \
  19          BIT(NFP_CCM_TYPE_CRYPTO_UPDATE))
  20 
  21 #define NFP_NET_TLS_OPCODE_MASK_RX                      \
  22         BIT(NFP_NET_CRYPTO_OP_TLS_1_2_AES_GCM_128_DEC)
  23 
  24 #define NFP_NET_TLS_OPCODE_MASK_TX                      \
  25         BIT(NFP_NET_CRYPTO_OP_TLS_1_2_AES_GCM_128_ENC)
  26 
  27 #define NFP_NET_TLS_OPCODE_MASK                                         \
  28         (NFP_NET_TLS_OPCODE_MASK_RX | NFP_NET_TLS_OPCODE_MASK_TX)
  29 
  30 static void nfp_net_crypto_set_op(struct nfp_net *nn, u8 opcode, bool on)
  31 {
  32         u32 off, val;
  33 
  34         off = nn->tlv_caps.crypto_enable_off + round_down(opcode / 8, 4);
  35 
  36         val = nn_readl(nn, off);
  37         if (on)
  38                 val |= BIT(opcode & 31);
  39         else
  40                 val &= ~BIT(opcode & 31);
  41         nn_writel(nn, off, val);
  42 }
  43 
  44 static bool
  45 __nfp_net_tls_conn_cnt_changed(struct nfp_net *nn, int add,
  46                                enum tls_offload_ctx_dir direction)
  47 {
  48         u8 opcode;
  49         int cnt;
  50 
  51         if (direction == TLS_OFFLOAD_CTX_DIR_TX) {
  52                 opcode = NFP_NET_CRYPTO_OP_TLS_1_2_AES_GCM_128_ENC;
  53                 nn->ktls_tx_conn_cnt += add;
  54                 cnt = nn->ktls_tx_conn_cnt;
  55                 nn->dp.ktls_tx = !!nn->ktls_tx_conn_cnt;
  56         } else {
  57                 opcode = NFP_NET_CRYPTO_OP_TLS_1_2_AES_GCM_128_DEC;
  58                 nn->ktls_rx_conn_cnt += add;
  59                 cnt = nn->ktls_rx_conn_cnt;
  60         }
  61 
  62         /* Care only about 0 -> 1 and 1 -> 0 transitions */
  63         if (cnt > 1)
  64                 return false;
  65 
  66         nfp_net_crypto_set_op(nn, opcode, cnt);
  67         return true;
  68 }
  69 
  70 static int
  71 nfp_net_tls_conn_cnt_changed(struct nfp_net *nn, int add,
  72                              enum tls_offload_ctx_dir direction)
  73 {
  74         int ret = 0;
  75 
  76         /* Use the BAR lock to protect the connection counts */
  77         nn_ctrl_bar_lock(nn);
  78         if (__nfp_net_tls_conn_cnt_changed(nn, add, direction)) {
  79                 ret = __nfp_net_reconfig(nn, NFP_NET_CFG_UPDATE_CRYPTO);
  80                 /* Undo the cnt adjustment if failed */
  81                 if (ret)
  82                         __nfp_net_tls_conn_cnt_changed(nn, -add, direction);
  83         }
  84         nn_ctrl_bar_unlock(nn);
  85 
  86         return ret;
  87 }
  88 
  89 static int
  90 nfp_net_tls_conn_add(struct nfp_net *nn, enum tls_offload_ctx_dir direction)
  91 {
  92         return nfp_net_tls_conn_cnt_changed(nn, 1, direction);
  93 }
  94 
  95 static int
  96 nfp_net_tls_conn_remove(struct nfp_net *nn, enum tls_offload_ctx_dir direction)
  97 {
  98         return nfp_net_tls_conn_cnt_changed(nn, -1, direction);
  99 }
 100 
 101 static struct sk_buff *
 102 nfp_net_tls_alloc_simple(struct nfp_net *nn, size_t req_sz, gfp_t flags)
 103 {
 104         return nfp_ccm_mbox_msg_alloc(nn, req_sz,
 105                                       sizeof(struct nfp_crypto_reply_simple),
 106                                       flags);
 107 }
 108 
 109 static int
 110 nfp_net_tls_communicate_simple(struct nfp_net *nn, struct sk_buff *skb,
 111                                const char *name, enum nfp_ccm_type type)
 112 {
 113         struct nfp_crypto_reply_simple *reply;
 114         int err;
 115 
 116         err = __nfp_ccm_mbox_communicate(nn, skb, type,
 117                                          sizeof(*reply), sizeof(*reply),
 118                                          type == NFP_CCM_TYPE_CRYPTO_DEL);
 119         if (err) {
 120                 nn_dp_warn(&nn->dp, "failed to %s TLS: %d\n", name, err);
 121                 return err;
 122         }
 123 
 124         reply = (void *)skb->data;
 125         err = -be32_to_cpu(reply->error);
 126         if (err)
 127                 nn_dp_warn(&nn->dp, "failed to %s TLS, fw replied: %d\n",
 128                            name, err);
 129         dev_consume_skb_any(skb);
 130 
 131         return err;
 132 }
 133 
 134 static void nfp_net_tls_del_fw(struct nfp_net *nn, __be32 *fw_handle)
 135 {
 136         struct nfp_crypto_req_del *req;
 137         struct sk_buff *skb;
 138 
 139         skb = nfp_net_tls_alloc_simple(nn, sizeof(*req), GFP_KERNEL);
 140         if (!skb)
 141                 return;
 142 
 143         req = (void *)skb->data;
 144         req->ep_id = 0;
 145         memcpy(req->handle, fw_handle, sizeof(req->handle));
 146 
 147         nfp_net_tls_communicate_simple(nn, skb, "delete",
 148                                        NFP_CCM_TYPE_CRYPTO_DEL);
 149 }
 150 
 151 static void
 152 nfp_net_tls_set_ipver_vlan(struct nfp_crypto_req_add_front *front, u8 ipver)
 153 {
 154         front->ipver_vlan = cpu_to_be16(FIELD_PREP(NFP_NET_TLS_IPVER, ipver) |
 155                                         FIELD_PREP(NFP_NET_TLS_VLAN,
 156                                                    NFP_NET_TLS_VLAN_UNUSED));
 157 }
 158 
 159 static void
 160 nfp_net_tls_assign_conn_id(struct nfp_net *nn,
 161                            struct nfp_crypto_req_add_front *front)
 162 {
 163         u32 len;
 164         u64 id;
 165 
 166         id = atomic64_inc_return(&nn->ktls_conn_id_gen);
 167         len = front->key_len - NFP_NET_TLS_NON_ADDR_KEY_LEN;
 168 
 169         memcpy(front->l3_addrs, &id, sizeof(id));
 170         memset(front->l3_addrs + sizeof(id), 0, len - sizeof(id));
 171 }
 172 
 173 static struct nfp_crypto_req_add_back *
 174 nfp_net_tls_set_ipv4(struct nfp_net *nn, struct nfp_crypto_req_add_v4 *req,
 175                      struct sock *sk, int direction)
 176 {
 177         struct inet_sock *inet = inet_sk(sk);
 178 
 179         req->front.key_len += sizeof(__be32) * 2;
 180 
 181         if (direction == TLS_OFFLOAD_CTX_DIR_TX) {
 182                 nfp_net_tls_assign_conn_id(nn, &req->front);
 183         } else {
 184                 req->src_ip = inet->inet_daddr;
 185                 req->dst_ip = inet->inet_saddr;
 186         }
 187 
 188         return &req->back;
 189 }
 190 
 191 static struct nfp_crypto_req_add_back *
 192 nfp_net_tls_set_ipv6(struct nfp_net *nn, struct nfp_crypto_req_add_v6 *req,
 193                      struct sock *sk, int direction)
 194 {
 195 #if IS_ENABLED(CONFIG_IPV6)
 196         struct ipv6_pinfo *np = inet6_sk(sk);
 197 
 198         req->front.key_len += sizeof(struct in6_addr) * 2;
 199 
 200         if (direction == TLS_OFFLOAD_CTX_DIR_TX) {
 201                 nfp_net_tls_assign_conn_id(nn, &req->front);
 202         } else {
 203                 memcpy(req->src_ip, &sk->sk_v6_daddr, sizeof(req->src_ip));
 204                 memcpy(req->dst_ip, &np->saddr, sizeof(req->dst_ip));
 205         }
 206 
 207 #endif
 208         return &req->back;
 209 }
 210 
 211 static void
 212 nfp_net_tls_set_l4(struct nfp_crypto_req_add_front *front,
 213                    struct nfp_crypto_req_add_back *back, struct sock *sk,
 214                    int direction)
 215 {
 216         struct inet_sock *inet = inet_sk(sk);
 217 
 218         front->l4_proto = IPPROTO_TCP;
 219 
 220         if (direction == TLS_OFFLOAD_CTX_DIR_TX) {
 221                 back->src_port = 0;
 222                 back->dst_port = 0;
 223         } else {
 224                 back->src_port = inet->inet_dport;
 225                 back->dst_port = inet->inet_sport;
 226         }
 227 }
 228 
 229 static u8 nfp_tls_1_2_dir_to_opcode(enum tls_offload_ctx_dir direction)
 230 {
 231         switch (direction) {
 232         case TLS_OFFLOAD_CTX_DIR_TX:
 233                 return NFP_NET_CRYPTO_OP_TLS_1_2_AES_GCM_128_ENC;
 234         case TLS_OFFLOAD_CTX_DIR_RX:
 235                 return NFP_NET_CRYPTO_OP_TLS_1_2_AES_GCM_128_DEC;
 236         default:
 237                 WARN_ON_ONCE(1);
 238                 return 0;
 239         }
 240 }
 241 
 242 static bool
 243 nfp_net_cipher_supported(struct nfp_net *nn, u16 cipher_type,
 244                          enum tls_offload_ctx_dir direction)
 245 {
 246         u8 bit;
 247 
 248         switch (cipher_type) {
 249         case TLS_CIPHER_AES_GCM_128:
 250                 if (direction == TLS_OFFLOAD_CTX_DIR_TX)
 251                         bit = NFP_NET_CRYPTO_OP_TLS_1_2_AES_GCM_128_ENC;
 252                 else
 253                         bit = NFP_NET_CRYPTO_OP_TLS_1_2_AES_GCM_128_DEC;
 254                 break;
 255         default:
 256                 return false;
 257         }
 258 
 259         return nn->tlv_caps.crypto_ops & BIT(bit);
 260 }
 261 
 262 static int
 263 nfp_net_tls_add(struct net_device *netdev, struct sock *sk,
 264                 enum tls_offload_ctx_dir direction,
 265                 struct tls_crypto_info *crypto_info,
 266                 u32 start_offload_tcp_sn)
 267 {
 268         struct tls12_crypto_info_aes_gcm_128 *tls_ci;
 269         struct nfp_net *nn = netdev_priv(netdev);
 270         struct nfp_crypto_req_add_front *front;
 271         struct nfp_net_tls_offload_ctx *ntls;
 272         struct nfp_crypto_req_add_back *back;
 273         struct nfp_crypto_reply_add *reply;
 274         struct sk_buff *skb;
 275         size_t req_sz;
 276         void *req;
 277         bool ipv6;
 278         int err;
 279 
 280         BUILD_BUG_ON(sizeof(struct nfp_net_tls_offload_ctx) >
 281                      TLS_DRIVER_STATE_SIZE_TX);
 282         BUILD_BUG_ON(offsetof(struct nfp_net_tls_offload_ctx, rx_end) >
 283                      TLS_DRIVER_STATE_SIZE_RX);
 284 
 285         if (!nfp_net_cipher_supported(nn, crypto_info->cipher_type, direction))
 286                 return -EOPNOTSUPP;
 287 
 288         switch (sk->sk_family) {
 289 #if IS_ENABLED(CONFIG_IPV6)
 290         case AF_INET6:
 291                 if (sk->sk_ipv6only ||
 292                     ipv6_addr_type(&sk->sk_v6_daddr) != IPV6_ADDR_MAPPED) {
 293                         req_sz = sizeof(struct nfp_crypto_req_add_v6);
 294                         ipv6 = true;
 295                         break;
 296                 }
 297 #endif
 298                 /* fall through */
 299         case AF_INET:
 300                 req_sz = sizeof(struct nfp_crypto_req_add_v4);
 301                 ipv6 = false;
 302                 break;
 303         default:
 304                 return -EOPNOTSUPP;
 305         }
 306 
 307         err = nfp_net_tls_conn_add(nn, direction);
 308         if (err)
 309                 return err;
 310 
 311         skb = nfp_ccm_mbox_msg_alloc(nn, req_sz, sizeof(*reply), GFP_KERNEL);
 312         if (!skb) {
 313                 err = -ENOMEM;
 314                 goto err_conn_remove;
 315         }
 316 
 317         front = (void *)skb->data;
 318         front->ep_id = 0;
 319         front->key_len = NFP_NET_TLS_NON_ADDR_KEY_LEN;
 320         front->opcode = nfp_tls_1_2_dir_to_opcode(direction);
 321         memset(front->resv, 0, sizeof(front->resv));
 322 
 323         nfp_net_tls_set_ipver_vlan(front, ipv6 ? 6 : 4);
 324 
 325         req = (void *)skb->data;
 326         if (ipv6)
 327                 back = nfp_net_tls_set_ipv6(nn, req, sk, direction);
 328         else
 329                 back = nfp_net_tls_set_ipv4(nn, req, sk, direction);
 330 
 331         nfp_net_tls_set_l4(front, back, sk, direction);
 332 
 333         back->counter = 0;
 334         back->tcp_seq = cpu_to_be32(start_offload_tcp_sn);
 335 
 336         tls_ci = (struct tls12_crypto_info_aes_gcm_128 *)crypto_info;
 337         memcpy(back->key, tls_ci->key, TLS_CIPHER_AES_GCM_128_KEY_SIZE);
 338         memset(&back->key[TLS_CIPHER_AES_GCM_128_KEY_SIZE / 4], 0,
 339                sizeof(back->key) - TLS_CIPHER_AES_GCM_128_KEY_SIZE);
 340         memcpy(back->iv, tls_ci->iv, TLS_CIPHER_AES_GCM_128_IV_SIZE);
 341         memcpy(&back->salt, tls_ci->salt, TLS_CIPHER_AES_GCM_128_SALT_SIZE);
 342         memcpy(back->rec_no, tls_ci->rec_seq, sizeof(tls_ci->rec_seq));
 343 
 344         /* Get an extra ref on the skb so we can wipe the key after */
 345         skb_get(skb);
 346 
 347         err = nfp_ccm_mbox_communicate(nn, skb, NFP_CCM_TYPE_CRYPTO_ADD,
 348                                        sizeof(*reply), sizeof(*reply));
 349         reply = (void *)skb->data;
 350 
 351         /* We depend on CCM MBOX code not reallocating skb we sent
 352          * so we can clear the key material out of the memory.
 353          */
 354         if (!WARN_ON_ONCE((u8 *)back < skb->head ||
 355                           (u8 *)back > skb_end_pointer(skb)) &&
 356             !WARN_ON_ONCE((u8 *)&reply[1] > (u8 *)back))
 357                 memzero_explicit(back, sizeof(*back));
 358         dev_consume_skb_any(skb); /* the extra ref from skb_get() above */
 359 
 360         if (err) {
 361                 nn_dp_warn(&nn->dp, "failed to add TLS: %d (%d)\n",
 362                            err, direction == TLS_OFFLOAD_CTX_DIR_TX);
 363                 /* communicate frees skb on error */
 364                 goto err_conn_remove;
 365         }
 366 
 367         err = -be32_to_cpu(reply->error);
 368         if (err) {
 369                 if (err == -ENOSPC) {
 370                         if (!atomic_fetch_inc(&nn->ktls_no_space))
 371                                 nn_info(nn, "HW TLS table full\n");
 372                 } else {
 373                         nn_dp_warn(&nn->dp,
 374                                    "failed to add TLS, FW replied: %d\n", err);
 375                 }
 376                 goto err_free_skb;
 377         }
 378 
 379         if (!reply->handle[0] && !reply->handle[1]) {
 380                 nn_dp_warn(&nn->dp, "FW returned NULL handle\n");
 381                 err = -EINVAL;
 382                 goto err_fw_remove;
 383         }
 384 
 385         ntls = tls_driver_ctx(sk, direction);
 386         memcpy(ntls->fw_handle, reply->handle, sizeof(ntls->fw_handle));
 387         if (direction == TLS_OFFLOAD_CTX_DIR_TX)
 388                 ntls->next_seq = start_offload_tcp_sn;
 389         dev_consume_skb_any(skb);
 390 
 391         if (direction == TLS_OFFLOAD_CTX_DIR_TX)
 392                 return 0;
 393 
 394         tls_offload_rx_resync_set_type(sk,
 395                                        TLS_OFFLOAD_SYNC_TYPE_CORE_NEXT_HINT);
 396         return 0;
 397 
 398 err_fw_remove:
 399         nfp_net_tls_del_fw(nn, reply->handle);
 400 err_free_skb:
 401         dev_consume_skb_any(skb);
 402 err_conn_remove:
 403         nfp_net_tls_conn_remove(nn, direction);
 404         return err;
 405 }
 406 
 407 static void
 408 nfp_net_tls_del(struct net_device *netdev, struct tls_context *tls_ctx,
 409                 enum tls_offload_ctx_dir direction)
 410 {
 411         struct nfp_net *nn = netdev_priv(netdev);
 412         struct nfp_net_tls_offload_ctx *ntls;
 413 
 414         nfp_net_tls_conn_remove(nn, direction);
 415 
 416         ntls = __tls_driver_ctx(tls_ctx, direction);
 417         nfp_net_tls_del_fw(nn, ntls->fw_handle);
 418 }
 419 
 420 static int
 421 nfp_net_tls_resync(struct net_device *netdev, struct sock *sk, u32 seq,
 422                    u8 *rcd_sn, enum tls_offload_ctx_dir direction)
 423 {
 424         struct nfp_net *nn = netdev_priv(netdev);
 425         struct nfp_net_tls_offload_ctx *ntls;
 426         struct nfp_crypto_req_update *req;
 427         struct sk_buff *skb;
 428         gfp_t flags;
 429         int err;
 430 
 431         flags = direction == TLS_OFFLOAD_CTX_DIR_TX ? GFP_KERNEL : GFP_ATOMIC;
 432         skb = nfp_net_tls_alloc_simple(nn, sizeof(*req), flags);
 433         if (!skb)
 434                 return -ENOMEM;
 435 
 436         ntls = tls_driver_ctx(sk, direction);
 437         req = (void *)skb->data;
 438         req->ep_id = 0;
 439         req->opcode = nfp_tls_1_2_dir_to_opcode(direction);
 440         memset(req->resv, 0, sizeof(req->resv));
 441         memcpy(req->handle, ntls->fw_handle, sizeof(ntls->fw_handle));
 442         req->tcp_seq = cpu_to_be32(seq);
 443         memcpy(req->rec_no, rcd_sn, sizeof(req->rec_no));
 444 
 445         if (direction == TLS_OFFLOAD_CTX_DIR_TX) {
 446                 err = nfp_net_tls_communicate_simple(nn, skb, "sync",
 447                                                      NFP_CCM_TYPE_CRYPTO_UPDATE);
 448                 if (err)
 449                         return err;
 450                 ntls->next_seq = seq;
 451         } else {
 452                 nfp_ccm_mbox_post(nn, skb, NFP_CCM_TYPE_CRYPTO_UPDATE,
 453                                   sizeof(struct nfp_crypto_reply_simple));
 454         }
 455 
 456         return 0;
 457 }
 458 
 459 static const struct tlsdev_ops nfp_net_tls_ops = {
 460         .tls_dev_add = nfp_net_tls_add,
 461         .tls_dev_del = nfp_net_tls_del,
 462         .tls_dev_resync = nfp_net_tls_resync,
 463 };
 464 
 465 static int nfp_net_tls_reset(struct nfp_net *nn)
 466 {
 467         struct nfp_crypto_req_reset *req;
 468         struct sk_buff *skb;
 469 
 470         skb = nfp_net_tls_alloc_simple(nn, sizeof(*req), GFP_KERNEL);
 471         if (!skb)
 472                 return -ENOMEM;
 473 
 474         req = (void *)skb->data;
 475         req->ep_id = 0;
 476 
 477         return nfp_net_tls_communicate_simple(nn, skb, "reset",
 478                                               NFP_CCM_TYPE_CRYPTO_RESET);
 479 }
 480 
 481 int nfp_net_tls_init(struct nfp_net *nn)
 482 {
 483         struct net_device *netdev = nn->dp.netdev;
 484         int err;
 485 
 486         if (!(nn->tlv_caps.crypto_ops & NFP_NET_TLS_OPCODE_MASK))
 487                 return 0;
 488 
 489         if ((nn->tlv_caps.mbox_cmsg_types & NFP_NET_TLS_CCM_MBOX_OPS_MASK) !=
 490             NFP_NET_TLS_CCM_MBOX_OPS_MASK)
 491                 return 0;
 492 
 493         if (!nfp_ccm_mbox_fits(nn, sizeof(struct nfp_crypto_req_add_v6))) {
 494                 nn_warn(nn, "disabling TLS offload - mbox too small: %d\n",
 495                         nn->tlv_caps.mbox_len);
 496                 return 0;
 497         }
 498 
 499         err = nfp_net_tls_reset(nn);
 500         if (err)
 501                 return err;
 502 
 503         nn_ctrl_bar_lock(nn);
 504         nn_writel(nn, nn->tlv_caps.crypto_enable_off, 0);
 505         err = __nfp_net_reconfig(nn, NFP_NET_CFG_UPDATE_CRYPTO);
 506         nn_ctrl_bar_unlock(nn);
 507         if (err)
 508                 return err;
 509 
 510         if (nn->tlv_caps.crypto_ops & NFP_NET_TLS_OPCODE_MASK_RX) {
 511                 netdev->hw_features |= NETIF_F_HW_TLS_RX;
 512                 netdev->features |= NETIF_F_HW_TLS_RX;
 513         }
 514         if (nn->tlv_caps.crypto_ops & NFP_NET_TLS_OPCODE_MASK_TX) {
 515                 netdev->hw_features |= NETIF_F_HW_TLS_TX;
 516                 netdev->features |= NETIF_F_HW_TLS_TX;
 517         }
 518 
 519         netdev->tlsdev_ops = &nfp_net_tls_ops;
 520 
 521         return 0;
 522 }

/* [<][>][^][v][top][bottom][index][help] */