1/* 2 * Copyright (c) 2013-2015, Mellanox Technologies. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 */ 32 33#include <linux/highmem.h> 34#include <linux/module.h> 35#include <linux/init.h> 36#include <linux/errno.h> 37#include <linux/pci.h> 38#include <linux/dma-mapping.h> 39#include <linux/slab.h> 40#include <linux/io-mapping.h> 41#include <linux/interrupt.h> 42#include <linux/delay.h> 43#include <linux/mlx5/driver.h> 44#include <linux/mlx5/cq.h> 45#include <linux/mlx5/qp.h> 46#include <linux/mlx5/srq.h> 47#include <linux/debugfs.h> 48#include <linux/kmod.h> 49#include <linux/delay.h> 50#include <linux/mlx5/mlx5_ifc.h> 51#include "mlx5_core.h" 52 53MODULE_AUTHOR("Eli Cohen <eli@mellanox.com>"); 54MODULE_DESCRIPTION("Mellanox Connect-IB, ConnectX-4 core driver"); 55MODULE_LICENSE("Dual BSD/GPL"); 56MODULE_VERSION(DRIVER_VERSION); 57 58int mlx5_core_debug_mask; 59module_param_named(debug_mask, mlx5_core_debug_mask, int, 0644); 60MODULE_PARM_DESC(debug_mask, "debug mask: 1 = dump cmd data, 2 = dump cmd exec time, 3 = both. Default=0"); 61 62#define MLX5_DEFAULT_PROF 2 63static int prof_sel = MLX5_DEFAULT_PROF; 64module_param_named(prof_sel, prof_sel, int, 0444); 65MODULE_PARM_DESC(prof_sel, "profile selector. Valid range 0 - 2"); 66 67static LIST_HEAD(intf_list); 68static LIST_HEAD(dev_list); 69static DEFINE_MUTEX(intf_mutex); 70 71struct mlx5_device_context { 72 struct list_head list; 73 struct mlx5_interface *intf; 74 void *context; 75}; 76 77static struct mlx5_profile profile[] = { 78 [0] = { 79 .mask = 0, 80 }, 81 [1] = { 82 .mask = MLX5_PROF_MASK_QP_SIZE, 83 .log_max_qp = 12, 84 }, 85 [2] = { 86 .mask = MLX5_PROF_MASK_QP_SIZE | 87 MLX5_PROF_MASK_MR_CACHE, 88 .log_max_qp = 17, 89 .mr_cache[0] = { 90 .size = 500, 91 .limit = 250 92 }, 93 .mr_cache[1] = { 94 .size = 500, 95 .limit = 250 96 }, 97 .mr_cache[2] = { 98 .size = 500, 99 .limit = 250 100 }, 101 .mr_cache[3] = { 102 .size = 500, 103 .limit = 250 104 }, 105 .mr_cache[4] = { 106 .size = 500, 107 .limit = 250 108 }, 109 .mr_cache[5] = { 110 .size = 500, 111 .limit = 250 112 }, 113 .mr_cache[6] = { 114 .size = 500, 115 .limit = 250 116 }, 117 .mr_cache[7] = { 118 .size = 500, 119 .limit = 250 120 }, 121 .mr_cache[8] = { 122 .size = 500, 123 .limit = 250 124 }, 125 .mr_cache[9] = { 126 .size = 500, 127 .limit = 250 128 }, 129 .mr_cache[10] = { 130 .size = 500, 131 .limit = 250 132 }, 133 .mr_cache[11] = { 134 .size = 500, 135 .limit = 250 136 }, 137 .mr_cache[12] = { 138 .size = 64, 139 .limit = 32 140 }, 141 .mr_cache[13] = { 142 .size = 32, 143 .limit = 16 144 }, 145 .mr_cache[14] = { 146 .size = 16, 147 .limit = 8 148 }, 149 .mr_cache[15] = { 150 .size = 8, 151 .limit = 4 152 }, 153 }, 154}; 155 156#define FW_INIT_TIMEOUT_MILI 2000 157#define FW_INIT_WAIT_MS 2 158 159static int wait_fw_init(struct mlx5_core_dev *dev, u32 max_wait_mili) 160{ 161 unsigned long end = jiffies + msecs_to_jiffies(max_wait_mili); 162 int err = 0; 163 164 while (fw_initializing(dev)) { 165 if (time_after(jiffies, end)) { 166 err = -EBUSY; 167 break; 168 } 169 msleep(FW_INIT_WAIT_MS); 170 } 171 172 return err; 173} 174 175static int set_dma_caps(struct pci_dev *pdev) 176{ 177 int err; 178 179 err = pci_set_dma_mask(pdev, DMA_BIT_MASK(64)); 180 if (err) { 181 dev_warn(&pdev->dev, "Warning: couldn't set 64-bit PCI DMA mask\n"); 182 err = pci_set_dma_mask(pdev, DMA_BIT_MASK(32)); 183 if (err) { 184 dev_err(&pdev->dev, "Can't set PCI DMA mask, aborting\n"); 185 return err; 186 } 187 } 188 189 err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64)); 190 if (err) { 191 dev_warn(&pdev->dev, 192 "Warning: couldn't set 64-bit consistent PCI DMA mask\n"); 193 err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32)); 194 if (err) { 195 dev_err(&pdev->dev, 196 "Can't set consistent PCI DMA mask, aborting\n"); 197 return err; 198 } 199 } 200 201 dma_set_max_seg_size(&pdev->dev, 2u * 1024 * 1024 * 1024); 202 return err; 203} 204 205static int mlx5_pci_enable_device(struct mlx5_core_dev *dev) 206{ 207 struct pci_dev *pdev = dev->pdev; 208 int err = 0; 209 210 mutex_lock(&dev->pci_status_mutex); 211 if (dev->pci_status == MLX5_PCI_STATUS_DISABLED) { 212 err = pci_enable_device(pdev); 213 if (!err) 214 dev->pci_status = MLX5_PCI_STATUS_ENABLED; 215 } 216 mutex_unlock(&dev->pci_status_mutex); 217 218 return err; 219} 220 221static void mlx5_pci_disable_device(struct mlx5_core_dev *dev) 222{ 223 struct pci_dev *pdev = dev->pdev; 224 225 mutex_lock(&dev->pci_status_mutex); 226 if (dev->pci_status == MLX5_PCI_STATUS_ENABLED) { 227 pci_disable_device(pdev); 228 dev->pci_status = MLX5_PCI_STATUS_DISABLED; 229 } 230 mutex_unlock(&dev->pci_status_mutex); 231} 232 233static int request_bar(struct pci_dev *pdev) 234{ 235 int err = 0; 236 237 if (!(pci_resource_flags(pdev, 0) & IORESOURCE_MEM)) { 238 dev_err(&pdev->dev, "Missing registers BAR, aborting\n"); 239 return -ENODEV; 240 } 241 242 err = pci_request_regions(pdev, DRIVER_NAME); 243 if (err) 244 dev_err(&pdev->dev, "Couldn't get PCI resources, aborting\n"); 245 246 return err; 247} 248 249static void release_bar(struct pci_dev *pdev) 250{ 251 pci_release_regions(pdev); 252} 253 254static int mlx5_enable_msix(struct mlx5_core_dev *dev) 255{ 256 struct mlx5_priv *priv = &dev->priv; 257 struct mlx5_eq_table *table = &priv->eq_table; 258 int num_eqs = 1 << MLX5_CAP_GEN(dev, log_max_eq); 259 int nvec; 260 int i; 261 262 nvec = MLX5_CAP_GEN(dev, num_ports) * num_online_cpus() + 263 MLX5_EQ_VEC_COMP_BASE; 264 nvec = min_t(int, nvec, num_eqs); 265 if (nvec <= MLX5_EQ_VEC_COMP_BASE) 266 return -ENOMEM; 267 268 priv->msix_arr = kcalloc(nvec, sizeof(*priv->msix_arr), GFP_KERNEL); 269 270 priv->irq_info = kcalloc(nvec, sizeof(*priv->irq_info), GFP_KERNEL); 271 if (!priv->msix_arr || !priv->irq_info) 272 goto err_free_msix; 273 274 for (i = 0; i < nvec; i++) 275 priv->msix_arr[i].entry = i; 276 277 nvec = pci_enable_msix_range(dev->pdev, priv->msix_arr, 278 MLX5_EQ_VEC_COMP_BASE + 1, nvec); 279 if (nvec < 0) 280 return nvec; 281 282 table->num_comp_vectors = nvec - MLX5_EQ_VEC_COMP_BASE; 283 284 return 0; 285 286err_free_msix: 287 kfree(priv->irq_info); 288 kfree(priv->msix_arr); 289 return -ENOMEM; 290} 291 292static void mlx5_disable_msix(struct mlx5_core_dev *dev) 293{ 294 struct mlx5_priv *priv = &dev->priv; 295 296 pci_disable_msix(dev->pdev); 297 kfree(priv->irq_info); 298 kfree(priv->msix_arr); 299} 300 301struct mlx5_reg_host_endianess { 302 u8 he; 303 u8 rsvd[15]; 304}; 305 306 307#define CAP_MASK(pos, size) ((u64)((1 << (size)) - 1) << (pos)) 308 309enum { 310 MLX5_CAP_BITS_RW_MASK = CAP_MASK(MLX5_CAP_OFF_CMDIF_CSUM, 2) | 311 MLX5_DEV_CAP_FLAG_DCT, 312}; 313 314static u16 to_fw_pkey_sz(u32 size) 315{ 316 switch (size) { 317 case 128: 318 return 0; 319 case 256: 320 return 1; 321 case 512: 322 return 2; 323 case 1024: 324 return 3; 325 case 2048: 326 return 4; 327 case 4096: 328 return 5; 329 default: 330 pr_warn("invalid pkey table size %d\n", size); 331 return 0; 332 } 333} 334 335int mlx5_core_get_caps(struct mlx5_core_dev *dev, enum mlx5_cap_type cap_type, 336 enum mlx5_cap_mode cap_mode) 337{ 338 u8 in[MLX5_ST_SZ_BYTES(query_hca_cap_in)]; 339 int out_sz = MLX5_ST_SZ_BYTES(query_hca_cap_out); 340 void *out, *hca_caps; 341 u16 opmod = (cap_type << 1) | (cap_mode & 0x01); 342 int err; 343 344 memset(in, 0, sizeof(in)); 345 out = kzalloc(out_sz, GFP_KERNEL); 346 if (!out) 347 return -ENOMEM; 348 349 MLX5_SET(query_hca_cap_in, in, opcode, MLX5_CMD_OP_QUERY_HCA_CAP); 350 MLX5_SET(query_hca_cap_in, in, op_mod, opmod); 351 err = mlx5_cmd_exec(dev, in, sizeof(in), out, out_sz); 352 if (err) 353 goto query_ex; 354 355 err = mlx5_cmd_status_to_err_v2(out); 356 if (err) { 357 mlx5_core_warn(dev, 358 "QUERY_HCA_CAP : type(%x) opmode(%x) Failed(%d)\n", 359 cap_type, cap_mode, err); 360 goto query_ex; 361 } 362 363 hca_caps = MLX5_ADDR_OF(query_hca_cap_out, out, capability); 364 365 switch (cap_mode) { 366 case HCA_CAP_OPMOD_GET_MAX: 367 memcpy(dev->hca_caps_max[cap_type], hca_caps, 368 MLX5_UN_SZ_BYTES(hca_cap_union)); 369 break; 370 case HCA_CAP_OPMOD_GET_CUR: 371 memcpy(dev->hca_caps_cur[cap_type], hca_caps, 372 MLX5_UN_SZ_BYTES(hca_cap_union)); 373 break; 374 default: 375 mlx5_core_warn(dev, 376 "Tried to query dev cap type(%x) with wrong opmode(%x)\n", 377 cap_type, cap_mode); 378 err = -EINVAL; 379 break; 380 } 381query_ex: 382 kfree(out); 383 return err; 384} 385 386static int set_caps(struct mlx5_core_dev *dev, void *in, int in_sz) 387{ 388 u32 out[MLX5_ST_SZ_DW(set_hca_cap_out)]; 389 int err; 390 391 memset(out, 0, sizeof(out)); 392 393 MLX5_SET(set_hca_cap_in, in, opcode, MLX5_CMD_OP_SET_HCA_CAP); 394 err = mlx5_cmd_exec(dev, in, in_sz, out, sizeof(out)); 395 if (err) 396 return err; 397 398 err = mlx5_cmd_status_to_err_v2(out); 399 400 return err; 401} 402 403static int handle_hca_cap(struct mlx5_core_dev *dev) 404{ 405 void *set_ctx = NULL; 406 struct mlx5_profile *prof = dev->profile; 407 int err = -ENOMEM; 408 int set_sz = MLX5_ST_SZ_BYTES(set_hca_cap_in); 409 void *set_hca_cap; 410 411 set_ctx = kzalloc(set_sz, GFP_KERNEL); 412 if (!set_ctx) 413 goto query_ex; 414 415 err = mlx5_core_get_caps(dev, MLX5_CAP_GENERAL, HCA_CAP_OPMOD_GET_MAX); 416 if (err) 417 goto query_ex; 418 419 err = mlx5_core_get_caps(dev, MLX5_CAP_GENERAL, HCA_CAP_OPMOD_GET_CUR); 420 if (err) 421 goto query_ex; 422 423 set_hca_cap = MLX5_ADDR_OF(set_hca_cap_in, set_ctx, 424 capability); 425 memcpy(set_hca_cap, dev->hca_caps_cur[MLX5_CAP_GENERAL], 426 MLX5_ST_SZ_BYTES(cmd_hca_cap)); 427 428 mlx5_core_dbg(dev, "Current Pkey table size %d Setting new size %d\n", 429 mlx5_to_sw_pkey_sz(MLX5_CAP_GEN(dev, pkey_table_size)), 430 128); 431 /* we limit the size of the pkey table to 128 entries for now */ 432 MLX5_SET(cmd_hca_cap, set_hca_cap, pkey_table_size, 433 to_fw_pkey_sz(128)); 434 435 if (prof->mask & MLX5_PROF_MASK_QP_SIZE) 436 MLX5_SET(cmd_hca_cap, set_hca_cap, log_max_qp, 437 prof->log_max_qp); 438 439 /* disable cmdif checksum */ 440 MLX5_SET(cmd_hca_cap, set_hca_cap, cmdif_checksum, 0); 441 442 MLX5_SET(cmd_hca_cap, set_hca_cap, log_uar_page_sz, PAGE_SHIFT - 12); 443 444 err = set_caps(dev, set_ctx, set_sz); 445 446query_ex: 447 kfree(set_ctx); 448 return err; 449} 450 451static int set_hca_ctrl(struct mlx5_core_dev *dev) 452{ 453 struct mlx5_reg_host_endianess he_in; 454 struct mlx5_reg_host_endianess he_out; 455 int err; 456 457 memset(&he_in, 0, sizeof(he_in)); 458 he_in.he = MLX5_SET_HOST_ENDIANNESS; 459 err = mlx5_core_access_reg(dev, &he_in, sizeof(he_in), 460 &he_out, sizeof(he_out), 461 MLX5_REG_HOST_ENDIANNESS, 0, 1); 462 return err; 463} 464 465static int mlx5_core_enable_hca(struct mlx5_core_dev *dev) 466{ 467 int err; 468 struct mlx5_enable_hca_mbox_in in; 469 struct mlx5_enable_hca_mbox_out out; 470 471 memset(&in, 0, sizeof(in)); 472 memset(&out, 0, sizeof(out)); 473 in.hdr.opcode = cpu_to_be16(MLX5_CMD_OP_ENABLE_HCA); 474 err = mlx5_cmd_exec(dev, &in, sizeof(in), &out, sizeof(out)); 475 if (err) 476 return err; 477 478 if (out.hdr.status) 479 return mlx5_cmd_status_to_err(&out.hdr); 480 481 return 0; 482} 483 484static int mlx5_core_disable_hca(struct mlx5_core_dev *dev) 485{ 486 int err; 487 struct mlx5_disable_hca_mbox_in in; 488 struct mlx5_disable_hca_mbox_out out; 489 490 memset(&in, 0, sizeof(in)); 491 memset(&out, 0, sizeof(out)); 492 in.hdr.opcode = cpu_to_be16(MLX5_CMD_OP_DISABLE_HCA); 493 err = mlx5_cmd_exec(dev, &in, sizeof(in), &out, sizeof(out)); 494 if (err) 495 return err; 496 497 if (out.hdr.status) 498 return mlx5_cmd_status_to_err(&out.hdr); 499 500 return 0; 501} 502 503static int mlx5_irq_set_affinity_hint(struct mlx5_core_dev *mdev, int i) 504{ 505 struct mlx5_priv *priv = &mdev->priv; 506 struct msix_entry *msix = priv->msix_arr; 507 int irq = msix[i + MLX5_EQ_VEC_COMP_BASE].vector; 508 int numa_node = priv->numa_node; 509 int err; 510 511 if (!zalloc_cpumask_var(&priv->irq_info[i].mask, GFP_KERNEL)) { 512 mlx5_core_warn(mdev, "zalloc_cpumask_var failed"); 513 return -ENOMEM; 514 } 515 516 cpumask_set_cpu(cpumask_local_spread(i, numa_node), 517 priv->irq_info[i].mask); 518 519 err = irq_set_affinity_hint(irq, priv->irq_info[i].mask); 520 if (err) { 521 mlx5_core_warn(mdev, "irq_set_affinity_hint failed,irq 0x%.4x", 522 irq); 523 goto err_clear_mask; 524 } 525 526 return 0; 527 528err_clear_mask: 529 free_cpumask_var(priv->irq_info[i].mask); 530 return err; 531} 532 533static void mlx5_irq_clear_affinity_hint(struct mlx5_core_dev *mdev, int i) 534{ 535 struct mlx5_priv *priv = &mdev->priv; 536 struct msix_entry *msix = priv->msix_arr; 537 int irq = msix[i + MLX5_EQ_VEC_COMP_BASE].vector; 538 539 irq_set_affinity_hint(irq, NULL); 540 free_cpumask_var(priv->irq_info[i].mask); 541} 542 543static int mlx5_irq_set_affinity_hints(struct mlx5_core_dev *mdev) 544{ 545 int err; 546 int i; 547 548 for (i = 0; i < mdev->priv.eq_table.num_comp_vectors; i++) { 549 err = mlx5_irq_set_affinity_hint(mdev, i); 550 if (err) 551 goto err_out; 552 } 553 554 return 0; 555 556err_out: 557 for (i--; i >= 0; i--) 558 mlx5_irq_clear_affinity_hint(mdev, i); 559 560 return err; 561} 562 563static void mlx5_irq_clear_affinity_hints(struct mlx5_core_dev *mdev) 564{ 565 int i; 566 567 for (i = 0; i < mdev->priv.eq_table.num_comp_vectors; i++) 568 mlx5_irq_clear_affinity_hint(mdev, i); 569} 570 571int mlx5_vector2eqn(struct mlx5_core_dev *dev, int vector, int *eqn, 572 unsigned int *irqn) 573{ 574 struct mlx5_eq_table *table = &dev->priv.eq_table; 575 struct mlx5_eq *eq, *n; 576 int err = -ENOENT; 577 578 spin_lock(&table->lock); 579 list_for_each_entry_safe(eq, n, &table->comp_eqs_list, list) { 580 if (eq->index == vector) { 581 *eqn = eq->eqn; 582 *irqn = eq->irqn; 583 err = 0; 584 break; 585 } 586 } 587 spin_unlock(&table->lock); 588 589 return err; 590} 591EXPORT_SYMBOL(mlx5_vector2eqn); 592 593static void free_comp_eqs(struct mlx5_core_dev *dev) 594{ 595 struct mlx5_eq_table *table = &dev->priv.eq_table; 596 struct mlx5_eq *eq, *n; 597 598 spin_lock(&table->lock); 599 list_for_each_entry_safe(eq, n, &table->comp_eqs_list, list) { 600 list_del(&eq->list); 601 spin_unlock(&table->lock); 602 if (mlx5_destroy_unmap_eq(dev, eq)) 603 mlx5_core_warn(dev, "failed to destroy EQ 0x%x\n", 604 eq->eqn); 605 kfree(eq); 606 spin_lock(&table->lock); 607 } 608 spin_unlock(&table->lock); 609} 610 611static int alloc_comp_eqs(struct mlx5_core_dev *dev) 612{ 613 struct mlx5_eq_table *table = &dev->priv.eq_table; 614 char name[MLX5_MAX_IRQ_NAME]; 615 struct mlx5_eq *eq; 616 int ncomp_vec; 617 int nent; 618 int err; 619 int i; 620 621 INIT_LIST_HEAD(&table->comp_eqs_list); 622 ncomp_vec = table->num_comp_vectors; 623 nent = MLX5_COMP_EQ_SIZE; 624 for (i = 0; i < ncomp_vec; i++) { 625 eq = kzalloc(sizeof(*eq), GFP_KERNEL); 626 if (!eq) { 627 err = -ENOMEM; 628 goto clean; 629 } 630 631 snprintf(name, MLX5_MAX_IRQ_NAME, "mlx5_comp%d", i); 632 err = mlx5_create_map_eq(dev, eq, 633 i + MLX5_EQ_VEC_COMP_BASE, nent, 0, 634 name, &dev->priv.uuari.uars[0]); 635 if (err) { 636 kfree(eq); 637 goto clean; 638 } 639 mlx5_core_dbg(dev, "allocated completion EQN %d\n", eq->eqn); 640 eq->index = i; 641 spin_lock(&table->lock); 642 list_add_tail(&eq->list, &table->comp_eqs_list); 643 spin_unlock(&table->lock); 644 } 645 646 return 0; 647 648clean: 649 free_comp_eqs(dev); 650 return err; 651} 652 653#ifdef CONFIG_MLX5_CORE_EN 654static int mlx5_core_set_issi(struct mlx5_core_dev *dev) 655{ 656 u32 query_in[MLX5_ST_SZ_DW(query_issi_in)]; 657 u32 query_out[MLX5_ST_SZ_DW(query_issi_out)]; 658 u32 set_in[MLX5_ST_SZ_DW(set_issi_in)]; 659 u32 set_out[MLX5_ST_SZ_DW(set_issi_out)]; 660 int err; 661 u32 sup_issi; 662 663 memset(query_in, 0, sizeof(query_in)); 664 memset(query_out, 0, sizeof(query_out)); 665 666 MLX5_SET(query_issi_in, query_in, opcode, MLX5_CMD_OP_QUERY_ISSI); 667 668 err = mlx5_cmd_exec_check_status(dev, query_in, sizeof(query_in), 669 query_out, sizeof(query_out)); 670 if (err) { 671 if (((struct mlx5_outbox_hdr *)query_out)->status == 672 MLX5_CMD_STAT_BAD_OP_ERR) { 673 pr_debug("Only ISSI 0 is supported\n"); 674 return 0; 675 } 676 677 pr_err("failed to query ISSI\n"); 678 return err; 679 } 680 681 sup_issi = MLX5_GET(query_issi_out, query_out, supported_issi_dw0); 682 683 if (sup_issi & (1 << 1)) { 684 memset(set_in, 0, sizeof(set_in)); 685 memset(set_out, 0, sizeof(set_out)); 686 687 MLX5_SET(set_issi_in, set_in, opcode, MLX5_CMD_OP_SET_ISSI); 688 MLX5_SET(set_issi_in, set_in, current_issi, 1); 689 690 err = mlx5_cmd_exec_check_status(dev, set_in, sizeof(set_in), 691 set_out, sizeof(set_out)); 692 if (err) { 693 pr_err("failed to set ISSI=1\n"); 694 return err; 695 } 696 697 dev->issi = 1; 698 699 return 0; 700 } else if (sup_issi & (1 << 0) || !sup_issi) { 701 return 0; 702 } 703 704 return -ENOTSUPP; 705} 706#endif 707 708static int map_bf_area(struct mlx5_core_dev *dev) 709{ 710 resource_size_t bf_start = pci_resource_start(dev->pdev, 0); 711 resource_size_t bf_len = pci_resource_len(dev->pdev, 0); 712 713 dev->priv.bf_mapping = io_mapping_create_wc(bf_start, bf_len); 714 715 return dev->priv.bf_mapping ? 0 : -ENOMEM; 716} 717 718static void unmap_bf_area(struct mlx5_core_dev *dev) 719{ 720 if (dev->priv.bf_mapping) 721 io_mapping_free(dev->priv.bf_mapping); 722} 723 724static void mlx5_add_device(struct mlx5_interface *intf, struct mlx5_priv *priv) 725{ 726 struct mlx5_device_context *dev_ctx; 727 struct mlx5_core_dev *dev = container_of(priv, struct mlx5_core_dev, priv); 728 729 dev_ctx = kmalloc(sizeof(*dev_ctx), GFP_KERNEL); 730 if (!dev_ctx) 731 return; 732 733 dev_ctx->intf = intf; 734 dev_ctx->context = intf->add(dev); 735 736 if (dev_ctx->context) { 737 spin_lock_irq(&priv->ctx_lock); 738 list_add_tail(&dev_ctx->list, &priv->ctx_list); 739 spin_unlock_irq(&priv->ctx_lock); 740 } else { 741 kfree(dev_ctx); 742 } 743} 744 745static void mlx5_remove_device(struct mlx5_interface *intf, struct mlx5_priv *priv) 746{ 747 struct mlx5_device_context *dev_ctx; 748 struct mlx5_core_dev *dev = container_of(priv, struct mlx5_core_dev, priv); 749 750 list_for_each_entry(dev_ctx, &priv->ctx_list, list) 751 if (dev_ctx->intf == intf) { 752 spin_lock_irq(&priv->ctx_lock); 753 list_del(&dev_ctx->list); 754 spin_unlock_irq(&priv->ctx_lock); 755 756 intf->remove(dev, dev_ctx->context); 757 kfree(dev_ctx); 758 return; 759 } 760} 761 762static int mlx5_register_device(struct mlx5_core_dev *dev) 763{ 764 struct mlx5_priv *priv = &dev->priv; 765 struct mlx5_interface *intf; 766 767 mutex_lock(&intf_mutex); 768 list_add_tail(&priv->dev_list, &dev_list); 769 list_for_each_entry(intf, &intf_list, list) 770 mlx5_add_device(intf, priv); 771 mutex_unlock(&intf_mutex); 772 773 return 0; 774} 775 776static void mlx5_unregister_device(struct mlx5_core_dev *dev) 777{ 778 struct mlx5_priv *priv = &dev->priv; 779 struct mlx5_interface *intf; 780 781 mutex_lock(&intf_mutex); 782 list_for_each_entry(intf, &intf_list, list) 783 mlx5_remove_device(intf, priv); 784 list_del(&priv->dev_list); 785 mutex_unlock(&intf_mutex); 786} 787 788int mlx5_register_interface(struct mlx5_interface *intf) 789{ 790 struct mlx5_priv *priv; 791 792 if (!intf->add || !intf->remove) 793 return -EINVAL; 794 795 mutex_lock(&intf_mutex); 796 list_add_tail(&intf->list, &intf_list); 797 list_for_each_entry(priv, &dev_list, dev_list) 798 mlx5_add_device(intf, priv); 799 mutex_unlock(&intf_mutex); 800 801 return 0; 802} 803EXPORT_SYMBOL(mlx5_register_interface); 804 805void mlx5_unregister_interface(struct mlx5_interface *intf) 806{ 807 struct mlx5_priv *priv; 808 809 mutex_lock(&intf_mutex); 810 list_for_each_entry(priv, &dev_list, dev_list) 811 mlx5_remove_device(intf, priv); 812 list_del(&intf->list); 813 mutex_unlock(&intf_mutex); 814} 815EXPORT_SYMBOL(mlx5_unregister_interface); 816 817void *mlx5_get_protocol_dev(struct mlx5_core_dev *mdev, int protocol) 818{ 819 struct mlx5_priv *priv = &mdev->priv; 820 struct mlx5_device_context *dev_ctx; 821 unsigned long flags; 822 void *result = NULL; 823 824 spin_lock_irqsave(&priv->ctx_lock, flags); 825 826 list_for_each_entry(dev_ctx, &mdev->priv.ctx_list, list) 827 if ((dev_ctx->intf->protocol == protocol) && 828 dev_ctx->intf->get_dev) { 829 result = dev_ctx->intf->get_dev(dev_ctx->context); 830 break; 831 } 832 833 spin_unlock_irqrestore(&priv->ctx_lock, flags); 834 835 return result; 836} 837EXPORT_SYMBOL(mlx5_get_protocol_dev); 838 839static int mlx5_pci_init(struct mlx5_core_dev *dev, struct mlx5_priv *priv) 840{ 841 struct pci_dev *pdev = dev->pdev; 842 int err = 0; 843 844 pci_set_drvdata(dev->pdev, dev); 845 strncpy(priv->name, dev_name(&pdev->dev), MLX5_MAX_NAME_LEN); 846 priv->name[MLX5_MAX_NAME_LEN - 1] = 0; 847 848 mutex_init(&priv->pgdir_mutex); 849 INIT_LIST_HEAD(&priv->pgdir_list); 850 spin_lock_init(&priv->mkey_lock); 851 852 mutex_init(&priv->alloc_mutex); 853 854 priv->numa_node = dev_to_node(&dev->pdev->dev); 855 856 priv->dbg_root = debugfs_create_dir(dev_name(&pdev->dev), mlx5_debugfs_root); 857 if (!priv->dbg_root) 858 return -ENOMEM; 859 860 err = mlx5_pci_enable_device(dev); 861 if (err) { 862 dev_err(&pdev->dev, "Cannot enable PCI device, aborting\n"); 863 goto err_dbg; 864 } 865 866 err = request_bar(pdev); 867 if (err) { 868 dev_err(&pdev->dev, "error requesting BARs, aborting\n"); 869 goto err_disable; 870 } 871 872 pci_set_master(pdev); 873 874 err = set_dma_caps(pdev); 875 if (err) { 876 dev_err(&pdev->dev, "Failed setting DMA capabilities mask, aborting\n"); 877 goto err_clr_master; 878 } 879 880 dev->iseg_base = pci_resource_start(dev->pdev, 0); 881 dev->iseg = ioremap(dev->iseg_base, sizeof(*dev->iseg)); 882 if (!dev->iseg) { 883 err = -ENOMEM; 884 dev_err(&pdev->dev, "Failed mapping initialization segment, aborting\n"); 885 goto err_clr_master; 886 } 887 888 return 0; 889 890err_clr_master: 891 pci_clear_master(dev->pdev); 892 release_bar(dev->pdev); 893err_disable: 894 mlx5_pci_disable_device(dev); 895 896err_dbg: 897 debugfs_remove(priv->dbg_root); 898 return err; 899} 900 901static void mlx5_pci_close(struct mlx5_core_dev *dev, struct mlx5_priv *priv) 902{ 903 iounmap(dev->iseg); 904 pci_clear_master(dev->pdev); 905 release_bar(dev->pdev); 906 mlx5_pci_disable_device(dev); 907 debugfs_remove(priv->dbg_root); 908} 909 910#define MLX5_IB_MOD "mlx5_ib" 911static int mlx5_load_one(struct mlx5_core_dev *dev, struct mlx5_priv *priv) 912{ 913 struct pci_dev *pdev = dev->pdev; 914 int err; 915 916 mutex_lock(&dev->intf_state_mutex); 917 if (dev->interface_state == MLX5_INTERFACE_STATE_UP) { 918 dev_warn(&dev->pdev->dev, "%s: interface is up, NOP\n", 919 __func__); 920 goto out; 921 } 922 923 dev_info(&pdev->dev, "firmware version: %d.%d.%d\n", fw_rev_maj(dev), 924 fw_rev_min(dev), fw_rev_sub(dev)); 925 926 /* on load removing any previous indication of internal error, device is 927 * up 928 */ 929 dev->state = MLX5_DEVICE_STATE_UP; 930 931 err = mlx5_cmd_init(dev); 932 if (err) { 933 dev_err(&pdev->dev, "Failed initializing command interface, aborting\n"); 934 goto out_err; 935 } 936 937 err = wait_fw_init(dev, FW_INIT_TIMEOUT_MILI); 938 if (err) { 939 dev_err(&dev->pdev->dev, "Firmware over %d MS in initializing state, aborting\n", 940 FW_INIT_TIMEOUT_MILI); 941 goto out_err; 942 } 943 944 mlx5_pagealloc_init(dev); 945 946 err = mlx5_core_enable_hca(dev); 947 if (err) { 948 dev_err(&pdev->dev, "enable hca failed\n"); 949 goto err_pagealloc_cleanup; 950 } 951 952#ifdef CONFIG_MLX5_CORE_EN 953 err = mlx5_core_set_issi(dev); 954 if (err) { 955 dev_err(&pdev->dev, "failed to set issi\n"); 956 goto err_disable_hca; 957 } 958#endif 959 960 err = mlx5_satisfy_startup_pages(dev, 1); 961 if (err) { 962 dev_err(&pdev->dev, "failed to allocate boot pages\n"); 963 goto err_disable_hca; 964 } 965 966 err = set_hca_ctrl(dev); 967 if (err) { 968 dev_err(&pdev->dev, "set_hca_ctrl failed\n"); 969 goto reclaim_boot_pages; 970 } 971 972 err = handle_hca_cap(dev); 973 if (err) { 974 dev_err(&pdev->dev, "handle_hca_cap failed\n"); 975 goto reclaim_boot_pages; 976 } 977 978 err = mlx5_satisfy_startup_pages(dev, 0); 979 if (err) { 980 dev_err(&pdev->dev, "failed to allocate init pages\n"); 981 goto reclaim_boot_pages; 982 } 983 984 err = mlx5_pagealloc_start(dev); 985 if (err) { 986 dev_err(&pdev->dev, "mlx5_pagealloc_start failed\n"); 987 goto reclaim_boot_pages; 988 } 989 990 err = mlx5_cmd_init_hca(dev); 991 if (err) { 992 dev_err(&pdev->dev, "init hca failed\n"); 993 goto err_pagealloc_stop; 994 } 995 996 mlx5_start_health_poll(dev); 997 998 err = mlx5_query_hca_caps(dev); 999 if (err) { 1000 dev_err(&pdev->dev, "query hca failed\n"); 1001 goto err_stop_poll; 1002 } 1003 1004 err = mlx5_query_board_id(dev); 1005 if (err) { 1006 dev_err(&pdev->dev, "query board id failed\n"); 1007 goto err_stop_poll; 1008 } 1009 1010 err = mlx5_enable_msix(dev); 1011 if (err) { 1012 dev_err(&pdev->dev, "enable msix failed\n"); 1013 goto err_stop_poll; 1014 } 1015 1016 err = mlx5_eq_init(dev); 1017 if (err) { 1018 dev_err(&pdev->dev, "failed to initialize eq\n"); 1019 goto disable_msix; 1020 } 1021 1022 err = mlx5_alloc_uuars(dev, &priv->uuari); 1023 if (err) { 1024 dev_err(&pdev->dev, "Failed allocating uar, aborting\n"); 1025 goto err_eq_cleanup; 1026 } 1027 1028 err = mlx5_start_eqs(dev); 1029 if (err) { 1030 dev_err(&pdev->dev, "Failed to start pages and async EQs\n"); 1031 goto err_free_uar; 1032 } 1033 1034 err = alloc_comp_eqs(dev); 1035 if (err) { 1036 dev_err(&pdev->dev, "Failed to alloc completion EQs\n"); 1037 goto err_stop_eqs; 1038 } 1039 1040 if (map_bf_area(dev)) 1041 dev_err(&pdev->dev, "Failed to map blue flame area\n"); 1042 1043 err = mlx5_irq_set_affinity_hints(dev); 1044 if (err) { 1045 dev_err(&pdev->dev, "Failed to alloc affinity hint cpumask\n"); 1046 goto err_unmap_bf_area; 1047 } 1048 1049 MLX5_INIT_DOORBELL_LOCK(&priv->cq_uar_lock); 1050 1051 mlx5_init_cq_table(dev); 1052 mlx5_init_qp_table(dev); 1053 mlx5_init_srq_table(dev); 1054 mlx5_init_mr_table(dev); 1055 1056 err = mlx5_register_device(dev); 1057 if (err) { 1058 dev_err(&pdev->dev, "mlx5_register_device failed %d\n", err); 1059 goto err_reg_dev; 1060 } 1061 1062 err = request_module_nowait(MLX5_IB_MOD); 1063 if (err) 1064 pr_info("failed request module on %s\n", MLX5_IB_MOD); 1065 1066 dev->interface_state = MLX5_INTERFACE_STATE_UP; 1067out: 1068 mutex_unlock(&dev->intf_state_mutex); 1069 1070 return 0; 1071 1072err_reg_dev: 1073 mlx5_cleanup_mr_table(dev); 1074 mlx5_cleanup_srq_table(dev); 1075 mlx5_cleanup_qp_table(dev); 1076 mlx5_cleanup_cq_table(dev); 1077 mlx5_irq_clear_affinity_hints(dev); 1078 1079err_unmap_bf_area: 1080 unmap_bf_area(dev); 1081 1082 free_comp_eqs(dev); 1083 1084err_stop_eqs: 1085 mlx5_stop_eqs(dev); 1086 1087err_free_uar: 1088 mlx5_free_uuars(dev, &priv->uuari); 1089 1090err_eq_cleanup: 1091 mlx5_eq_cleanup(dev); 1092 1093disable_msix: 1094 mlx5_disable_msix(dev); 1095 1096err_stop_poll: 1097 mlx5_stop_health_poll(dev); 1098 if (mlx5_cmd_teardown_hca(dev)) { 1099 dev_err(&dev->pdev->dev, "tear_down_hca failed, skip cleanup\n"); 1100 goto out_err; 1101 } 1102 1103err_pagealloc_stop: 1104 mlx5_pagealloc_stop(dev); 1105 1106reclaim_boot_pages: 1107 mlx5_reclaim_startup_pages(dev); 1108 1109err_disable_hca: 1110 mlx5_core_disable_hca(dev); 1111 1112err_pagealloc_cleanup: 1113 mlx5_pagealloc_cleanup(dev); 1114 mlx5_cmd_cleanup(dev); 1115 1116out_err: 1117 dev->state = MLX5_DEVICE_STATE_INTERNAL_ERROR; 1118 mutex_unlock(&dev->intf_state_mutex); 1119 1120 return err; 1121} 1122 1123static int mlx5_unload_one(struct mlx5_core_dev *dev, struct mlx5_priv *priv) 1124{ 1125 int err = 0; 1126 1127 mutex_lock(&dev->intf_state_mutex); 1128 if (dev->interface_state == MLX5_INTERFACE_STATE_DOWN) { 1129 dev_warn(&dev->pdev->dev, "%s: interface is down, NOP\n", 1130 __func__); 1131 goto out; 1132 } 1133 mlx5_unregister_device(dev); 1134 mlx5_cleanup_mr_table(dev); 1135 mlx5_cleanup_srq_table(dev); 1136 mlx5_cleanup_qp_table(dev); 1137 mlx5_cleanup_cq_table(dev); 1138 mlx5_irq_clear_affinity_hints(dev); 1139 unmap_bf_area(dev); 1140 free_comp_eqs(dev); 1141 mlx5_stop_eqs(dev); 1142 mlx5_free_uuars(dev, &priv->uuari); 1143 mlx5_eq_cleanup(dev); 1144 mlx5_disable_msix(dev); 1145 mlx5_stop_health_poll(dev); 1146 err = mlx5_cmd_teardown_hca(dev); 1147 if (err) { 1148 dev_err(&dev->pdev->dev, "tear_down_hca failed, skip cleanup\n"); 1149 goto out; 1150 } 1151 mlx5_pagealloc_stop(dev); 1152 mlx5_reclaim_startup_pages(dev); 1153 mlx5_core_disable_hca(dev); 1154 mlx5_pagealloc_cleanup(dev); 1155 mlx5_cmd_cleanup(dev); 1156 1157out: 1158 dev->interface_state = MLX5_INTERFACE_STATE_DOWN; 1159 mutex_unlock(&dev->intf_state_mutex); 1160 return err; 1161} 1162 1163void mlx5_core_event(struct mlx5_core_dev *dev, enum mlx5_dev_event event, 1164 unsigned long param) 1165{ 1166 struct mlx5_priv *priv = &dev->priv; 1167 struct mlx5_device_context *dev_ctx; 1168 unsigned long flags; 1169 1170 spin_lock_irqsave(&priv->ctx_lock, flags); 1171 1172 list_for_each_entry(dev_ctx, &priv->ctx_list, list) 1173 if (dev_ctx->intf->event) 1174 dev_ctx->intf->event(dev, dev_ctx->context, event, param); 1175 1176 spin_unlock_irqrestore(&priv->ctx_lock, flags); 1177} 1178 1179struct mlx5_core_event_handler { 1180 void (*event)(struct mlx5_core_dev *dev, 1181 enum mlx5_dev_event event, 1182 void *data); 1183}; 1184 1185 1186static int init_one(struct pci_dev *pdev, 1187 const struct pci_device_id *id) 1188{ 1189 struct mlx5_core_dev *dev; 1190 struct mlx5_priv *priv; 1191 int err; 1192 1193 dev = kzalloc(sizeof(*dev), GFP_KERNEL); 1194 if (!dev) { 1195 dev_err(&pdev->dev, "kzalloc failed\n"); 1196 return -ENOMEM; 1197 } 1198 priv = &dev->priv; 1199 1200 pci_set_drvdata(pdev, dev); 1201 1202 if (prof_sel < 0 || prof_sel >= ARRAY_SIZE(profile)) { 1203 pr_warn("selected profile out of range, selecting default (%d)\n", 1204 MLX5_DEFAULT_PROF); 1205 prof_sel = MLX5_DEFAULT_PROF; 1206 } 1207 dev->profile = &profile[prof_sel]; 1208 dev->pdev = pdev; 1209 dev->event = mlx5_core_event; 1210 1211 INIT_LIST_HEAD(&priv->ctx_list); 1212 spin_lock_init(&priv->ctx_lock); 1213 mutex_init(&dev->pci_status_mutex); 1214 mutex_init(&dev->intf_state_mutex); 1215 err = mlx5_pci_init(dev, priv); 1216 if (err) { 1217 dev_err(&pdev->dev, "mlx5_pci_init failed with error code %d\n", err); 1218 goto clean_dev; 1219 } 1220 1221 err = mlx5_health_init(dev); 1222 if (err) { 1223 dev_err(&pdev->dev, "mlx5_health_init failed with error code %d\n", err); 1224 goto close_pci; 1225 } 1226 1227 err = mlx5_load_one(dev, priv); 1228 if (err) { 1229 dev_err(&pdev->dev, "mlx5_load_one failed with error code %d\n", err); 1230 goto clean_health; 1231 } 1232 1233 return 0; 1234 1235clean_health: 1236 mlx5_health_cleanup(dev); 1237close_pci: 1238 mlx5_pci_close(dev, priv); 1239clean_dev: 1240 pci_set_drvdata(pdev, NULL); 1241 kfree(dev); 1242 1243 return err; 1244} 1245 1246static void remove_one(struct pci_dev *pdev) 1247{ 1248 struct mlx5_core_dev *dev = pci_get_drvdata(pdev); 1249 struct mlx5_priv *priv = &dev->priv; 1250 1251 if (mlx5_unload_one(dev, priv)) { 1252 dev_err(&dev->pdev->dev, "mlx5_unload_one failed\n"); 1253 mlx5_health_cleanup(dev); 1254 return; 1255 } 1256 mlx5_health_cleanup(dev); 1257 mlx5_pci_close(dev, priv); 1258 pci_set_drvdata(pdev, NULL); 1259 kfree(dev); 1260} 1261 1262static pci_ers_result_t mlx5_pci_err_detected(struct pci_dev *pdev, 1263 pci_channel_state_t state) 1264{ 1265 struct mlx5_core_dev *dev = pci_get_drvdata(pdev); 1266 struct mlx5_priv *priv = &dev->priv; 1267 1268 dev_info(&pdev->dev, "%s was called\n", __func__); 1269 mlx5_enter_error_state(dev); 1270 mlx5_unload_one(dev, priv); 1271 mlx5_pci_disable_device(dev); 1272 return state == pci_channel_io_perm_failure ? 1273 PCI_ERS_RESULT_DISCONNECT : PCI_ERS_RESULT_NEED_RESET; 1274} 1275 1276static pci_ers_result_t mlx5_pci_slot_reset(struct pci_dev *pdev) 1277{ 1278 struct mlx5_core_dev *dev = pci_get_drvdata(pdev); 1279 int err = 0; 1280 1281 dev_info(&pdev->dev, "%s was called\n", __func__); 1282 1283 err = mlx5_pci_enable_device(dev); 1284 if (err) { 1285 dev_err(&pdev->dev, "%s: mlx5_pci_enable_device failed with error code: %d\n" 1286 , __func__, err); 1287 return PCI_ERS_RESULT_DISCONNECT; 1288 } 1289 pci_set_master(pdev); 1290 pci_set_power_state(pdev, PCI_D0); 1291 pci_restore_state(pdev); 1292 1293 return err ? PCI_ERS_RESULT_DISCONNECT : PCI_ERS_RESULT_RECOVERED; 1294} 1295 1296void mlx5_disable_device(struct mlx5_core_dev *dev) 1297{ 1298 mlx5_pci_err_detected(dev->pdev, 0); 1299} 1300 1301/* wait for the device to show vital signs. For now we check 1302 * that we can read the device ID and that the health buffer 1303 * shows a non zero value which is different than 0xffffffff 1304 */ 1305static void wait_vital(struct pci_dev *pdev) 1306{ 1307 struct mlx5_core_dev *dev = pci_get_drvdata(pdev); 1308 struct mlx5_core_health *health = &dev->priv.health; 1309 const int niter = 100; 1310 u32 count; 1311 u16 did; 1312 int i; 1313 1314 /* Wait for firmware to be ready after reset */ 1315 msleep(1000); 1316 for (i = 0; i < niter; i++) { 1317 if (pci_read_config_word(pdev, 2, &did)) { 1318 dev_warn(&pdev->dev, "failed reading config word\n"); 1319 break; 1320 } 1321 if (did == pdev->device) { 1322 dev_info(&pdev->dev, "device ID correctly read after %d iterations\n", i); 1323 break; 1324 } 1325 msleep(50); 1326 } 1327 if (i == niter) 1328 dev_warn(&pdev->dev, "%s-%d: could not read device ID\n", __func__, __LINE__); 1329 1330 for (i = 0; i < niter; i++) { 1331 count = ioread32be(health->health_counter); 1332 if (count && count != 0xffffffff) { 1333 dev_info(&pdev->dev, "Counter value 0x%x after %d iterations\n", count, i); 1334 break; 1335 } 1336 msleep(50); 1337 } 1338 1339 if (i == niter) 1340 dev_warn(&pdev->dev, "%s-%d: could not read device ID\n", __func__, __LINE__); 1341} 1342 1343static void mlx5_pci_resume(struct pci_dev *pdev) 1344{ 1345 struct mlx5_core_dev *dev = pci_get_drvdata(pdev); 1346 struct mlx5_priv *priv = &dev->priv; 1347 int err; 1348 1349 dev_info(&pdev->dev, "%s was called\n", __func__); 1350 1351 pci_save_state(pdev); 1352 wait_vital(pdev); 1353 1354 err = mlx5_load_one(dev, priv); 1355 if (err) 1356 dev_err(&pdev->dev, "%s: mlx5_load_one failed with error code: %d\n" 1357 , __func__, err); 1358 else 1359 dev_info(&pdev->dev, "%s: device recovered\n", __func__); 1360} 1361 1362static const struct pci_error_handlers mlx5_err_handler = { 1363 .error_detected = mlx5_pci_err_detected, 1364 .slot_reset = mlx5_pci_slot_reset, 1365 .resume = mlx5_pci_resume 1366}; 1367 1368static const struct pci_device_id mlx5_core_pci_table[] = { 1369 { PCI_VDEVICE(MELLANOX, 0x1011) }, /* Connect-IB */ 1370 { PCI_VDEVICE(MELLANOX, 0x1012) }, /* Connect-IB VF */ 1371 { PCI_VDEVICE(MELLANOX, 0x1013) }, /* ConnectX-4 */ 1372 { PCI_VDEVICE(MELLANOX, 0x1014) }, /* ConnectX-4 VF */ 1373 { PCI_VDEVICE(MELLANOX, 0x1015) }, /* ConnectX-4LX */ 1374 { PCI_VDEVICE(MELLANOX, 0x1016) }, /* ConnectX-4LX VF */ 1375 { 0, } 1376}; 1377 1378MODULE_DEVICE_TABLE(pci, mlx5_core_pci_table); 1379 1380static struct pci_driver mlx5_core_driver = { 1381 .name = DRIVER_NAME, 1382 .id_table = mlx5_core_pci_table, 1383 .probe = init_one, 1384 .remove = remove_one, 1385 .err_handler = &mlx5_err_handler 1386}; 1387 1388static int __init init(void) 1389{ 1390 int err; 1391 1392 mlx5_register_debugfs(); 1393 1394 err = pci_register_driver(&mlx5_core_driver); 1395 if (err) 1396 goto err_debug; 1397 1398#ifdef CONFIG_MLX5_CORE_EN 1399 mlx5e_init(); 1400#endif 1401 1402 return 0; 1403 1404err_debug: 1405 mlx5_unregister_debugfs(); 1406 return err; 1407} 1408 1409static void __exit cleanup(void) 1410{ 1411#ifdef CONFIG_MLX5_CORE_EN 1412 mlx5e_cleanup(); 1413#endif 1414 pci_unregister_driver(&mlx5_core_driver); 1415 mlx5_unregister_debugfs(); 1416} 1417 1418module_init(init); 1419module_exit(cleanup); 1420