root/drivers/infiniband/core/device.c

DEFINITIONS

1. ib_client_put
2. rdma_dev_access_netns
3. xan_find_marked
4. __ibdev_printk
5. ibdev_printk
6. ib_device_check_mandatory
7. ib_device_get_by_index
8. ib_device_put
9. __ib_device_get_by_name
10. ib_device_get_by_name
11. rename_compat_devs
12. ib_device_rename
13. ib_device_set_dim
14. alloc_name
15. ib_device_release
16. ib_device_uevent
17. net_namespace
18. rdma_init_coredev
19. _ib_alloc_device
20. ib_dealloc_device
21. add_client_context
22. remove_client_context
23. alloc_port_data
24. verify_immutable
25. setup_port_data
26. ib_get_device_fw_str
27. ib_policy_change_task
28. ib_security_change
29. compatdev_release
30. add_one_compat_dev
31. remove_one_compat_dev
32. remove_compat_devs
33. add_compat_devs
34. remove_all_compat_devs
35. add_all_compat_devs
36. rdma_compatdev_set
37. rdma_dev_exit_net
38. rdma_dev_init_net
39. assign_name
40. setup_dma_device
41. setup_device
42. disable_device
43. enable_device_and_get
44. ib_register_device
45. __ib_unregister_device
46. ib_unregister_device
47. ib_unregister_device_and_put
48. ib_unregister_driver
49. ib_unregister_work
50. ib_unregister_device_queued
51. rdma_dev_change_netns
52. ib_device_set_netns_put
53. assign_client_id
54. remove_client_id
55. ib_register_client
56. ib_unregister_client
57. __ib_get_global_client_nl_info
58. __ib_get_client_nl_info
59. ib_get_client_nl_info
60. ib_set_client_data
61. ib_register_event_handler
62. ib_unregister_event_handler
63. ib_dispatch_event_clients
64. iw_query_port
65. __ib_query_port
66. ib_query_port
67. add_ndev_hash
68. ib_device_set_netdev
69. free_netdevs
70. ib_device_get_netdev
71. ib_device_get_by_netdev
72. ib_enum_roce_netdev
73. ib_enum_all_roce_netdevs
74. ib_enum_all_devs
75. ib_query_pkey
76. ib_modify_device
77. ib_modify_port
78. ib_find_gid
79. ib_find_pkey
80. ib_get_net_dev_by_params
81. ib_set_device_ops
82. ib_core_init
83. ib_core_cleanup

   1 /*
   2  * Copyright (c) 2004 Topspin Communications.  All rights reserved.
   3  * Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved.
   4  *
   5  * This software is available to you under a choice of one of two
   6  * licenses.  You may choose to be licensed under the terms of the GNU
   7  * General Public License (GPL) Version 2, available from the file
   8  * COPYING in the main directory of this source tree, or the
   9  * OpenIB.org BSD license below:
  10  *
  11  *     Redistribution and use in source and binary forms, with or
  12  *     without modification, are permitted provided that the following
  13  *     conditions are met:
  14  *
  15  *      - Redistributions of source code must retain the above
  16  *        copyright notice, this list of conditions and the following
  17  *        disclaimer.
  18  *
  19  *      - Redistributions in binary form must reproduce the above
  20  *        copyright notice, this list of conditions and the following
  21  *        disclaimer in the documentation and/or other materials
  22  *        provided with the distribution.
  23  *
  24  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  25  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  26  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  27  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  28  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  29  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  30  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  31  * SOFTWARE.
  32  */
  33 
  34 #include <linux/module.h>
  35 #include <linux/string.h>
  36 #include <linux/errno.h>
  37 #include <linux/kernel.h>
  38 #include <linux/slab.h>
  39 #include <linux/init.h>
  40 #include <linux/netdevice.h>
  41 #include <net/net_namespace.h>
  42 #include <linux/security.h>
  43 #include <linux/notifier.h>
  44 #include <linux/hashtable.h>
  45 #include <rdma/rdma_netlink.h>
  46 #include <rdma/ib_addr.h>
  47 #include <rdma/ib_cache.h>
  48 #include <rdma/rdma_counter.h>
  49 
  50 #include "core_priv.h"
  51 #include "restrack.h"
  52 
  53 MODULE_AUTHOR("Roland Dreier");
  54 MODULE_DESCRIPTION("core kernel InfiniBand API");
  55 MODULE_LICENSE("Dual BSD/GPL");
  56 
  57 struct workqueue_struct *ib_comp_wq;
  58 struct workqueue_struct *ib_comp_unbound_wq;
  59 struct workqueue_struct *ib_wq;
  60 EXPORT_SYMBOL_GPL(ib_wq);
  61 
  62 /*
  63  * Each of the three rwsem locks (devices, clients, client_data) protects the
  64  * xarray of the same name. Specifically it allows the caller to assert that
  65  * the MARK will/will not be changing under the lock, and for devices and
  66  * clients, that the value in the xarray is still a valid pointer. Change of
  67  * the MARK is linked to the object state, so holding the lock and testing the
  68  * MARK also asserts that the contained object is in a certain state.
  69  *
  70  * This is used to build a two stage register/unregister flow where objects
  71  * can continue to be in the xarray even though they are still in progress to
  72  * register/unregister.
  73  *
  74  * The xarray itself provides additional locking, and restartable iteration,
  75  * which is also relied on.
  76  *
  77  * Locks should not be nested, with the exception of client_data, which is
  78  * allowed to nest under the read side of the other two locks.
  79  *
  80  * The devices_rwsem also protects the device name list, any change or
  81  * assignment of device name must also hold the write side to guarantee unique
  82  * names.
  83  */
  84 
  85 /*
  86  * devices contains devices that have had their names assigned. The
  87  * devices may not be registered. Users that care about the registration
  88  * status need to call ib_device_try_get() on the device to ensure it is
  89  * registered, and keep it registered, for the required duration.
  90  *
  91  */
  92 static DEFINE_XARRAY_FLAGS(devices, XA_FLAGS_ALLOC);
  93 static DECLARE_RWSEM(devices_rwsem);
  94 #define DEVICE_REGISTERED XA_MARK_1
  95 
  96 static u32 highest_client_id;
  97 #define CLIENT_REGISTERED XA_MARK_1
  98 static DEFINE_XARRAY_FLAGS(clients, XA_FLAGS_ALLOC);
  99 static DECLARE_RWSEM(clients_rwsem);
 100 
 101 static void ib_client_put(struct ib_client *client)
 102 {
 103         if (refcount_dec_and_test(&client->uses))
 104                 complete(&client->uses_zero);
 105 }
 106 
 107 /*
 108  * If client_data is registered then the corresponding client must also still
 109  * be registered.
 110  */
 111 #define CLIENT_DATA_REGISTERED XA_MARK_1
 112 
 113 unsigned int rdma_dev_net_id;
 114 
 115 /*
 116  * A list of net namespaces is maintained in an xarray. This is necessary
 117  * because we can't get the locking right using the existing net ns list. We
 118  * would require a init_net callback after the list is updated.
 119  */
 120 static DEFINE_XARRAY_FLAGS(rdma_nets, XA_FLAGS_ALLOC);
 121 /*
 122  * rwsem to protect accessing the rdma_nets xarray entries.
 123  */
 124 static DECLARE_RWSEM(rdma_nets_rwsem);
 125 
 126 bool ib_devices_shared_netns = true;
 127 module_param_named(netns_mode, ib_devices_shared_netns, bool, 0444);
 128 MODULE_PARM_DESC(netns_mode,
 129                  "Share device among net namespaces; default=1 (shared)");
 130 /**
 131  * rdma_dev_access_netns() - Return whether a rdma device can be accessed
 132  *                           from a specified net namespace or not.
 133  * @device:     Pointer to rdma device which needs to be checked
 134  * @net:        Pointer to net namesapce for which access to be checked
 135  *
 136  * rdma_dev_access_netns() - Return whether a rdma device can be accessed
 137  *                           from a specified net namespace or not. When
 138  *                           rdma device is in shared mode, it ignores the
 139  *                           net namespace. When rdma device is exclusive
 140  *                           to a net namespace, rdma device net namespace is
 141  *                           checked against the specified one.
 142  */
 143 bool rdma_dev_access_netns(const struct ib_device *dev, const struct net *net)
 144 {
 145         return (ib_devices_shared_netns ||
 146                 net_eq(read_pnet(&dev->coredev.rdma_net), net));
 147 }
 148 EXPORT_SYMBOL(rdma_dev_access_netns);
 149 
 150 /*
 151  * xarray has this behavior where it won't iterate over NULL values stored in
 152  * allocated arrays.  So we need our own iterator to see all values stored in
 153  * the array. This does the same thing as xa_for_each except that it also
 154  * returns NULL valued entries if the array is allocating. Simplified to only
 155  * work on simple xarrays.
 156  */
 157 static void *xan_find_marked(struct xarray *xa, unsigned long *indexp,
 158                              xa_mark_t filter)
 159 {
 160         XA_STATE(xas, xa, *indexp);
 161         void *entry;
 162 
 163         rcu_read_lock();
 164         do {
 165                 entry = xas_find_marked(&xas, ULONG_MAX, filter);
 166                 if (xa_is_zero(entry))
 167                         break;
 168         } while (xas_retry(&xas, entry));
 169         rcu_read_unlock();
 170 
 171         if (entry) {
 172                 *indexp = xas.xa_index;
 173                 if (xa_is_zero(entry))
 174                         return NULL;
 175                 return entry;
 176         }
 177         return XA_ERROR(-ENOENT);
 178 }
 179 #define xan_for_each_marked(xa, index, entry, filter)                          \
 180         for (index = 0, entry = xan_find_marked(xa, &(index), filter);         \
 181              !xa_is_err(entry);                                                \
 182              (index)++, entry = xan_find_marked(xa, &(index), filter))
 183 
 184 /* RCU hash table mapping netdevice pointers to struct ib_port_data */
 185 static DEFINE_SPINLOCK(ndev_hash_lock);
 186 static DECLARE_HASHTABLE(ndev_hash, 5);
 187 
 188 static void free_netdevs(struct ib_device *ib_dev);
 189 static void ib_unregister_work(struct work_struct *work);
 190 static void __ib_unregister_device(struct ib_device *device);
 191 static int ib_security_change(struct notifier_block *nb, unsigned long event,
 192                               void *lsm_data);
 193 static void ib_policy_change_task(struct work_struct *work);
 194 static DECLARE_WORK(ib_policy_change_work, ib_policy_change_task);
 195 
 196 static void __ibdev_printk(const char *level, const struct ib_device *ibdev,
 197                            struct va_format *vaf)
 198 {
 199         if (ibdev && ibdev->dev.parent)
 200                 dev_printk_emit(level[1] - '0',
 201                                 ibdev->dev.parent,
 202                                 "%s %s %s: %pV",
 203                                 dev_driver_string(ibdev->dev.parent),
 204                                 dev_name(ibdev->dev.parent),
 205                                 dev_name(&ibdev->dev),
 206                                 vaf);
 207         else if (ibdev)
 208                 printk("%s%s: %pV",
 209                        level, dev_name(&ibdev->dev), vaf);
 210         else
 211                 printk("%s(NULL ib_device): %pV", level, vaf);
 212 }
 213 
 214 void ibdev_printk(const char *level, const struct ib_device *ibdev,
 215                   const char *format, ...)
 216 {
 217         struct va_format vaf;
 218         va_list args;
 219 
 220         va_start(args, format);
 221 
 222         vaf.fmt = format;
 223         vaf.va = &args;
 224 
 225         __ibdev_printk(level, ibdev, &vaf);
 226 
 227         va_end(args);
 228 }
 229 EXPORT_SYMBOL(ibdev_printk);
 230 
 231 #define define_ibdev_printk_level(func, level)                  \
 232 void func(const struct ib_device *ibdev, const char *fmt, ...)  \
 233 {                                                               \
 234         struct va_format vaf;                                   \
 235         va_list args;                                           \
 236                                                                 \
 237         va_start(args, fmt);                                    \
 238                                                                 \
 239         vaf.fmt = fmt;                                          \
 240         vaf.va = &args;                                         \
 241                                                                 \
 242         __ibdev_printk(level, ibdev, &vaf);                     \
 243                                                                 \
 244         va_end(args);                                           \
 245 }                                                               \
 246 EXPORT_SYMBOL(func);
 247 
 248 define_ibdev_printk_level(ibdev_emerg, KERN_EMERG);
 249 define_ibdev_printk_level(ibdev_alert, KERN_ALERT);
 250 define_ibdev_printk_level(ibdev_crit, KERN_CRIT);
 251 define_ibdev_printk_level(ibdev_err, KERN_ERR);
 252 define_ibdev_printk_level(ibdev_warn, KERN_WARNING);
 253 define_ibdev_printk_level(ibdev_notice, KERN_NOTICE);
 254 define_ibdev_printk_level(ibdev_info, KERN_INFO);
 255 
 256 static struct notifier_block ibdev_lsm_nb = {
 257         .notifier_call = ib_security_change,
 258 };
 259 
 260 static int rdma_dev_change_netns(struct ib_device *device, struct net *cur_net,
 261                                  struct net *net);
 262 
 263 /* Pointer to the RCU head at the start of the ib_port_data array */
 264 struct ib_port_data_rcu {
 265         struct rcu_head rcu_head;
 266         struct ib_port_data pdata[];
 267 };
 268 
 269 static void ib_device_check_mandatory(struct ib_device *device)
 270 {
 271 #define IB_MANDATORY_FUNC(x) { offsetof(struct ib_device_ops, x), #x }
 272         static const struct {
 273                 size_t offset;
 274                 char  *name;
 275         } mandatory_table[] = {
 276                 IB_MANDATORY_FUNC(query_device),
 277                 IB_MANDATORY_FUNC(query_port),
 278                 IB_MANDATORY_FUNC(query_pkey),
 279                 IB_MANDATORY_FUNC(alloc_pd),
 280                 IB_MANDATORY_FUNC(dealloc_pd),
 281                 IB_MANDATORY_FUNC(create_qp),
 282                 IB_MANDATORY_FUNC(modify_qp),
 283                 IB_MANDATORY_FUNC(destroy_qp),
 284                 IB_MANDATORY_FUNC(post_send),
 285                 IB_MANDATORY_FUNC(post_recv),
 286                 IB_MANDATORY_FUNC(create_cq),
 287                 IB_MANDATORY_FUNC(destroy_cq),
 288                 IB_MANDATORY_FUNC(poll_cq),
 289                 IB_MANDATORY_FUNC(req_notify_cq),
 290                 IB_MANDATORY_FUNC(get_dma_mr),
 291                 IB_MANDATORY_FUNC(dereg_mr),
 292                 IB_MANDATORY_FUNC(get_port_immutable)
 293         };
 294         int i;
 295 
 296         device->kverbs_provider = true;
 297         for (i = 0; i < ARRAY_SIZE(mandatory_table); ++i) {
 298                 if (!*(void **) ((void *) &device->ops +
 299                                  mandatory_table[i].offset)) {
 300                         device->kverbs_provider = false;
 301                         break;
 302                 }
 303         }
 304 }
 305 
 306 /*
 307  * Caller must perform ib_device_put() to return the device reference count
 308  * when ib_device_get_by_index() returns valid device pointer.
 309  */
 310 struct ib_device *ib_device_get_by_index(const struct net *net, u32 index)
 311 {
 312         struct ib_device *device;
 313 
 314         down_read(&devices_rwsem);
 315         device = xa_load(&devices, index);
 316         if (device) {
 317                 if (!rdma_dev_access_netns(device, net)) {
 318                         device = NULL;
 319                         goto out;
 320                 }
 321 
 322                 if (!ib_device_try_get(device))
 323                         device = NULL;
 324         }
 325 out:
 326         up_read(&devices_rwsem);
 327         return device;
 328 }
 329 
 330 /**
 331  * ib_device_put - Release IB device reference
 332  * @device: device whose reference to be released
 333  *
 334  * ib_device_put() releases reference to the IB device to allow it to be
 335  * unregistered and eventually free.
 336  */
 337 void ib_device_put(struct ib_device *device)
 338 {
 339         if (refcount_dec_and_test(&device->refcount))
 340                 complete(&device->unreg_completion);
 341 }
 342 EXPORT_SYMBOL(ib_device_put);
 343 
 344 static struct ib_device *__ib_device_get_by_name(const char *name)
 345 {
 346         struct ib_device *device;
 347         unsigned long index;
 348 
 349         xa_for_each (&devices, index, device)
 350                 if (!strcmp(name, dev_name(&device->dev)))
 351                         return device;
 352 
 353         return NULL;
 354 }
 355 
 356 /**
 357  * ib_device_get_by_name - Find an IB device by name
 358  * @name: The name to look for
 359  * @driver_id: The driver ID that must match (RDMA_DRIVER_UNKNOWN matches all)
 360  *
 361  * Find and hold an ib_device by its name. The caller must call
 362  * ib_device_put() on the returned pointer.
 363  */
 364 struct ib_device *ib_device_get_by_name(const char *name,
 365                                         enum rdma_driver_id driver_id)
 366 {
 367         struct ib_device *device;
 368 
 369         down_read(&devices_rwsem);
 370         device = __ib_device_get_by_name(name);
 371         if (device && driver_id != RDMA_DRIVER_UNKNOWN &&
 372             device->ops.driver_id != driver_id)
 373                 device = NULL;
 374 
 375         if (device) {
 376                 if (!ib_device_try_get(device))
 377                         device = NULL;
 378         }
 379         up_read(&devices_rwsem);
 380         return device;
 381 }
 382 EXPORT_SYMBOL(ib_device_get_by_name);
 383 
 384 static int rename_compat_devs(struct ib_device *device)
 385 {
 386         struct ib_core_device *cdev;
 387         unsigned long index;
 388         int ret = 0;
 389 
 390         mutex_lock(&device->compat_devs_mutex);
 391         xa_for_each (&device->compat_devs, index, cdev) {
 392                 ret = device_rename(&cdev->dev, dev_name(&device->dev));
 393                 if (ret) {
 394                         dev_warn(&cdev->dev,
 395                                  "Fail to rename compatdev to new name %s\n",
 396                                  dev_name(&device->dev));
 397                         break;
 398                 }
 399         }
 400         mutex_unlock(&device->compat_devs_mutex);
 401         return ret;
 402 }
 403 
 404 int ib_device_rename(struct ib_device *ibdev, const char *name)
 405 {
 406         unsigned long index;
 407         void *client_data;
 408         int ret;
 409 
 410         down_write(&devices_rwsem);
 411         if (!strcmp(name, dev_name(&ibdev->dev))) {
 412                 up_write(&devices_rwsem);
 413                 return 0;
 414         }
 415 
 416         if (__ib_device_get_by_name(name)) {
 417                 up_write(&devices_rwsem);
 418                 return -EEXIST;
 419         }
 420 
 421         ret = device_rename(&ibdev->dev, name);
 422         if (ret) {
 423                 up_write(&devices_rwsem);
 424                 return ret;
 425         }
 426 
 427         strlcpy(ibdev->name, name, IB_DEVICE_NAME_MAX);
 428         ret = rename_compat_devs(ibdev);
 429 
 430         downgrade_write(&devices_rwsem);
 431         down_read(&ibdev->client_data_rwsem);
 432         xan_for_each_marked(&ibdev->client_data, index, client_data,
 433                             CLIENT_DATA_REGISTERED) {
 434                 struct ib_client *client = xa_load(&clients, index);
 435 
 436                 if (!client || !client->rename)
 437                         continue;
 438 
 439                 client->rename(ibdev, client_data);
 440         }
 441         up_read(&ibdev->client_data_rwsem);
 442         up_read(&devices_rwsem);
 443         return 0;
 444 }
 445 
 446 int ib_device_set_dim(struct ib_device *ibdev, u8 use_dim)
 447 {
 448         if (use_dim > 1)
 449                 return -EINVAL;
 450         ibdev->use_cq_dim = use_dim;
 451 
 452         return 0;
 453 }
 454 
 455 static int alloc_name(struct ib_device *ibdev, const char *name)
 456 {
 457         struct ib_device *device;
 458         unsigned long index;
 459         struct ida inuse;
 460         int rc;
 461         int i;
 462 
 463         lockdep_assert_held_write(&devices_rwsem);
 464         ida_init(&inuse);
 465         xa_for_each (&devices, index, device) {
 466                 char buf[IB_DEVICE_NAME_MAX];
 467 
 468                 if (sscanf(dev_name(&device->dev), name, &i) != 1)
 469                         continue;
 470                 if (i < 0 || i >= INT_MAX)
 471                         continue;
 472                 snprintf(buf, sizeof buf, name, i);
 473                 if (strcmp(buf, dev_name(&device->dev)) != 0)
 474                         continue;
 475 
 476                 rc = ida_alloc_range(&inuse, i, i, GFP_KERNEL);
 477                 if (rc < 0)
 478                         goto out;
 479         }
 480 
 481         rc = ida_alloc(&inuse, GFP_KERNEL);
 482         if (rc < 0)
 483                 goto out;
 484 
 485         rc = dev_set_name(&ibdev->dev, name, rc);
 486 out:
 487         ida_destroy(&inuse);
 488         return rc;
 489 }
 490 
 491 static void ib_device_release(struct device *device)
 492 {
 493         struct ib_device *dev = container_of(device, struct ib_device, dev);
 494 
 495         free_netdevs(dev);
 496         WARN_ON(refcount_read(&dev->refcount));
 497         if (dev->port_data) {
 498                 ib_cache_release_one(dev);
 499                 ib_security_release_port_pkey_list(dev);
 500                 rdma_counter_release(dev);
 501                 kfree_rcu(container_of(dev->port_data, struct ib_port_data_rcu,
 502                                        pdata[0]),
 503                           rcu_head);
 504         }
 505 
 506         mutex_destroy(&dev->unregistration_lock);
 507         mutex_destroy(&dev->compat_devs_mutex);
 508 
 509         xa_destroy(&dev->compat_devs);
 510         xa_destroy(&dev->client_data);
 511         kfree_rcu(dev, rcu_head);
 512 }
 513 
 514 static int ib_device_uevent(struct device *device,
 515                             struct kobj_uevent_env *env)
 516 {
 517         if (add_uevent_var(env, "NAME=%s", dev_name(device)))
 518                 return -ENOMEM;
 519 
 520         /*
 521          * It would be nice to pass the node GUID with the event...
 522          */
 523 
 524         return 0;
 525 }
 526 
 527 static const void *net_namespace(struct device *d)
 528 {
 529         struct ib_core_device *coredev =
 530                         container_of(d, struct ib_core_device, dev);
 531 
 532         return read_pnet(&coredev->rdma_net);
 533 }
 534 
 535 static struct class ib_class = {
 536         .name    = "infiniband",
 537         .dev_release = ib_device_release,
 538         .dev_uevent = ib_device_uevent,
 539         .ns_type = &net_ns_type_operations,
 540         .namespace = net_namespace,
 541 };
 542 
 543 static void rdma_init_coredev(struct ib_core_device *coredev,
 544                               struct ib_device *dev, struct net *net)
 545 {
 546         /* This BUILD_BUG_ON is intended to catch layout change
 547          * of union of ib_core_device and device.
 548          * dev must be the first element as ib_core and providers
 549          * driver uses it. Adding anything in ib_core_device before
 550          * device will break this assumption.
 551          */
 552         BUILD_BUG_ON(offsetof(struct ib_device, coredev.dev) !=
 553                      offsetof(struct ib_device, dev));
 554 
 555         coredev->dev.class = &ib_class;
 556         coredev->dev.groups = dev->groups;
 557         device_initialize(&coredev->dev);
 558         coredev->owner = dev;
 559         INIT_LIST_HEAD(&coredev->port_list);
 560         write_pnet(&coredev->rdma_net, net);
 561 }
 562 
 563 /**
 564  * _ib_alloc_device - allocate an IB device struct
 565  * @size:size of structure to allocate
 566  *
 567  * Low-level drivers should use ib_alloc_device() to allocate &struct
 568  * ib_device.  @size is the size of the structure to be allocated,
 569  * including any private data used by the low-level driver.
 570  * ib_dealloc_device() must be used to free structures allocated with
 571  * ib_alloc_device().
 572  */
 573 struct ib_device *_ib_alloc_device(size_t size)
 574 {
 575         struct ib_device *device;
 576 
 577         if (WARN_ON(size < sizeof(struct ib_device)))
 578                 return NULL;
 579 
 580         device = kzalloc(size, GFP_KERNEL);
 581         if (!device)
 582                 return NULL;
 583 
 584         if (rdma_restrack_init(device)) {
 585                 kfree(device);
 586                 return NULL;
 587         }
 588 
 589         device->groups[0] = &ib_dev_attr_group;
 590         rdma_init_coredev(&device->coredev, device, &init_net);
 591 
 592         INIT_LIST_HEAD(&device->event_handler_list);
 593         spin_lock_init(&device->event_handler_lock);
 594         init_rwsem(&device->event_handler_rwsem);
 595         mutex_init(&device->unregistration_lock);
 596         /*
 597          * client_data needs to be alloc because we don't want our mark to be
 598          * destroyed if the user stores NULL in the client data.
 599          */
 600         xa_init_flags(&device->client_data, XA_FLAGS_ALLOC);
 601         init_rwsem(&device->client_data_rwsem);
 602         xa_init_flags(&device->compat_devs, XA_FLAGS_ALLOC);
 603         mutex_init(&device->compat_devs_mutex);
 604         init_completion(&device->unreg_completion);
 605         INIT_WORK(&device->unregistration_work, ib_unregister_work);
 606 
 607         return device;
 608 }
 609 EXPORT_SYMBOL(_ib_alloc_device);
 610 
 611 /**
 612  * ib_dealloc_device - free an IB device struct
 613  * @device:structure to free
 614  *
 615  * Free a structure allocated with ib_alloc_device().
 616  */
 617 void ib_dealloc_device(struct ib_device *device)
 618 {
 619         if (device->ops.dealloc_driver)
 620                 device->ops.dealloc_driver(device);
 621 
 622         /*
 623          * ib_unregister_driver() requires all devices to remain in the xarray
 624          * while their ops are callable. The last op we call is dealloc_driver
 625          * above.  This is needed to create a fence on op callbacks prior to
 626          * allowing the driver module to unload.
 627          */
 628         down_write(&devices_rwsem);
 629         if (xa_load(&devices, device->index) == device)
 630                 xa_erase(&devices, device->index);
 631         up_write(&devices_rwsem);
 632 
 633         /* Expedite releasing netdev references */
 634         free_netdevs(device);
 635 
 636         WARN_ON(!xa_empty(&device->compat_devs));
 637         WARN_ON(!xa_empty(&device->client_data));
 638         WARN_ON(refcount_read(&device->refcount));
 639         rdma_restrack_clean(device);
 640         /* Balances with device_initialize */
 641         put_device(&device->dev);
 642 }
 643 EXPORT_SYMBOL(ib_dealloc_device);
 644 
 645 /*
 646  * add_client_context() and remove_client_context() must be safe against
 647  * parallel calls on the same device - registration/unregistration of both the
 648  * device and client can be occurring in parallel.
 649  *
 650  * The routines need to be a fence, any caller must not return until the add
 651  * or remove is fully completed.
 652  */
 653 static int add_client_context(struct ib_device *device,
 654                               struct ib_client *client)
 655 {
 656         int ret = 0;
 657 
 658         if (!device->kverbs_provider && !client->no_kverbs_req)
 659                 return 0;
 660 
 661         down_write(&device->client_data_rwsem);
 662         /*
 663          * So long as the client is registered hold both the client and device
 664          * unregistration locks.
 665          */
 666         if (!refcount_inc_not_zero(&client->uses))
 667                 goto out_unlock;
 668         refcount_inc(&device->refcount);
 669 
 670         /*
 671          * Another caller to add_client_context got here first and has already
 672          * completely initialized context.
 673          */
 674         if (xa_get_mark(&device->client_data, client->client_id,
 675                     CLIENT_DATA_REGISTERED))
 676                 goto out;
 677 
 678         ret = xa_err(xa_store(&device->client_data, client->client_id, NULL,
 679                               GFP_KERNEL));
 680         if (ret)
 681                 goto out;
 682         downgrade_write(&device->client_data_rwsem);
 683         if (client->add)
 684                 client->add(device);
 685 
 686         /* Readers shall not see a client until add has been completed */
 687         xa_set_mark(&device->client_data, client->client_id,
 688                     CLIENT_DATA_REGISTERED);
 689         up_read(&device->client_data_rwsem);
 690         return 0;
 691 
 692 out:
 693         ib_device_put(device);
 694         ib_client_put(client);
 695 out_unlock:
 696         up_write(&device->client_data_rwsem);
 697         return ret;
 698 }
 699 
 700 static void remove_client_context(struct ib_device *device,
 701                                   unsigned int client_id)
 702 {
 703         struct ib_client *client;
 704         void *client_data;
 705 
 706         down_write(&device->client_data_rwsem);
 707         if (!xa_get_mark(&device->client_data, client_id,
 708                          CLIENT_DATA_REGISTERED)) {
 709                 up_write(&device->client_data_rwsem);
 710                 return;
 711         }
 712         client_data = xa_load(&device->client_data, client_id);
 713         xa_clear_mark(&device->client_data, client_id, CLIENT_DATA_REGISTERED);
 714         client = xa_load(&clients, client_id);
 715         up_write(&device->client_data_rwsem);
 716 
 717         /*
 718          * Notice we cannot be holding any exclusive locks when calling the
 719          * remove callback as the remove callback can recurse back into any
 720          * public functions in this module and thus try for any locks those
 721          * functions take.
 722          *
 723          * For this reason clients and drivers should not call the
 724          * unregistration functions will holdling any locks.
 725          */
 726         if (client->remove)
 727                 client->remove(device, client_data);
 728 
 729         xa_erase(&device->client_data, client_id);
 730         ib_device_put(device);
 731         ib_client_put(client);
 732 }
 733 
 734 static int alloc_port_data(struct ib_device *device)
 735 {
 736         struct ib_port_data_rcu *pdata_rcu;
 737         unsigned int port;
 738 
 739         if (device->port_data)
 740                 return 0;
 741 
 742         /* This can only be called once the physical port range is defined */
 743         if (WARN_ON(!device->phys_port_cnt))
 744                 return -EINVAL;
 745 
 746         /*
 747          * device->port_data is indexed directly by the port number to make
 748          * access to this data as efficient as possible.
 749          *
 750          * Therefore port_data is declared as a 1 based array with potential
 751          * empty slots at the beginning.
 752          */
 753         pdata_rcu = kzalloc(struct_size(pdata_rcu, pdata,
 754                                         rdma_end_port(device) + 1),
 755                             GFP_KERNEL);
 756         if (!pdata_rcu)
 757                 return -ENOMEM;
 758         /*
 759          * The rcu_head is put in front of the port data array and the stored
 760          * pointer is adjusted since we never need to see that member until
 761          * kfree_rcu.
 762          */
 763         device->port_data = pdata_rcu->pdata;
 764 
 765         rdma_for_each_port (device, port) {
 766                 struct ib_port_data *pdata = &device->port_data[port];
 767 
 768                 pdata->ib_dev = device;
 769                 spin_lock_init(&pdata->pkey_list_lock);
 770                 INIT_LIST_HEAD(&pdata->pkey_list);
 771                 spin_lock_init(&pdata->netdev_lock);
 772                 INIT_HLIST_NODE(&pdata->ndev_hash_link);
 773         }
 774         return 0;
 775 }
 776 
 777 static int verify_immutable(const struct ib_device *dev, u8 port)
 778 {
 779         return WARN_ON(!rdma_cap_ib_mad(dev, port) &&
 780                             rdma_max_mad_size(dev, port) != 0);
 781 }
 782 
 783 static int setup_port_data(struct ib_device *device)
 784 {
 785         unsigned int port;
 786         int ret;
 787 
 788         ret = alloc_port_data(device);
 789         if (ret)
 790                 return ret;
 791 
 792         rdma_for_each_port (device, port) {
 793                 struct ib_port_data *pdata = &device->port_data[port];
 794 
 795                 ret = device->ops.get_port_immutable(device, port,
 796                                                      &pdata->immutable);
 797                 if (ret)
 798                         return ret;
 799 
 800                 if (verify_immutable(device, port))
 801                         return -EINVAL;
 802         }
 803         return 0;
 804 }
 805 
 806 void ib_get_device_fw_str(struct ib_device *dev, char *str)
 807 {
 808         if (dev->ops.get_dev_fw_str)
 809                 dev->ops.get_dev_fw_str(dev, str);
 810         else
 811                 str[0] = '\0';
 812 }
 813 EXPORT_SYMBOL(ib_get_device_fw_str);
 814 
 815 static void ib_policy_change_task(struct work_struct *work)
 816 {
 817         struct ib_device *dev;
 818         unsigned long index;
 819 
 820         down_read(&devices_rwsem);
 821         xa_for_each_marked (&devices, index, dev, DEVICE_REGISTERED) {
 822                 unsigned int i;
 823 
 824                 rdma_for_each_port (dev, i) {
 825                         u64 sp;
 826                         int ret = ib_get_cached_subnet_prefix(dev,
 827                                                               i,
 828                                                               &sp);
 829 
 830                         WARN_ONCE(ret,
 831                                   "ib_get_cached_subnet_prefix err: %d, this should never happen here\n",
 832                                   ret);
 833                         if (!ret)
 834                                 ib_security_cache_change(dev, i, sp);
 835                 }
 836         }
 837         up_read(&devices_rwsem);
 838 }
 839 
 840 static int ib_security_change(struct notifier_block *nb, unsigned long event,
 841                               void *lsm_data)
 842 {
 843         if (event != LSM_POLICY_CHANGE)
 844                 return NOTIFY_DONE;
 845 
 846         schedule_work(&ib_policy_change_work);
 847         ib_mad_agent_security_change();
 848 
 849         return NOTIFY_OK;
 850 }
 851 
 852 static void compatdev_release(struct device *dev)
 853 {
 854         struct ib_core_device *cdev =
 855                 container_of(dev, struct ib_core_device, dev);
 856 
 857         kfree(cdev);
 858 }
 859 
 860 static int add_one_compat_dev(struct ib_device *device,
 861                               struct rdma_dev_net *rnet)
 862 {
 863         struct ib_core_device *cdev;
 864         int ret;
 865 
 866         lockdep_assert_held(&rdma_nets_rwsem);
 867         if (!ib_devices_shared_netns)
 868                 return 0;
 869 
 870         /*
 871          * Create and add compat device in all namespaces other than where it
 872          * is currently bound to.
 873          */
 874         if (net_eq(read_pnet(&rnet->net),
 875                    read_pnet(&device->coredev.rdma_net)))
 876                 return 0;
 877 
 878         /*
 879          * The first of init_net() or ib_register_device() to take the
 880          * compat_devs_mutex wins and gets to add the device. Others will wait
 881          * for completion here.
 882          */
 883         mutex_lock(&device->compat_devs_mutex);
 884         cdev = xa_load(&device->compat_devs, rnet->id);
 885         if (cdev) {
 886                 ret = 0;
 887                 goto done;
 888         }
 889         ret = xa_reserve(&device->compat_devs, rnet->id, GFP_KERNEL);
 890         if (ret)
 891                 goto done;
 892 
 893         cdev = kzalloc(sizeof(*cdev), GFP_KERNEL);
 894         if (!cdev) {
 895                 ret = -ENOMEM;
 896                 goto cdev_err;
 897         }
 898 
 899         cdev->dev.parent = device->dev.parent;
 900         rdma_init_coredev(cdev, device, read_pnet(&rnet->net));
 901         cdev->dev.release = compatdev_release;
 902         ret = dev_set_name(&cdev->dev, "%s", dev_name(&device->dev));
 903         if (ret)
 904                 goto add_err;
 905 
 906         ret = device_add(&cdev->dev);
 907         if (ret)
 908                 goto add_err;
 909         ret = ib_setup_port_attrs(cdev);
 910         if (ret)
 911                 goto port_err;
 912 
 913         ret = xa_err(xa_store(&device->compat_devs, rnet->id,
 914                               cdev, GFP_KERNEL));
 915         if (ret)
 916                 goto insert_err;
 917 
 918         mutex_unlock(&device->compat_devs_mutex);
 919         return 0;
 920 
 921 insert_err:
 922         ib_free_port_attrs(cdev);
 923 port_err:
 924         device_del(&cdev->dev);
 925 add_err:
 926         put_device(&cdev->dev);
 927 cdev_err:
 928         xa_release(&device->compat_devs, rnet->id);
 929 done:
 930         mutex_unlock(&device->compat_devs_mutex);
 931         return ret;
 932 }
 933 
 934 static void remove_one_compat_dev(struct ib_device *device, u32 id)
 935 {
 936         struct ib_core_device *cdev;
 937 
 938         mutex_lock(&device->compat_devs_mutex);
 939         cdev = xa_erase(&device->compat_devs, id);
 940         mutex_unlock(&device->compat_devs_mutex);
 941         if (cdev) {
 942                 ib_free_port_attrs(cdev);
 943                 device_del(&cdev->dev);
 944                 put_device(&cdev->dev);
 945         }
 946 }
 947 
 948 static void remove_compat_devs(struct ib_device *device)
 949 {
 950         struct ib_core_device *cdev;
 951         unsigned long index;
 952 
 953         xa_for_each (&device->compat_devs, index, cdev)
 954                 remove_one_compat_dev(device, index);
 955 }
 956 
 957 static int add_compat_devs(struct ib_device *device)
 958 {
 959         struct rdma_dev_net *rnet;
 960         unsigned long index;
 961         int ret = 0;
 962 
 963         lockdep_assert_held(&devices_rwsem);
 964 
 965         down_read(&rdma_nets_rwsem);
 966         xa_for_each (&rdma_nets, index, rnet) {
 967                 ret = add_one_compat_dev(device, rnet);
 968                 if (ret)
 969                         break;
 970         }
 971         up_read(&rdma_nets_rwsem);
 972         return ret;
 973 }
 974 
 975 static void remove_all_compat_devs(void)
 976 {
 977         struct ib_compat_device *cdev;
 978         struct ib_device *dev;
 979         unsigned long index;
 980 
 981         down_read(&devices_rwsem);
 982         xa_for_each (&devices, index, dev) {
 983                 unsigned long c_index = 0;
 984 
 985                 /* Hold nets_rwsem so that any other thread modifying this
 986                  * system param can sync with this thread.
 987                  */
 988                 down_read(&rdma_nets_rwsem);
 989                 xa_for_each (&dev->compat_devs, c_index, cdev)
 990                         remove_one_compat_dev(dev, c_index);
 991                 up_read(&rdma_nets_rwsem);
 992         }
 993         up_read(&devices_rwsem);
 994 }
 995 
 996 static int add_all_compat_devs(void)
 997 {
 998         struct rdma_dev_net *rnet;
 999         struct ib_device *dev;
1000         unsigned long index;
1001         int ret = 0;
1002 
1003         down_read(&devices_rwsem);
1004         xa_for_each_marked (&devices, index, dev, DEVICE_REGISTERED) {
1005                 unsigned long net_index = 0;
1006 
1007                 /* Hold nets_rwsem so that any other thread modifying this
1008                  * system param can sync with this thread.
1009                  */
1010                 down_read(&rdma_nets_rwsem);
1011                 xa_for_each (&rdma_nets, net_index, rnet) {
1012                         ret = add_one_compat_dev(dev, rnet);
1013                         if (ret)
1014                                 break;
1015                 }
1016                 up_read(&rdma_nets_rwsem);
1017         }
1018         up_read(&devices_rwsem);
1019         if (ret)
1020                 remove_all_compat_devs();
1021         return ret;
1022 }
1023 
1024 int rdma_compatdev_set(u8 enable)
1025 {
1026         struct rdma_dev_net *rnet;
1027         unsigned long index;
1028         int ret = 0;
1029 
1030         down_write(&rdma_nets_rwsem);
1031         if (ib_devices_shared_netns == enable) {
1032                 up_write(&rdma_nets_rwsem);
1033                 return 0;
1034         }
1035 
1036         /* enable/disable of compat devices is not supported
1037          * when more than default init_net exists.
1038          */
1039         xa_for_each (&rdma_nets, index, rnet) {
1040                 ret++;
1041                 break;
1042         }
1043         if (!ret)
1044                 ib_devices_shared_netns = enable;
1045         up_write(&rdma_nets_rwsem);
1046         if (ret)
1047                 return -EBUSY;
1048 
1049         if (enable)
1050                 ret = add_all_compat_devs();
1051         else
1052                 remove_all_compat_devs();
1053         return ret;
1054 }
1055 
1056 static void rdma_dev_exit_net(struct net *net)
1057 {
1058         struct rdma_dev_net *rnet = rdma_net_to_dev_net(net);
1059         struct ib_device *dev;
1060         unsigned long index;
1061         int ret;
1062 
1063         down_write(&rdma_nets_rwsem);
1064         /*
1065          * Prevent the ID from being re-used and hide the id from xa_for_each.
1066          */
1067         ret = xa_err(xa_store(&rdma_nets, rnet->id, NULL, GFP_KERNEL));
1068         WARN_ON(ret);
1069         up_write(&rdma_nets_rwsem);
1070 
1071         down_read(&devices_rwsem);
1072         xa_for_each (&devices, index, dev) {
1073                 get_device(&dev->dev);
1074                 /*
1075                  * Release the devices_rwsem so that pontentially blocking
1076                  * device_del, doesn't hold the devices_rwsem for too long.
1077                  */
1078                 up_read(&devices_rwsem);
1079 
1080                 remove_one_compat_dev(dev, rnet->id);
1081 
1082                 /*
1083                  * If the real device is in the NS then move it back to init.
1084                  */
1085                 rdma_dev_change_netns(dev, net, &init_net);
1086 
1087                 put_device(&dev->dev);
1088                 down_read(&devices_rwsem);
1089         }
1090         up_read(&devices_rwsem);
1091 
1092         rdma_nl_net_exit(rnet);
1093         xa_erase(&rdma_nets, rnet->id);
1094 }
1095 
1096 static __net_init int rdma_dev_init_net(struct net *net)
1097 {
1098         struct rdma_dev_net *rnet = rdma_net_to_dev_net(net);
1099         unsigned long index;
1100         struct ib_device *dev;
1101         int ret;
1102 
1103         write_pnet(&rnet->net, net);
1104 
1105         ret = rdma_nl_net_init(rnet);
1106         if (ret)
1107                 return ret;
1108 
1109         /* No need to create any compat devices in default init_net. */
1110         if (net_eq(net, &init_net))
1111                 return 0;
1112 
1113         ret = xa_alloc(&rdma_nets, &rnet->id, rnet, xa_limit_32b, GFP_KERNEL);
1114         if (ret) {
1115                 rdma_nl_net_exit(rnet);
1116                 return ret;
1117         }
1118 
1119         down_read(&devices_rwsem);
1120         xa_for_each_marked (&devices, index, dev, DEVICE_REGISTERED) {
1121                 /* Hold nets_rwsem so that netlink command cannot change
1122                  * system configuration for device sharing mode.
1123                  */
1124                 down_read(&rdma_nets_rwsem);
1125                 ret = add_one_compat_dev(dev, rnet);
1126                 up_read(&rdma_nets_rwsem);
1127                 if (ret)
1128                         break;
1129         }
1130         up_read(&devices_rwsem);
1131 
1132         if (ret)
1133                 rdma_dev_exit_net(net);
1134 
1135         return ret;
1136 }
1137 
1138 /*
1139  * Assign the unique string device name and the unique device index. This is
1140  * undone by ib_dealloc_device.
1141  */
1142 static int assign_name(struct ib_device *device, const char *name)
1143 {
1144         static u32 last_id;
1145         int ret;
1146 
1147         down_write(&devices_rwsem);
1148         /* Assign a unique name to the device */
1149         if (strchr(name, '%'))
1150                 ret = alloc_name(device, name);
1151         else
1152                 ret = dev_set_name(&device->dev, name);
1153         if (ret)
1154                 goto out;
1155 
1156         if (__ib_device_get_by_name(dev_name(&device->dev))) {
1157                 ret = -ENFILE;
1158                 goto out;
1159         }
1160         strlcpy(device->name, dev_name(&device->dev), IB_DEVICE_NAME_MAX);
1161 
1162         ret = xa_alloc_cyclic(&devices, &device->index, device, xa_limit_31b,
1163                         &last_id, GFP_KERNEL);
1164         if (ret > 0)
1165                 ret = 0;
1166 
1167 out:
1168         up_write(&devices_rwsem);
1169         return ret;
1170 }
1171 
1172 static void setup_dma_device(struct ib_device *device)
1173 {
1174         struct device *parent = device->dev.parent;
1175 
1176         WARN_ON_ONCE(device->dma_device);
1177         if (device->dev.dma_ops) {
1178                 /*
1179                  * The caller provided custom DMA operations. Copy the
1180                  * DMA-related fields that are used by e.g. dma_alloc_coherent()
1181                  * into device->dev.
1182                  */
1183                 device->dma_device = &device->dev;
1184                 if (!device->dev.dma_mask) {
1185                         if (parent)
1186                                 device->dev.dma_mask = parent->dma_mask;
1187                         else
1188                                 WARN_ON_ONCE(true);
1189                 }
1190                 if (!device->dev.coherent_dma_mask) {
1191                         if (parent)
1192                                 device->dev.coherent_dma_mask =
1193                                         parent->coherent_dma_mask;
1194                         else
1195                                 WARN_ON_ONCE(true);
1196                 }
1197         } else {
1198                 /*
1199                  * The caller did not provide custom DMA operations. Use the
1200                  * DMA mapping operations of the parent device.
1201                  */
1202                 WARN_ON_ONCE(!parent);
1203                 device->dma_device = parent;
1204         }
1205 
1206         if (!device->dev.dma_parms) {
1207                 if (parent) {
1208                         /*
1209                          * The caller did not provide DMA parameters, so
1210                          * 'parent' probably represents a PCI device. The PCI
1211                          * core sets the maximum segment size to 64
1212                          * KB. Increase this parameter to 2 GB.
1213                          */
1214                         device->dev.dma_parms = parent->dma_parms;
1215                         dma_set_max_seg_size(device->dma_device, SZ_2G);
1216                 } else {
1217                         WARN_ON_ONCE(true);
1218                 }
1219         }
1220 }
1221 
1222 /*
1223  * setup_device() allocates memory and sets up data that requires calling the
1224  * device ops, this is the only reason these actions are not done during
1225  * ib_alloc_device. It is undone by ib_dealloc_device().
1226  */
1227 static int setup_device(struct ib_device *device)
1228 {
1229         struct ib_udata uhw = {.outlen = 0, .inlen = 0};
1230         int ret;
1231 
1232         setup_dma_device(device);
1233         ib_device_check_mandatory(device);
1234 
1235         ret = setup_port_data(device);
1236         if (ret) {
1237                 dev_warn(&device->dev, "Couldn't create per-port data\n");
1238                 return ret;
1239         }
1240 
1241         memset(&device->attrs, 0, sizeof(device->attrs));
1242         ret = device->ops.query_device(device, &device->attrs, &uhw);
1243         if (ret) {
1244                 dev_warn(&device->dev,
1245                          "Couldn't query the device attributes\n");
1246                 return ret;
1247         }
1248 
1249         return 0;
1250 }
1251 
1252 static void disable_device(struct ib_device *device)
1253 {
1254         u32 cid;
1255 
1256         WARN_ON(!refcount_read(&device->refcount));
1257 
1258         down_write(&devices_rwsem);
1259         xa_clear_mark(&devices, device->index, DEVICE_REGISTERED);
1260         up_write(&devices_rwsem);
1261 
1262         /*
1263          * Remove clients in LIFO order, see assign_client_id. This could be
1264          * more efficient if xarray learns to reverse iterate. Since no new
1265          * clients can be added to this ib_device past this point we only need
1266          * the maximum possible client_id value here.
1267          */
1268         down_read(&clients_rwsem);
1269         cid = highest_client_id;
1270         up_read(&clients_rwsem);
1271         while (cid) {
1272                 cid--;
1273                 remove_client_context(device, cid);
1274         }
1275 
1276         /* Pairs with refcount_set in enable_device */
1277         ib_device_put(device);
1278         wait_for_completion(&device->unreg_completion);
1279 
1280         /*
1281          * compat devices must be removed after device refcount drops to zero.
1282          * Otherwise init_net() may add more compatdevs after removing compat
1283          * devices and before device is disabled.
1284          */
1285         remove_compat_devs(device);
1286 }
1287 
1288 /*
1289  * An enabled device is visible to all clients and to all the public facing
1290  * APIs that return a device pointer. This always returns with a new get, even
1291  * if it fails.
1292  */
1293 static int enable_device_and_get(struct ib_device *device)
1294 {
1295         struct ib_client *client;
1296         unsigned long index;
1297         int ret = 0;
1298 
1299         /*
1300          * One ref belongs to the xa and the other belongs to this
1301          * thread. This is needed to guard against parallel unregistration.
1302          */
1303         refcount_set(&device->refcount, 2);
1304         down_write(&devices_rwsem);
1305         xa_set_mark(&devices, device->index, DEVICE_REGISTERED);
1306 
1307         /*
1308          * By using downgrade_write() we ensure that no other thread can clear
1309          * DEVICE_REGISTERED while we are completing the client setup.
1310          */
1311         downgrade_write(&devices_rwsem);
1312 
1313         if (device->ops.enable_driver) {
1314                 ret = device->ops.enable_driver(device);
1315                 if (ret)
1316                         goto out;
1317         }
1318 
1319         down_read(&clients_rwsem);
1320         xa_for_each_marked (&clients, index, client, CLIENT_REGISTERED) {
1321                 ret = add_client_context(device, client);
1322                 if (ret)
1323                         break;
1324         }
1325         up_read(&clients_rwsem);
1326         if (!ret)
1327                 ret = add_compat_devs(device);
1328 out:
1329         up_read(&devices_rwsem);
1330         return ret;
1331 }
1332 
1333 /**
1334  * ib_register_device - Register an IB device with IB core
1335  * @device:Device to register
1336  *
1337  * Low-level drivers use ib_register_device() to register their
1338  * devices with the IB core.  All registered clients will receive a
1339  * callback for each device that is added. @device must be allocated
1340  * with ib_alloc_device().
1341  *
1342  * If the driver uses ops.dealloc_driver and calls any ib_unregister_device()
1343  * asynchronously then the device pointer may become freed as soon as this
1344  * function returns.
1345  */
1346 int ib_register_device(struct ib_device *device, const char *name)
1347 {
1348         int ret;
1349 
1350         ret = assign_name(device, name);
1351         if (ret)
1352                 return ret;
1353 
1354         ret = setup_device(device);
1355         if (ret)
1356                 return ret;
1357 
1358         ret = ib_cache_setup_one(device);
1359         if (ret) {
1360                 dev_warn(&device->dev,
1361                          "Couldn't set up InfiniBand P_Key/GID cache\n");
1362                 return ret;
1363         }
1364 
1365         ib_device_register_rdmacg(device);
1366 
1367         rdma_counter_init(device);
1368 
1369         /*
1370          * Ensure that ADD uevent is not fired because it
1371          * is too early amd device is not initialized yet.
1372          */
1373         dev_set_uevent_suppress(&device->dev, true);
1374         ret = device_add(&device->dev);
1375         if (ret)
1376                 goto cg_cleanup;
1377 
1378         ret = ib_device_register_sysfs(device);
1379         if (ret) {
1380                 dev_warn(&device->dev,
1381                          "Couldn't register device with driver model\n");
1382                 goto dev_cleanup;
1383         }
1384 
1385         ret = enable_device_and_get(device);
1386         dev_set_uevent_suppress(&device->dev, false);
1387         /* Mark for userspace that device is ready */
1388         kobject_uevent(&device->dev.kobj, KOBJ_ADD);
1389         if (ret) {
1390                 void (*dealloc_fn)(struct ib_device *);
1391 
1392                 /*
1393                  * If we hit this error flow then we don't want to
1394                  * automatically dealloc the device since the caller is
1395                  * expected to call ib_dealloc_device() after
1396                  * ib_register_device() fails. This is tricky due to the
1397                  * possibility for a parallel unregistration along with this
1398                  * error flow. Since we have a refcount here we know any
1399                  * parallel flow is stopped in disable_device and will see the
1400                  * NULL pointers, causing the responsibility to
1401                  * ib_dealloc_device() to revert back to this thread.
1402                  */
1403                 dealloc_fn = device->ops.dealloc_driver;
1404                 device->ops.dealloc_driver = NULL;
1405                 ib_device_put(device);
1406                 __ib_unregister_device(device);
1407                 device->ops.dealloc_driver = dealloc_fn;
1408                 return ret;
1409         }
1410         ib_device_put(device);
1411 
1412         return 0;
1413 
1414 dev_cleanup:
1415         device_del(&device->dev);
1416 cg_cleanup:
1417         dev_set_uevent_suppress(&device->dev, false);
1418         ib_device_unregister_rdmacg(device);
1419         ib_cache_cleanup_one(device);
1420         return ret;
1421 }
1422 EXPORT_SYMBOL(ib_register_device);
1423 
1424 /* Callers must hold a get on the device. */
1425 static void __ib_unregister_device(struct ib_device *ib_dev)
1426 {
1427         /*
1428          * We have a registration lock so that all the calls to unregister are
1429          * fully fenced, once any unregister returns the device is truely
1430          * unregistered even if multiple callers are unregistering it at the
1431          * same time. This also interacts with the registration flow and
1432          * provides sane semantics if register and unregister are racing.
1433          */
1434         mutex_lock(&ib_dev->unregistration_lock);
1435         if (!refcount_read(&ib_dev->refcount))
1436                 goto out;
1437 
1438         disable_device(ib_dev);
1439 
1440         /* Expedite removing unregistered pointers from the hash table */
1441         free_netdevs(ib_dev);
1442 
1443         ib_device_unregister_sysfs(ib_dev);
1444         device_del(&ib_dev->dev);
1445         ib_device_unregister_rdmacg(ib_dev);
1446         ib_cache_cleanup_one(ib_dev);
1447 
1448         /*
1449          * Drivers using the new flow may not call ib_dealloc_device except
1450          * in error unwind prior to registration success.
1451          */
1452         if (ib_dev->ops.dealloc_driver) {
1453                 WARN_ON(kref_read(&ib_dev->dev.kobj.kref) <= 1);
1454                 ib_dealloc_device(ib_dev);
1455         }
1456 out:
1457         mutex_unlock(&ib_dev->unregistration_lock);
1458 }
1459 
1460 /**
1461  * ib_unregister_device - Unregister an IB device
1462  * @device: The device to unregister
1463  *
1464  * Unregister an IB device.  All clients will receive a remove callback.
1465  *
1466  * Callers should call this routine only once, and protect against races with
1467  * registration. Typically it should only be called as part of a remove
1468  * callback in an implementation of driver core's struct device_driver and
1469  * related.
1470  *
1471  * If ops.dealloc_driver is used then ib_dev will be freed upon return from
1472  * this function.
1473  */
1474 void ib_unregister_device(struct ib_device *ib_dev)
1475 {
1476         get_device(&ib_dev->dev);
1477         __ib_unregister_device(ib_dev);
1478         put_device(&ib_dev->dev);
1479 }
1480 EXPORT_SYMBOL(ib_unregister_device);
1481 
1482 /**
1483  * ib_unregister_device_and_put - Unregister a device while holding a 'get'
1484  * device: The device to unregister
1485  *
1486  * This is the same as ib_unregister_device(), except it includes an internal
1487  * ib_device_put() that should match a 'get' obtained by the caller.
1488  *
1489  * It is safe to call this routine concurrently from multiple threads while
1490  * holding the 'get'. When the function returns the device is fully
1491  * unregistered.
1492  *
1493  * Drivers using this flow MUST use the driver_unregister callback to clean up
1494  * their resources associated with the device and dealloc it.
1495  */
1496 void ib_unregister_device_and_put(struct ib_device *ib_dev)
1497 {
1498         WARN_ON(!ib_dev->ops.dealloc_driver);
1499         get_device(&ib_dev->dev);
1500         ib_device_put(ib_dev);
1501         __ib_unregister_device(ib_dev);
1502         put_device(&ib_dev->dev);
1503 }
1504 EXPORT_SYMBOL(ib_unregister_device_and_put);
1505 
1506 /**
1507  * ib_unregister_driver - Unregister all IB devices for a driver
1508  * @driver_id: The driver to unregister
1509  *
1510  * This implements a fence for device unregistration. It only returns once all
1511  * devices associated with the driver_id have fully completed their
1512  * unregistration and returned from ib_unregister_device*().
1513  *
1514  * If device's are not yet unregistered it goes ahead and starts unregistering
1515  * them.
1516  *
1517  * This does not block creation of new devices with the given driver_id, that
1518  * is the responsibility of the caller.
1519  */
1520 void ib_unregister_driver(enum rdma_driver_id driver_id)
1521 {
1522         struct ib_device *ib_dev;
1523         unsigned long index;
1524 
1525         down_read(&devices_rwsem);
1526         xa_for_each (&devices, index, ib_dev) {
1527                 if (ib_dev->ops.driver_id != driver_id)
1528                         continue;
1529 
1530                 get_device(&ib_dev->dev);
1531                 up_read(&devices_rwsem);
1532 
1533                 WARN_ON(!ib_dev->ops.dealloc_driver);
1534                 __ib_unregister_device(ib_dev);
1535 
1536                 put_device(&ib_dev->dev);
1537                 down_read(&devices_rwsem);
1538         }
1539         up_read(&devices_rwsem);
1540 }
1541 EXPORT_SYMBOL(ib_unregister_driver);
1542 
1543 static void ib_unregister_work(struct work_struct *work)
1544 {
1545         struct ib_device *ib_dev =
1546                 container_of(work, struct ib_device, unregistration_work);
1547 
1548         __ib_unregister_device(ib_dev);
1549         put_device(&ib_dev->dev);
1550 }
1551 
1552 /**
1553  * ib_unregister_device_queued - Unregister a device using a work queue
1554  * device: The device to unregister
1555  *
1556  * This schedules an asynchronous unregistration using a WQ for the device. A
1557  * driver should use this to avoid holding locks while doing unregistration,
1558  * such as holding the RTNL lock.
1559  *
1560  * Drivers using this API must use ib_unregister_driver before module unload
1561  * to ensure that all scheduled unregistrations have completed.
1562  */
1563 void ib_unregister_device_queued(struct ib_device *ib_dev)
1564 {
1565         WARN_ON(!refcount_read(&ib_dev->refcount));
1566         WARN_ON(!ib_dev->ops.dealloc_driver);
1567         get_device(&ib_dev->dev);
1568         if (!queue_work(system_unbound_wq, &ib_dev->unregistration_work))
1569                 put_device(&ib_dev->dev);
1570 }
1571 EXPORT_SYMBOL(ib_unregister_device_queued);
1572 
1573 /*
1574  * The caller must pass in a device that has the kref held and the refcount
1575  * released. If the device is in cur_net and still registered then it is moved
1576  * into net.
1577  */
1578 static int rdma_dev_change_netns(struct ib_device *device, struct net *cur_net,
1579                                  struct net *net)
1580 {
1581         int ret2 = -EINVAL;
1582         int ret;
1583 
1584         mutex_lock(&device->unregistration_lock);
1585 
1586         /*
1587          * If a device not under ib_device_get() or if the unregistration_lock
1588          * is not held, the namespace can be changed, or it can be unregistered.
1589          * Check again under the lock.
1590          */
1591         if (refcount_read(&device->refcount) == 0 ||
1592             !net_eq(cur_net, read_pnet(&device->coredev.rdma_net))) {
1593                 ret = -ENODEV;
1594                 goto out;
1595         }
1596 
1597         kobject_uevent(&device->dev.kobj, KOBJ_REMOVE);
1598         disable_device(device);
1599 
1600         /*
1601          * At this point no one can be using the device, so it is safe to
1602          * change the namespace.
1603          */
1604         write_pnet(&device->coredev.rdma_net, net);
1605 
1606         down_read(&devices_rwsem);
1607         /*
1608          * Currently rdma devices are system wide unique. So the device name
1609          * is guaranteed free in the new namespace. Publish the new namespace
1610          * at the sysfs level.
1611          */
1612         ret = device_rename(&device->dev, dev_name(&device->dev));
1613         up_read(&devices_rwsem);
1614         if (ret) {
1615                 dev_warn(&device->dev,
1616                          "%s: Couldn't rename device after namespace change\n",
1617                          __func__);
1618                 /* Try and put things back and re-enable the device */
1619                 write_pnet(&device->coredev.rdma_net, cur_net);
1620         }
1621 
1622         ret2 = enable_device_and_get(device);
1623         if (ret2) {
1624                 /*
1625                  * This shouldn't really happen, but if it does, let the user
1626                  * retry at later point. So don't disable the device.
1627                  */
1628                 dev_warn(&device->dev,
1629                          "%s: Couldn't re-enable device after namespace change\n",
1630                          __func__);
1631         }
1632         kobject_uevent(&device->dev.kobj, KOBJ_ADD);
1633 
1634         ib_device_put(device);
1635 out:
1636         mutex_unlock(&device->unregistration_lock);
1637         if (ret)
1638                 return ret;
1639         return ret2;
1640 }
1641 
1642 int ib_device_set_netns_put(struct sk_buff *skb,
1643                             struct ib_device *dev, u32 ns_fd)
1644 {
1645         struct net *net;
1646         int ret;
1647 
1648         net = get_net_ns_by_fd(ns_fd);
1649         if (IS_ERR(net)) {
1650                 ret = PTR_ERR(net);
1651                 goto net_err;
1652         }
1653 
1654         if (!netlink_ns_capable(skb, net->user_ns, CAP_NET_ADMIN)) {
1655                 ret = -EPERM;
1656                 goto ns_err;
1657         }
1658 
1659         /*
1660          * Currently supported only for those providers which support
1661          * disassociation and don't do port specific sysfs init. Once a
1662          * port_cleanup infrastructure is implemented, this limitation will be
1663          * removed.
1664          */
1665         if (!dev->ops.disassociate_ucontext || dev->ops.init_port ||
1666             ib_devices_shared_netns) {
1667                 ret = -EOPNOTSUPP;
1668                 goto ns_err;
1669         }
1670 
1671         get_device(&dev->dev);
1672         ib_device_put(dev);
1673         ret = rdma_dev_change_netns(dev, current->nsproxy->net_ns, net);
1674         put_device(&dev->dev);
1675 
1676         put_net(net);
1677         return ret;
1678 
1679 ns_err:
1680         put_net(net);
1681 net_err:
1682         ib_device_put(dev);
1683         return ret;
1684 }
1685 
1686 static struct pernet_operations rdma_dev_net_ops = {
1687         .init = rdma_dev_init_net,
1688         .exit = rdma_dev_exit_net,
1689         .id = &rdma_dev_net_id,
1690         .size = sizeof(struct rdma_dev_net),
1691 };
1692 
1693 static int assign_client_id(struct ib_client *client)
1694 {
1695         int ret;
1696 
1697         down_write(&clients_rwsem);
1698         /*
1699          * The add/remove callbacks must be called in FIFO/LIFO order. To
1700          * achieve this we assign client_ids so they are sorted in
1701          * registration order.
1702          */
1703         client->client_id = highest_client_id;
1704         ret = xa_insert(&clients, client->client_id, client, GFP_KERNEL);
1705         if (ret)
1706                 goto out;
1707 
1708         highest_client_id++;
1709         xa_set_mark(&clients, client->client_id, CLIENT_REGISTERED);
1710 
1711 out:
1712         up_write(&clients_rwsem);
1713         return ret;
1714 }
1715 
1716 static void remove_client_id(struct ib_client *client)
1717 {
1718         down_write(&clients_rwsem);
1719         xa_erase(&clients, client->client_id);
1720         for (; highest_client_id; highest_client_id--)
1721                 if (xa_load(&clients, highest_client_id - 1))
1722                         break;
1723         up_write(&clients_rwsem);
1724 }
1725 
1726 /**
1727  * ib_register_client - Register an IB client
1728  * @client:Client to register
1729  *
1730  * Upper level users of the IB drivers can use ib_register_client() to
1731  * register callbacks for IB device addition and removal.  When an IB
1732  * device is added, each registered client's add method will be called
1733  * (in the order the clients were registered), and when a device is
1734  * removed, each client's remove method will be called (in the reverse
1735  * order that clients were registered).  In addition, when
1736  * ib_register_client() is called, the client will receive an add
1737  * callback for all devices already registered.
1738  */
1739 int ib_register_client(struct ib_client *client)
1740 {
1741         struct ib_device *device;
1742         unsigned long index;
1743         int ret;
1744 
1745         refcount_set(&client->uses, 1);
1746         init_completion(&client->uses_zero);
1747         ret = assign_client_id(client);
1748         if (ret)
1749                 return ret;
1750 
1751         down_read(&devices_rwsem);
1752         xa_for_each_marked (&devices, index, device, DEVICE_REGISTERED) {
1753                 ret = add_client_context(device, client);
1754                 if (ret) {
1755                         up_read(&devices_rwsem);
1756                         ib_unregister_client(client);
1757                         return ret;
1758                 }
1759         }
1760         up_read(&devices_rwsem);
1761         return 0;
1762 }
1763 EXPORT_SYMBOL(ib_register_client);
1764 
1765 /**
1766  * ib_unregister_client - Unregister an IB client
1767  * @client:Client to unregister
1768  *
1769  * Upper level users use ib_unregister_client() to remove their client
1770  * registration.  When ib_unregister_client() is called, the client
1771  * will receive a remove callback for each IB device still registered.
1772  *
1773  * This is a full fence, once it returns no client callbacks will be called,
1774  * or are running in another thread.
1775  */
1776 void ib_unregister_client(struct ib_client *client)
1777 {
1778         struct ib_device *device;
1779         unsigned long index;
1780 
1781         down_write(&clients_rwsem);
1782         ib_client_put(client);
1783         xa_clear_mark(&clients, client->client_id, CLIENT_REGISTERED);
1784         up_write(&clients_rwsem);
1785 
1786         /* We do not want to have locks while calling client->remove() */
1787         rcu_read_lock();
1788         xa_for_each (&devices, index, device) {
1789                 if (!ib_device_try_get(device))
1790                         continue;
1791                 rcu_read_unlock();
1792 
1793                 remove_client_context(device, client->client_id);
1794 
1795                 ib_device_put(device);
1796                 rcu_read_lock();
1797         }
1798         rcu_read_unlock();
1799 
1800         /*
1801          * remove_client_context() is not a fence, it can return even though a
1802          * removal is ongoing. Wait until all removals are completed.
1803          */
1804         wait_for_completion(&client->uses_zero);
1805         remove_client_id(client);
1806 }
1807 EXPORT_SYMBOL(ib_unregister_client);
1808 
1809 static int __ib_get_global_client_nl_info(const char *client_name,
1810                                           struct ib_client_nl_info *res)
1811 {
1812         struct ib_client *client;
1813         unsigned long index;
1814         int ret = -ENOENT;
1815 
1816         down_read(&clients_rwsem);
1817         xa_for_each_marked (&clients, index, client, CLIENT_REGISTERED) {
1818                 if (strcmp(client->name, client_name) != 0)
1819                         continue;
1820                 if (!client->get_global_nl_info) {
1821                         ret = -EOPNOTSUPP;
1822                         break;
1823                 }
1824                 ret = client->get_global_nl_info(res);
1825                 if (WARN_ON(ret == -ENOENT))
1826                         ret = -EINVAL;
1827                 if (!ret && res->cdev)
1828                         get_device(res->cdev);
1829                 break;
1830         }
1831         up_read(&clients_rwsem);
1832         return ret;
1833 }
1834 
1835 static int __ib_get_client_nl_info(struct ib_device *ibdev,
1836                                    const char *client_name,
1837                                    struct ib_client_nl_info *res)
1838 {
1839         unsigned long index;
1840         void *client_data;
1841         int ret = -ENOENT;
1842 
1843         down_read(&ibdev->client_data_rwsem);
1844         xan_for_each_marked (&ibdev->client_data, index, client_data,
1845                              CLIENT_DATA_REGISTERED) {
1846                 struct ib_client *client = xa_load(&clients, index);
1847 
1848                 if (!client || strcmp(client->name, client_name) != 0)
1849                         continue;
1850                 if (!client->get_nl_info) {
1851                         ret = -EOPNOTSUPP;
1852                         break;
1853                 }
1854                 ret = client->get_nl_info(ibdev, client_data, res);
1855                 if (WARN_ON(ret == -ENOENT))
1856                         ret = -EINVAL;
1857 
1858                 /*
1859                  * The cdev is guaranteed valid as long as we are inside the
1860                  * client_data_rwsem as remove_one can't be called. Keep it
1861                  * valid for the caller.
1862                  */
1863                 if (!ret && res->cdev)
1864                         get_device(res->cdev);
1865                 break;
1866         }
1867         up_read(&ibdev->client_data_rwsem);
1868 
1869         return ret;
1870 }
1871 
1872 /**
1873  * ib_get_client_nl_info - Fetch the nl_info from a client
1874  * @device - IB device
1875  * @client_name - Name of the client
1876  * @res - Result of the query
1877  */
1878 int ib_get_client_nl_info(struct ib_device *ibdev, const char *client_name,
1879                           struct ib_client_nl_info *res)
1880 {
1881         int ret;
1882 
1883         if (ibdev)
1884                 ret = __ib_get_client_nl_info(ibdev, client_name, res);
1885         else
1886                 ret = __ib_get_global_client_nl_info(client_name, res);
1887 #ifdef CONFIG_MODULES
1888         if (ret == -ENOENT) {
1889                 request_module("rdma-client-%s", client_name);
1890                 if (ibdev)
1891                         ret = __ib_get_client_nl_info(ibdev, client_name, res);
1892                 else
1893                         ret = __ib_get_global_client_nl_info(client_name, res);
1894         }
1895 #endif
1896         if (ret) {
1897                 if (ret == -ENOENT)
1898                         return -EOPNOTSUPP;
1899                 return ret;
1900         }
1901 
1902         if (WARN_ON(!res->cdev))
1903                 return -EINVAL;
1904         return 0;
1905 }
1906 
1907 /**
1908  * ib_set_client_data - Set IB client context
1909  * @device:Device to set context for
1910  * @client:Client to set context for
1911  * @data:Context to set
1912  *
1913  * ib_set_client_data() sets client context data that can be retrieved with
1914  * ib_get_client_data(). This can only be called while the client is
1915  * registered to the device, once the ib_client remove() callback returns this
1916  * cannot be called.
1917  */
1918 void ib_set_client_data(struct ib_device *device, struct ib_client *client,
1919                         void *data)
1920 {
1921         void *rc;
1922 
1923         if (WARN_ON(IS_ERR(data)))
1924                 data = NULL;
1925 
1926         rc = xa_store(&device->client_data, client->client_id, data,
1927                       GFP_KERNEL);
1928         WARN_ON(xa_is_err(rc));
1929 }
1930 EXPORT_SYMBOL(ib_set_client_data);
1931 
1932 /**
1933  * ib_register_event_handler - Register an IB event handler
1934  * @event_handler:Handler to register
1935  *
1936  * ib_register_event_handler() registers an event handler that will be
1937  * called back when asynchronous IB events occur (as defined in
1938  * chapter 11 of the InfiniBand Architecture Specification). This
1939  * callback occurs in workqueue context.
1940  */
1941 void ib_register_event_handler(struct ib_event_handler *event_handler)
1942 {
1943         down_write(&event_handler->device->event_handler_rwsem);
1944         list_add_tail(&event_handler->list,
1945                       &event_handler->device->event_handler_list);
1946         up_write(&event_handler->device->event_handler_rwsem);
1947 }
1948 EXPORT_SYMBOL(ib_register_event_handler);
1949 
1950 /**
1951  * ib_unregister_event_handler - Unregister an event handler
1952  * @event_handler:Handler to unregister
1953  *
1954  * Unregister an event handler registered with
1955  * ib_register_event_handler().
1956  */
1957 void ib_unregister_event_handler(struct ib_event_handler *event_handler)
1958 {
1959         down_write(&event_handler->device->event_handler_rwsem);
1960         list_del(&event_handler->list);
1961         up_write(&event_handler->device->event_handler_rwsem);
1962 }
1963 EXPORT_SYMBOL(ib_unregister_event_handler);
1964 
1965 void ib_dispatch_event_clients(struct ib_event *event)
1966 {
1967         struct ib_event_handler *handler;
1968 
1969         down_read(&event->device->event_handler_rwsem);
1970 
1971         list_for_each_entry(handler, &event->device->event_handler_list, list)
1972                 handler->handler(handler, event);
1973 
1974         up_read(&event->device->event_handler_rwsem);
1975 }
1976 
1977 static int iw_query_port(struct ib_device *device,
1978                            u8 port_num,
1979                            struct ib_port_attr *port_attr)
1980 {
1981         struct in_device *inetdev;
1982         struct net_device *netdev;
1983         int err;
1984 
1985         memset(port_attr, 0, sizeof(*port_attr));
1986 
1987         netdev = ib_device_get_netdev(device, port_num);
1988         if (!netdev)
1989                 return -ENODEV;
1990 
1991         port_attr->max_mtu = IB_MTU_4096;
1992         port_attr->active_mtu = ib_mtu_int_to_enum(netdev->mtu);
1993 
1994         if (!netif_carrier_ok(netdev)) {
1995                 port_attr->state = IB_PORT_DOWN;
1996                 port_attr->phys_state = IB_PORT_PHYS_STATE_DISABLED;
1997         } else {
1998                 rcu_read_lock();
1999                 inetdev = __in_dev_get_rcu(netdev);
2000 
2001                 if (inetdev && inetdev->ifa_list) {
2002                         port_attr->state = IB_PORT_ACTIVE;
2003                         port_attr->phys_state = IB_PORT_PHYS_STATE_LINK_UP;
2004                 } else {
2005                         port_attr->state = IB_PORT_INIT;
2006                         port_attr->phys_state =
2007                                 IB_PORT_PHYS_STATE_PORT_CONFIGURATION_TRAINING;
2008                 }
2009 
2010                 rcu_read_unlock();
2011         }
2012 
2013         dev_put(netdev);
2014         err = device->ops.query_port(device, port_num, port_attr);
2015         if (err)
2016                 return err;
2017 
2018         return 0;
2019 }
2020 
2021 static int __ib_query_port(struct ib_device *device,
2022                            u8 port_num,
2023                            struct ib_port_attr *port_attr)
2024 {
2025         union ib_gid gid = {};
2026         int err;
2027 
2028         memset(port_attr, 0, sizeof(*port_attr));
2029 
2030         err = device->ops.query_port(device, port_num, port_attr);
2031         if (err || port_attr->subnet_prefix)
2032                 return err;
2033 
2034         if (rdma_port_get_link_layer(device, port_num) !=
2035             IB_LINK_LAYER_INFINIBAND)
2036                 return 0;
2037 
2038         err = device->ops.query_gid(device, port_num, 0, &gid);
2039         if (err)
2040                 return err;
2041 
2042         port_attr->subnet_prefix = be64_to_cpu(gid.global.subnet_prefix);
2043         return 0;
2044 }
2045 
2046 /**
2047  * ib_query_port - Query IB port attributes
2048  * @device:Device to query
2049  * @port_num:Port number to query
2050  * @port_attr:Port attributes
2051  *
2052  * ib_query_port() returns the attributes of a port through the
2053  * @port_attr pointer.
2054  */
2055 int ib_query_port(struct ib_device *device,
2056                   u8 port_num,
2057                   struct ib_port_attr *port_attr)
2058 {
2059         if (!rdma_is_port_valid(device, port_num))
2060                 return -EINVAL;
2061 
2062         if (rdma_protocol_iwarp(device, port_num))
2063                 return iw_query_port(device, port_num, port_attr);
2064         else
2065                 return __ib_query_port(device, port_num, port_attr);
2066 }
2067 EXPORT_SYMBOL(ib_query_port);
2068 
2069 static void add_ndev_hash(struct ib_port_data *pdata)
2070 {
2071         unsigned long flags;
2072 
2073         might_sleep();
2074 
2075         spin_lock_irqsave(&ndev_hash_lock, flags);
2076         if (hash_hashed(&pdata->ndev_hash_link)) {
2077                 hash_del_rcu(&pdata->ndev_hash_link);
2078                 spin_unlock_irqrestore(&ndev_hash_lock, flags);
2079                 /*
2080                  * We cannot do hash_add_rcu after a hash_del_rcu until the
2081                  * grace period
2082                  */
2083                 synchronize_rcu();
2084                 spin_lock_irqsave(&ndev_hash_lock, flags);
2085         }
2086         if (pdata->netdev)
2087                 hash_add_rcu(ndev_hash, &pdata->ndev_hash_link,
2088                              (uintptr_t)pdata->netdev);
2089         spin_unlock_irqrestore(&ndev_hash_lock, flags);
2090 }
2091 
2092 /**
2093  * ib_device_set_netdev - Associate the ib_dev with an underlying net_device
2094  * @ib_dev: Device to modify
2095  * @ndev: net_device to affiliate, may be NULL
2096  * @port: IB port the net_device is connected to
2097  *
2098  * Drivers should use this to link the ib_device to a netdev so the netdev
2099  * shows up in interfaces like ib_enum_roce_netdev. Only one netdev may be
2100  * affiliated with any port.
2101  *
2102  * The caller must ensure that the given ndev is not unregistered or
2103  * unregistering, and that either the ib_device is unregistered or
2104  * ib_device_set_netdev() is called with NULL when the ndev sends a
2105  * NETDEV_UNREGISTER event.
2106  */
2107 int ib_device_set_netdev(struct ib_device *ib_dev, struct net_device *ndev,
2108                          unsigned int port)
2109 {
2110         struct net_device *old_ndev;
2111         struct ib_port_data *pdata;
2112         unsigned long flags;
2113         int ret;
2114 
2115         /*
2116          * Drivers wish to call this before ib_register_driver, so we have to
2117          * setup the port data early.
2118          */
2119         ret = alloc_port_data(ib_dev);
2120         if (ret)
2121                 return ret;
2122 
2123         if (!rdma_is_port_valid(ib_dev, port))
2124                 return -EINVAL;
2125 
2126         pdata = &ib_dev->port_data[port];
2127         spin_lock_irqsave(&pdata->netdev_lock, flags);
2128         old_ndev = rcu_dereference_protected(
2129                 pdata->netdev, lockdep_is_held(&pdata->netdev_lock));
2130         if (old_ndev == ndev) {
2131                 spin_unlock_irqrestore(&pdata->netdev_lock, flags);
2132                 return 0;
2133         }
2134 
2135         if (ndev)
2136                 dev_hold(ndev);
2137         rcu_assign_pointer(pdata->netdev, ndev);
2138         spin_unlock_irqrestore(&pdata->netdev_lock, flags);
2139 
2140         add_ndev_hash(pdata);
2141         if (old_ndev)
2142                 dev_put(old_ndev);
2143 
2144         return 0;
2145 }
2146 EXPORT_SYMBOL(ib_device_set_netdev);
2147 
2148 static void free_netdevs(struct ib_device *ib_dev)
2149 {
2150         unsigned long flags;
2151         unsigned int port;
2152 
2153         if (!ib_dev->port_data)
2154                 return;
2155 
2156         rdma_for_each_port (ib_dev, port) {
2157                 struct ib_port_data *pdata = &ib_dev->port_data[port];
2158                 struct net_device *ndev;
2159 
2160                 spin_lock_irqsave(&pdata->netdev_lock, flags);
2161                 ndev = rcu_dereference_protected(
2162                         pdata->netdev, lockdep_is_held(&pdata->netdev_lock));
2163                 if (ndev) {
2164                         spin_lock(&ndev_hash_lock);
2165                         hash_del_rcu(&pdata->ndev_hash_link);
2166                         spin_unlock(&ndev_hash_lock);
2167 
2168                         /*
2169                          * If this is the last dev_put there is still a
2170                          * synchronize_rcu before the netdev is kfreed, so we
2171                          * can continue to rely on unlocked pointer
2172                          * comparisons after the put
2173                          */
2174                         rcu_assign_pointer(pdata->netdev, NULL);
2175                         dev_put(ndev);
2176                 }
2177                 spin_unlock_irqrestore(&pdata->netdev_lock, flags);
2178         }
2179 }
2180 
2181 struct net_device *ib_device_get_netdev(struct ib_device *ib_dev,
2182                                         unsigned int port)
2183 {
2184         struct ib_port_data *pdata;
2185         struct net_device *res;
2186 
2187         if (!rdma_is_port_valid(ib_dev, port))
2188                 return NULL;
2189 
2190         pdata = &ib_dev->port_data[port];
2191 
2192         /*
2193          * New drivers should use ib_device_set_netdev() not the legacy
2194          * get_netdev().
2195          */
2196         if (ib_dev->ops.get_netdev)
2197                 res = ib_dev->ops.get_netdev(ib_dev, port);
2198         else {
2199                 spin_lock(&pdata->netdev_lock);
2200                 res = rcu_dereference_protected(
2201                         pdata->netdev, lockdep_is_held(&pdata->netdev_lock));
2202                 if (res)
2203                         dev_hold(res);
2204                 spin_unlock(&pdata->netdev_lock);
2205         }
2206 
2207         /*
2208          * If we are starting to unregister expedite things by preventing
2209          * propagation of an unregistering netdev.
2210          */
2211         if (res && res->reg_state != NETREG_REGISTERED) {
2212                 dev_put(res);
2213                 return NULL;
2214         }
2215 
2216         return res;
2217 }
2218 
2219 /**
2220  * ib_device_get_by_netdev - Find an IB device associated with a netdev
2221  * @ndev: netdev to locate
2222  * @driver_id: The driver ID that must match (RDMA_DRIVER_UNKNOWN matches all)
2223  *
2224  * Find and hold an ib_device that is associated with a netdev via
2225  * ib_device_set_netdev(). The caller must call ib_device_put() on the
2226  * returned pointer.
2227  */
2228 struct ib_device *ib_device_get_by_netdev(struct net_device *ndev,
2229                                           enum rdma_driver_id driver_id)
2230 {
2231         struct ib_device *res = NULL;
2232         struct ib_port_data *cur;
2233 
2234         rcu_read_lock();
2235         hash_for_each_possible_rcu (ndev_hash, cur, ndev_hash_link,
2236                                     (uintptr_t)ndev) {
2237                 if (rcu_access_pointer(cur->netdev) == ndev &&
2238                     (driver_id == RDMA_DRIVER_UNKNOWN ||
2239                      cur->ib_dev->ops.driver_id == driver_id) &&
2240                     ib_device_try_get(cur->ib_dev)) {
2241                         res = cur->ib_dev;
2242                         break;
2243                 }
2244         }
2245         rcu_read_unlock();
2246 
2247         return res;
2248 }
2249 EXPORT_SYMBOL(ib_device_get_by_netdev);
2250 
2251 /**
2252  * ib_enum_roce_netdev - enumerate all RoCE ports
2253  * @ib_dev : IB device we want to query
2254  * @filter: Should we call the callback?
2255  * @filter_cookie: Cookie passed to filter
2256  * @cb: Callback to call for each found RoCE ports
2257  * @cookie: Cookie passed back to the callback
2258  *
2259  * Enumerates all of the physical RoCE ports of ib_dev
2260  * which are related to netdevice and calls callback() on each
2261  * device for which filter() function returns non zero.
2262  */
2263 void ib_enum_roce_netdev(struct ib_device *ib_dev,
2264                          roce_netdev_filter filter,
2265                          void *filter_cookie,
2266                          roce_netdev_callback cb,
2267                          void *cookie)
2268 {
2269         unsigned int port;
2270 
2271         rdma_for_each_port (ib_dev, port)
2272                 if (rdma_protocol_roce(ib_dev, port)) {
2273                         struct net_device *idev =
2274                                 ib_device_get_netdev(ib_dev, port);
2275 
2276                         if (filter(ib_dev, port, idev, filter_cookie))
2277                                 cb(ib_dev, port, idev, cookie);
2278 
2279                         if (idev)
2280                                 dev_put(idev);
2281                 }
2282 }
2283 
2284 /**
2285  * ib_enum_all_roce_netdevs - enumerate all RoCE devices
2286  * @filter: Should we call the callback?
2287  * @filter_cookie: Cookie passed to filter
2288  * @cb: Callback to call for each found RoCE ports
2289  * @cookie: Cookie passed back to the callback
2290  *
2291  * Enumerates all RoCE devices' physical ports which are related
2292  * to netdevices and calls callback() on each device for which
2293  * filter() function returns non zero.
2294  */
2295 void ib_enum_all_roce_netdevs(roce_netdev_filter filter,
2296                               void *filter_cookie,
2297                               roce_netdev_callback cb,
2298                               void *cookie)
2299 {
2300         struct ib_device *dev;
2301         unsigned long index;
2302 
2303         down_read(&devices_rwsem);
2304         xa_for_each_marked (&devices, index, dev, DEVICE_REGISTERED)
2305                 ib_enum_roce_netdev(dev, filter, filter_cookie, cb, cookie);
2306         up_read(&devices_rwsem);
2307 }
2308 
2309 /**
2310  * ib_enum_all_devs - enumerate all ib_devices
2311  * @cb: Callback to call for each found ib_device
2312  *
2313  * Enumerates all ib_devices and calls callback() on each device.
2314  */
2315 int ib_enum_all_devs(nldev_callback nldev_cb, struct sk_buff *skb,
2316                      struct netlink_callback *cb)
2317 {
2318         unsigned long index;
2319         struct ib_device *dev;
2320         unsigned int idx = 0;
2321         int ret = 0;
2322 
2323         down_read(&devices_rwsem);
2324         xa_for_each_marked (&devices, index, dev, DEVICE_REGISTERED) {
2325                 if (!rdma_dev_access_netns(dev, sock_net(skb->sk)))
2326                         continue;
2327 
2328                 ret = nldev_cb(dev, skb, cb, idx);
2329                 if (ret)
2330                         break;
2331                 idx++;
2332         }
2333         up_read(&devices_rwsem);
2334         return ret;
2335 }
2336 
2337 /**
2338  * ib_query_pkey - Get P_Key table entry
2339  * @device:Device to query
2340  * @port_num:Port number to query
2341  * @index:P_Key table index to query
2342  * @pkey:Returned P_Key
2343  *
2344  * ib_query_pkey() fetches the specified P_Key table entry.
2345  */
2346 int ib_query_pkey(struct ib_device *device,
2347                   u8 port_num, u16 index, u16 *pkey)
2348 {
2349         if (!rdma_is_port_valid(device, port_num))
2350                 return -EINVAL;
2351 
2352         return device->ops.query_pkey(device, port_num, index, pkey);
2353 }
2354 EXPORT_SYMBOL(ib_query_pkey);
2355 
2356 /**
2357  * ib_modify_device - Change IB device attributes
2358  * @device:Device to modify
2359  * @device_modify_mask:Mask of attributes to change
2360  * @device_modify:New attribute values
2361  *
2362  * ib_modify_device() changes a device's attributes as specified by
2363  * the @device_modify_mask and @device_modify structure.
2364  */
2365 int ib_modify_device(struct ib_device *device,
2366                      int device_modify_mask,
2367                      struct ib_device_modify *device_modify)
2368 {
2369         if (!device->ops.modify_device)
2370                 return -ENOSYS;
2371 
2372         return device->ops.modify_device(device, device_modify_mask,
2373                                          device_modify);
2374 }
2375 EXPORT_SYMBOL(ib_modify_device);
2376 
2377 /**
2378  * ib_modify_port - Modifies the attributes for the specified port.
2379  * @device: The device to modify.
2380  * @port_num: The number of the port to modify.
2381  * @port_modify_mask: Mask used to specify which attributes of the port
2382  *   to change.
2383  * @port_modify: New attribute values for the port.
2384  *
2385  * ib_modify_port() changes a port's attributes as specified by the
2386  * @port_modify_mask and @port_modify structure.
2387  */
2388 int ib_modify_port(struct ib_device *device,
2389                    u8 port_num, int port_modify_mask,
2390                    struct ib_port_modify *port_modify)
2391 {
2392         int rc;
2393 
2394         if (!rdma_is_port_valid(device, port_num))
2395                 return -EINVAL;
2396 
2397         if (device->ops.modify_port)
2398                 rc = device->ops.modify_port(device, port_num,
2399                                              port_modify_mask,
2400                                              port_modify);
2401         else if (rdma_protocol_roce(device, port_num) &&
2402                  ((port_modify->set_port_cap_mask & ~IB_PORT_CM_SUP) == 0 ||
2403                   (port_modify->clr_port_cap_mask & ~IB_PORT_CM_SUP) == 0))
2404                 rc = 0;
2405         else
2406                 rc = -EOPNOTSUPP;
2407         return rc;
2408 }
2409 EXPORT_SYMBOL(ib_modify_port);
2410 
2411 /**
2412  * ib_find_gid - Returns the port number and GID table index where
2413  *   a specified GID value occurs. Its searches only for IB link layer.
2414  * @device: The device to query.
2415  * @gid: The GID value to search for.
2416  * @port_num: The port number of the device where the GID value was found.
2417  * @index: The index into the GID table where the GID was found.  This
2418  *   parameter may be NULL.
2419  */
2420 int ib_find_gid(struct ib_device *device, union ib_gid *gid,
2421                 u8 *port_num, u16 *index)
2422 {
2423         union ib_gid tmp_gid;
2424         unsigned int port;
2425         int ret, i;
2426 
2427         rdma_for_each_port (device, port) {
2428                 if (!rdma_protocol_ib(device, port))
2429                         continue;
2430 
2431                 for (i = 0; i < device->port_data[port].immutable.gid_tbl_len;
2432                      ++i) {
2433                         ret = rdma_query_gid(device, port, i, &tmp_gid);
2434                         if (ret)
2435                                 return ret;
2436                         if (!memcmp(&tmp_gid, gid, sizeof *gid)) {
2437                                 *port_num = port;
2438                                 if (index)
2439                                         *index = i;
2440                                 return 0;
2441                         }
2442                 }
2443         }
2444 
2445         return -ENOENT;
2446 }
2447 EXPORT_SYMBOL(ib_find_gid);
2448 
2449 /**
2450  * ib_find_pkey - Returns the PKey table index where a specified
2451  *   PKey value occurs.
2452  * @device: The device to query.
2453  * @port_num: The port number of the device to search for the PKey.
2454  * @pkey: The PKey value to search for.
2455  * @index: The index into the PKey table where the PKey was found.
2456  */
2457 int ib_find_pkey(struct ib_device *device,
2458                  u8 port_num, u16 pkey, u16 *index)
2459 {
2460         int ret, i;
2461         u16 tmp_pkey;
2462         int partial_ix = -1;
2463 
2464         for (i = 0; i < device->port_data[port_num].immutable.pkey_tbl_len;
2465              ++i) {
2466                 ret = ib_query_pkey(device, port_num, i, &tmp_pkey);
2467                 if (ret)
2468                         return ret;
2469                 if ((pkey & 0x7fff) == (tmp_pkey & 0x7fff)) {
2470                         /* if there is full-member pkey take it.*/
2471                         if (tmp_pkey & 0x8000) {
2472                                 *index = i;
2473                                 return 0;
2474                         }
2475                         if (partial_ix < 0)
2476                                 partial_ix = i;
2477                 }
2478         }
2479 
2480         /*no full-member, if exists take the limited*/
2481         if (partial_ix >= 0) {
2482                 *index = partial_ix;
2483                 return 0;
2484         }
2485         return -ENOENT;
2486 }
2487 EXPORT_SYMBOL(ib_find_pkey);
2488 
2489 /**
2490  * ib_get_net_dev_by_params() - Return the appropriate net_dev
2491  * for a received CM request
2492  * @dev:        An RDMA device on which the request has been received.
2493  * @port:       Port number on the RDMA device.
2494  * @pkey:       The Pkey the request came on.
2495  * @gid:        A GID that the net_dev uses to communicate.
2496  * @addr:       Contains the IP address that the request specified as its
2497  *              destination.
2498  *
2499  */
2500 struct net_device *ib_get_net_dev_by_params(struct ib_device *dev,
2501                                             u8 port,
2502                                             u16 pkey,
2503                                             const union ib_gid *gid,
2504                                             const struct sockaddr *addr)
2505 {
2506         struct net_device *net_dev = NULL;
2507         unsigned long index;
2508         void *client_data;
2509 
2510         if (!rdma_protocol_ib(dev, port))
2511                 return NULL;
2512 
2513         /*
2514          * Holding the read side guarantees that the client will not become
2515          * unregistered while we are calling get_net_dev_by_params()
2516          */
2517         down_read(&dev->client_data_rwsem);
2518         xan_for_each_marked (&dev->client_data, index, client_data,
2519                              CLIENT_DATA_REGISTERED) {
2520                 struct ib_client *client = xa_load(&clients, index);
2521 
2522                 if (!client || !client->get_net_dev_by_params)
2523                         continue;
2524 
2525                 net_dev = client->get_net_dev_by_params(dev, port, pkey, gid,
2526                                                         addr, client_data);
2527                 if (net_dev)
2528                         break;
2529         }
2530         up_read(&dev->client_data_rwsem);
2531 
2532         return net_dev;
2533 }
2534 EXPORT_SYMBOL(ib_get_net_dev_by_params);
2535 
2536 void ib_set_device_ops(struct ib_device *dev, const struct ib_device_ops *ops)
2537 {
2538         struct ib_device_ops *dev_ops = &dev->ops;
2539 #define SET_DEVICE_OP(ptr, name)                                               \
2540         do {                                                                   \
2541                 if (ops->name)                                                 \
2542                         if (!((ptr)->name))                                    \
2543                                 (ptr)->name = ops->name;                       \
2544         } while (0)
2545 
2546 #define SET_OBJ_SIZE(ptr, name) SET_DEVICE_OP(ptr, size_##name)
2547 
2548         if (ops->driver_id != RDMA_DRIVER_UNKNOWN) {
2549                 WARN_ON(dev_ops->driver_id != RDMA_DRIVER_UNKNOWN &&
2550                         dev_ops->driver_id != ops->driver_id);
2551                 dev_ops->driver_id = ops->driver_id;
2552         }
2553         if (ops->owner) {
2554                 WARN_ON(dev_ops->owner && dev_ops->owner != ops->owner);
2555                 dev_ops->owner = ops->owner;
2556         }
2557         if (ops->uverbs_abi_ver)
2558                 dev_ops->uverbs_abi_ver = ops->uverbs_abi_ver;
2559 
2560         dev_ops->uverbs_no_driver_id_binding |=
2561                 ops->uverbs_no_driver_id_binding;
2562 
2563         SET_DEVICE_OP(dev_ops, add_gid);
2564         SET_DEVICE_OP(dev_ops, advise_mr);
2565         SET_DEVICE_OP(dev_ops, alloc_dm);
2566         SET_DEVICE_OP(dev_ops, alloc_fmr);
2567         SET_DEVICE_OP(dev_ops, alloc_hw_stats);
2568         SET_DEVICE_OP(dev_ops, alloc_mr);
2569         SET_DEVICE_OP(dev_ops, alloc_mr_integrity);
2570         SET_DEVICE_OP(dev_ops, alloc_mw);
2571         SET_DEVICE_OP(dev_ops, alloc_pd);
2572         SET_DEVICE_OP(dev_ops, alloc_rdma_netdev);
2573         SET_DEVICE_OP(dev_ops, alloc_ucontext);
2574         SET_DEVICE_OP(dev_ops, alloc_xrcd);
2575         SET_DEVICE_OP(dev_ops, attach_mcast);
2576         SET_DEVICE_OP(dev_ops, check_mr_status);
2577         SET_DEVICE_OP(dev_ops, counter_alloc_stats);
2578         SET_DEVICE_OP(dev_ops, counter_bind_qp);
2579         SET_DEVICE_OP(dev_ops, counter_dealloc);
2580         SET_DEVICE_OP(dev_ops, counter_unbind_qp);
2581         SET_DEVICE_OP(dev_ops, counter_update_stats);
2582         SET_DEVICE_OP(dev_ops, create_ah);
2583         SET_DEVICE_OP(dev_ops, create_counters);
2584         SET_DEVICE_OP(dev_ops, create_cq);
2585         SET_DEVICE_OP(dev_ops, create_flow);
2586         SET_DEVICE_OP(dev_ops, create_flow_action_esp);
2587         SET_DEVICE_OP(dev_ops, create_qp);
2588         SET_DEVICE_OP(dev_ops, create_rwq_ind_table);
2589         SET_DEVICE_OP(dev_ops, create_srq);
2590         SET_DEVICE_OP(dev_ops, create_wq);
2591         SET_DEVICE_OP(dev_ops, dealloc_dm);
2592         SET_DEVICE_OP(dev_ops, dealloc_driver);
2593         SET_DEVICE_OP(dev_ops, dealloc_fmr);
2594         SET_DEVICE_OP(dev_ops, dealloc_mw);
2595         SET_DEVICE_OP(dev_ops, dealloc_pd);
2596         SET_DEVICE_OP(dev_ops, dealloc_ucontext);
2597         SET_DEVICE_OP(dev_ops, dealloc_xrcd);
2598         SET_DEVICE_OP(dev_ops, del_gid);
2599         SET_DEVICE_OP(dev_ops, dereg_mr);
2600         SET_DEVICE_OP(dev_ops, destroy_ah);
2601         SET_DEVICE_OP(dev_ops, destroy_counters);
2602         SET_DEVICE_OP(dev_ops, destroy_cq);
2603         SET_DEVICE_OP(dev_ops, destroy_flow);
2604         SET_DEVICE_OP(dev_ops, destroy_flow_action);
2605         SET_DEVICE_OP(dev_ops, destroy_qp);
2606         SET_DEVICE_OP(dev_ops, destroy_rwq_ind_table);
2607         SET_DEVICE_OP(dev_ops, destroy_srq);
2608         SET_DEVICE_OP(dev_ops, destroy_wq);
2609         SET_DEVICE_OP(dev_ops, detach_mcast);
2610         SET_DEVICE_OP(dev_ops, disassociate_ucontext);
2611         SET_DEVICE_OP(dev_ops, drain_rq);
2612         SET_DEVICE_OP(dev_ops, drain_sq);
2613         SET_DEVICE_OP(dev_ops, enable_driver);
2614         SET_DEVICE_OP(dev_ops, fill_res_entry);
2615         SET_DEVICE_OP(dev_ops, get_dev_fw_str);
2616         SET_DEVICE_OP(dev_ops, get_dma_mr);
2617         SET_DEVICE_OP(dev_ops, get_hw_stats);
2618         SET_DEVICE_OP(dev_ops, get_link_layer);
2619         SET_DEVICE_OP(dev_ops, get_netdev);
2620         SET_DEVICE_OP(dev_ops, get_port_immutable);
2621         SET_DEVICE_OP(dev_ops, get_vector_affinity);
2622         SET_DEVICE_OP(dev_ops, get_vf_config);
2623         SET_DEVICE_OP(dev_ops, get_vf_stats);
2624         SET_DEVICE_OP(dev_ops, init_port);
2625         SET_DEVICE_OP(dev_ops, invalidate_range);
2626         SET_DEVICE_OP(dev_ops, iw_accept);
2627         SET_DEVICE_OP(dev_ops, iw_add_ref);
2628         SET_DEVICE_OP(dev_ops, iw_connect);
2629         SET_DEVICE_OP(dev_ops, iw_create_listen);
2630         SET_DEVICE_OP(dev_ops, iw_destroy_listen);
2631         SET_DEVICE_OP(dev_ops, iw_get_qp);
2632         SET_DEVICE_OP(dev_ops, iw_reject);
2633         SET_DEVICE_OP(dev_ops, iw_rem_ref);
2634         SET_DEVICE_OP(dev_ops, map_mr_sg);
2635         SET_DEVICE_OP(dev_ops, map_mr_sg_pi);
2636         SET_DEVICE_OP(dev_ops, map_phys_fmr);
2637         SET_DEVICE_OP(dev_ops, mmap);
2638         SET_DEVICE_OP(dev_ops, modify_ah);
2639         SET_DEVICE_OP(dev_ops, modify_cq);
2640         SET_DEVICE_OP(dev_ops, modify_device);
2641         SET_DEVICE_OP(dev_ops, modify_flow_action_esp);
2642         SET_DEVICE_OP(dev_ops, modify_port);
2643         SET_DEVICE_OP(dev_ops, modify_qp);
2644         SET_DEVICE_OP(dev_ops, modify_srq);
2645         SET_DEVICE_OP(dev_ops, modify_wq);
2646         SET_DEVICE_OP(dev_ops, peek_cq);
2647         SET_DEVICE_OP(dev_ops, poll_cq);
2648         SET_DEVICE_OP(dev_ops, post_recv);
2649         SET_DEVICE_OP(dev_ops, post_send);
2650         SET_DEVICE_OP(dev_ops, post_srq_recv);
2651         SET_DEVICE_OP(dev_ops, process_mad);
2652         SET_DEVICE_OP(dev_ops, query_ah);
2653         SET_DEVICE_OP(dev_ops, query_device);
2654         SET_DEVICE_OP(dev_ops, query_gid);
2655         SET_DEVICE_OP(dev_ops, query_pkey);
2656         SET_DEVICE_OP(dev_ops, query_port);
2657         SET_DEVICE_OP(dev_ops, query_qp);
2658         SET_DEVICE_OP(dev_ops, query_srq);
2659         SET_DEVICE_OP(dev_ops, rdma_netdev_get_params);
2660         SET_DEVICE_OP(dev_ops, read_counters);
2661         SET_DEVICE_OP(dev_ops, reg_dm_mr);
2662         SET_DEVICE_OP(dev_ops, reg_user_mr);
2663         SET_DEVICE_OP(dev_ops, req_ncomp_notif);
2664         SET_DEVICE_OP(dev_ops, req_notify_cq);
2665         SET_DEVICE_OP(dev_ops, rereg_user_mr);
2666         SET_DEVICE_OP(dev_ops, resize_cq);
2667         SET_DEVICE_OP(dev_ops, set_vf_guid);
2668         SET_DEVICE_OP(dev_ops, set_vf_link_state);
2669         SET_DEVICE_OP(dev_ops, unmap_fmr);
2670 
2671         SET_OBJ_SIZE(dev_ops, ib_ah);
2672         SET_OBJ_SIZE(dev_ops, ib_cq);
2673         SET_OBJ_SIZE(dev_ops, ib_pd);
2674         SET_OBJ_SIZE(dev_ops, ib_srq);
2675         SET_OBJ_SIZE(dev_ops, ib_ucontext);
2676 }
2677 EXPORT_SYMBOL(ib_set_device_ops);
2678 
2679 static const struct rdma_nl_cbs ibnl_ls_cb_table[RDMA_NL_LS_NUM_OPS] = {
2680         [RDMA_NL_LS_OP_RESOLVE] = {
2681                 .doit = ib_nl_handle_resolve_resp,
2682                 .flags = RDMA_NL_ADMIN_PERM,
2683         },
2684         [RDMA_NL_LS_OP_SET_TIMEOUT] = {
2685                 .doit = ib_nl_handle_set_timeout,
2686                 .flags = RDMA_NL_ADMIN_PERM,
2687         },
2688         [RDMA_NL_LS_OP_IP_RESOLVE] = {
2689                 .doit = ib_nl_handle_ip_res_resp,
2690                 .flags = RDMA_NL_ADMIN_PERM,
2691         },
2692 };
2693 
2694 static int __init ib_core_init(void)
2695 {
2696         int ret;
2697 
2698         ib_wq = alloc_workqueue("infiniband", 0, 0);
2699         if (!ib_wq)
2700                 return -ENOMEM;
2701 
2702         ib_comp_wq = alloc_workqueue("ib-comp-wq",
2703                         WQ_HIGHPRI | WQ_MEM_RECLAIM | WQ_SYSFS, 0);
2704         if (!ib_comp_wq) {
2705                 ret = -ENOMEM;
2706                 goto err;
2707         }
2708 
2709         ib_comp_unbound_wq =
2710                 alloc_workqueue("ib-comp-unb-wq",
2711                                 WQ_UNBOUND | WQ_HIGHPRI | WQ_MEM_RECLAIM |
2712                                 WQ_SYSFS, WQ_UNBOUND_MAX_ACTIVE);
2713         if (!ib_comp_unbound_wq) {
2714                 ret = -ENOMEM;
2715                 goto err_comp;
2716         }
2717 
2718         ret = class_register(&ib_class);
2719         if (ret) {
2720                 pr_warn("Couldn't create InfiniBand device class\n");
2721                 goto err_comp_unbound;
2722         }
2723 
2724         rdma_nl_init();
2725 
2726         ret = addr_init();
2727         if (ret) {
2728                 pr_warn("Could't init IB address resolution\n");
2729                 goto err_ibnl;
2730         }
2731 
2732         ret = ib_mad_init();
2733         if (ret) {
2734                 pr_warn("Couldn't init IB MAD\n");
2735                 goto err_addr;
2736         }
2737 
2738         ret = ib_sa_init();
2739         if (ret) {
2740                 pr_warn("Couldn't init SA\n");
2741                 goto err_mad;
2742         }
2743 
2744         ret = register_blocking_lsm_notifier(&ibdev_lsm_nb);
2745         if (ret) {
2746                 pr_warn("Couldn't register LSM notifier. ret %d\n", ret);
2747                 goto err_sa;
2748         }
2749 
2750         ret = register_pernet_device(&rdma_dev_net_ops);
2751         if (ret) {
2752                 pr_warn("Couldn't init compat dev. ret %d\n", ret);
2753                 goto err_compat;
2754         }
2755 
2756         nldev_init();
2757         rdma_nl_register(RDMA_NL_LS, ibnl_ls_cb_table);
2758         roce_gid_mgmt_init();
2759 
2760         return 0;
2761 
2762 err_compat:
2763         unregister_blocking_lsm_notifier(&ibdev_lsm_nb);
2764 err_sa:
2765         ib_sa_cleanup();
2766 err_mad:
2767         ib_mad_cleanup();
2768 err_addr:
2769         addr_cleanup();
2770 err_ibnl:
2771         class_unregister(&ib_class);
2772 err_comp_unbound:
2773         destroy_workqueue(ib_comp_unbound_wq);
2774 err_comp:
2775         destroy_workqueue(ib_comp_wq);
2776 err:
2777         destroy_workqueue(ib_wq);
2778         return ret;
2779 }
2780 
2781 static void __exit ib_core_cleanup(void)
2782 {
2783         roce_gid_mgmt_cleanup();
2784         nldev_exit();
2785         rdma_nl_unregister(RDMA_NL_LS);
2786         unregister_pernet_device(&rdma_dev_net_ops);
2787         unregister_blocking_lsm_notifier(&ibdev_lsm_nb);
2788         ib_sa_cleanup();
2789         ib_mad_cleanup();
2790         addr_cleanup();
2791         rdma_nl_exit();
2792         class_unregister(&ib_class);
2793         destroy_workqueue(ib_comp_unbound_wq);
2794         destroy_workqueue(ib_comp_wq);
2795         /* Make sure that any pending umem accounting work is done. */
2796         destroy_workqueue(ib_wq);
2797         flush_workqueue(system_unbound_wq);
2798         WARN_ON(!xa_empty(&clients));
2799         WARN_ON(!xa_empty(&devices));
2800 }
2801 
2802 MODULE_ALIAS_RDMA_NETLINK(RDMA_NL_LS, 4);
2803 
2804 /* ib core relies on netdev stack to first register net_ns_type_operations
2805  * ns kobject type before ib_core initialization.
2806  */
2807 fs_initcall(ib_core_init);
2808 module_exit(ib_core_cleanup);