root/drivers/infiniband/core/counters.c

DEFINITIONS

1. __counter_set_mode
2. rdma_counter_set_auto_mode
3. rdma_counter_alloc
4. rdma_counter_free
5. auto_mode_init_counter
6. auto_mode_match
7. __rdma_counter_bind_qp
8. __rdma_counter_unbind_qp
9. counter_history_stat_update
10. rdma_get_counter_auto_mode
11. rdma_counter_res_add
12. counter_release
13. rdma_counter_bind_qp_auto
14. rdma_counter_unbind_qp
15. rdma_counter_query_stats
16. get_running_counters_hwstat_sum
17. rdma_counter_get_hwstat_value
18. rdma_counter_get_qp
19. rdma_counter_bind_qp_manual
20. rdma_get_counter_by_id
21. rdma_counter_bind_qpn
22. rdma_counter_bind_qpn_alloc
23. rdma_counter_unbind_qpn
24. rdma_counter_get_mode
25. rdma_counter_init
26. rdma_counter_release

   1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
   2 /*
   3  * Copyright (c) 2019 Mellanox Technologies. All rights reserved.
   4  */
   5 #include <rdma/ib_verbs.h>
   6 #include <rdma/rdma_counter.h>
   7 
   8 #include "core_priv.h"
   9 #include "restrack.h"
  10 
  11 #define ALL_AUTO_MODE_MASKS (RDMA_COUNTER_MASK_QP_TYPE)
  12 
  13 static int __counter_set_mode(struct rdma_counter_mode *curr,
  14                               enum rdma_nl_counter_mode new_mode,
  15                               enum rdma_nl_counter_mask new_mask)
  16 {
  17         if ((new_mode == RDMA_COUNTER_MODE_AUTO) &&
  18             ((new_mask & (~ALL_AUTO_MODE_MASKS)) ||
  19              (curr->mode != RDMA_COUNTER_MODE_NONE)))
  20                 return -EINVAL;
  21 
  22         curr->mode = new_mode;
  23         curr->mask = new_mask;
  24         return 0;
  25 }
  26 
  27 /**
  28  * rdma_counter_set_auto_mode() - Turn on/off per-port auto mode
  29  *
  30  * When @on is true, the @mask must be set; When @on is false, it goes
  31  * into manual mode if there's any counter, so that the user is able to
  32  * manually access them.
  33  */
  34 int rdma_counter_set_auto_mode(struct ib_device *dev, u8 port,
  35                                bool on, enum rdma_nl_counter_mask mask)
  36 {
  37         struct rdma_port_counter *port_counter;
  38         int ret;
  39 
  40         port_counter = &dev->port_data[port].port_counter;
  41         if (!port_counter->hstats)
  42                 return -EOPNOTSUPP;
  43 
  44         mutex_lock(&port_counter->lock);
  45         if (on) {
  46                 ret = __counter_set_mode(&port_counter->mode,
  47                                          RDMA_COUNTER_MODE_AUTO, mask);
  48         } else {
  49                 if (port_counter->mode.mode != RDMA_COUNTER_MODE_AUTO) {
  50                         ret = -EINVAL;
  51                         goto out;
  52                 }
  53 
  54                 if (port_counter->num_counters)
  55                         ret = __counter_set_mode(&port_counter->mode,
  56                                                  RDMA_COUNTER_MODE_MANUAL, 0);
  57                 else
  58                         ret = __counter_set_mode(&port_counter->mode,
  59                                                  RDMA_COUNTER_MODE_NONE, 0);
  60         }
  61 
  62 out:
  63         mutex_unlock(&port_counter->lock);
  64         return ret;
  65 }
  66 
  67 static struct rdma_counter *rdma_counter_alloc(struct ib_device *dev, u8 port,
  68                                                enum rdma_nl_counter_mode mode)
  69 {
  70         struct rdma_port_counter *port_counter;
  71         struct rdma_counter *counter;
  72         int ret;
  73 
  74         if (!dev->ops.counter_dealloc || !dev->ops.counter_alloc_stats)
  75                 return NULL;
  76 
  77         counter = kzalloc(sizeof(*counter), GFP_KERNEL);
  78         if (!counter)
  79                 return NULL;
  80 
  81         counter->device    = dev;
  82         counter->port      = port;
  83         counter->res.type  = RDMA_RESTRACK_COUNTER;
  84         counter->stats     = dev->ops.counter_alloc_stats(counter);
  85         if (!counter->stats)
  86                 goto err_stats;
  87 
  88         port_counter = &dev->port_data[port].port_counter;
  89         mutex_lock(&port_counter->lock);
  90         if (mode == RDMA_COUNTER_MODE_MANUAL) {
  91                 ret = __counter_set_mode(&port_counter->mode,
  92                                          RDMA_COUNTER_MODE_MANUAL, 0);
  93                 if (ret)
  94                         goto err_mode;
  95         }
  96 
  97         port_counter->num_counters++;
  98         mutex_unlock(&port_counter->lock);
  99 
 100         counter->mode.mode = mode;
 101         kref_init(&counter->kref);
 102         mutex_init(&counter->lock);
 103 
 104         return counter;
 105 
 106 err_mode:
 107         mutex_unlock(&port_counter->lock);
 108         kfree(counter->stats);
 109 err_stats:
 110         kfree(counter);
 111         return NULL;
 112 }
 113 
 114 static void rdma_counter_free(struct rdma_counter *counter)
 115 {
 116         struct rdma_port_counter *port_counter;
 117 
 118         port_counter = &counter->device->port_data[counter->port].port_counter;
 119         mutex_lock(&port_counter->lock);
 120         port_counter->num_counters--;
 121         if (!port_counter->num_counters &&
 122             (port_counter->mode.mode == RDMA_COUNTER_MODE_MANUAL))
 123                 __counter_set_mode(&port_counter->mode, RDMA_COUNTER_MODE_NONE,
 124                                    0);
 125 
 126         mutex_unlock(&port_counter->lock);
 127 
 128         rdma_restrack_del(&counter->res);
 129         kfree(counter->stats);
 130         kfree(counter);
 131 }
 132 
 133 static void auto_mode_init_counter(struct rdma_counter *counter,
 134                                    const struct ib_qp *qp,
 135                                    enum rdma_nl_counter_mask new_mask)
 136 {
 137         struct auto_mode_param *param = &counter->mode.param;
 138 
 139         counter->mode.mode = RDMA_COUNTER_MODE_AUTO;
 140         counter->mode.mask = new_mask;
 141 
 142         if (new_mask & RDMA_COUNTER_MASK_QP_TYPE)
 143                 param->qp_type = qp->qp_type;
 144 }
 145 
 146 static bool auto_mode_match(struct ib_qp *qp, struct rdma_counter *counter,
 147                             enum rdma_nl_counter_mask auto_mask)
 148 {
 149         struct auto_mode_param *param = &counter->mode.param;
 150         bool match = true;
 151 
 152         if (!rdma_is_visible_in_pid_ns(&qp->res))
 153                 return false;
 154 
 155         /* Ensure that counter belongs to the right PID */
 156         if (task_pid_nr(counter->res.task) != task_pid_nr(qp->res.task))
 157                 return false;
 158 
 159         if (auto_mask & RDMA_COUNTER_MASK_QP_TYPE)
 160                 match &= (param->qp_type == qp->qp_type);
 161 
 162         return match;
 163 }
 164 
 165 static int __rdma_counter_bind_qp(struct rdma_counter *counter,
 166                                   struct ib_qp *qp)
 167 {
 168         int ret;
 169 
 170         if (qp->counter)
 171                 return -EINVAL;
 172 
 173         if (!qp->device->ops.counter_bind_qp)
 174                 return -EOPNOTSUPP;
 175 
 176         mutex_lock(&counter->lock);
 177         ret = qp->device->ops.counter_bind_qp(counter, qp);
 178         mutex_unlock(&counter->lock);
 179 
 180         return ret;
 181 }
 182 
 183 static int __rdma_counter_unbind_qp(struct ib_qp *qp)
 184 {
 185         struct rdma_counter *counter = qp->counter;
 186         int ret;
 187 
 188         if (!qp->device->ops.counter_unbind_qp)
 189                 return -EOPNOTSUPP;
 190 
 191         mutex_lock(&counter->lock);
 192         ret = qp->device->ops.counter_unbind_qp(qp);
 193         mutex_unlock(&counter->lock);
 194 
 195         return ret;
 196 }
 197 
 198 static void counter_history_stat_update(const struct rdma_counter *counter)
 199 {
 200         struct ib_device *dev = counter->device;
 201         struct rdma_port_counter *port_counter;
 202         int i;
 203 
 204         port_counter = &dev->port_data[counter->port].port_counter;
 205         if (!port_counter->hstats)
 206                 return;
 207 
 208         for (i = 0; i < counter->stats->num_counters; i++)
 209                 port_counter->hstats->value[i] += counter->stats->value[i];
 210 }
 211 
 212 /**
 213  * rdma_get_counter_auto_mode - Find the counter that @qp should be bound
 214  *     with in auto mode
 215  *
 216  * Return: The counter (with ref-count increased) if found
 217  */
 218 static struct rdma_counter *rdma_get_counter_auto_mode(struct ib_qp *qp,
 219                                                        u8 port)
 220 {
 221         struct rdma_port_counter *port_counter;
 222         struct rdma_counter *counter = NULL;
 223         struct ib_device *dev = qp->device;
 224         struct rdma_restrack_entry *res;
 225         struct rdma_restrack_root *rt;
 226         unsigned long id = 0;
 227 
 228         port_counter = &dev->port_data[port].port_counter;
 229         rt = &dev->res[RDMA_RESTRACK_COUNTER];
 230         xa_lock(&rt->xa);
 231         xa_for_each(&rt->xa, id, res) {
 232                 if (!rdma_is_visible_in_pid_ns(res))
 233                         continue;
 234 
 235                 counter = container_of(res, struct rdma_counter, res);
 236                 if ((counter->device != qp->device) || (counter->port != port))
 237                         goto next;
 238 
 239                 if (auto_mode_match(qp, counter, port_counter->mode.mask))
 240                         break;
 241 next:
 242                 counter = NULL;
 243         }
 244 
 245         if (counter && !kref_get_unless_zero(&counter->kref))
 246                 counter = NULL;
 247 
 248         xa_unlock(&rt->xa);
 249         return counter;
 250 }
 251 
 252 static void rdma_counter_res_add(struct rdma_counter *counter,
 253                                  struct ib_qp *qp)
 254 {
 255         if (rdma_is_kernel_res(&qp->res)) {
 256                 rdma_restrack_set_task(&counter->res, qp->res.kern_name);
 257                 rdma_restrack_kadd(&counter->res);
 258         } else {
 259                 rdma_restrack_attach_task(&counter->res, qp->res.task);
 260                 rdma_restrack_uadd(&counter->res);
 261         }
 262 }
 263 
 264 static void counter_release(struct kref *kref)
 265 {
 266         struct rdma_counter *counter;
 267 
 268         counter = container_of(kref, struct rdma_counter, kref);
 269         counter_history_stat_update(counter);
 270         counter->device->ops.counter_dealloc(counter);
 271         rdma_counter_free(counter);
 272 }
 273 
 274 /**
 275  * rdma_counter_bind_qp_auto - Check and bind the QP to a counter base on
 276  *   the auto-mode rule
 277  */
 278 int rdma_counter_bind_qp_auto(struct ib_qp *qp, u8 port)
 279 {
 280         struct rdma_port_counter *port_counter;
 281         struct ib_device *dev = qp->device;
 282         struct rdma_counter *counter;
 283         int ret;
 284 
 285         if (!qp->res.valid)
 286                 return 0;
 287 
 288         if (!rdma_is_port_valid(dev, port))
 289                 return -EINVAL;
 290 
 291         port_counter = &dev->port_data[port].port_counter;
 292         if (port_counter->mode.mode != RDMA_COUNTER_MODE_AUTO)
 293                 return 0;
 294 
 295         counter = rdma_get_counter_auto_mode(qp, port);
 296         if (counter) {
 297                 ret = __rdma_counter_bind_qp(counter, qp);
 298                 if (ret) {
 299                         kref_put(&counter->kref, counter_release);
 300                         return ret;
 301                 }
 302         } else {
 303                 counter = rdma_counter_alloc(dev, port, RDMA_COUNTER_MODE_AUTO);
 304                 if (!counter)
 305                         return -ENOMEM;
 306 
 307                 auto_mode_init_counter(counter, qp, port_counter->mode.mask);
 308 
 309                 ret = __rdma_counter_bind_qp(counter, qp);
 310                 if (ret) {
 311                         rdma_counter_free(counter);
 312                         return ret;
 313                 }
 314 
 315                 rdma_counter_res_add(counter, qp);
 316         }
 317 
 318         return 0;
 319 }
 320 
 321 /**
 322  * rdma_counter_unbind_qp - Unbind a qp from a counter
 323  * @force:
 324  *   true - Decrease the counter ref-count anyway (e.g., qp destroy)
 325  */
 326 int rdma_counter_unbind_qp(struct ib_qp *qp, bool force)
 327 {
 328         struct rdma_counter *counter = qp->counter;
 329         int ret;
 330 
 331         if (!counter)
 332                 return -EINVAL;
 333 
 334         ret = __rdma_counter_unbind_qp(qp);
 335         if (ret && !force)
 336                 return ret;
 337 
 338         kref_put(&counter->kref, counter_release);
 339         return 0;
 340 }
 341 
 342 int rdma_counter_query_stats(struct rdma_counter *counter)
 343 {
 344         struct ib_device *dev = counter->device;
 345         int ret;
 346 
 347         if (!dev->ops.counter_update_stats)
 348                 return -EINVAL;
 349 
 350         mutex_lock(&counter->lock);
 351         ret = dev->ops.counter_update_stats(counter);
 352         mutex_unlock(&counter->lock);
 353 
 354         return ret;
 355 }
 356 
 357 static u64 get_running_counters_hwstat_sum(struct ib_device *dev,
 358                                            u8 port, u32 index)
 359 {
 360         struct rdma_restrack_entry *res;
 361         struct rdma_restrack_root *rt;
 362         struct rdma_counter *counter;
 363         unsigned long id = 0;
 364         u64 sum = 0;
 365 
 366         rt = &dev->res[RDMA_RESTRACK_COUNTER];
 367         xa_lock(&rt->xa);
 368         xa_for_each(&rt->xa, id, res) {
 369                 if (!rdma_restrack_get(res))
 370                         continue;
 371 
 372                 xa_unlock(&rt->xa);
 373 
 374                 counter = container_of(res, struct rdma_counter, res);
 375                 if ((counter->device != dev) || (counter->port != port) ||
 376                     rdma_counter_query_stats(counter))
 377                         goto next;
 378 
 379                 sum += counter->stats->value[index];
 380 
 381 next:
 382                 xa_lock(&rt->xa);
 383                 rdma_restrack_put(res);
 384         }
 385 
 386         xa_unlock(&rt->xa);
 387         return sum;
 388 }
 389 
 390 /**
 391  * rdma_counter_get_hwstat_value() - Get the sum value of all counters on a
 392  *   specific port, including the running ones and history data
 393  */
 394 u64 rdma_counter_get_hwstat_value(struct ib_device *dev, u8 port, u32 index)
 395 {
 396         struct rdma_port_counter *port_counter;
 397         u64 sum;
 398 
 399         port_counter = &dev->port_data[port].port_counter;
 400         if (!port_counter->hstats)
 401                 return 0;
 402 
 403         sum = get_running_counters_hwstat_sum(dev, port, index);
 404         sum += port_counter->hstats->value[index];
 405 
 406         return sum;
 407 }
 408 
 409 static struct ib_qp *rdma_counter_get_qp(struct ib_device *dev, u32 qp_num)
 410 {
 411         struct rdma_restrack_entry *res = NULL;
 412         struct ib_qp *qp = NULL;
 413 
 414         res = rdma_restrack_get_byid(dev, RDMA_RESTRACK_QP, qp_num);
 415         if (IS_ERR(res))
 416                 return NULL;
 417 
 418         if (!rdma_is_visible_in_pid_ns(res))
 419                 goto err;
 420 
 421         qp = container_of(res, struct ib_qp, res);
 422         if (qp->qp_type == IB_QPT_RAW_PACKET && !capable(CAP_NET_RAW))
 423                 goto err;
 424 
 425         return qp;
 426 
 427 err:
 428         rdma_restrack_put(res);
 429         return NULL;
 430 }
 431 
 432 static int rdma_counter_bind_qp_manual(struct rdma_counter *counter,
 433                                        struct ib_qp *qp)
 434 {
 435         if ((counter->device != qp->device) || (counter->port != qp->port))
 436                 return -EINVAL;
 437 
 438         return __rdma_counter_bind_qp(counter, qp);
 439 }
 440 
 441 static struct rdma_counter *rdma_get_counter_by_id(struct ib_device *dev,
 442                                                    u32 counter_id)
 443 {
 444         struct rdma_restrack_entry *res;
 445         struct rdma_counter *counter;
 446 
 447         res = rdma_restrack_get_byid(dev, RDMA_RESTRACK_COUNTER, counter_id);
 448         if (IS_ERR(res))
 449                 return NULL;
 450 
 451         if (!rdma_is_visible_in_pid_ns(res)) {
 452                 rdma_restrack_put(res);
 453                 return NULL;
 454         }
 455 
 456         counter = container_of(res, struct rdma_counter, res);
 457         kref_get(&counter->kref);
 458         rdma_restrack_put(res);
 459 
 460         return counter;
 461 }
 462 
 463 /**
 464  * rdma_counter_bind_qpn() - Bind QP @qp_num to counter @counter_id
 465  */
 466 int rdma_counter_bind_qpn(struct ib_device *dev, u8 port,
 467                           u32 qp_num, u32 counter_id)
 468 {
 469         struct rdma_port_counter *port_counter;
 470         struct rdma_counter *counter;
 471         struct ib_qp *qp;
 472         int ret;
 473 
 474         port_counter = &dev->port_data[port].port_counter;
 475         if (port_counter->mode.mode == RDMA_COUNTER_MODE_AUTO)
 476                 return -EINVAL;
 477 
 478         qp = rdma_counter_get_qp(dev, qp_num);
 479         if (!qp)
 480                 return -ENOENT;
 481 
 482         counter = rdma_get_counter_by_id(dev, counter_id);
 483         if (!counter) {
 484                 ret = -ENOENT;
 485                 goto err;
 486         }
 487 
 488         if (counter->res.task != qp->res.task) {
 489                 ret = -EINVAL;
 490                 goto err_task;
 491         }
 492 
 493         ret = rdma_counter_bind_qp_manual(counter, qp);
 494         if (ret)
 495                 goto err_task;
 496 
 497         rdma_restrack_put(&qp->res);
 498         return 0;
 499 
 500 err_task:
 501         kref_put(&counter->kref, counter_release);
 502 err:
 503         rdma_restrack_put(&qp->res);
 504         return ret;
 505 }
 506 
 507 /**
 508  * rdma_counter_bind_qpn_alloc() - Alloc a counter and bind QP @qp_num to it
 509  *   The id of new counter is returned in @counter_id
 510  */
 511 int rdma_counter_bind_qpn_alloc(struct ib_device *dev, u8 port,
 512                                 u32 qp_num, u32 *counter_id)
 513 {
 514         struct rdma_port_counter *port_counter;
 515         struct rdma_counter *counter;
 516         struct ib_qp *qp;
 517         int ret;
 518 
 519         if (!rdma_is_port_valid(dev, port))
 520                 return -EINVAL;
 521 
 522         port_counter = &dev->port_data[port].port_counter;
 523         if (!port_counter->hstats)
 524                 return -EOPNOTSUPP;
 525 
 526         if (port_counter->mode.mode == RDMA_COUNTER_MODE_AUTO)
 527                 return -EINVAL;
 528 
 529         qp = rdma_counter_get_qp(dev, qp_num);
 530         if (!qp)
 531                 return -ENOENT;
 532 
 533         if (rdma_is_port_valid(dev, qp->port) && (qp->port != port)) {
 534                 ret = -EINVAL;
 535                 goto err;
 536         }
 537 
 538         counter = rdma_counter_alloc(dev, port, RDMA_COUNTER_MODE_MANUAL);
 539         if (!counter) {
 540                 ret = -ENOMEM;
 541                 goto err;
 542         }
 543 
 544         ret = rdma_counter_bind_qp_manual(counter, qp);
 545         if (ret)
 546                 goto err_bind;
 547 
 548         if (counter_id)
 549                 *counter_id = counter->id;
 550 
 551         rdma_counter_res_add(counter, qp);
 552 
 553         rdma_restrack_put(&qp->res);
 554         return ret;
 555 
 556 err_bind:
 557         rdma_counter_free(counter);
 558 err:
 559         rdma_restrack_put(&qp->res);
 560         return ret;
 561 }
 562 
 563 /**
 564  * rdma_counter_unbind_qpn() - Unbind QP @qp_num from a counter
 565  */
 566 int rdma_counter_unbind_qpn(struct ib_device *dev, u8 port,
 567                             u32 qp_num, u32 counter_id)
 568 {
 569         struct rdma_port_counter *port_counter;
 570         struct ib_qp *qp;
 571         int ret;
 572 
 573         if (!rdma_is_port_valid(dev, port))
 574                 return -EINVAL;
 575 
 576         qp = rdma_counter_get_qp(dev, qp_num);
 577         if (!qp)
 578                 return -ENOENT;
 579 
 580         if (rdma_is_port_valid(dev, qp->port) && (qp->port != port)) {
 581                 ret = -EINVAL;
 582                 goto out;
 583         }
 584 
 585         port_counter = &dev->port_data[port].port_counter;
 586         if (!qp->counter || qp->counter->id != counter_id ||
 587             port_counter->mode.mode != RDMA_COUNTER_MODE_MANUAL) {
 588                 ret = -EINVAL;
 589                 goto out;
 590         }
 591 
 592         ret = rdma_counter_unbind_qp(qp, false);
 593 
 594 out:
 595         rdma_restrack_put(&qp->res);
 596         return ret;
 597 }
 598 
 599 int rdma_counter_get_mode(struct ib_device *dev, u8 port,
 600                           enum rdma_nl_counter_mode *mode,
 601                           enum rdma_nl_counter_mask *mask)
 602 {
 603         struct rdma_port_counter *port_counter;
 604 
 605         port_counter = &dev->port_data[port].port_counter;
 606         *mode = port_counter->mode.mode;
 607         *mask = port_counter->mode.mask;
 608 
 609         return 0;
 610 }
 611 
 612 void rdma_counter_init(struct ib_device *dev)
 613 {
 614         struct rdma_port_counter *port_counter;
 615         u32 port, i;
 616 
 617         if (!dev->port_data)
 618                 return;
 619 
 620         rdma_for_each_port(dev, port) {
 621                 port_counter = &dev->port_data[port].port_counter;
 622                 port_counter->mode.mode = RDMA_COUNTER_MODE_NONE;
 623                 mutex_init(&port_counter->lock);
 624 
 625                 if (!dev->ops.alloc_hw_stats)
 626                         continue;
 627 
 628                 port_counter->hstats = dev->ops.alloc_hw_stats(dev, port);
 629                 if (!port_counter->hstats)
 630                         goto fail;
 631         }
 632 
 633         return;
 634 
 635 fail:
 636         for (i = port; i >= rdma_start_port(dev); i--) {
 637                 port_counter = &dev->port_data[port].port_counter;
 638                 kfree(port_counter->hstats);
 639                 port_counter->hstats = NULL;
 640                 mutex_destroy(&port_counter->lock);
 641         }
 642 }
 643 
 644 void rdma_counter_release(struct ib_device *dev)
 645 {
 646         struct rdma_port_counter *port_counter;
 647         u32 port;
 648 
 649         rdma_for_each_port(dev, port) {
 650                 port_counter = &dev->port_data[port].port_counter;
 651                 kfree(port_counter->hstats);
 652                 mutex_destroy(&port_counter->lock);
 653         }
 654 }