root/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_icm_pool.c

DEFINITIONS

1. dr_icm_create_dm_mkey
2. dr_icm_pool_mr_create
3. dr_icm_pool_mr_destroy
4. dr_icm_chunk_ste_init
5. dr_icm_chunks_create
6. dr_icm_chunk_ste_cleanup
7. dr_icm_chunk_destroy
8. dr_icm_bucket_init
9. dr_icm_bucket_cleanup
10. dr_icm_hot_mem_size
11. dr_icm_reuse_hot_entries
12. dr_icm_chill_bucket_start
13. dr_icm_chill_bucket_end
14. dr_icm_chill_bucket_abort
15. dr_icm_chill_buckets_start
16. dr_icm_chill_buckets_end
17. dr_icm_chill_buckets_abort
18. mlx5dr_icm_alloc_chunk
19. mlx5dr_icm_free_chunk
20. mlx5dr_icm_pool_create
21. mlx5dr_icm_pool_destroy

   1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
   2 /* Copyright (c) 2019 Mellanox Technologies. */
   3 
   4 #include "dr_types.h"
   5 
   6 #define DR_ICM_MODIFY_HDR_ALIGN_BASE 64
   7 #define DR_ICM_SYNC_THRESHOLD (64 * 1024 * 1024)
   8 
   9 struct mlx5dr_icm_pool;
  10 
  11 struct mlx5dr_icm_bucket {
  12         struct mlx5dr_icm_pool *pool;
  13 
  14         /* Chunks that aren't visible to HW not directly and not in cache */
  15         struct list_head free_list;
  16         unsigned int free_list_count;
  17 
  18         /* Used chunks, HW may be accessing this memory */
  19         struct list_head used_list;
  20         unsigned int used_list_count;
  21 
  22         /* HW may be accessing this memory but at some future,
  23          * undetermined time, it might cease to do so. Before deciding to call
  24          * sync_ste, this list is moved to sync_list
  25          */
  26         struct list_head hot_list;
  27         unsigned int hot_list_count;
  28 
  29         /* Pending sync list, entries from the hot list are moved to this list.
  30          * sync_ste is executed and then sync_list is concatenated to the free list
  31          */
  32         struct list_head sync_list;
  33         unsigned int sync_list_count;
  34 
  35         u32 total_chunks;
  36         u32 num_of_entries;
  37         u32 entry_size;
  38         /* protect the ICM bucket */
  39         struct mutex mutex;
  40 };
  41 
  42 struct mlx5dr_icm_pool {
  43         struct mlx5dr_icm_bucket *buckets;
  44         enum mlx5dr_icm_type icm_type;
  45         enum mlx5dr_icm_chunk_size max_log_chunk_sz;
  46         enum mlx5dr_icm_chunk_size num_of_buckets;
  47         struct list_head icm_mr_list;
  48         /* protect the ICM MR list */
  49         struct mutex mr_mutex;
  50         struct mlx5dr_domain *dmn;
  51 };
  52 
  53 struct mlx5dr_icm_dm {
  54         u32 obj_id;
  55         enum mlx5_sw_icm_type type;
  56         phys_addr_t addr;
  57         size_t length;
  58 };
  59 
  60 struct mlx5dr_icm_mr {
  61         struct mlx5dr_icm_pool *pool;
  62         struct mlx5_core_mkey mkey;
  63         struct mlx5dr_icm_dm dm;
  64         size_t used_length;
  65         size_t length;
  66         u64 icm_start_addr;
  67         struct list_head mr_list;
  68 };
  69 
  70 static int dr_icm_create_dm_mkey(struct mlx5_core_dev *mdev,
  71                                  u32 pd, u64 length, u64 start_addr, int mode,
  72                                  struct mlx5_core_mkey *mkey)
  73 {
  74         u32 inlen = MLX5_ST_SZ_BYTES(create_mkey_in);
  75         u32 in[MLX5_ST_SZ_DW(create_mkey_in)] = {};
  76         void *mkc;
  77 
  78         mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
  79 
  80         MLX5_SET(mkc, mkc, access_mode_1_0, mode);
  81         MLX5_SET(mkc, mkc, access_mode_4_2, (mode >> 2) & 0x7);
  82         MLX5_SET(mkc, mkc, lw, 1);
  83         MLX5_SET(mkc, mkc, lr, 1);
  84         if (mode == MLX5_MKC_ACCESS_MODE_SW_ICM) {
  85                 MLX5_SET(mkc, mkc, rw, 1);
  86                 MLX5_SET(mkc, mkc, rr, 1);
  87         }
  88 
  89         MLX5_SET64(mkc, mkc, len, length);
  90         MLX5_SET(mkc, mkc, pd, pd);
  91         MLX5_SET(mkc, mkc, qpn, 0xffffff);
  92         MLX5_SET64(mkc, mkc, start_addr, start_addr);
  93 
  94         return mlx5_core_create_mkey(mdev, mkey, in, inlen);
  95 }
  96 
  97 static struct mlx5dr_icm_mr *
  98 dr_icm_pool_mr_create(struct mlx5dr_icm_pool *pool,
  99                       enum mlx5_sw_icm_type type,
 100                       size_t align_base)
 101 {
 102         struct mlx5_core_dev *mdev = pool->dmn->mdev;
 103         struct mlx5dr_icm_mr *icm_mr;
 104         size_t align_diff;
 105         int err;
 106 
 107         icm_mr = kvzalloc(sizeof(*icm_mr), GFP_KERNEL);
 108         if (!icm_mr)
 109                 return NULL;
 110 
 111         icm_mr->pool = pool;
 112         INIT_LIST_HEAD(&icm_mr->mr_list);
 113 
 114         icm_mr->dm.type = type;
 115 
 116         /* 2^log_biggest_table * entry-size * double-for-alignment */
 117         icm_mr->dm.length = mlx5dr_icm_pool_chunk_size_to_byte(pool->max_log_chunk_sz,
 118                                                                pool->icm_type) * 2;
 119 
 120         err = mlx5_dm_sw_icm_alloc(mdev, icm_mr->dm.type, icm_mr->dm.length, 0,
 121                                    &icm_mr->dm.addr, &icm_mr->dm.obj_id);
 122         if (err) {
 123                 mlx5dr_err(pool->dmn, "Failed to allocate SW ICM memory, err (%d)\n", err);
 124                 goto free_icm_mr;
 125         }
 126 
 127         /* Register device memory */
 128         err = dr_icm_create_dm_mkey(mdev, pool->dmn->pdn,
 129                                     icm_mr->dm.length,
 130                                     icm_mr->dm.addr,
 131                                     MLX5_MKC_ACCESS_MODE_SW_ICM,
 132                                     &icm_mr->mkey);
 133         if (err) {
 134                 mlx5dr_err(pool->dmn, "Failed to create SW ICM MKEY, err (%d)\n", err);
 135                 goto free_dm;
 136         }
 137 
 138         icm_mr->icm_start_addr = icm_mr->dm.addr;
 139 
 140         /* align_base is always a power of 2 */
 141         align_diff = icm_mr->icm_start_addr & (align_base - 1);
 142         if (align_diff)
 143                 icm_mr->used_length = align_base - align_diff;
 144 
 145         list_add_tail(&icm_mr->mr_list, &pool->icm_mr_list);
 146 
 147         return icm_mr;
 148 
 149 free_dm:
 150         mlx5_dm_sw_icm_dealloc(mdev, icm_mr->dm.type, icm_mr->dm.length, 0,
 151                                icm_mr->dm.addr, icm_mr->dm.obj_id);
 152 free_icm_mr:
 153         kvfree(icm_mr);
 154         return NULL;
 155 }
 156 
 157 static void dr_icm_pool_mr_destroy(struct mlx5dr_icm_mr *icm_mr)
 158 {
 159         struct mlx5_core_dev *mdev = icm_mr->pool->dmn->mdev;
 160         struct mlx5dr_icm_dm *dm = &icm_mr->dm;
 161 
 162         list_del(&icm_mr->mr_list);
 163         mlx5_core_destroy_mkey(mdev, &icm_mr->mkey);
 164         mlx5_dm_sw_icm_dealloc(mdev, dm->type, dm->length, 0,
 165                                dm->addr, dm->obj_id);
 166         kvfree(icm_mr);
 167 }
 168 
 169 static int dr_icm_chunk_ste_init(struct mlx5dr_icm_chunk *chunk)
 170 {
 171         struct mlx5dr_icm_bucket *bucket = chunk->bucket;
 172 
 173         chunk->ste_arr = kvzalloc(bucket->num_of_entries *
 174                                   sizeof(chunk->ste_arr[0]), GFP_KERNEL);
 175         if (!chunk->ste_arr)
 176                 return -ENOMEM;
 177 
 178         chunk->hw_ste_arr = kvzalloc(bucket->num_of_entries *
 179                                      DR_STE_SIZE_REDUCED, GFP_KERNEL);
 180         if (!chunk->hw_ste_arr)
 181                 goto out_free_ste_arr;
 182 
 183         chunk->miss_list = kvmalloc(bucket->num_of_entries *
 184                                     sizeof(chunk->miss_list[0]), GFP_KERNEL);
 185         if (!chunk->miss_list)
 186                 goto out_free_hw_ste_arr;
 187 
 188         return 0;
 189 
 190 out_free_hw_ste_arr:
 191         kvfree(chunk->hw_ste_arr);
 192 out_free_ste_arr:
 193         kvfree(chunk->ste_arr);
 194         return -ENOMEM;
 195 }
 196 
 197 static int dr_icm_chunks_create(struct mlx5dr_icm_bucket *bucket)
 198 {
 199         size_t mr_free_size, mr_req_size, mr_row_size;
 200         struct mlx5dr_icm_pool *pool = bucket->pool;
 201         struct mlx5dr_icm_mr *icm_mr = NULL;
 202         struct mlx5dr_icm_chunk *chunk;
 203         enum mlx5_sw_icm_type dm_type;
 204         size_t align_base;
 205         int i, err = 0;
 206 
 207         mr_req_size = bucket->num_of_entries * bucket->entry_size;
 208         mr_row_size = mlx5dr_icm_pool_chunk_size_to_byte(pool->max_log_chunk_sz,
 209                                                          pool->icm_type);
 210 
 211         if (pool->icm_type == DR_ICM_TYPE_STE) {
 212                 dm_type = MLX5_SW_ICM_TYPE_STEERING;
 213                 /* Align base is the biggest chunk size / row size */
 214                 align_base = mr_row_size;
 215         } else {
 216                 dm_type = MLX5_SW_ICM_TYPE_HEADER_MODIFY;
 217                 /* Align base is 64B */
 218                 align_base = DR_ICM_MODIFY_HDR_ALIGN_BASE;
 219         }
 220 
 221         mutex_lock(&pool->mr_mutex);
 222         if (!list_empty(&pool->icm_mr_list)) {
 223                 icm_mr = list_last_entry(&pool->icm_mr_list,
 224                                          struct mlx5dr_icm_mr, mr_list);
 225 
 226                 if (icm_mr)
 227                         mr_free_size = icm_mr->dm.length - icm_mr->used_length;
 228         }
 229 
 230         if (!icm_mr || mr_free_size < mr_row_size) {
 231                 icm_mr = dr_icm_pool_mr_create(pool, dm_type, align_base);
 232                 if (!icm_mr) {
 233                         err = -ENOMEM;
 234                         goto out_err;
 235                 }
 236         }
 237 
 238         /* Create memory aligned chunks */
 239         for (i = 0; i < mr_row_size / mr_req_size; i++) {
 240                 chunk = kvzalloc(sizeof(*chunk), GFP_KERNEL);
 241                 if (!chunk) {
 242                         err = -ENOMEM;
 243                         goto out_err;
 244                 }
 245 
 246                 chunk->bucket = bucket;
 247                 chunk->rkey = icm_mr->mkey.key;
 248                 /* mr start addr is zero based */
 249                 chunk->mr_addr = icm_mr->used_length;
 250                 chunk->icm_addr = (uintptr_t)icm_mr->icm_start_addr + icm_mr->used_length;
 251                 icm_mr->used_length += mr_req_size;
 252                 chunk->num_of_entries = bucket->num_of_entries;
 253                 chunk->byte_size = chunk->num_of_entries * bucket->entry_size;
 254 
 255                 if (pool->icm_type == DR_ICM_TYPE_STE) {
 256                         err = dr_icm_chunk_ste_init(chunk);
 257                         if (err)
 258                                 goto out_free_chunk;
 259                 }
 260 
 261                 INIT_LIST_HEAD(&chunk->chunk_list);
 262                 list_add(&chunk->chunk_list, &bucket->free_list);
 263                 bucket->free_list_count++;
 264                 bucket->total_chunks++;
 265         }
 266         mutex_unlock(&pool->mr_mutex);
 267         return 0;
 268 
 269 out_free_chunk:
 270         kvfree(chunk);
 271 out_err:
 272         mutex_unlock(&pool->mr_mutex);
 273         return err;
 274 }
 275 
 276 static void dr_icm_chunk_ste_cleanup(struct mlx5dr_icm_chunk *chunk)
 277 {
 278         kvfree(chunk->miss_list);
 279         kvfree(chunk->hw_ste_arr);
 280         kvfree(chunk->ste_arr);
 281 }
 282 
 283 static void dr_icm_chunk_destroy(struct mlx5dr_icm_chunk *chunk)
 284 {
 285         struct mlx5dr_icm_bucket *bucket = chunk->bucket;
 286 
 287         list_del(&chunk->chunk_list);
 288         bucket->total_chunks--;
 289 
 290         if (bucket->pool->icm_type == DR_ICM_TYPE_STE)
 291                 dr_icm_chunk_ste_cleanup(chunk);
 292 
 293         kvfree(chunk);
 294 }
 295 
 296 static void dr_icm_bucket_init(struct mlx5dr_icm_pool *pool,
 297                                struct mlx5dr_icm_bucket *bucket,
 298                                enum mlx5dr_icm_chunk_size chunk_size)
 299 {
 300         if (pool->icm_type == DR_ICM_TYPE_STE)
 301                 bucket->entry_size = DR_STE_SIZE;
 302         else
 303                 bucket->entry_size = DR_MODIFY_ACTION_SIZE;
 304 
 305         bucket->num_of_entries = mlx5dr_icm_pool_chunk_size_to_entries(chunk_size);
 306         bucket->pool = pool;
 307         mutex_init(&bucket->mutex);
 308         INIT_LIST_HEAD(&bucket->free_list);
 309         INIT_LIST_HEAD(&bucket->used_list);
 310         INIT_LIST_HEAD(&bucket->hot_list);
 311         INIT_LIST_HEAD(&bucket->sync_list);
 312 }
 313 
 314 static void dr_icm_bucket_cleanup(struct mlx5dr_icm_bucket *bucket)
 315 {
 316         struct mlx5dr_icm_chunk *chunk, *next;
 317 
 318         mutex_destroy(&bucket->mutex);
 319         list_splice_tail_init(&bucket->sync_list, &bucket->free_list);
 320         list_splice_tail_init(&bucket->hot_list, &bucket->free_list);
 321 
 322         list_for_each_entry_safe(chunk, next, &bucket->free_list, chunk_list)
 323                 dr_icm_chunk_destroy(chunk);
 324 
 325         WARN_ON(bucket->total_chunks != 0);
 326 
 327         /* Cleanup of unreturned chunks */
 328         list_for_each_entry_safe(chunk, next, &bucket->used_list, chunk_list)
 329                 dr_icm_chunk_destroy(chunk);
 330 }
 331 
 332 static u64 dr_icm_hot_mem_size(struct mlx5dr_icm_pool *pool)
 333 {
 334         u64 hot_size = 0;
 335         int chunk_order;
 336 
 337         for (chunk_order = 0; chunk_order < pool->num_of_buckets; chunk_order++)
 338                 hot_size += pool->buckets[chunk_order].hot_list_count *
 339                             mlx5dr_icm_pool_chunk_size_to_byte(chunk_order, pool->icm_type);
 340 
 341         return hot_size;
 342 }
 343 
 344 static bool dr_icm_reuse_hot_entries(struct mlx5dr_icm_pool *pool,
 345                                      struct mlx5dr_icm_bucket *bucket)
 346 {
 347         u64 bytes_for_sync;
 348 
 349         bytes_for_sync = dr_icm_hot_mem_size(pool);
 350         if (bytes_for_sync < DR_ICM_SYNC_THRESHOLD || !bucket->hot_list_count)
 351                 return false;
 352 
 353         return true;
 354 }
 355 
 356 static void dr_icm_chill_bucket_start(struct mlx5dr_icm_bucket *bucket)
 357 {
 358         list_splice_tail_init(&bucket->hot_list, &bucket->sync_list);
 359         bucket->sync_list_count += bucket->hot_list_count;
 360         bucket->hot_list_count = 0;
 361 }
 362 
 363 static void dr_icm_chill_bucket_end(struct mlx5dr_icm_bucket *bucket)
 364 {
 365         list_splice_tail_init(&bucket->sync_list, &bucket->free_list);
 366         bucket->free_list_count += bucket->sync_list_count;
 367         bucket->sync_list_count = 0;
 368 }
 369 
 370 static void dr_icm_chill_bucket_abort(struct mlx5dr_icm_bucket *bucket)
 371 {
 372         list_splice_tail_init(&bucket->sync_list, &bucket->hot_list);
 373         bucket->hot_list_count += bucket->sync_list_count;
 374         bucket->sync_list_count = 0;
 375 }
 376 
 377 static void dr_icm_chill_buckets_start(struct mlx5dr_icm_pool *pool,
 378                                        struct mlx5dr_icm_bucket *cb,
 379                                        bool buckets[DR_CHUNK_SIZE_MAX])
 380 {
 381         struct mlx5dr_icm_bucket *bucket;
 382         int i;
 383 
 384         for (i = 0; i < pool->num_of_buckets; i++) {
 385                 bucket = &pool->buckets[i];
 386                 if (bucket == cb) {
 387                         dr_icm_chill_bucket_start(bucket);
 388                         continue;
 389                 }
 390 
 391                 /* Freeing the mutex is done at the end of that process, after
 392                  * sync_ste was executed at dr_icm_chill_buckets_end func.
 393                  */
 394                 if (mutex_trylock(&bucket->mutex)) {
 395                         dr_icm_chill_bucket_start(bucket);
 396                         buckets[i] = true;
 397                 }
 398         }
 399 }
 400 
 401 static void dr_icm_chill_buckets_end(struct mlx5dr_icm_pool *pool,
 402                                      struct mlx5dr_icm_bucket *cb,
 403                                      bool buckets[DR_CHUNK_SIZE_MAX])
 404 {
 405         struct mlx5dr_icm_bucket *bucket;
 406         int i;
 407 
 408         for (i = 0; i < pool->num_of_buckets; i++) {
 409                 bucket = &pool->buckets[i];
 410                 if (bucket == cb) {
 411                         dr_icm_chill_bucket_end(bucket);
 412                         continue;
 413                 }
 414 
 415                 if (!buckets[i])
 416                         continue;
 417 
 418                 dr_icm_chill_bucket_end(bucket);
 419                 mutex_unlock(&bucket->mutex);
 420         }
 421 }
 422 
 423 static void dr_icm_chill_buckets_abort(struct mlx5dr_icm_pool *pool,
 424                                        struct mlx5dr_icm_bucket *cb,
 425                                        bool buckets[DR_CHUNK_SIZE_MAX])
 426 {
 427         struct mlx5dr_icm_bucket *bucket;
 428         int i;
 429 
 430         for (i = 0; i < pool->num_of_buckets; i++) {
 431                 bucket = &pool->buckets[i];
 432                 if (bucket == cb) {
 433                         dr_icm_chill_bucket_abort(bucket);
 434                         continue;
 435                 }
 436 
 437                 if (!buckets[i])
 438                         continue;
 439 
 440                 dr_icm_chill_bucket_abort(bucket);
 441                 mutex_unlock(&bucket->mutex);
 442         }
 443 }
 444 
 445 /* Allocate an ICM chunk, each chunk holds a piece of ICM memory and
 446  * also memory used for HW STE management for optimizations.
 447  */
 448 struct mlx5dr_icm_chunk *
 449 mlx5dr_icm_alloc_chunk(struct mlx5dr_icm_pool *pool,
 450                        enum mlx5dr_icm_chunk_size chunk_size)
 451 {
 452         struct mlx5dr_icm_chunk *chunk = NULL; /* Fix compilation warning */
 453         bool buckets[DR_CHUNK_SIZE_MAX] = {};
 454         struct mlx5dr_icm_bucket *bucket;
 455         int err;
 456 
 457         if (chunk_size > pool->max_log_chunk_sz)
 458                 return NULL;
 459 
 460         bucket = &pool->buckets[chunk_size];
 461 
 462         mutex_lock(&bucket->mutex);
 463 
 464         /* Take chunk from pool if available, otherwise allocate new chunks */
 465         if (list_empty(&bucket->free_list)) {
 466                 if (dr_icm_reuse_hot_entries(pool, bucket)) {
 467                         dr_icm_chill_buckets_start(pool, bucket, buckets);
 468                         err = mlx5dr_cmd_sync_steering(pool->dmn->mdev);
 469                         if (err) {
 470                                 dr_icm_chill_buckets_abort(pool, bucket, buckets);
 471                                 mlx5dr_dbg(pool->dmn, "Sync_steering failed\n");
 472                                 chunk = NULL;
 473                                 goto out;
 474                         }
 475                         dr_icm_chill_buckets_end(pool, bucket, buckets);
 476                 } else {
 477                         dr_icm_chunks_create(bucket);
 478                 }
 479         }
 480 
 481         if (!list_empty(&bucket->free_list)) {
 482                 chunk = list_last_entry(&bucket->free_list,
 483                                         struct mlx5dr_icm_chunk,
 484                                         chunk_list);
 485                 if (chunk) {
 486                         list_del_init(&chunk->chunk_list);
 487                         list_add_tail(&chunk->chunk_list, &bucket->used_list);
 488                         bucket->free_list_count--;
 489                         bucket->used_list_count++;
 490                 }
 491         }
 492 out:
 493         mutex_unlock(&bucket->mutex);
 494         return chunk;
 495 }
 496 
 497 void mlx5dr_icm_free_chunk(struct mlx5dr_icm_chunk *chunk)
 498 {
 499         struct mlx5dr_icm_bucket *bucket = chunk->bucket;
 500 
 501         if (bucket->pool->icm_type == DR_ICM_TYPE_STE) {
 502                 memset(chunk->ste_arr, 0,
 503                        bucket->num_of_entries * sizeof(chunk->ste_arr[0]));
 504                 memset(chunk->hw_ste_arr, 0,
 505                        bucket->num_of_entries * DR_STE_SIZE_REDUCED);
 506         }
 507 
 508         mutex_lock(&bucket->mutex);
 509         list_del_init(&chunk->chunk_list);
 510         list_add_tail(&chunk->chunk_list, &bucket->hot_list);
 511         bucket->hot_list_count++;
 512         bucket->used_list_count--;
 513         mutex_unlock(&bucket->mutex);
 514 }
 515 
 516 struct mlx5dr_icm_pool *mlx5dr_icm_pool_create(struct mlx5dr_domain *dmn,
 517                                                enum mlx5dr_icm_type icm_type)
 518 {
 519         enum mlx5dr_icm_chunk_size max_log_chunk_sz;
 520         struct mlx5dr_icm_pool *pool;
 521         int i;
 522 
 523         if (icm_type == DR_ICM_TYPE_STE)
 524                 max_log_chunk_sz = dmn->info.max_log_sw_icm_sz;
 525         else
 526                 max_log_chunk_sz = dmn->info.max_log_action_icm_sz;
 527 
 528         pool = kvzalloc(sizeof(*pool), GFP_KERNEL);
 529         if (!pool)
 530                 return NULL;
 531 
 532         pool->buckets = kcalloc(max_log_chunk_sz + 1,
 533                                 sizeof(pool->buckets[0]),
 534                                 GFP_KERNEL);
 535         if (!pool->buckets)
 536                 goto free_pool;
 537 
 538         pool->dmn = dmn;
 539         pool->icm_type = icm_type;
 540         pool->max_log_chunk_sz = max_log_chunk_sz;
 541         pool->num_of_buckets = max_log_chunk_sz + 1;
 542         INIT_LIST_HEAD(&pool->icm_mr_list);
 543 
 544         for (i = 0; i < pool->num_of_buckets; i++)
 545                 dr_icm_bucket_init(pool, &pool->buckets[i], i);
 546 
 547         mutex_init(&pool->mr_mutex);
 548 
 549         return pool;
 550 
 551 free_pool:
 552         kvfree(pool);
 553         return NULL;
 554 }
 555 
 556 void mlx5dr_icm_pool_destroy(struct mlx5dr_icm_pool *pool)
 557 {
 558         struct mlx5dr_icm_mr *icm_mr, *next;
 559         int i;
 560 
 561         mutex_destroy(&pool->mr_mutex);
 562 
 563         list_for_each_entry_safe(icm_mr, next, &pool->icm_mr_list, mr_list)
 564                 dr_icm_pool_mr_destroy(icm_mr);
 565 
 566         for (i = 0; i < pool->num_of_buckets; i++)
 567                 dr_icm_bucket_cleanup(&pool->buckets[i]);
 568 
 569         kfree(pool->buckets);
 570         kvfree(pool);
 571 }