root/drivers/infiniband/hw/i40iw/i40iw_pble.c

DEFINITIONS

1. i40iw_destroy_pble_pool
2. i40iw_hmc_init_pble
3. get_sd_pd_idx
4. add_sd_direct
5. i40iw_free_vmalloc_mem
6. i40iw_get_vmalloc_mem
7. fpm_to_idx
8. add_bp_pages
9. add_pble_pool
10. free_lvl2
11. get_lvl2_pble
12. get_lvl1_pble
13. get_lvl1_lvl2_pble
14. i40iw_get_pble
15. i40iw_free_pble

   1 /*******************************************************************************
   2 *
   3 * Copyright (c) 2015-2016 Intel Corporation.  All rights reserved.
   4 *
   5 * This software is available to you under a choice of one of two
   6 * licenses.  You may choose to be licensed under the terms of the GNU
   7 * General Public License (GPL) Version 2, available from the file
   8 * COPYING in the main directory of this source tree, or the
   9 * OpenFabrics.org BSD license below:
  10 *
  11 *   Redistribution and use in source and binary forms, with or
  12 *   without modification, are permitted provided that the following
  13 *   conditions are met:
  14 *
  15 *    - Redistributions of source code must retain the above
  16 *       copyright notice, this list of conditions and the following
  17 *       disclaimer.
  18 *
  19 *    - Redistributions in binary form must reproduce the above
  20 *       copyright notice, this list of conditions and the following
  21 *       disclaimer in the documentation and/or other materials
  22 *       provided with the distribution.
  23 *
  24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  31 * SOFTWARE.
  32 *
  33 *******************************************************************************/
  34 
  35 #include "i40iw_status.h"
  36 #include "i40iw_osdep.h"
  37 #include "i40iw_register.h"
  38 #include "i40iw_hmc.h"
  39 
  40 #include "i40iw_d.h"
  41 #include "i40iw_type.h"
  42 #include "i40iw_p.h"
  43 
  44 #include <linux/pci.h>
  45 #include <linux/genalloc.h>
  46 #include <linux/vmalloc.h>
  47 #include "i40iw_pble.h"
  48 #include "i40iw.h"
  49 
  50 struct i40iw_device;
  51 static enum i40iw_status_code add_pble_pool(struct i40iw_sc_dev *dev,
  52                                             struct i40iw_hmc_pble_rsrc *pble_rsrc);
  53 static void i40iw_free_vmalloc_mem(struct i40iw_hw *hw, struct i40iw_chunk *chunk);
  54 
  55 /**
  56  * i40iw_destroy_pble_pool - destroy pool during module unload
  57  * @pble_rsrc:  pble resources
  58  */
  59 void i40iw_destroy_pble_pool(struct i40iw_sc_dev *dev, struct i40iw_hmc_pble_rsrc *pble_rsrc)
  60 {
  61         struct list_head *clist;
  62         struct list_head *tlist;
  63         struct i40iw_chunk *chunk;
  64         struct i40iw_pble_pool *pinfo = &pble_rsrc->pinfo;
  65 
  66         if (pinfo->pool) {
  67                 list_for_each_safe(clist, tlist, &pinfo->clist) {
  68                         chunk = list_entry(clist, struct i40iw_chunk, list);
  69                         if (chunk->type == I40IW_VMALLOC)
  70                                 i40iw_free_vmalloc_mem(dev->hw, chunk);
  71                         kfree(chunk);
  72                 }
  73                 gen_pool_destroy(pinfo->pool);
  74         }
  75 }
  76 
  77 /**
  78  * i40iw_hmc_init_pble - Initialize pble resources during module load
  79  * @dev: i40iw_sc_dev struct
  80  * @pble_rsrc:  pble resources
  81  */
  82 enum i40iw_status_code i40iw_hmc_init_pble(struct i40iw_sc_dev *dev,
  83                                            struct i40iw_hmc_pble_rsrc *pble_rsrc)
  84 {
  85         struct i40iw_hmc_info *hmc_info;
  86         u32 fpm_idx = 0;
  87 
  88         hmc_info = dev->hmc_info;
  89         pble_rsrc->fpm_base_addr = hmc_info->hmc_obj[I40IW_HMC_IW_PBLE].base;
  90         /* Now start the pble' on 4k boundary */
  91         if (pble_rsrc->fpm_base_addr & 0xfff)
  92                 fpm_idx = (PAGE_SIZE - (pble_rsrc->fpm_base_addr & 0xfff)) >> 3;
  93 
  94         pble_rsrc->unallocated_pble =
  95             hmc_info->hmc_obj[I40IW_HMC_IW_PBLE].cnt - fpm_idx;
  96         pble_rsrc->next_fpm_addr = pble_rsrc->fpm_base_addr + (fpm_idx << 3);
  97 
  98         pble_rsrc->pinfo.pool_shift = POOL_SHIFT;
  99         pble_rsrc->pinfo.pool = gen_pool_create(pble_rsrc->pinfo.pool_shift, -1);
 100         INIT_LIST_HEAD(&pble_rsrc->pinfo.clist);
 101         if (!pble_rsrc->pinfo.pool)
 102                 goto error;
 103 
 104         if (add_pble_pool(dev, pble_rsrc))
 105                 goto error;
 106 
 107         return 0;
 108 
 109  error:i40iw_destroy_pble_pool(dev, pble_rsrc);
 110         return I40IW_ERR_NO_MEMORY;
 111 }
 112 
 113 /**
 114  * get_sd_pd_idx -  Returns sd index, pd index and rel_pd_idx from fpm address
 115  * @ pble_rsrc: structure containing fpm address
 116  * @ idx: where to return indexes
 117  */
 118 static inline void get_sd_pd_idx(struct i40iw_hmc_pble_rsrc *pble_rsrc,
 119                                  struct sd_pd_idx *idx)
 120 {
 121         idx->sd_idx = (u32)(pble_rsrc->next_fpm_addr) / I40IW_HMC_DIRECT_BP_SIZE;
 122         idx->pd_idx = (u32)(pble_rsrc->next_fpm_addr) / I40IW_HMC_PAGED_BP_SIZE;
 123         idx->rel_pd_idx = (idx->pd_idx % I40IW_HMC_PD_CNT_IN_SD);
 124 }
 125 
 126 /**
 127  * add_sd_direct - add sd direct for pble
 128  * @dev: hardware control device structure
 129  * @pble_rsrc: pble resource ptr
 130  * @info: page info for sd
 131  */
 132 static enum i40iw_status_code add_sd_direct(struct i40iw_sc_dev *dev,
 133                                             struct i40iw_hmc_pble_rsrc *pble_rsrc,
 134                                             struct i40iw_add_page_info *info)
 135 {
 136         enum i40iw_status_code ret_code = 0;
 137         struct sd_pd_idx *idx = &info->idx;
 138         struct i40iw_chunk *chunk = info->chunk;
 139         struct i40iw_hmc_info *hmc_info = info->hmc_info;
 140         struct i40iw_hmc_sd_entry *sd_entry = info->sd_entry;
 141         u32 offset = 0;
 142 
 143         if (!sd_entry->valid) {
 144                 if (dev->is_pf) {
 145                         ret_code = i40iw_add_sd_table_entry(dev->hw, hmc_info,
 146                                                             info->idx.sd_idx,
 147                                                             I40IW_SD_TYPE_DIRECT,
 148                                                             I40IW_HMC_DIRECT_BP_SIZE);
 149                         if (ret_code)
 150                                 return ret_code;
 151                         chunk->type = I40IW_DMA_COHERENT;
 152                 }
 153         }
 154         offset = idx->rel_pd_idx << I40IW_HMC_PAGED_BP_SHIFT;
 155         chunk->size = info->pages << I40IW_HMC_PAGED_BP_SHIFT;
 156         chunk->vaddr = ((u8 *)sd_entry->u.bp.addr.va + offset);
 157         chunk->fpm_addr = pble_rsrc->next_fpm_addr;
 158         i40iw_debug(dev, I40IW_DEBUG_PBLE, "chunk_size[%d] = 0x%x vaddr=%p fpm_addr = %llx\n",
 159                     chunk->size, chunk->size, chunk->vaddr, chunk->fpm_addr);
 160         return 0;
 161 }
 162 
 163 /**
 164  * i40iw_free_vmalloc_mem - free vmalloc during close
 165  * @hw: hw struct
 166  * @chunk: chunk information for vmalloc
 167  */
 168 static void i40iw_free_vmalloc_mem(struct i40iw_hw *hw, struct i40iw_chunk *chunk)
 169 {
 170         struct pci_dev *pcidev = (struct pci_dev *)hw->dev_context;
 171         int i;
 172 
 173         if (!chunk->pg_cnt)
 174                 goto done;
 175         for (i = 0; i < chunk->pg_cnt; i++)
 176                 dma_unmap_page(&pcidev->dev, chunk->dmaaddrs[i], PAGE_SIZE, DMA_BIDIRECTIONAL);
 177 
 178  done:
 179         kfree(chunk->dmaaddrs);
 180         chunk->dmaaddrs = NULL;
 181         vfree(chunk->vaddr);
 182         chunk->vaddr = NULL;
 183         chunk->type = 0;
 184 }
 185 
 186 /**
 187  * i40iw_get_vmalloc_mem - get 2M page for sd
 188  * @hw: hardware address
 189  * @chunk: chunk to adf
 190  * @pg_cnt: #of 4 K pages
 191  */
 192 static enum i40iw_status_code i40iw_get_vmalloc_mem(struct i40iw_hw *hw,
 193                                                     struct i40iw_chunk *chunk,
 194                                                     int pg_cnt)
 195 {
 196         struct pci_dev *pcidev = (struct pci_dev *)hw->dev_context;
 197         struct page *page;
 198         u8 *addr;
 199         u32 size;
 200         int i;
 201 
 202         chunk->dmaaddrs = kzalloc(pg_cnt << 3, GFP_KERNEL);
 203         if (!chunk->dmaaddrs)
 204                 return I40IW_ERR_NO_MEMORY;
 205         size = PAGE_SIZE * pg_cnt;
 206         chunk->vaddr = vmalloc(size);
 207         if (!chunk->vaddr) {
 208                 kfree(chunk->dmaaddrs);
 209                 chunk->dmaaddrs = NULL;
 210                 return I40IW_ERR_NO_MEMORY;
 211         }
 212         chunk->size = size;
 213         addr = (u8 *)chunk->vaddr;
 214         for (i = 0; i < pg_cnt; i++) {
 215                 page = vmalloc_to_page((void *)addr);
 216                 if (!page)
 217                         break;
 218                 chunk->dmaaddrs[i] = dma_map_page(&pcidev->dev, page, 0,
 219                                                   PAGE_SIZE, DMA_BIDIRECTIONAL);
 220                 if (dma_mapping_error(&pcidev->dev, chunk->dmaaddrs[i]))
 221                         break;
 222                 addr += PAGE_SIZE;
 223         }
 224 
 225         chunk->pg_cnt = i;
 226         chunk->type = I40IW_VMALLOC;
 227         if (i == pg_cnt)
 228                 return 0;
 229 
 230         i40iw_free_vmalloc_mem(hw, chunk);
 231         return I40IW_ERR_NO_MEMORY;
 232 }
 233 
 234 /**
 235  * fpm_to_idx - given fpm address, get pble index
 236  * @pble_rsrc: pble resource management
 237  * @addr: fpm address for index
 238  */
 239 static inline u32 fpm_to_idx(struct i40iw_hmc_pble_rsrc *pble_rsrc, u64 addr)
 240 {
 241         return (addr - (pble_rsrc->fpm_base_addr)) >> 3;
 242 }
 243 
 244 /**
 245  * add_bp_pages - add backing pages for sd
 246  * @dev: hardware control device structure
 247  * @pble_rsrc: pble resource management
 248  * @info: page info for sd
 249  */
 250 static enum i40iw_status_code add_bp_pages(struct i40iw_sc_dev *dev,
 251                                            struct i40iw_hmc_pble_rsrc *pble_rsrc,
 252                                            struct i40iw_add_page_info *info)
 253 {
 254         u8 *addr;
 255         struct i40iw_dma_mem mem;
 256         struct i40iw_hmc_pd_entry *pd_entry;
 257         struct i40iw_hmc_sd_entry *sd_entry = info->sd_entry;
 258         struct i40iw_hmc_info *hmc_info = info->hmc_info;
 259         struct i40iw_chunk *chunk = info->chunk;
 260         struct i40iw_manage_vf_pble_info vf_pble_info;
 261         enum i40iw_status_code status = 0;
 262         u32 rel_pd_idx = info->idx.rel_pd_idx;
 263         u32 pd_idx = info->idx.pd_idx;
 264         u32 i;
 265 
 266         status = i40iw_get_vmalloc_mem(dev->hw, chunk, info->pages);
 267         if (status)
 268                 return I40IW_ERR_NO_MEMORY;
 269         status = i40iw_add_sd_table_entry(dev->hw, hmc_info,
 270                                           info->idx.sd_idx, I40IW_SD_TYPE_PAGED,
 271                                           I40IW_HMC_DIRECT_BP_SIZE);
 272         if (status)
 273                 goto error;
 274         if (!dev->is_pf) {
 275                 status = i40iw_vchnl_vf_add_hmc_objs(dev, I40IW_HMC_IW_PBLE,
 276                                                      fpm_to_idx(pble_rsrc,
 277                                                                 pble_rsrc->next_fpm_addr),
 278                                                      (info->pages << PBLE_512_SHIFT));
 279                 if (status) {
 280                         i40iw_pr_err("allocate PBLEs in the PF.  Error %i\n", status);
 281                         goto error;
 282                 }
 283         }
 284         addr = chunk->vaddr;
 285         for (i = 0; i < info->pages; i++) {
 286                 mem.pa = chunk->dmaaddrs[i];
 287                 mem.size = PAGE_SIZE;
 288                 mem.va = (void *)(addr);
 289                 pd_entry = &sd_entry->u.pd_table.pd_entry[rel_pd_idx++];
 290                 if (!pd_entry->valid) {
 291                         status = i40iw_add_pd_table_entry(dev->hw, hmc_info, pd_idx++, &mem);
 292                         if (status)
 293                                 goto error;
 294                         addr += PAGE_SIZE;
 295                 } else {
 296                         i40iw_pr_err("pd entry is valid expecting to be invalid\n");
 297                 }
 298         }
 299         if (!dev->is_pf) {
 300                 vf_pble_info.first_pd_index = info->idx.rel_pd_idx;
 301                 vf_pble_info.inv_pd_ent = false;
 302                 vf_pble_info.pd_entry_cnt = PBLE_PER_PAGE;
 303                 vf_pble_info.pd_pl_pba = sd_entry->u.pd_table.pd_page_addr.pa;
 304                 vf_pble_info.sd_index = info->idx.sd_idx;
 305                 status = i40iw_hw_manage_vf_pble_bp(dev->back_dev,
 306                                                     &vf_pble_info, true);
 307                 if (status) {
 308                         i40iw_pr_err("CQP manage VF PBLE BP failed.  %i\n", status);
 309                         goto error;
 310                 }
 311         }
 312         chunk->fpm_addr = pble_rsrc->next_fpm_addr;
 313         return 0;
 314 error:
 315         i40iw_free_vmalloc_mem(dev->hw, chunk);
 316         return status;
 317 }
 318 
 319 /**
 320  * add_pble_pool - add a sd entry for pble resoure
 321  * @dev: hardware control device structure
 322  * @pble_rsrc: pble resource management
 323  */
 324 static enum i40iw_status_code add_pble_pool(struct i40iw_sc_dev *dev,
 325                                             struct i40iw_hmc_pble_rsrc *pble_rsrc)
 326 {
 327         struct i40iw_hmc_sd_entry *sd_entry;
 328         struct i40iw_hmc_info *hmc_info;
 329         struct i40iw_chunk *chunk;
 330         struct i40iw_add_page_info info;
 331         struct sd_pd_idx *idx = &info.idx;
 332         enum i40iw_status_code ret_code = 0;
 333         enum i40iw_sd_entry_type sd_entry_type;
 334         u64 sd_reg_val = 0;
 335         u32 pages;
 336 
 337         if (pble_rsrc->unallocated_pble < PBLE_PER_PAGE)
 338                 return I40IW_ERR_NO_MEMORY;
 339         if (pble_rsrc->next_fpm_addr & 0xfff) {
 340                 i40iw_pr_err("next fpm_addr %llx\n", pble_rsrc->next_fpm_addr);
 341                 return I40IW_ERR_INVALID_PAGE_DESC_INDEX;
 342         }
 343         chunk = kzalloc(sizeof(*chunk), GFP_KERNEL);
 344         if (!chunk)
 345                 return I40IW_ERR_NO_MEMORY;
 346         hmc_info = dev->hmc_info;
 347         chunk->fpm_addr = pble_rsrc->next_fpm_addr;
 348         get_sd_pd_idx(pble_rsrc, idx);
 349         sd_entry = &hmc_info->sd_table.sd_entry[idx->sd_idx];
 350         pages = (idx->rel_pd_idx) ? (I40IW_HMC_PD_CNT_IN_SD -
 351                         idx->rel_pd_idx) : I40IW_HMC_PD_CNT_IN_SD;
 352         pages = min(pages, pble_rsrc->unallocated_pble >> PBLE_512_SHIFT);
 353         info.chunk = chunk;
 354         info.hmc_info = hmc_info;
 355         info.pages = pages;
 356         info.sd_entry = sd_entry;
 357         if (!sd_entry->valid) {
 358                 sd_entry_type = (!idx->rel_pd_idx &&
 359                                  (pages == I40IW_HMC_PD_CNT_IN_SD) &&
 360                                  dev->is_pf) ? I40IW_SD_TYPE_DIRECT : I40IW_SD_TYPE_PAGED;
 361         } else {
 362                 sd_entry_type = sd_entry->entry_type;
 363         }
 364         i40iw_debug(dev, I40IW_DEBUG_PBLE,
 365                     "pages = %d, unallocated_pble[%u] current_fpm_addr = %llx\n",
 366                     pages, pble_rsrc->unallocated_pble, pble_rsrc->next_fpm_addr);
 367         i40iw_debug(dev, I40IW_DEBUG_PBLE, "sd_entry_type = %d sd_entry valid = %d\n",
 368                     sd_entry_type, sd_entry->valid);
 369 
 370         if (sd_entry_type == I40IW_SD_TYPE_DIRECT)
 371                 ret_code = add_sd_direct(dev, pble_rsrc, &info);
 372         if (ret_code)
 373                 sd_entry_type = I40IW_SD_TYPE_PAGED;
 374         else
 375                 pble_rsrc->stats_direct_sds++;
 376 
 377         if (sd_entry_type == I40IW_SD_TYPE_PAGED) {
 378                 ret_code = add_bp_pages(dev, pble_rsrc, &info);
 379                 if (ret_code)
 380                         goto error;
 381                 else
 382                         pble_rsrc->stats_paged_sds++;
 383         }
 384 
 385         if (gen_pool_add_virt(pble_rsrc->pinfo.pool, (unsigned long)chunk->vaddr,
 386                               (phys_addr_t)chunk->fpm_addr, chunk->size, -1)) {
 387                 i40iw_pr_err("could not allocate memory by gen_pool_addr_virt()\n");
 388                 ret_code = I40IW_ERR_NO_MEMORY;
 389                 goto error;
 390         }
 391         pble_rsrc->next_fpm_addr += chunk->size;
 392         i40iw_debug(dev, I40IW_DEBUG_PBLE, "next_fpm_addr = %llx chunk_size[%u] = 0x%x\n",
 393                     pble_rsrc->next_fpm_addr, chunk->size, chunk->size);
 394         pble_rsrc->unallocated_pble -= (chunk->size >> 3);
 395         list_add(&chunk->list, &pble_rsrc->pinfo.clist);
 396         sd_reg_val = (sd_entry_type == I40IW_SD_TYPE_PAGED) ?
 397                         sd_entry->u.pd_table.pd_page_addr.pa : sd_entry->u.bp.addr.pa;
 398         if (sd_entry->valid)
 399                 return 0;
 400         if (dev->is_pf) {
 401                 ret_code = i40iw_hmc_sd_one(dev, hmc_info->hmc_fn_id,
 402                                             sd_reg_val, idx->sd_idx,
 403                                             sd_entry->entry_type, true);
 404                 if (ret_code) {
 405                         i40iw_pr_err("cqp cmd failed for sd (pbles)\n");
 406                         goto error;
 407                 }
 408         }
 409 
 410         sd_entry->valid = true;
 411         return 0;
 412  error:
 413         kfree(chunk);
 414         return ret_code;
 415 }
 416 
 417 /**
 418  * free_lvl2 - fee level 2 pble
 419  * @pble_rsrc: pble resource management
 420  * @palloc: level 2 pble allocation
 421  */
 422 static void free_lvl2(struct i40iw_hmc_pble_rsrc *pble_rsrc,
 423                       struct i40iw_pble_alloc *palloc)
 424 {
 425         u32 i;
 426         struct gen_pool *pool;
 427         struct i40iw_pble_level2 *lvl2 = &palloc->level2;
 428         struct i40iw_pble_info *root = &lvl2->root;
 429         struct i40iw_pble_info *leaf = lvl2->leaf;
 430 
 431         pool = pble_rsrc->pinfo.pool;
 432 
 433         for (i = 0; i < lvl2->leaf_cnt; i++, leaf++) {
 434                 if (leaf->addr)
 435                         gen_pool_free(pool, leaf->addr, (leaf->cnt << 3));
 436                 else
 437                         break;
 438         }
 439 
 440         if (root->addr)
 441                 gen_pool_free(pool, root->addr, (root->cnt << 3));
 442 
 443         kfree(lvl2->leaf);
 444         lvl2->leaf = NULL;
 445 }
 446 
 447 /**
 448  * get_lvl2_pble - get level 2 pble resource
 449  * @pble_rsrc: pble resource management
 450  * @palloc: level 2 pble allocation
 451  * @pool: pool pointer
 452  */
 453 static enum i40iw_status_code get_lvl2_pble(struct i40iw_hmc_pble_rsrc *pble_rsrc,
 454                                             struct i40iw_pble_alloc *palloc,
 455                                             struct gen_pool *pool)
 456 {
 457         u32 lf4k, lflast, total, i;
 458         u32 pblcnt = PBLE_PER_PAGE;
 459         u64 *addr;
 460         struct i40iw_pble_level2 *lvl2 = &palloc->level2;
 461         struct i40iw_pble_info *root = &lvl2->root;
 462         struct i40iw_pble_info *leaf;
 463 
 464         /* number of full 512 (4K) leafs) */
 465         lf4k = palloc->total_cnt >> 9;
 466         lflast = palloc->total_cnt % PBLE_PER_PAGE;
 467         total = (lflast == 0) ? lf4k : lf4k + 1;
 468         lvl2->leaf_cnt = total;
 469 
 470         leaf = kzalloc((sizeof(*leaf) * total), GFP_ATOMIC);
 471         if (!leaf)
 472                 return I40IW_ERR_NO_MEMORY;
 473         lvl2->leaf = leaf;
 474         /* allocate pbles for the root */
 475         root->addr = gen_pool_alloc(pool, (total << 3));
 476         if (!root->addr) {
 477                 kfree(lvl2->leaf);
 478                 lvl2->leaf = NULL;
 479                 return I40IW_ERR_NO_MEMORY;
 480         }
 481         root->idx = fpm_to_idx(pble_rsrc,
 482                                (u64)gen_pool_virt_to_phys(pool, root->addr));
 483         root->cnt = total;
 484         addr = (u64 *)root->addr;
 485         for (i = 0; i < total; i++, leaf++) {
 486                 pblcnt = (lflast && ((i + 1) == total)) ? lflast : PBLE_PER_PAGE;
 487                 leaf->addr = gen_pool_alloc(pool, (pblcnt << 3));
 488                 if (!leaf->addr)
 489                         goto error;
 490                 leaf->idx = fpm_to_idx(pble_rsrc, (u64)gen_pool_virt_to_phys(pool, leaf->addr));
 491 
 492                 leaf->cnt = pblcnt;
 493                 *addr = (u64)leaf->idx;
 494                 addr++;
 495         }
 496         palloc->level = I40IW_LEVEL_2;
 497         pble_rsrc->stats_lvl2++;
 498         return 0;
 499  error:
 500         free_lvl2(pble_rsrc, palloc);
 501         return I40IW_ERR_NO_MEMORY;
 502 }
 503 
 504 /**
 505  * get_lvl1_pble - get level 1 pble resource
 506  * @dev: hardware control device structure
 507  * @pble_rsrc: pble resource management
 508  * @palloc: level 1 pble allocation
 509  */
 510 static enum i40iw_status_code get_lvl1_pble(struct i40iw_sc_dev *dev,
 511                                             struct i40iw_hmc_pble_rsrc *pble_rsrc,
 512                                             struct i40iw_pble_alloc *palloc)
 513 {
 514         u64 *addr;
 515         struct gen_pool *pool;
 516         struct i40iw_pble_info *lvl1 = &palloc->level1;
 517 
 518         pool = pble_rsrc->pinfo.pool;
 519         addr = (u64 *)gen_pool_alloc(pool, (palloc->total_cnt << 3));
 520 
 521         if (!addr)
 522                 return I40IW_ERR_NO_MEMORY;
 523 
 524         palloc->level = I40IW_LEVEL_1;
 525         lvl1->addr = (unsigned long)addr;
 526         lvl1->idx = fpm_to_idx(pble_rsrc, (u64)gen_pool_virt_to_phys(pool,
 527                                (unsigned long)addr));
 528         lvl1->cnt = palloc->total_cnt;
 529         pble_rsrc->stats_lvl1++;
 530         return 0;
 531 }
 532 
 533 /**
 534  * get_lvl1_lvl2_pble - calls get_lvl1 and get_lvl2 pble routine
 535  * @dev: i40iw_sc_dev struct
 536  * @pble_rsrc:  pble resources
 537  * @palloc: contains all inforamtion regarding pble (idx + pble addr)
 538  * @pool: pointer to general purpose special memory pool descriptor
 539  */
 540 static inline enum i40iw_status_code get_lvl1_lvl2_pble(struct i40iw_sc_dev *dev,
 541                                                         struct i40iw_hmc_pble_rsrc *pble_rsrc,
 542                                                         struct i40iw_pble_alloc *palloc,
 543                                                         struct gen_pool *pool)
 544 {
 545         enum i40iw_status_code status = 0;
 546 
 547         status = get_lvl1_pble(dev, pble_rsrc, palloc);
 548         if (status && (palloc->total_cnt > PBLE_PER_PAGE))
 549                 status = get_lvl2_pble(pble_rsrc, palloc, pool);
 550         return status;
 551 }
 552 
 553 /**
 554  * i40iw_get_pble - allocate pbles from the pool
 555  * @dev: i40iw_sc_dev struct
 556  * @pble_rsrc:  pble resources
 557  * @palloc: contains all inforamtion regarding pble (idx + pble addr)
 558  * @pble_cnt: #of pbles requested
 559  */
 560 enum i40iw_status_code i40iw_get_pble(struct i40iw_sc_dev *dev,
 561                                       struct i40iw_hmc_pble_rsrc *pble_rsrc,
 562                                       struct i40iw_pble_alloc *palloc,
 563                                       u32 pble_cnt)
 564 {
 565         struct gen_pool *pool;
 566         enum i40iw_status_code status = 0;
 567         u32 max_sds = 0;
 568         int i;
 569 
 570         pool = pble_rsrc->pinfo.pool;
 571         palloc->total_cnt = pble_cnt;
 572         palloc->level = I40IW_LEVEL_0;
 573         /*check first to see if we can get pble's without acquiring additional sd's */
 574         status = get_lvl1_lvl2_pble(dev, pble_rsrc, palloc, pool);
 575         if (!status)
 576                 goto exit;
 577         max_sds = (palloc->total_cnt >> 18) + 1;
 578         for (i = 0; i < max_sds; i++) {
 579                 status = add_pble_pool(dev, pble_rsrc);
 580                 if (status)
 581                         break;
 582                 status = get_lvl1_lvl2_pble(dev, pble_rsrc, palloc, pool);
 583                 if (!status)
 584                         break;
 585         }
 586 exit:
 587         if (!status)
 588                 pble_rsrc->stats_alloc_ok++;
 589         else
 590                 pble_rsrc->stats_alloc_fail++;
 591 
 592         return status;
 593 }
 594 
 595 /**
 596  * i40iw_free_pble - put pbles back into pool
 597  * @pble_rsrc:  pble resources
 598  * @palloc: contains all inforamtion regarding pble resource being freed
 599  */
 600 void i40iw_free_pble(struct i40iw_hmc_pble_rsrc *pble_rsrc,
 601                      struct i40iw_pble_alloc *palloc)
 602 {
 603         struct gen_pool *pool;
 604 
 605         pool = pble_rsrc->pinfo.pool;
 606         if (palloc->level == I40IW_LEVEL_2)
 607                 free_lvl2(pble_rsrc, palloc);
 608         else
 609                 gen_pool_free(pool, palloc->level1.addr,
 610                               (palloc->level1.cnt << 3));
 611         pble_rsrc->stats_alloc_freed++;
 612 }