drivers/infiniband/hw/mlx5/odp.c

/* [<][>][^][v][top][bottom][index][help] */
This source file includes following definitions.
check_parent
mr_to_per_mm
odp_next
odp_lookup
mlx5_odp_populate_klm
mr_leaf_free_action
mlx5_ib_invalidate_range
mlx5_ib_internal_fill_odp_caps
mlx5_ib_page_fault_resume
implicit_mr_alloc
implicit_mr_get_data
mlx5_ib_alloc_implicit_mr
mlx5_ib_free_implicit_mr
pagefault_mr
mkey_is_eq
get_indirect_num_descs
pagefault_single_data_segment
pagefault_data_segments
mlx5_ib_mr_initiator_pfault_handler
mlx5_ib_mr_responder_pfault_handler_srq
mlx5_ib_mr_responder_pfault_handler_rq
odp_get_rsc
res_to_qp
res_to_srq
mlx5_ib_mr_wqe_pfault_handler
pages_in_range
mlx5_ib_mr_rdma_pfault_handler
mlx5_ib_pfault
mlx5_ib_eqe_pf_action
mlx5_ib_eq_pf_process
mlx5_ib_eq_pf_int
mempool_refill
mlx5_ib_eq_pf_action
mlx5_ib_create_pf_eq
mlx5_ib_destroy_pf_eq
mlx5_odp_init_mr_cache_entry
mlx5_ib_odp_init_one
mlx5_ib_odp_cleanup_one
mlx5_ib_odp_init
num_pending_prefetch_dec
num_pending_prefetch_inc
mlx5_ib_prefetch_sg_list
mlx5_ib_prefetch_mr_work
mlx5_ib_advise_mr_prefetch
   1 /*
   2  * Copyright (c) 2013-2015, Mellanox Technologies. All rights reserved.
   3  *
   4  * This software is available to you under a choice of one of two
   5  * licenses.  You may choose to be licensed under the terms of the GNU
   6  * General Public License (GPL) Version 2, available from the file
   7  * COPYING in the main directory of this source tree, or the
   8  * OpenIB.org BSD license below:
   9  *
  10  *     Redistribution and use in source and binary forms, with or
  11  *     without modification, are permitted provided that the following
  12  *     conditions are met:
  13  *
  14  *      - Redistributions of source code must retain the above
  15  *        copyright notice, this list of conditions and the following
  16  *        disclaimer.
  17  *
  18  *      - Redistributions in binary form must reproduce the above
  19  *        copyright notice, this list of conditions and the following
  20  *        disclaimer in the documentation and/or other materials
  21  *        provided with the distribution.
  22  *
  23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30  * SOFTWARE.
  31  */
  32 
  33 #include <rdma/ib_umem.h>
  34 #include <rdma/ib_umem_odp.h>
  35 #include <linux/kernel.h>
  36 
  37 #include "mlx5_ib.h"
  38 #include "cmd.h"
  39 
  40 #include <linux/mlx5/eq.h>
  41 
  42 /* Contains the details of a pagefault. */
  43 struct mlx5_pagefault {
  44         u32                     bytes_committed;
  45         u32                     token;
  46         u8                      event_subtype;
  47         u8                      type;
  48         union {
  49                 /* Initiator or send message responder pagefault details. */
  50                 struct {
  51                         /* Received packet size, only valid for responders. */
  52                         u32     packet_size;
  53                         /*
  54                          * Number of resource holding WQE, depends on type.
  55                          */
  56                         u32     wq_num;
  57                         /*
  58                          * WQE index. Refers to either the send queue or
  59                          * receive queue, according to event_subtype.
  60                          */
  61                         u16     wqe_index;
  62                 } wqe;
  63                 /* RDMA responder pagefault details */
  64                 struct {
  65                         u32     r_key;
  66                         /*
  67                          * Received packet size, minimal size page fault
  68                          * resolution required for forward progress.
  69                          */
  70                         u32     packet_size;
  71                         u32     rdma_op_len;
  72                         u64     rdma_va;
  73                 } rdma;
  74         };
  75 
  76         struct mlx5_ib_pf_eq    *eq;
  77         struct work_struct      work;
  78 };
  79 
  80 #define MAX_PREFETCH_LEN (4*1024*1024U)
  81 
  82 /* Timeout in ms to wait for an active mmu notifier to complete when handling
  83  * a pagefault. */
  84 #define MMU_NOTIFIER_TIMEOUT 1000
  85 
  86 #define MLX5_IMR_MTT_BITS (30 - PAGE_SHIFT)
  87 #define MLX5_IMR_MTT_SHIFT (MLX5_IMR_MTT_BITS + PAGE_SHIFT)
  88 #define MLX5_IMR_MTT_ENTRIES BIT_ULL(MLX5_IMR_MTT_BITS)
  89 #define MLX5_IMR_MTT_SIZE BIT_ULL(MLX5_IMR_MTT_SHIFT)
  90 #define MLX5_IMR_MTT_MASK (~(MLX5_IMR_MTT_SIZE - 1))
  91 
  92 #define MLX5_KSM_PAGE_SHIFT MLX5_IMR_MTT_SHIFT
  93 
  94 static u64 mlx5_imr_ksm_entries;
  95 
  96 static int check_parent(struct ib_umem_odp *odp,
  97                                struct mlx5_ib_mr *parent)
  98 {
  99         struct mlx5_ib_mr *mr = odp->private;
 100 
 101         return mr && mr->parent == parent && !odp->dying;
 102 }
 103 
 104 static struct ib_ucontext_per_mm *mr_to_per_mm(struct mlx5_ib_mr *mr)
 105 {
 106         if (WARN_ON(!mr || !is_odp_mr(mr)))
 107                 return NULL;
 108 
 109         return to_ib_umem_odp(mr->umem)->per_mm;
 110 }
 111 
 112 static struct ib_umem_odp *odp_next(struct ib_umem_odp *odp)
 113 {
 114         struct mlx5_ib_mr *mr = odp->private, *parent = mr->parent;
 115         struct ib_ucontext_per_mm *per_mm = odp->per_mm;
 116         struct rb_node *rb;
 117 
 118         down_read(&per_mm->umem_rwsem);
 119         while (1) {
 120                 rb = rb_next(&odp->interval_tree.rb);
 121                 if (!rb)
 122                         goto not_found;
 123                 odp = rb_entry(rb, struct ib_umem_odp, interval_tree.rb);
 124                 if (check_parent(odp, parent))
 125                         goto end;
 126         }
 127 not_found:
 128         odp = NULL;
 129 end:
 130         up_read(&per_mm->umem_rwsem);
 131         return odp;
 132 }
 133 
 134 static struct ib_umem_odp *odp_lookup(u64 start, u64 length,
 135                                       struct mlx5_ib_mr *parent)
 136 {
 137         struct ib_ucontext_per_mm *per_mm = mr_to_per_mm(parent);
 138         struct ib_umem_odp *odp;
 139         struct rb_node *rb;
 140 
 141         down_read(&per_mm->umem_rwsem);
 142         odp = rbt_ib_umem_lookup(&per_mm->umem_tree, start, length);
 143         if (!odp)
 144                 goto end;
 145 
 146         while (1) {
 147                 if (check_parent(odp, parent))
 148                         goto end;
 149                 rb = rb_next(&odp->interval_tree.rb);
 150                 if (!rb)
 151                         goto not_found;
 152                 odp = rb_entry(rb, struct ib_umem_odp, interval_tree.rb);
 153                 if (ib_umem_start(odp) > start + length)
 154                         goto not_found;
 155         }
 156 not_found:
 157         odp = NULL;
 158 end:
 159         up_read(&per_mm->umem_rwsem);
 160         return odp;
 161 }
 162 
 163 void mlx5_odp_populate_klm(struct mlx5_klm *pklm, size_t offset,
 164                            size_t nentries, struct mlx5_ib_mr *mr, int flags)
 165 {
 166         struct ib_pd *pd = mr->ibmr.pd;
 167         struct mlx5_ib_dev *dev = to_mdev(pd->device);
 168         struct ib_umem_odp *odp;
 169         unsigned long va;
 170         int i;
 171 
 172         if (flags & MLX5_IB_UPD_XLT_ZAP) {
 173                 for (i = 0; i < nentries; i++, pklm++) {
 174                         pklm->bcount = cpu_to_be32(MLX5_IMR_MTT_SIZE);
 175                         pklm->key = cpu_to_be32(dev->null_mkey);
 176                         pklm->va = 0;
 177                 }
 178                 return;
 179         }
 180 
 181         /*
 182          * The locking here is pretty subtle. Ideally the implicit children
 183          * list would be protected by the umem_mutex, however that is not
 184          * possible. Instead this uses a weaker update-then-lock pattern:
 185          *
 186          *  srcu_read_lock()
 187          *    <change children list>
 188          *    mutex_lock(umem_mutex)
 189          *     mlx5_ib_update_xlt()
 190          *    mutex_unlock(umem_mutex)
 191          *    destroy lkey
 192          *
 193          * ie any change the children list must be followed by the locked
 194          * update_xlt before destroying.
 195          *
 196          * The umem_mutex provides the acquire/release semantic needed to make
 197          * the children list visible to a racing thread. While SRCU is not
 198          * technically required, using it gives consistent use of the SRCU
 199          * locking around the children list.
 200          */
 201         lockdep_assert_held(&to_ib_umem_odp(mr->umem)->umem_mutex);
 202         lockdep_assert_held(&mr->dev->mr_srcu);
 203 
 204         odp = odp_lookup(offset * MLX5_IMR_MTT_SIZE,
 205                          nentries * MLX5_IMR_MTT_SIZE, mr);
 206 
 207         for (i = 0; i < nentries; i++, pklm++) {
 208                 pklm->bcount = cpu_to_be32(MLX5_IMR_MTT_SIZE);
 209                 va = (offset + i) * MLX5_IMR_MTT_SIZE;
 210                 if (odp && ib_umem_start(odp) == va) {
 211                         struct mlx5_ib_mr *mtt = odp->private;
 212 
 213                         pklm->key = cpu_to_be32(mtt->ibmr.lkey);
 214                         odp = odp_next(odp);
 215                 } else {
 216                         pklm->key = cpu_to_be32(dev->null_mkey);
 217                 }
 218                 mlx5_ib_dbg(dev, "[%d] va %lx key %x\n",
 219                             i, va, be32_to_cpu(pklm->key));
 220         }
 221 }
 222 
 223 static void mr_leaf_free_action(struct work_struct *work)
 224 {
 225         struct ib_umem_odp *odp = container_of(work, struct ib_umem_odp, work);
 226         int idx = ib_umem_start(odp) >> MLX5_IMR_MTT_SHIFT;
 227         struct mlx5_ib_mr *mr = odp->private, *imr = mr->parent;
 228         struct ib_umem_odp *odp_imr = to_ib_umem_odp(imr->umem);
 229         int srcu_key;
 230 
 231         mr->parent = NULL;
 232         synchronize_srcu(&mr->dev->mr_srcu);
 233 
 234         if (smp_load_acquire(&imr->live)) {
 235                 srcu_key = srcu_read_lock(&mr->dev->mr_srcu);
 236                 mutex_lock(&odp_imr->umem_mutex);
 237                 mlx5_ib_update_xlt(imr, idx, 1, 0,
 238                                    MLX5_IB_UPD_XLT_INDIRECT |
 239                                    MLX5_IB_UPD_XLT_ATOMIC);
 240                 mutex_unlock(&odp_imr->umem_mutex);
 241                 srcu_read_unlock(&mr->dev->mr_srcu, srcu_key);
 242         }
 243         ib_umem_odp_release(odp);
 244         mlx5_mr_cache_free(mr->dev, mr);
 245 
 246         if (atomic_dec_and_test(&imr->num_leaf_free))
 247                 wake_up(&imr->q_leaf_free);
 248 }
 249 
 250 void mlx5_ib_invalidate_range(struct ib_umem_odp *umem_odp, unsigned long start,
 251                               unsigned long end)
 252 {
 253         struct mlx5_ib_mr *mr;
 254         const u64 umr_block_mask = (MLX5_UMR_MTT_ALIGNMENT /
 255                                     sizeof(struct mlx5_mtt)) - 1;
 256         u64 idx = 0, blk_start_idx = 0;
 257         int in_block = 0;
 258         u64 addr;
 259 
 260         if (!umem_odp) {
 261                 pr_err("invalidation called on NULL umem or non-ODP umem\n");
 262                 return;
 263         }
 264 
 265         mr = umem_odp->private;
 266 
 267         if (!mr || !mr->ibmr.pd)
 268                 return;
 269 
 270         start = max_t(u64, ib_umem_start(umem_odp), start);
 271         end = min_t(u64, ib_umem_end(umem_odp), end);
 272 
 273         /*
 274          * Iteration one - zap the HW's MTTs. The notifiers_count ensures that
 275          * while we are doing the invalidation, no page fault will attempt to
 276          * overwrite the same MTTs.  Concurent invalidations might race us,
 277          * but they will write 0s as well, so no difference in the end result.
 278          */
 279         mutex_lock(&umem_odp->umem_mutex);
 280         for (addr = start; addr < end; addr += BIT(umem_odp->page_shift)) {
 281                 idx = (addr - ib_umem_start(umem_odp)) >> umem_odp->page_shift;
 282                 /*
 283                  * Strive to write the MTTs in chunks, but avoid overwriting
 284                  * non-existing MTTs. The huristic here can be improved to
 285                  * estimate the cost of another UMR vs. the cost of bigger
 286                  * UMR.
 287                  */
 288                 if (umem_odp->dma_list[idx] &
 289                     (ODP_READ_ALLOWED_BIT | ODP_WRITE_ALLOWED_BIT)) {
 290                         if (!in_block) {
 291                                 blk_start_idx = idx;
 292                                 in_block = 1;
 293                         }
 294                 } else {
 295                         u64 umr_offset = idx & umr_block_mask;
 296 
 297                         if (in_block && umr_offset == 0) {
 298                                 mlx5_ib_update_xlt(mr, blk_start_idx,
 299                                                    idx - blk_start_idx, 0,
 300                                                    MLX5_IB_UPD_XLT_ZAP |
 301                                                    MLX5_IB_UPD_XLT_ATOMIC);
 302                                 in_block = 0;
 303                         }
 304                 }
 305         }
 306         if (in_block)
 307                 mlx5_ib_update_xlt(mr, blk_start_idx,
 308                                    idx - blk_start_idx + 1, 0,
 309                                    MLX5_IB_UPD_XLT_ZAP |
 310                                    MLX5_IB_UPD_XLT_ATOMIC);
 311         /*
 312          * We are now sure that the device will not access the
 313          * memory. We can safely unmap it, and mark it as dirty if
 314          * needed.
 315          */
 316 
 317         ib_umem_odp_unmap_dma_pages(umem_odp, start, end);
 318 
 319         if (unlikely(!umem_odp->npages && mr->parent &&
 320                      !umem_odp->dying)) {
 321                 WRITE_ONCE(mr->live, 0);
 322                 umem_odp->dying = 1;
 323                 atomic_inc(&mr->parent->num_leaf_free);
 324                 schedule_work(&umem_odp->work);
 325         }
 326         mutex_unlock(&umem_odp->umem_mutex);
 327 }
 328 
 329 void mlx5_ib_internal_fill_odp_caps(struct mlx5_ib_dev *dev)
 330 {
 331         struct ib_odp_caps *caps = &dev->odp_caps;
 332 
 333         memset(caps, 0, sizeof(*caps));
 334 
 335         if (!MLX5_CAP_GEN(dev->mdev, pg) ||
 336             !mlx5_ib_can_use_umr(dev, true))
 337                 return;
 338 
 339         caps->general_caps = IB_ODP_SUPPORT;
 340 
 341         if (MLX5_CAP_GEN(dev->mdev, umr_extended_translation_offset))
 342                 dev->odp_max_size = U64_MAX;
 343         else
 344                 dev->odp_max_size = BIT_ULL(MLX5_MAX_UMR_SHIFT + PAGE_SHIFT);
 345 
 346         if (MLX5_CAP_ODP(dev->mdev, ud_odp_caps.send))
 347                 caps->per_transport_caps.ud_odp_caps |= IB_ODP_SUPPORT_SEND;
 348 
 349         if (MLX5_CAP_ODP(dev->mdev, ud_odp_caps.srq_receive))
 350                 caps->per_transport_caps.ud_odp_caps |= IB_ODP_SUPPORT_SRQ_RECV;
 351 
 352         if (MLX5_CAP_ODP(dev->mdev, rc_odp_caps.send))
 353                 caps->per_transport_caps.rc_odp_caps |= IB_ODP_SUPPORT_SEND;
 354 
 355         if (MLX5_CAP_ODP(dev->mdev, rc_odp_caps.receive))
 356                 caps->per_transport_caps.rc_odp_caps |= IB_ODP_SUPPORT_RECV;
 357 
 358         if (MLX5_CAP_ODP(dev->mdev, rc_odp_caps.write))
 359                 caps->per_transport_caps.rc_odp_caps |= IB_ODP_SUPPORT_WRITE;
 360 
 361         if (MLX5_CAP_ODP(dev->mdev, rc_odp_caps.read))
 362                 caps->per_transport_caps.rc_odp_caps |= IB_ODP_SUPPORT_READ;
 363 
 364         if (MLX5_CAP_ODP(dev->mdev, rc_odp_caps.atomic))
 365                 caps->per_transport_caps.rc_odp_caps |= IB_ODP_SUPPORT_ATOMIC;
 366 
 367         if (MLX5_CAP_ODP(dev->mdev, rc_odp_caps.srq_receive))
 368                 caps->per_transport_caps.rc_odp_caps |= IB_ODP_SUPPORT_SRQ_RECV;
 369 
 370         if (MLX5_CAP_ODP(dev->mdev, xrc_odp_caps.send))
 371                 caps->per_transport_caps.xrc_odp_caps |= IB_ODP_SUPPORT_SEND;
 372 
 373         if (MLX5_CAP_ODP(dev->mdev, xrc_odp_caps.receive))
 374                 caps->per_transport_caps.xrc_odp_caps |= IB_ODP_SUPPORT_RECV;
 375 
 376         if (MLX5_CAP_ODP(dev->mdev, xrc_odp_caps.write))
 377                 caps->per_transport_caps.xrc_odp_caps |= IB_ODP_SUPPORT_WRITE;
 378 
 379         if (MLX5_CAP_ODP(dev->mdev, xrc_odp_caps.read))
 380                 caps->per_transport_caps.xrc_odp_caps |= IB_ODP_SUPPORT_READ;
 381 
 382         if (MLX5_CAP_ODP(dev->mdev, xrc_odp_caps.atomic))
 383                 caps->per_transport_caps.xrc_odp_caps |= IB_ODP_SUPPORT_ATOMIC;
 384 
 385         if (MLX5_CAP_ODP(dev->mdev, xrc_odp_caps.srq_receive))
 386                 caps->per_transport_caps.xrc_odp_caps |= IB_ODP_SUPPORT_SRQ_RECV;
 387 
 388         if (MLX5_CAP_GEN(dev->mdev, fixed_buffer_size) &&
 389             MLX5_CAP_GEN(dev->mdev, null_mkey) &&
 390             MLX5_CAP_GEN(dev->mdev, umr_extended_translation_offset) &&
 391             !MLX5_CAP_GEN(dev->mdev, umr_indirect_mkey_disabled))
 392                 caps->general_caps |= IB_ODP_SUPPORT_IMPLICIT;
 393 
 394         return;
 395 }
 396 
 397 static void mlx5_ib_page_fault_resume(struct mlx5_ib_dev *dev,
 398                                       struct mlx5_pagefault *pfault,
 399                                       int error)
 400 {
 401         int wq_num = pfault->event_subtype == MLX5_PFAULT_SUBTYPE_WQE ?
 402                      pfault->wqe.wq_num : pfault->token;
 403         u32 out[MLX5_ST_SZ_DW(page_fault_resume_out)] = { };
 404         u32 in[MLX5_ST_SZ_DW(page_fault_resume_in)]   = { };
 405         int err;
 406 
 407         MLX5_SET(page_fault_resume_in, in, opcode, MLX5_CMD_OP_PAGE_FAULT_RESUME);
 408         MLX5_SET(page_fault_resume_in, in, page_fault_type, pfault->type);
 409         MLX5_SET(page_fault_resume_in, in, token, pfault->token);
 410         MLX5_SET(page_fault_resume_in, in, wq_number, wq_num);
 411         MLX5_SET(page_fault_resume_in, in, error, !!error);
 412 
 413         err = mlx5_cmd_exec(dev->mdev, in, sizeof(in), out, sizeof(out));
 414         if (err)
 415                 mlx5_ib_err(dev, "Failed to resolve the page fault on WQ 0x%x err %d\n",
 416                             wq_num, err);
 417 }
 418 
 419 static struct mlx5_ib_mr *implicit_mr_alloc(struct ib_pd *pd,
 420                                             struct ib_umem_odp *umem_odp,
 421                                             bool ksm, int access_flags)
 422 {
 423         struct mlx5_ib_dev *dev = to_mdev(pd->device);
 424         struct mlx5_ib_mr *mr;
 425         int err;
 426 
 427         mr = mlx5_mr_cache_alloc(dev, ksm ? MLX5_IMR_KSM_CACHE_ENTRY :
 428                                             MLX5_IMR_MTT_CACHE_ENTRY);
 429 
 430         if (IS_ERR(mr))
 431                 return mr;
 432 
 433         mr->ibmr.pd = pd;
 434 
 435         mr->dev = dev;
 436         mr->access_flags = access_flags;
 437         mr->mmkey.iova = 0;
 438         mr->umem = &umem_odp->umem;
 439 
 440         if (ksm) {
 441                 err = mlx5_ib_update_xlt(mr, 0,
 442                                          mlx5_imr_ksm_entries,
 443                                          MLX5_KSM_PAGE_SHIFT,
 444                                          MLX5_IB_UPD_XLT_INDIRECT |
 445                                          MLX5_IB_UPD_XLT_ZAP |
 446                                          MLX5_IB_UPD_XLT_ENABLE);
 447 
 448         } else {
 449                 err = mlx5_ib_update_xlt(mr, 0,
 450                                          MLX5_IMR_MTT_ENTRIES,
 451                                          PAGE_SHIFT,
 452                                          MLX5_IB_UPD_XLT_ZAP |
 453                                          MLX5_IB_UPD_XLT_ENABLE |
 454                                          MLX5_IB_UPD_XLT_ATOMIC);
 455         }
 456 
 457         if (err)
 458                 goto fail;
 459 
 460         mr->ibmr.lkey = mr->mmkey.key;
 461         mr->ibmr.rkey = mr->mmkey.key;
 462 
 463         mlx5_ib_dbg(dev, "key %x dev %p mr %p\n",
 464                     mr->mmkey.key, dev->mdev, mr);
 465 
 466         return mr;
 467 
 468 fail:
 469         mlx5_ib_err(dev, "Failed to register MKEY %d\n", err);
 470         mlx5_mr_cache_free(dev, mr);
 471 
 472         return ERR_PTR(err);
 473 }
 474 
 475 static struct ib_umem_odp *implicit_mr_get_data(struct mlx5_ib_mr *mr,
 476                                                 u64 io_virt, size_t bcnt)
 477 {
 478         struct mlx5_ib_dev *dev = to_mdev(mr->ibmr.pd->device);
 479         struct ib_umem_odp *odp, *result = NULL;
 480         struct ib_umem_odp *odp_mr = to_ib_umem_odp(mr->umem);
 481         u64 addr = io_virt & MLX5_IMR_MTT_MASK;
 482         int nentries = 0, start_idx = 0, ret;
 483         struct mlx5_ib_mr *mtt;
 484 
 485         mutex_lock(&odp_mr->umem_mutex);
 486         odp = odp_lookup(addr, 1, mr);
 487 
 488         mlx5_ib_dbg(dev, "io_virt:%llx bcnt:%zx addr:%llx odp:%p\n",
 489                     io_virt, bcnt, addr, odp);
 490 
 491 next_mr:
 492         if (likely(odp)) {
 493                 if (nentries)
 494                         nentries++;
 495         } else {
 496                 odp = ib_umem_odp_alloc_child(odp_mr, addr, MLX5_IMR_MTT_SIZE);
 497                 if (IS_ERR(odp)) {
 498                         mutex_unlock(&odp_mr->umem_mutex);
 499                         return ERR_CAST(odp);
 500                 }
 501 
 502                 mtt = implicit_mr_alloc(mr->ibmr.pd, odp, 0,
 503                                         mr->access_flags);
 504                 if (IS_ERR(mtt)) {
 505                         mutex_unlock(&odp_mr->umem_mutex);
 506                         ib_umem_odp_release(odp);
 507                         return ERR_CAST(mtt);
 508                 }
 509 
 510                 odp->private = mtt;
 511                 mtt->umem = &odp->umem;
 512                 mtt->mmkey.iova = addr;
 513                 mtt->parent = mr;
 514                 INIT_WORK(&odp->work, mr_leaf_free_action);
 515 
 516                 smp_store_release(&mtt->live, 1);
 517 
 518                 if (!nentries)
 519                         start_idx = addr >> MLX5_IMR_MTT_SHIFT;
 520                 nentries++;
 521         }
 522 
 523         /* Return first odp if region not covered by single one */
 524         if (likely(!result))
 525                 result = odp;
 526 
 527         addr += MLX5_IMR_MTT_SIZE;
 528         if (unlikely(addr < io_virt + bcnt)) {
 529                 odp = odp_next(odp);
 530                 if (odp && ib_umem_start(odp) != addr)
 531                         odp = NULL;
 532                 goto next_mr;
 533         }
 534 
 535         if (unlikely(nentries)) {
 536                 ret = mlx5_ib_update_xlt(mr, start_idx, nentries, 0,
 537                                          MLX5_IB_UPD_XLT_INDIRECT |
 538                                          MLX5_IB_UPD_XLT_ATOMIC);
 539                 if (ret) {
 540                         mlx5_ib_err(dev, "Failed to update PAS\n");
 541                         result = ERR_PTR(ret);
 542                 }
 543         }
 544 
 545         mutex_unlock(&odp_mr->umem_mutex);
 546         return result;
 547 }
 548 
 549 struct mlx5_ib_mr *mlx5_ib_alloc_implicit_mr(struct mlx5_ib_pd *pd,
 550                                              struct ib_udata *udata,
 551                                              int access_flags)
 552 {
 553         struct mlx5_ib_mr *imr;
 554         struct ib_umem_odp *umem_odp;
 555 
 556         umem_odp = ib_umem_odp_alloc_implicit(udata, access_flags);
 557         if (IS_ERR(umem_odp))
 558                 return ERR_CAST(umem_odp);
 559 
 560         imr = implicit_mr_alloc(&pd->ibpd, umem_odp, 1, access_flags);
 561         if (IS_ERR(imr)) {
 562                 ib_umem_odp_release(umem_odp);
 563                 return ERR_CAST(imr);
 564         }
 565 
 566         imr->umem = &umem_odp->umem;
 567         init_waitqueue_head(&imr->q_leaf_free);
 568         atomic_set(&imr->num_leaf_free, 0);
 569         atomic_set(&imr->num_pending_prefetch, 0);
 570         smp_store_release(&imr->live, 1);
 571 
 572         return imr;
 573 }
 574 
 575 void mlx5_ib_free_implicit_mr(struct mlx5_ib_mr *imr)
 576 {
 577         struct ib_ucontext_per_mm *per_mm = mr_to_per_mm(imr);
 578         struct rb_node *node;
 579 
 580         down_read(&per_mm->umem_rwsem);
 581         for (node = rb_first_cached(&per_mm->umem_tree); node;
 582              node = rb_next(node)) {
 583                 struct ib_umem_odp *umem_odp =
 584                         rb_entry(node, struct ib_umem_odp, interval_tree.rb);
 585                 struct mlx5_ib_mr *mr = umem_odp->private;
 586 
 587                 if (mr->parent != imr)
 588                         continue;
 589 
 590                 mutex_lock(&umem_odp->umem_mutex);
 591                 ib_umem_odp_unmap_dma_pages(umem_odp, ib_umem_start(umem_odp),
 592                                             ib_umem_end(umem_odp));
 593 
 594                 if (umem_odp->dying) {
 595                         mutex_unlock(&umem_odp->umem_mutex);
 596                         continue;
 597                 }
 598 
 599                 umem_odp->dying = 1;
 600                 atomic_inc(&imr->num_leaf_free);
 601                 schedule_work(&umem_odp->work);
 602                 mutex_unlock(&umem_odp->umem_mutex);
 603         }
 604         up_read(&per_mm->umem_rwsem);
 605 
 606         wait_event(imr->q_leaf_free, !atomic_read(&imr->num_leaf_free));
 607 }
 608 
 609 #define MLX5_PF_FLAGS_PREFETCH  BIT(0)
 610 #define MLX5_PF_FLAGS_DOWNGRADE BIT(1)
 611 static int pagefault_mr(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr,
 612                         u64 io_virt, size_t bcnt, u32 *bytes_mapped,
 613                         u32 flags)
 614 {
 615         int npages = 0, current_seq, page_shift, ret, np;
 616         struct ib_umem_odp *odp_mr = to_ib_umem_odp(mr->umem);
 617         bool downgrade = flags & MLX5_PF_FLAGS_DOWNGRADE;
 618         bool prefetch = flags & MLX5_PF_FLAGS_PREFETCH;
 619         u64 access_mask;
 620         u64 start_idx, page_mask;
 621         struct ib_umem_odp *odp;
 622         size_t size;
 623 
 624         if (odp_mr->is_implicit_odp) {
 625                 odp = implicit_mr_get_data(mr, io_virt, bcnt);
 626 
 627                 if (IS_ERR(odp))
 628                         return PTR_ERR(odp);
 629                 mr = odp->private;
 630         } else {
 631                 odp = odp_mr;
 632         }
 633 
 634 next_mr:
 635         size = min_t(size_t, bcnt, ib_umem_end(odp) - io_virt);
 636 
 637         page_shift = odp->page_shift;
 638         page_mask = ~(BIT(page_shift) - 1);
 639         start_idx = (io_virt - (mr->mmkey.iova & page_mask)) >> page_shift;
 640         access_mask = ODP_READ_ALLOWED_BIT;
 641 
 642         if (prefetch && !downgrade && !odp->umem.writable) {
 643                 /* prefetch with write-access must
 644                  * be supported by the MR
 645                  */
 646                 ret = -EINVAL;
 647                 goto out;
 648         }
 649 
 650         if (odp->umem.writable && !downgrade)
 651                 access_mask |= ODP_WRITE_ALLOWED_BIT;
 652 
 653         current_seq = READ_ONCE(odp->notifiers_seq);
 654         /*
 655          * Ensure the sequence number is valid for some time before we call
 656          * gup.
 657          */
 658         smp_rmb();
 659 
 660         ret = ib_umem_odp_map_dma_pages(odp, io_virt, size, access_mask,
 661                                         current_seq);
 662 
 663         if (ret < 0)
 664                 goto out;
 665 
 666         np = ret;
 667 
 668         mutex_lock(&odp->umem_mutex);
 669         if (!ib_umem_mmu_notifier_retry(odp, current_seq)) {
 670                 /*
 671                  * No need to check whether the MTTs really belong to
 672                  * this MR, since ib_umem_odp_map_dma_pages already
 673                  * checks this.
 674                  */
 675                 ret = mlx5_ib_update_xlt(mr, start_idx, np,
 676                                          page_shift, MLX5_IB_UPD_XLT_ATOMIC);
 677         } else {
 678                 ret = -EAGAIN;
 679         }
 680         mutex_unlock(&odp->umem_mutex);
 681 
 682         if (ret < 0) {
 683                 if (ret != -EAGAIN)
 684                         mlx5_ib_err(dev, "Failed to update mkey page tables\n");
 685                 goto out;
 686         }
 687 
 688         if (bytes_mapped) {
 689                 u32 new_mappings = (np << page_shift) -
 690                         (io_virt - round_down(io_virt, 1 << page_shift));
 691                 *bytes_mapped += min_t(u32, new_mappings, size);
 692         }
 693 
 694         npages += np << (page_shift - PAGE_SHIFT);
 695         bcnt -= size;
 696 
 697         if (unlikely(bcnt)) {
 698                 struct ib_umem_odp *next;
 699 
 700                 io_virt += size;
 701                 next = odp_next(odp);
 702                 if (unlikely(!next || ib_umem_start(next) != io_virt)) {
 703                         mlx5_ib_dbg(dev, "next implicit leaf removed at 0x%llx. got %p\n",
 704                                     io_virt, next);
 705                         return -EAGAIN;
 706                 }
 707                 odp = next;
 708                 mr = odp->private;
 709                 goto next_mr;
 710         }
 711 
 712         return npages;
 713 
 714 out:
 715         if (ret == -EAGAIN) {
 716                 unsigned long timeout = msecs_to_jiffies(MMU_NOTIFIER_TIMEOUT);
 717 
 718                 if (!wait_for_completion_timeout(&odp->notifier_completion,
 719                                                  timeout)) {
 720                         mlx5_ib_warn(
 721                                 dev,
 722                                 "timeout waiting for mmu notifier. seq %d against %d. notifiers_count=%d\n",
 723                                 current_seq, odp->notifiers_seq,
 724                                 odp->notifiers_count);
 725                 }
 726         }
 727 
 728         return ret;
 729 }
 730 
 731 struct pf_frame {
 732         struct pf_frame *next;
 733         u32 key;
 734         u64 io_virt;
 735         size_t bcnt;
 736         int depth;
 737 };
 738 
 739 static bool mkey_is_eq(struct mlx5_core_mkey *mmkey, u32 key)
 740 {
 741         if (!mmkey)
 742                 return false;
 743         if (mmkey->type == MLX5_MKEY_MW)
 744                 return mlx5_base_mkey(mmkey->key) == mlx5_base_mkey(key);
 745         return mmkey->key == key;
 746 }
 747 
 748 static int get_indirect_num_descs(struct mlx5_core_mkey *mmkey)
 749 {
 750         struct mlx5_ib_mw *mw;
 751         struct mlx5_ib_devx_mr *devx_mr;
 752 
 753         if (mmkey->type == MLX5_MKEY_MW) {
 754                 mw = container_of(mmkey, struct mlx5_ib_mw, mmkey);
 755                 return mw->ndescs;
 756         }
 757 
 758         devx_mr = container_of(mmkey, struct mlx5_ib_devx_mr,
 759                                mmkey);
 760         return devx_mr->ndescs;
 761 }
 762 
 763 /*
 764  * Handle a single data segment in a page-fault WQE or RDMA region.
 765  *
 766  * Returns number of OS pages retrieved on success. The caller may continue to
 767  * the next data segment.
 768  * Can return the following error codes:
 769  * -EAGAIN to designate a temporary error. The caller will abort handling the
 770  *  page fault and resolve it.
 771  * -EFAULT when there's an error mapping the requested pages. The caller will
 772  *  abort the page fault handling.
 773  */
 774 static int pagefault_single_data_segment(struct mlx5_ib_dev *dev,
 775                                          struct ib_pd *pd, u32 key,
 776                                          u64 io_virt, size_t bcnt,
 777                                          u32 *bytes_committed,
 778                                          u32 *bytes_mapped, u32 flags)
 779 {
 780         int npages = 0, srcu_key, ret, i, outlen, cur_outlen = 0, depth = 0;
 781         bool prefetch = flags & MLX5_PF_FLAGS_PREFETCH;
 782         struct pf_frame *head = NULL, *frame;
 783         struct mlx5_core_mkey *mmkey;
 784         struct mlx5_ib_mr *mr;
 785         struct mlx5_klm *pklm;
 786         u32 *out = NULL;
 787         size_t offset;
 788         int ndescs;
 789 
 790         srcu_key = srcu_read_lock(&dev->mr_srcu);
 791 
 792         io_virt += *bytes_committed;
 793         bcnt -= *bytes_committed;
 794 
 795 next_mr:
 796         mmkey = xa_load(&dev->mdev->priv.mkey_table, mlx5_base_mkey(key));
 797         if (!mkey_is_eq(mmkey, key)) {
 798                 mlx5_ib_dbg(dev, "failed to find mkey %x\n", key);
 799                 ret = -EFAULT;
 800                 goto srcu_unlock;
 801         }
 802 
 803         if (prefetch && mmkey->type != MLX5_MKEY_MR) {
 804                 mlx5_ib_dbg(dev, "prefetch is allowed only for MR\n");
 805                 ret = -EINVAL;
 806                 goto srcu_unlock;
 807         }
 808 
 809         switch (mmkey->type) {
 810         case MLX5_MKEY_MR:
 811                 mr = container_of(mmkey, struct mlx5_ib_mr, mmkey);
 812                 if (!smp_load_acquire(&mr->live) || !mr->ibmr.pd) {
 813                         mlx5_ib_dbg(dev, "got dead MR\n");
 814                         ret = -EFAULT;
 815                         goto srcu_unlock;
 816                 }
 817 
 818                 if (prefetch) {
 819                         if (!is_odp_mr(mr) ||
 820                             mr->ibmr.pd != pd) {
 821                                 mlx5_ib_dbg(dev, "Invalid prefetch request: %s\n",
 822                                             is_odp_mr(mr) ?  "MR is not ODP" :
 823                                             "PD is not of the MR");
 824                                 ret = -EINVAL;
 825                                 goto srcu_unlock;
 826                         }
 827                 }
 828 
 829                 if (!is_odp_mr(mr)) {
 830                         mlx5_ib_dbg(dev, "skipping non ODP MR (lkey=0x%06x) in page fault handler.\n",
 831                                     key);
 832                         if (bytes_mapped)
 833                                 *bytes_mapped += bcnt;
 834                         ret = 0;
 835                         goto srcu_unlock;
 836                 }
 837 
 838                 ret = pagefault_mr(dev, mr, io_virt, bcnt, bytes_mapped, flags);
 839                 if (ret < 0)
 840                         goto srcu_unlock;
 841 
 842                 npages += ret;
 843                 ret = 0;
 844                 break;
 845 
 846         case MLX5_MKEY_MW:
 847         case MLX5_MKEY_INDIRECT_DEVX:
 848                 ndescs = get_indirect_num_descs(mmkey);
 849 
 850                 if (depth >= MLX5_CAP_GEN(dev->mdev, max_indirection)) {
 851                         mlx5_ib_dbg(dev, "indirection level exceeded\n");
 852                         ret = -EFAULT;
 853                         goto srcu_unlock;
 854                 }
 855 
 856                 outlen = MLX5_ST_SZ_BYTES(query_mkey_out) +
 857                         sizeof(*pklm) * (ndescs - 2);
 858 
 859                 if (outlen > cur_outlen) {
 860                         kfree(out);
 861                         out = kzalloc(outlen, GFP_KERNEL);
 862                         if (!out) {
 863                                 ret = -ENOMEM;
 864                                 goto srcu_unlock;
 865                         }
 866                         cur_outlen = outlen;
 867                 }
 868 
 869                 pklm = (struct mlx5_klm *)MLX5_ADDR_OF(query_mkey_out, out,
 870                                                        bsf0_klm0_pas_mtt0_1);
 871 
 872                 ret = mlx5_core_query_mkey(dev->mdev, mmkey, out, outlen);
 873                 if (ret)
 874                         goto srcu_unlock;
 875 
 876                 offset = io_virt - MLX5_GET64(query_mkey_out, out,
 877                                               memory_key_mkey_entry.start_addr);
 878 
 879                 for (i = 0; bcnt && i < ndescs; i++, pklm++) {
 880                         if (offset >= be32_to_cpu(pklm->bcount)) {
 881                                 offset -= be32_to_cpu(pklm->bcount);
 882                                 continue;
 883                         }
 884 
 885                         frame = kzalloc(sizeof(*frame), GFP_KERNEL);
 886                         if (!frame) {
 887                                 ret = -ENOMEM;
 888                                 goto srcu_unlock;
 889                         }
 890 
 891                         frame->key = be32_to_cpu(pklm->key);
 892                         frame->io_virt = be64_to_cpu(pklm->va) + offset;
 893                         frame->bcnt = min_t(size_t, bcnt,
 894                                             be32_to_cpu(pklm->bcount) - offset);
 895                         frame->depth = depth + 1;
 896                         frame->next = head;
 897                         head = frame;
 898 
 899                         bcnt -= frame->bcnt;
 900                         offset = 0;
 901                 }
 902                 break;
 903 
 904         default:
 905                 mlx5_ib_dbg(dev, "wrong mkey type %d\n", mmkey->type);
 906                 ret = -EFAULT;
 907                 goto srcu_unlock;
 908         }
 909 
 910         if (head) {
 911                 frame = head;
 912                 head = frame->next;
 913 
 914                 key = frame->key;
 915                 io_virt = frame->io_virt;
 916                 bcnt = frame->bcnt;
 917                 depth = frame->depth;
 918                 kfree(frame);
 919 
 920                 goto next_mr;
 921         }
 922 
 923 srcu_unlock:
 924         while (head) {
 925                 frame = head;
 926                 head = frame->next;
 927                 kfree(frame);
 928         }
 929         kfree(out);
 930 
 931         srcu_read_unlock(&dev->mr_srcu, srcu_key);
 932         *bytes_committed = 0;
 933         return ret ? ret : npages;
 934 }
 935 
 936 /**
 937  * Parse a series of data segments for page fault handling.
 938  *
 939  * @pfault contains page fault information.
 940  * @wqe points at the first data segment in the WQE.
 941  * @wqe_end points after the end of the WQE.
 942  * @bytes_mapped receives the number of bytes that the function was able to
 943  *               map. This allows the caller to decide intelligently whether
 944  *               enough memory was mapped to resolve the page fault
 945  *               successfully (e.g. enough for the next MTU, or the entire
 946  *               WQE).
 947  * @total_wqe_bytes receives the total data size of this WQE in bytes (minus
 948  *                  the committed bytes).
 949  *
 950  * Returns the number of pages loaded if positive, zero for an empty WQE, or a
 951  * negative error code.
 952  */
 953 static int pagefault_data_segments(struct mlx5_ib_dev *dev,
 954                                    struct mlx5_pagefault *pfault,
 955                                    void *wqe,
 956                                    void *wqe_end, u32 *bytes_mapped,
 957                                    u32 *total_wqe_bytes, bool receive_queue)
 958 {
 959         int ret = 0, npages = 0;
 960         u64 io_virt;
 961         u32 key;
 962         u32 byte_count;
 963         size_t bcnt;
 964         int inline_segment;
 965 
 966         if (bytes_mapped)
 967                 *bytes_mapped = 0;
 968         if (total_wqe_bytes)
 969                 *total_wqe_bytes = 0;
 970 
 971         while (wqe < wqe_end) {
 972                 struct mlx5_wqe_data_seg *dseg = wqe;
 973 
 974                 io_virt = be64_to_cpu(dseg->addr);
 975                 key = be32_to_cpu(dseg->lkey);
 976                 byte_count = be32_to_cpu(dseg->byte_count);
 977                 inline_segment = !!(byte_count &  MLX5_INLINE_SEG);
 978                 bcnt           = byte_count & ~MLX5_INLINE_SEG;
 979 
 980                 if (inline_segment) {
 981                         bcnt = bcnt & MLX5_WQE_INLINE_SEG_BYTE_COUNT_MASK;
 982                         wqe += ALIGN(sizeof(struct mlx5_wqe_inline_seg) + bcnt,
 983                                      16);
 984                 } else {
 985                         wqe += sizeof(*dseg);
 986                 }
 987 
 988                 /* receive WQE end of sg list. */
 989                 if (receive_queue && bcnt == 0 && key == MLX5_INVALID_LKEY &&
 990                     io_virt == 0)
 991                         break;
 992 
 993                 if (!inline_segment && total_wqe_bytes) {
 994                         *total_wqe_bytes += bcnt - min_t(size_t, bcnt,
 995                                         pfault->bytes_committed);
 996                 }
 997 
 998                 /* A zero length data segment designates a length of 2GB. */
 999                 if (bcnt == 0)
1000                         bcnt = 1U << 31;
1001 
1002                 if (inline_segment || bcnt <= pfault->bytes_committed) {
1003                         pfault->bytes_committed -=
1004                                 min_t(size_t, bcnt,
1005                                       pfault->bytes_committed);
1006                         continue;
1007                 }
1008 
1009                 ret = pagefault_single_data_segment(dev, NULL, key,
1010                                                     io_virt, bcnt,
1011                                                     &pfault->bytes_committed,
1012                                                     bytes_mapped, 0);
1013                 if (ret < 0)
1014                         break;
1015                 npages += ret;
1016         }
1017 
1018         return ret < 0 ? ret : npages;
1019 }
1020 
1021 /*
1022  * Parse initiator WQE. Advances the wqe pointer to point at the
1023  * scatter-gather list, and set wqe_end to the end of the WQE.
1024  */
1025 static int mlx5_ib_mr_initiator_pfault_handler(
1026         struct mlx5_ib_dev *dev, struct mlx5_pagefault *pfault,
1027         struct mlx5_ib_qp *qp, void **wqe, void **wqe_end, int wqe_length)
1028 {
1029         struct mlx5_wqe_ctrl_seg *ctrl = *wqe;
1030         u16 wqe_index = pfault->wqe.wqe_index;
1031         struct mlx5_base_av *av;
1032         unsigned ds, opcode;
1033         u32 qpn = qp->trans_qp.base.mqp.qpn;
1034 
1035         ds = be32_to_cpu(ctrl->qpn_ds) & MLX5_WQE_CTRL_DS_MASK;
1036         if (ds * MLX5_WQE_DS_UNITS > wqe_length) {
1037                 mlx5_ib_err(dev, "Unable to read the complete WQE. ds = 0x%x, ret = 0x%x\n",
1038                             ds, wqe_length);
1039                 return -EFAULT;
1040         }
1041 
1042         if (ds == 0) {
1043                 mlx5_ib_err(dev, "Got WQE with zero DS. wqe_index=%x, qpn=%x\n",
1044                             wqe_index, qpn);
1045                 return -EFAULT;
1046         }
1047 
1048         *wqe_end = *wqe + ds * MLX5_WQE_DS_UNITS;
1049         *wqe += sizeof(*ctrl);
1050 
1051         opcode = be32_to_cpu(ctrl->opmod_idx_opcode) &
1052                  MLX5_WQE_CTRL_OPCODE_MASK;
1053 
1054         if (qp->ibqp.qp_type == IB_QPT_XRC_INI)
1055                 *wqe += sizeof(struct mlx5_wqe_xrc_seg);
1056 
1057         if (qp->ibqp.qp_type == IB_QPT_UD ||
1058             qp->qp_sub_type == MLX5_IB_QPT_DCI) {
1059                 av = *wqe;
1060                 if (av->dqp_dct & cpu_to_be32(MLX5_EXTENDED_UD_AV))
1061                         *wqe += sizeof(struct mlx5_av);
1062                 else
1063                         *wqe += sizeof(struct mlx5_base_av);
1064         }
1065 
1066         switch (opcode) {
1067         case MLX5_OPCODE_RDMA_WRITE:
1068         case MLX5_OPCODE_RDMA_WRITE_IMM:
1069         case MLX5_OPCODE_RDMA_READ:
1070                 *wqe += sizeof(struct mlx5_wqe_raddr_seg);
1071                 break;
1072         case MLX5_OPCODE_ATOMIC_CS:
1073         case MLX5_OPCODE_ATOMIC_FA:
1074                 *wqe += sizeof(struct mlx5_wqe_raddr_seg);
1075                 *wqe += sizeof(struct mlx5_wqe_atomic_seg);
1076                 break;
1077         }
1078 
1079         return 0;
1080 }
1081 
1082 /*
1083  * Parse responder WQE and set wqe_end to the end of the WQE.
1084  */
1085 static int mlx5_ib_mr_responder_pfault_handler_srq(struct mlx5_ib_dev *dev,
1086                                                    struct mlx5_ib_srq *srq,
1087                                                    void **wqe, void **wqe_end,
1088                                                    int wqe_length)
1089 {
1090         int wqe_size = 1 << srq->msrq.wqe_shift;
1091 
1092         if (wqe_size > wqe_length) {
1093                 mlx5_ib_err(dev, "Couldn't read all of the receive WQE's content\n");
1094                 return -EFAULT;
1095         }
1096 
1097         *wqe_end = *wqe + wqe_size;
1098         *wqe += sizeof(struct mlx5_wqe_srq_next_seg);
1099 
1100         return 0;
1101 }
1102 
1103 static int mlx5_ib_mr_responder_pfault_handler_rq(struct mlx5_ib_dev *dev,
1104                                                   struct mlx5_ib_qp *qp,
1105                                                   void *wqe, void **wqe_end,
1106                                                   int wqe_length)
1107 {
1108         struct mlx5_ib_wq *wq = &qp->rq;
1109         int wqe_size = 1 << wq->wqe_shift;
1110 
1111         if (qp->wq_sig) {
1112                 mlx5_ib_err(dev, "ODP fault with WQE signatures is not supported\n");
1113                 return -EFAULT;
1114         }
1115 
1116         if (wqe_size > wqe_length) {
1117                 mlx5_ib_err(dev, "Couldn't read all of the receive WQE's content\n");
1118                 return -EFAULT;
1119         }
1120 
1121         *wqe_end = wqe + wqe_size;
1122 
1123         return 0;
1124 }
1125 
1126 static inline struct mlx5_core_rsc_common *odp_get_rsc(struct mlx5_ib_dev *dev,
1127                                                        u32 wq_num, int pf_type)
1128 {
1129         struct mlx5_core_rsc_common *common = NULL;
1130         struct mlx5_core_srq *srq;
1131 
1132         switch (pf_type) {
1133         case MLX5_WQE_PF_TYPE_RMP:
1134                 srq = mlx5_cmd_get_srq(dev, wq_num);
1135                 if (srq)
1136                         common = &srq->common;
1137                 break;
1138         case MLX5_WQE_PF_TYPE_REQ_SEND_OR_WRITE:
1139         case MLX5_WQE_PF_TYPE_RESP:
1140         case MLX5_WQE_PF_TYPE_REQ_READ_OR_ATOMIC:
1141                 common = mlx5_core_res_hold(dev->mdev, wq_num, MLX5_RES_QP);
1142                 break;
1143         default:
1144                 break;
1145         }
1146 
1147         return common;
1148 }
1149 
1150 static inline struct mlx5_ib_qp *res_to_qp(struct mlx5_core_rsc_common *res)
1151 {
1152         struct mlx5_core_qp *mqp = (struct mlx5_core_qp *)res;
1153 
1154         return to_mibqp(mqp);
1155 }
1156 
1157 static inline struct mlx5_ib_srq *res_to_srq(struct mlx5_core_rsc_common *res)
1158 {
1159         struct mlx5_core_srq *msrq =
1160                 container_of(res, struct mlx5_core_srq, common);
1161 
1162         return to_mibsrq(msrq);
1163 }
1164 
1165 static void mlx5_ib_mr_wqe_pfault_handler(struct mlx5_ib_dev *dev,
1166                                           struct mlx5_pagefault *pfault)
1167 {
1168         bool sq = pfault->type & MLX5_PFAULT_REQUESTOR;
1169         u16 wqe_index = pfault->wqe.wqe_index;
1170         void *wqe, *wqe_start = NULL, *wqe_end = NULL;
1171         u32 bytes_mapped, total_wqe_bytes;
1172         struct mlx5_core_rsc_common *res;
1173         int resume_with_error = 1;
1174         struct mlx5_ib_qp *qp;
1175         size_t bytes_copied;
1176         int ret = 0;
1177 
1178         res = odp_get_rsc(dev, pfault->wqe.wq_num, pfault->type);
1179         if (!res) {
1180                 mlx5_ib_dbg(dev, "wqe page fault for missing resource %d\n", pfault->wqe.wq_num);
1181                 return;
1182         }
1183 
1184         if (res->res != MLX5_RES_QP && res->res != MLX5_RES_SRQ &&
1185             res->res != MLX5_RES_XSRQ) {
1186                 mlx5_ib_err(dev, "wqe page fault for unsupported type %d\n",
1187                             pfault->type);
1188                 goto resolve_page_fault;
1189         }
1190 
1191         wqe_start = (void *)__get_free_page(GFP_KERNEL);
1192         if (!wqe_start) {
1193                 mlx5_ib_err(dev, "Error allocating memory for IO page fault handling.\n");
1194                 goto resolve_page_fault;
1195         }
1196 
1197         wqe = wqe_start;
1198         qp = (res->res == MLX5_RES_QP) ? res_to_qp(res) : NULL;
1199         if (qp && sq) {
1200                 ret = mlx5_ib_read_user_wqe_sq(qp, wqe_index, wqe, PAGE_SIZE,
1201                                                &bytes_copied);
1202                 if (ret)
1203                         goto read_user;
1204                 ret = mlx5_ib_mr_initiator_pfault_handler(
1205                         dev, pfault, qp, &wqe, &wqe_end, bytes_copied);
1206         } else if (qp && !sq) {
1207                 ret = mlx5_ib_read_user_wqe_rq(qp, wqe_index, wqe, PAGE_SIZE,
1208                                                &bytes_copied);
1209                 if (ret)
1210                         goto read_user;
1211                 ret = mlx5_ib_mr_responder_pfault_handler_rq(
1212                         dev, qp, wqe, &wqe_end, bytes_copied);
1213         } else if (!qp) {
1214                 struct mlx5_ib_srq *srq = res_to_srq(res);
1215 
1216                 ret = mlx5_ib_read_user_wqe_srq(srq, wqe_index, wqe, PAGE_SIZE,
1217                                                 &bytes_copied);
1218                 if (ret)
1219                         goto read_user;
1220                 ret = mlx5_ib_mr_responder_pfault_handler_srq(
1221                         dev, srq, &wqe, &wqe_end, bytes_copied);
1222         }
1223 
1224         if (ret < 0 || wqe >= wqe_end)
1225                 goto resolve_page_fault;
1226 
1227         ret = pagefault_data_segments(dev, pfault, wqe, wqe_end, &bytes_mapped,
1228                                       &total_wqe_bytes, !sq);
1229         if (ret == -EAGAIN)
1230                 goto out;
1231 
1232         if (ret < 0 || total_wqe_bytes > bytes_mapped)
1233                 goto resolve_page_fault;
1234 
1235 out:
1236         ret = 0;
1237         resume_with_error = 0;
1238 
1239 read_user:
1240         if (ret)
1241                 mlx5_ib_err(
1242                         dev,
1243                         "Failed reading a WQE following page fault, error %d, wqe_index %x, qpn %x\n",
1244                         ret, wqe_index, pfault->token);
1245 
1246 resolve_page_fault:
1247         mlx5_ib_page_fault_resume(dev, pfault, resume_with_error);
1248         mlx5_ib_dbg(dev, "PAGE FAULT completed. QP 0x%x resume_with_error=%d, type: 0x%x\n",
1249                     pfault->wqe.wq_num, resume_with_error,
1250                     pfault->type);
1251         mlx5_core_res_put(res);
1252         free_page((unsigned long)wqe_start);
1253 }
1254 
1255 static int pages_in_range(u64 address, u32 length)
1256 {
1257         return (ALIGN(address + length, PAGE_SIZE) -
1258                 (address & PAGE_MASK)) >> PAGE_SHIFT;
1259 }
1260 
1261 static void mlx5_ib_mr_rdma_pfault_handler(struct mlx5_ib_dev *dev,
1262                                            struct mlx5_pagefault *pfault)
1263 {
1264         u64 address;
1265         u32 length;
1266         u32 prefetch_len = pfault->bytes_committed;
1267         int prefetch_activated = 0;
1268         u32 rkey = pfault->rdma.r_key;
1269         int ret;
1270 
1271         /* The RDMA responder handler handles the page fault in two parts.
1272          * First it brings the necessary pages for the current packet
1273          * (and uses the pfault context), and then (after resuming the QP)
1274          * prefetches more pages. The second operation cannot use the pfault
1275          * context and therefore uses the dummy_pfault context allocated on
1276          * the stack */
1277         pfault->rdma.rdma_va += pfault->bytes_committed;
1278         pfault->rdma.rdma_op_len -= min(pfault->bytes_committed,
1279                                          pfault->rdma.rdma_op_len);
1280         pfault->bytes_committed = 0;
1281 
1282         address = pfault->rdma.rdma_va;
1283         length  = pfault->rdma.rdma_op_len;
1284 
1285         /* For some operations, the hardware cannot tell the exact message
1286          * length, and in those cases it reports zero. Use prefetch
1287          * logic. */
1288         if (length == 0) {
1289                 prefetch_activated = 1;
1290                 length = pfault->rdma.packet_size;
1291                 prefetch_len = min(MAX_PREFETCH_LEN, prefetch_len);
1292         }
1293 
1294         ret = pagefault_single_data_segment(dev, NULL, rkey, address, length,
1295                                             &pfault->bytes_committed, NULL,
1296                                             0);
1297         if (ret == -EAGAIN) {
1298                 /* We're racing with an invalidation, don't prefetch */
1299                 prefetch_activated = 0;
1300         } else if (ret < 0 || pages_in_range(address, length) > ret) {
1301                 mlx5_ib_page_fault_resume(dev, pfault, 1);
1302                 if (ret != -ENOENT)
1303                         mlx5_ib_dbg(dev, "PAGE FAULT error %d. QP 0x%x, type: 0x%x\n",
1304                                     ret, pfault->token, pfault->type);
1305                 return;
1306         }
1307 
1308         mlx5_ib_page_fault_resume(dev, pfault, 0);
1309         mlx5_ib_dbg(dev, "PAGE FAULT completed. QP 0x%x, type: 0x%x, prefetch_activated: %d\n",
1310                     pfault->token, pfault->type,
1311                     prefetch_activated);
1312 
1313         /* At this point, there might be a new pagefault already arriving in
1314          * the eq, switch to the dummy pagefault for the rest of the
1315          * processing. We're still OK with the objects being alive as the
1316          * work-queue is being fenced. */
1317 
1318         if (prefetch_activated) {
1319                 u32 bytes_committed = 0;
1320 
1321                 ret = pagefault_single_data_segment(dev, NULL, rkey, address,
1322                                                     prefetch_len,
1323                                                     &bytes_committed, NULL,
1324                                                     0);
1325                 if (ret < 0 && ret != -EAGAIN) {
1326                         mlx5_ib_dbg(dev, "Prefetch failed. ret: %d, QP 0x%x, address: 0x%.16llx, length = 0x%.16x\n",
1327                                     ret, pfault->token, address, prefetch_len);
1328                 }
1329         }
1330 }
1331 
1332 static void mlx5_ib_pfault(struct mlx5_ib_dev *dev, struct mlx5_pagefault *pfault)
1333 {
1334         u8 event_subtype = pfault->event_subtype;
1335 
1336         switch (event_subtype) {
1337         case MLX5_PFAULT_SUBTYPE_WQE:
1338                 mlx5_ib_mr_wqe_pfault_handler(dev, pfault);
1339                 break;
1340         case MLX5_PFAULT_SUBTYPE_RDMA:
1341                 mlx5_ib_mr_rdma_pfault_handler(dev, pfault);
1342                 break;
1343         default:
1344                 mlx5_ib_err(dev, "Invalid page fault event subtype: 0x%x\n",
1345                             event_subtype);
1346                 mlx5_ib_page_fault_resume(dev, pfault, 1);
1347         }
1348 }
1349 
1350 static void mlx5_ib_eqe_pf_action(struct work_struct *work)
1351 {
1352         struct mlx5_pagefault *pfault = container_of(work,
1353                                                      struct mlx5_pagefault,
1354                                                      work);
1355         struct mlx5_ib_pf_eq *eq = pfault->eq;
1356 
1357         mlx5_ib_pfault(eq->dev, pfault);
1358         mempool_free(pfault, eq->pool);
1359 }
1360 
1361 static void mlx5_ib_eq_pf_process(struct mlx5_ib_pf_eq *eq)
1362 {
1363         struct mlx5_eqe_page_fault *pf_eqe;
1364         struct mlx5_pagefault *pfault;
1365         struct mlx5_eqe *eqe;
1366         int cc = 0;
1367 
1368         while ((eqe = mlx5_eq_get_eqe(eq->core, cc))) {
1369                 pfault = mempool_alloc(eq->pool, GFP_ATOMIC);
1370                 if (!pfault) {
1371                         schedule_work(&eq->work);
1372                         break;
1373                 }
1374 
1375                 pf_eqe = &eqe->data.page_fault;
1376                 pfault->event_subtype = eqe->sub_type;
1377                 pfault->bytes_committed = be32_to_cpu(pf_eqe->bytes_committed);
1378 
1379                 mlx5_ib_dbg(eq->dev,
1380                             "PAGE_FAULT: subtype: 0x%02x, bytes_committed: 0x%06x\n",
1381                             eqe->sub_type, pfault->bytes_committed);
1382 
1383                 switch (eqe->sub_type) {
1384                 case MLX5_PFAULT_SUBTYPE_RDMA:
1385                         /* RDMA based event */
1386                         pfault->type =
1387                                 be32_to_cpu(pf_eqe->rdma.pftype_token) >> 24;
1388                         pfault->token =
1389                                 be32_to_cpu(pf_eqe->rdma.pftype_token) &
1390                                 MLX5_24BIT_MASK;
1391                         pfault->rdma.r_key =
1392                                 be32_to_cpu(pf_eqe->rdma.r_key);
1393                         pfault->rdma.packet_size =
1394                                 be16_to_cpu(pf_eqe->rdma.packet_length);
1395                         pfault->rdma.rdma_op_len =
1396                                 be32_to_cpu(pf_eqe->rdma.rdma_op_len);
1397                         pfault->rdma.rdma_va =
1398                                 be64_to_cpu(pf_eqe->rdma.rdma_va);
1399                         mlx5_ib_dbg(eq->dev,
1400                                     "PAGE_FAULT: type:0x%x, token: 0x%06x, r_key: 0x%08x\n",
1401                                     pfault->type, pfault->token,
1402                                     pfault->rdma.r_key);
1403                         mlx5_ib_dbg(eq->dev,
1404                                     "PAGE_FAULT: rdma_op_len: 0x%08x, rdma_va: 0x%016llx\n",
1405                                     pfault->rdma.rdma_op_len,
1406                                     pfault->rdma.rdma_va);
1407                         break;
1408 
1409                 case MLX5_PFAULT_SUBTYPE_WQE:
1410                         /* WQE based event */
1411                         pfault->type =
1412                                 (be32_to_cpu(pf_eqe->wqe.pftype_wq) >> 24) & 0x7;
1413                         pfault->token =
1414                                 be32_to_cpu(pf_eqe->wqe.token);
1415                         pfault->wqe.wq_num =
1416                                 be32_to_cpu(pf_eqe->wqe.pftype_wq) &
1417                                 MLX5_24BIT_MASK;
1418                         pfault->wqe.wqe_index =
1419                                 be16_to_cpu(pf_eqe->wqe.wqe_index);
1420                         pfault->wqe.packet_size =
1421                                 be16_to_cpu(pf_eqe->wqe.packet_length);
1422                         mlx5_ib_dbg(eq->dev,
1423                                     "PAGE_FAULT: type:0x%x, token: 0x%06x, wq_num: 0x%06x, wqe_index: 0x%04x\n",
1424                                     pfault->type, pfault->token,
1425                                     pfault->wqe.wq_num,
1426                                     pfault->wqe.wqe_index);
1427                         break;
1428 
1429                 default:
1430                         mlx5_ib_warn(eq->dev,
1431                                      "Unsupported page fault event sub-type: 0x%02hhx\n",
1432                                      eqe->sub_type);
1433                         /* Unsupported page faults should still be
1434                          * resolved by the page fault handler
1435                          */
1436                 }
1437 
1438                 pfault->eq = eq;
1439                 INIT_WORK(&pfault->work, mlx5_ib_eqe_pf_action);
1440                 queue_work(eq->wq, &pfault->work);
1441 
1442                 cc = mlx5_eq_update_cc(eq->core, ++cc);
1443         }
1444 
1445         mlx5_eq_update_ci(eq->core, cc, 1);
1446 }
1447 
1448 static int mlx5_ib_eq_pf_int(struct notifier_block *nb, unsigned long type,
1449                              void *data)
1450 {
1451         struct mlx5_ib_pf_eq *eq =
1452                 container_of(nb, struct mlx5_ib_pf_eq, irq_nb);
1453         unsigned long flags;
1454 
1455         if (spin_trylock_irqsave(&eq->lock, flags)) {
1456                 mlx5_ib_eq_pf_process(eq);
1457                 spin_unlock_irqrestore(&eq->lock, flags);
1458         } else {
1459                 schedule_work(&eq->work);
1460         }
1461 
1462         return IRQ_HANDLED;
1463 }
1464 
1465 /* mempool_refill() was proposed but unfortunately wasn't accepted
1466  * http://lkml.iu.edu/hypermail/linux/kernel/1512.1/05073.html
1467  * Cheap workaround.
1468  */
1469 static void mempool_refill(mempool_t *pool)
1470 {
1471         while (pool->curr_nr < pool->min_nr)
1472                 mempool_free(mempool_alloc(pool, GFP_KERNEL), pool);
1473 }
1474 
1475 static void mlx5_ib_eq_pf_action(struct work_struct *work)
1476 {
1477         struct mlx5_ib_pf_eq *eq =
1478                 container_of(work, struct mlx5_ib_pf_eq, work);
1479 
1480         mempool_refill(eq->pool);
1481 
1482         spin_lock_irq(&eq->lock);
1483         mlx5_ib_eq_pf_process(eq);
1484         spin_unlock_irq(&eq->lock);
1485 }
1486 
1487 enum {
1488         MLX5_IB_NUM_PF_EQE      = 0x1000,
1489         MLX5_IB_NUM_PF_DRAIN    = 64,
1490 };
1491 
1492 static int
1493 mlx5_ib_create_pf_eq(struct mlx5_ib_dev *dev, struct mlx5_ib_pf_eq *eq)
1494 {
1495         struct mlx5_eq_param param = {};
1496         int err;
1497 
1498         INIT_WORK(&eq->work, mlx5_ib_eq_pf_action);
1499         spin_lock_init(&eq->lock);
1500         eq->dev = dev;
1501 
1502         eq->pool = mempool_create_kmalloc_pool(MLX5_IB_NUM_PF_DRAIN,
1503                                                sizeof(struct mlx5_pagefault));
1504         if (!eq->pool)
1505                 return -ENOMEM;
1506 
1507         eq->wq = alloc_workqueue("mlx5_ib_page_fault",
1508                                  WQ_HIGHPRI | WQ_UNBOUND | WQ_MEM_RECLAIM,
1509                                  MLX5_NUM_CMD_EQE);
1510         if (!eq->wq) {
1511                 err = -ENOMEM;
1512                 goto err_mempool;
1513         }
1514 
1515         eq->irq_nb.notifier_call = mlx5_ib_eq_pf_int;
1516         param = (struct mlx5_eq_param) {
1517                 .irq_index = 0,
1518                 .nent = MLX5_IB_NUM_PF_EQE,
1519         };
1520         param.mask[0] = 1ull << MLX5_EVENT_TYPE_PAGE_FAULT;
1521         eq->core = mlx5_eq_create_generic(dev->mdev, &param);
1522         if (IS_ERR(eq->core)) {
1523                 err = PTR_ERR(eq->core);
1524                 goto err_wq;
1525         }
1526         err = mlx5_eq_enable(dev->mdev, eq->core, &eq->irq_nb);
1527         if (err) {
1528                 mlx5_ib_err(dev, "failed to enable odp EQ %d\n", err);
1529                 goto err_eq;
1530         }
1531 
1532         return 0;
1533 err_eq:
1534         mlx5_eq_destroy_generic(dev->mdev, eq->core);
1535 err_wq:
1536         destroy_workqueue(eq->wq);
1537 err_mempool:
1538         mempool_destroy(eq->pool);
1539         return err;
1540 }
1541 
1542 static int
1543 mlx5_ib_destroy_pf_eq(struct mlx5_ib_dev *dev, struct mlx5_ib_pf_eq *eq)
1544 {
1545         int err;
1546 
1547         mlx5_eq_disable(dev->mdev, eq->core, &eq->irq_nb);
1548         err = mlx5_eq_destroy_generic(dev->mdev, eq->core);
1549         cancel_work_sync(&eq->work);
1550         destroy_workqueue(eq->wq);
1551         mempool_destroy(eq->pool);
1552 
1553         return err;
1554 }
1555 
1556 void mlx5_odp_init_mr_cache_entry(struct mlx5_cache_ent *ent)
1557 {
1558         if (!(ent->dev->odp_caps.general_caps & IB_ODP_SUPPORT_IMPLICIT))
1559                 return;
1560 
1561         switch (ent->order - 2) {
1562         case MLX5_IMR_MTT_CACHE_ENTRY:
1563                 ent->page = PAGE_SHIFT;
1564                 ent->xlt = MLX5_IMR_MTT_ENTRIES *
1565                            sizeof(struct mlx5_mtt) /
1566                            MLX5_IB_UMR_OCTOWORD;
1567                 ent->access_mode = MLX5_MKC_ACCESS_MODE_MTT;
1568                 ent->limit = 0;
1569                 break;
1570 
1571         case MLX5_IMR_KSM_CACHE_ENTRY:
1572                 ent->page = MLX5_KSM_PAGE_SHIFT;
1573                 ent->xlt = mlx5_imr_ksm_entries *
1574                            sizeof(struct mlx5_klm) /
1575                            MLX5_IB_UMR_OCTOWORD;
1576                 ent->access_mode = MLX5_MKC_ACCESS_MODE_KSM;
1577                 ent->limit = 0;
1578                 break;
1579         }
1580 }
1581 
1582 static const struct ib_device_ops mlx5_ib_dev_odp_ops = {
1583         .advise_mr = mlx5_ib_advise_mr,
1584         .invalidate_range = mlx5_ib_invalidate_range,
1585 };
1586 
1587 int mlx5_ib_odp_init_one(struct mlx5_ib_dev *dev)
1588 {
1589         int ret = 0;
1590 
1591         if (!(dev->odp_caps.general_caps & IB_ODP_SUPPORT))
1592                 return ret;
1593 
1594         ib_set_device_ops(&dev->ib_dev, &mlx5_ib_dev_odp_ops);
1595 
1596         if (dev->odp_caps.general_caps & IB_ODP_SUPPORT_IMPLICIT) {
1597                 ret = mlx5_cmd_null_mkey(dev->mdev, &dev->null_mkey);
1598                 if (ret) {
1599                         mlx5_ib_err(dev, "Error getting null_mkey %d\n", ret);
1600                         return ret;
1601                 }
1602         }
1603 
1604         ret = mlx5_ib_create_pf_eq(dev, &dev->odp_pf_eq);
1605 
1606         return ret;
1607 }
1608 
1609 void mlx5_ib_odp_cleanup_one(struct mlx5_ib_dev *dev)
1610 {
1611         if (!(dev->odp_caps.general_caps & IB_ODP_SUPPORT))
1612                 return;
1613 
1614         mlx5_ib_destroy_pf_eq(dev, &dev->odp_pf_eq);
1615 }
1616 
1617 int mlx5_ib_odp_init(void)
1618 {
1619         mlx5_imr_ksm_entries = BIT_ULL(get_order(TASK_SIZE) -
1620                                        MLX5_IMR_MTT_BITS);
1621 
1622         return 0;
1623 }
1624 
1625 struct prefetch_mr_work {
1626         struct work_struct work;
1627         struct ib_pd *pd;
1628         u32 pf_flags;
1629         u32 num_sge;
1630         struct ib_sge sg_list[0];
1631 };
1632 
1633 static void num_pending_prefetch_dec(struct mlx5_ib_dev *dev,
1634                                      struct ib_sge *sg_list, u32 num_sge,
1635                                      u32 from)
1636 {
1637         u32 i;
1638         int srcu_key;
1639 
1640         srcu_key = srcu_read_lock(&dev->mr_srcu);
1641 
1642         for (i = from; i < num_sge; ++i) {
1643                 struct mlx5_core_mkey *mmkey;
1644                 struct mlx5_ib_mr *mr;
1645 
1646                 mmkey = xa_load(&dev->mdev->priv.mkey_table,
1647                                 mlx5_base_mkey(sg_list[i].lkey));
1648                 mr = container_of(mmkey, struct mlx5_ib_mr, mmkey);
1649                 atomic_dec(&mr->num_pending_prefetch);
1650         }
1651 
1652         srcu_read_unlock(&dev->mr_srcu, srcu_key);
1653 }
1654 
1655 static bool num_pending_prefetch_inc(struct ib_pd *pd,
1656                                      struct ib_sge *sg_list, u32 num_sge)
1657 {
1658         struct mlx5_ib_dev *dev = to_mdev(pd->device);
1659         bool ret = true;
1660         u32 i;
1661 
1662         for (i = 0; i < num_sge; ++i) {
1663                 struct mlx5_core_mkey *mmkey;
1664                 struct mlx5_ib_mr *mr;
1665 
1666                 mmkey = xa_load(&dev->mdev->priv.mkey_table,
1667                                 mlx5_base_mkey(sg_list[i].lkey));
1668                 if (!mmkey || mmkey->key != sg_list[i].lkey) {
1669                         ret = false;
1670                         break;
1671                 }
1672 
1673                 if (mmkey->type != MLX5_MKEY_MR) {
1674                         ret = false;
1675                         break;
1676                 }
1677 
1678                 mr = container_of(mmkey, struct mlx5_ib_mr, mmkey);
1679 
1680                 if (!smp_load_acquire(&mr->live)) {
1681                         ret = false;
1682                         break;
1683                 }
1684 
1685                 if (mr->ibmr.pd != pd) {
1686                         ret = false;
1687                         break;
1688                 }
1689 
1690                 atomic_inc(&mr->num_pending_prefetch);
1691         }
1692 
1693         if (!ret)
1694                 num_pending_prefetch_dec(dev, sg_list, i, 0);
1695 
1696         return ret;
1697 }
1698 
1699 static int mlx5_ib_prefetch_sg_list(struct ib_pd *pd, u32 pf_flags,
1700                                     struct ib_sge *sg_list, u32 num_sge)
1701 {
1702         u32 i;
1703         int ret = 0;
1704         struct mlx5_ib_dev *dev = to_mdev(pd->device);
1705 
1706         for (i = 0; i < num_sge; ++i) {
1707                 struct ib_sge *sg = &sg_list[i];
1708                 int bytes_committed = 0;
1709 
1710                 ret = pagefault_single_data_segment(dev, pd, sg->lkey, sg->addr,
1711                                                     sg->length,
1712                                                     &bytes_committed, NULL,
1713                                                     pf_flags);
1714                 if (ret < 0)
1715                         break;
1716         }
1717 
1718         return ret < 0 ? ret : 0;
1719 }
1720 
1721 static void mlx5_ib_prefetch_mr_work(struct work_struct *work)
1722 {
1723         struct prefetch_mr_work *w =
1724                 container_of(work, struct prefetch_mr_work, work);
1725 
1726         if (ib_device_try_get(w->pd->device)) {
1727                 mlx5_ib_prefetch_sg_list(w->pd, w->pf_flags, w->sg_list,
1728                                          w->num_sge);
1729                 ib_device_put(w->pd->device);
1730         }
1731 
1732         num_pending_prefetch_dec(to_mdev(w->pd->device), w->sg_list,
1733                                  w->num_sge, 0);
1734         kvfree(w);
1735 }
1736 
1737 int mlx5_ib_advise_mr_prefetch(struct ib_pd *pd,
1738                                enum ib_uverbs_advise_mr_advice advice,
1739                                u32 flags, struct ib_sge *sg_list, u32 num_sge)
1740 {
1741         struct mlx5_ib_dev *dev = to_mdev(pd->device);
1742         u32 pf_flags = MLX5_PF_FLAGS_PREFETCH;
1743         struct prefetch_mr_work *work;
1744         bool valid_req;
1745         int srcu_key;
1746 
1747         if (advice == IB_UVERBS_ADVISE_MR_ADVICE_PREFETCH)
1748                 pf_flags |= MLX5_PF_FLAGS_DOWNGRADE;
1749 
1750         if (flags & IB_UVERBS_ADVISE_MR_FLAG_FLUSH)
1751                 return mlx5_ib_prefetch_sg_list(pd, pf_flags, sg_list,
1752                                                 num_sge);
1753 
1754         work = kvzalloc(struct_size(work, sg_list, num_sge), GFP_KERNEL);
1755         if (!work)
1756                 return -ENOMEM;
1757 
1758         memcpy(work->sg_list, sg_list, num_sge * sizeof(struct ib_sge));
1759 
1760         /* It is guaranteed that the pd when work is executed is the pd when
1761          * work was queued since pd can't be destroyed while it holds MRs and
1762          * destroying a MR leads to flushing the workquque
1763          */
1764         work->pd = pd;
1765         work->pf_flags = pf_flags;
1766         work->num_sge = num_sge;
1767 
1768         INIT_WORK(&work->work, mlx5_ib_prefetch_mr_work);
1769 
1770         srcu_key = srcu_read_lock(&dev->mr_srcu);
1771 
1772         valid_req = num_pending_prefetch_inc(pd, sg_list, num_sge);
1773         if (valid_req)
1774                 queue_work(system_unbound_wq, &work->work);
1775         else
1776                 kvfree(work);
1777 
1778         srcu_read_unlock(&dev->mr_srcu, srcu_key);
1779 
1780         return valid_req ? 0 : -EINVAL;
1781 }
/* [<][>][^][v][top][bottom][index][help] */
root/drivers/infiniband/hw/mlx5/odp.c

DEFINITIONS
