154 static void mlx5_ib_page_fault_resume(struct mlx5_ib_qp *qp,  in mlx5_ib_page_fault_resume()  argument
157 	struct mlx5_ib_dev *dev = to_mdev(qp->ibqp.pd->device);  in mlx5_ib_page_fault_resume()
158 	int ret = mlx5_core_page_fault_resume(dev->mdev, qp->mqp.qpn,  in mlx5_ib_page_fault_resume()
163 		       qp->mqp.qpn);  in mlx5_ib_page_fault_resume()
178 static int pagefault_single_data_segment(struct mlx5_ib_qp *qp,  in pagefault_single_data_segment()  argument
183 	struct mlx5_ib_dev *mib_dev = to_mdev(qp->ibqp.pd->device);  in pagefault_single_data_segment()
212 	if (mr->ibmr.pd != qp->ibqp.pd) {  in pagefault_single_data_segment()
310 static int pagefault_data_segments(struct mlx5_ib_qp *qp,  in pagefault_data_segments()  argument
323 	if (receive_queue && qp->ibqp.srq)  in pagefault_data_segments()
369 		ret = pagefault_single_data_segment(qp, pfault, key, io_virt,  in pagefault_data_segments()
384 	struct mlx5_ib_qp *qp, struct mlx5_ib_pfault *pfault,  in mlx5_ib_mr_initiator_pfault_handler()  argument
387 	struct mlx5_ib_dev *dev = to_mdev(qp->ibqp.pd->device);  in mlx5_ib_mr_initiator_pfault_handler()
404 			    wqe_index, qp->mqp.qpn);  in mlx5_ib_mr_initiator_pfault_handler()
414 			    wqe_index, qp->mqp.qpn,  in mlx5_ib_mr_initiator_pfault_handler()
421 	if (qp->mqp.qpn != ctrl_qpn) {  in mlx5_ib_mr_initiator_pfault_handler()
423 			    wqe_index, qp->mqp.qpn,  in mlx5_ib_mr_initiator_pfault_handler()
434 	switch (qp->ibqp.qp_type) {  in mlx5_ib_mr_initiator_pfault_handler()
477 			    qp->ibqp.qp_type, opcode);  in mlx5_ib_mr_initiator_pfault_handler()
489 	struct mlx5_ib_qp *qp, struct mlx5_ib_pfault *pfault,  in mlx5_ib_mr_responder_pfault_handler()  argument
492 	struct mlx5_ib_dev *dev = to_mdev(qp->ibqp.pd->device);  in mlx5_ib_mr_responder_pfault_handler()
493 	struct mlx5_ib_wq *wq = &qp->rq;  in mlx5_ib_mr_responder_pfault_handler()
496 	if (qp->ibqp.srq) {  in mlx5_ib_mr_responder_pfault_handler()
501 	if (qp->wq_sig) {  in mlx5_ib_mr_responder_pfault_handler()
511 	switch (qp->ibqp.qp_type) {  in mlx5_ib_mr_responder_pfault_handler()
520 			    qp->ibqp.qp_type);  in mlx5_ib_mr_responder_pfault_handler()
529 static void mlx5_ib_mr_wqe_pfault_handler(struct mlx5_ib_qp *qp,  in mlx5_ib_mr_wqe_pfault_handler()  argument
532 	struct mlx5_ib_dev *dev = to_mdev(qp->ibqp.pd->device);  in mlx5_ib_mr_wqe_pfault_handler()
548 	ret = mlx5_ib_read_user_wqe(qp, requestor, wqe_index, buffer,  in mlx5_ib_mr_wqe_pfault_handler()
552 			    -ret, wqe_index, qp->mqp.qpn);  in mlx5_ib_mr_wqe_pfault_handler()
559 		ret = mlx5_ib_mr_initiator_pfault_handler(qp, pfault, &wqe,  in mlx5_ib_mr_wqe_pfault_handler()
562 		ret = mlx5_ib_mr_responder_pfault_handler(qp, pfault, &wqe,  in mlx5_ib_mr_wqe_pfault_handler()
575 	ret = pagefault_data_segments(qp, pfault, wqe, wqe_end, &bytes_mapped,  in mlx5_ib_mr_wqe_pfault_handler()
587 	mlx5_ib_page_fault_resume(qp, pfault, resume_with_error);  in mlx5_ib_mr_wqe_pfault_handler()
589 		    qp->mqp.qpn, resume_with_error, pfault->mpfault.flags);  in mlx5_ib_mr_wqe_pfault_handler()
600 static void mlx5_ib_mr_rdma_pfault_handler(struct mlx5_ib_qp *qp,  in mlx5_ib_mr_rdma_pfault_handler()  argument
638 	ret = pagefault_single_data_segment(qp, pfault, rkey, address, length,  in mlx5_ib_mr_rdma_pfault_handler()
644 		mlx5_ib_page_fault_resume(qp, pfault, 1);  in mlx5_ib_mr_rdma_pfault_handler()
648 	mlx5_ib_page_fault_resume(qp, pfault, 0);  in mlx5_ib_mr_rdma_pfault_handler()
656 		ret = pagefault_single_data_segment(qp, &dummy_pfault, rkey,  in mlx5_ib_mr_rdma_pfault_handler()
663 				qp->ibqp.qp_num, address, prefetch_len);  in mlx5_ib_mr_rdma_pfault_handler()
668 void mlx5_ib_mr_pfault_handler(struct mlx5_ib_qp *qp,  in mlx5_ib_mr_pfault_handler()  argument
675 		mlx5_ib_mr_wqe_pfault_handler(qp, pfault);  in mlx5_ib_mr_pfault_handler()
678 		mlx5_ib_mr_rdma_pfault_handler(qp, pfault);  in mlx5_ib_mr_pfault_handler()
683 		mlx5_ib_page_fault_resume(qp, pfault, 1);  in mlx5_ib_mr_pfault_handler()
695 	struct mlx5_ib_qp *qp = container_of(pfault, struct mlx5_ib_qp,  in mlx5_ib_qp_pfault_action()  local
697 	mlx5_ib_mr_pfault_handler(qp, pfault);  in mlx5_ib_qp_pfault_action()
700 void mlx5_ib_qp_disable_pagefaults(struct mlx5_ib_qp *qp)  in mlx5_ib_qp_disable_pagefaults()  argument
704 	spin_lock_irqsave(&qp->disable_page_faults_lock, flags);  in mlx5_ib_qp_disable_pagefaults()
705 	qp->disable_page_faults = 1;  in mlx5_ib_qp_disable_pagefaults()
706 	spin_unlock_irqrestore(&qp->disable_page_faults_lock, flags);  in mlx5_ib_qp_disable_pagefaults()
716 void mlx5_ib_qp_enable_pagefaults(struct mlx5_ib_qp *qp)  in mlx5_ib_qp_enable_pagefaults()  argument
720 	spin_lock_irqsave(&qp->disable_page_faults_lock, flags);  in mlx5_ib_qp_enable_pagefaults()
721 	qp->disable_page_faults = 0;  in mlx5_ib_qp_enable_pagefaults()
722 	spin_unlock_irqrestore(&qp->disable_page_faults_lock, flags);  in mlx5_ib_qp_enable_pagefaults()
725 static void mlx5_ib_pfault_handler(struct mlx5_core_qp *qp,  in mlx5_ib_pfault_handler()  argument
735 	struct mlx5_ib_qp *mibqp = to_mibqp(qp);  in mlx5_ib_pfault_handler()
749 void mlx5_ib_odp_create_qp(struct mlx5_ib_qp *qp)  in mlx5_ib_odp_create_qp()  argument
753 	qp->disable_page_faults = 1;  in mlx5_ib_odp_create_qp()
754 	spin_lock_init(&qp->disable_page_faults_lock);  in mlx5_ib_odp_create_qp()
756 	qp->mqp.pfault_handler	= mlx5_ib_pfault_handler;  in mlx5_ib_odp_create_qp()
759 		INIT_WORK(&qp->pagefaults[i].work, mlx5_ib_qp_pfault_action);  in mlx5_ib_odp_create_qp()