2002 static void set_mkey_segment(struct mlx5_mkey_seg *seg, struct ib_send_wr *wr,  in set_mkey_segment()  argument
2005 	memset(seg, 0, sizeof(*seg));  in set_mkey_segment()
2007 		seg->status = MLX5_MKEY_STATUS_FREE;  in set_mkey_segment()
2011 	seg->flags = get_umr_flags(wr->wr.fast_reg.access_flags) |  in set_mkey_segment()
2013 	*writ = seg->flags & (MLX5_PERM_LOCAL_WRITE | IB_ACCESS_REMOTE_WRITE);  in set_mkey_segment()
2014 	seg->qpn_mkey7_0 = cpu_to_be32((wr->wr.fast_reg.rkey & 0xff) | 0xffffff00);  in set_mkey_segment()
2015 	seg->flags_pd = cpu_to_be32(MLX5_MKEY_REMOTE_INVAL);  in set_mkey_segment()
2016 	seg->start_addr = cpu_to_be64(wr->wr.fast_reg.iova_start);  in set_mkey_segment()
2017 	seg->len = cpu_to_be64(wr->wr.fast_reg.length);  in set_mkey_segment()
2018 	seg->xlt_oct_size = cpu_to_be32((wr->wr.fast_reg.page_list_len + 1) / 2);  in set_mkey_segment()
2019 	seg->log2_page_size = wr->wr.fast_reg.page_shift;  in set_mkey_segment()
2022 static void set_reg_mkey_segment(struct mlx5_mkey_seg *seg, struct ib_send_wr *wr)  in set_reg_mkey_segment()  argument
2026 	memset(seg, 0, sizeof(*seg));  in set_reg_mkey_segment()
2028 		seg->status = MLX5_MKEY_STATUS_FREE;  in set_reg_mkey_segment()
2032 	seg->flags = convert_access(umrwr->access_flags);  in set_reg_mkey_segment()
2034 		seg->flags_pd = cpu_to_be32(to_mpd(umrwr->pd)->pdn);  in set_reg_mkey_segment()
2035 		seg->start_addr = cpu_to_be64(umrwr->target.virt_addr);  in set_reg_mkey_segment()
2037 	seg->len = cpu_to_be64(umrwr->length);  in set_reg_mkey_segment()
2038 	seg->log2_page_size = umrwr->page_shift;  in set_reg_mkey_segment()
2039 	seg->qpn_mkey7_0 = cpu_to_be32(0xffffff00 |  in set_reg_mkey_segment()
2096 	struct mlx5_wqe_inline_seg *seg;  in set_data_inl_seg()  local
2104 	seg = wqe;  in set_data_inl_seg()
2105 	wqe += sizeof(*seg);  in set_data_inl_seg()
2125 	seg->byte_count = cpu_to_be32(inl | MLX5_INLINE_SEG);  in set_data_inl_seg()
2127 	*sz = ALIGN(inl + sizeof(seg->byte_count), 16) / 16;  in set_data_inl_seg()
2240 				void **seg, int *size)  in set_sig_data_segment()  argument
2265 		struct mlx5_klm *data_klm = *seg;  in set_sig_data_segment()
2293 		sblock_ctrl = *seg;  in set_sig_data_segment()
2322 	*seg += wqe_size;  in set_sig_data_segment()
2324 	if (unlikely((*seg == qp->sq.qend)))  in set_sig_data_segment()
2325 		*seg = mlx5_get_send_wqe(qp, 0);  in set_sig_data_segment()
2327 	bsf = *seg;  in set_sig_data_segment()
2332 	*seg += sizeof(*bsf);  in set_sig_data_segment()
2334 	if (unlikely((*seg == qp->sq.qend)))  in set_sig_data_segment()
2335 		*seg = mlx5_get_send_wqe(qp, 0);  in set_sig_data_segment()
2340 static void set_sig_mkey_segment(struct mlx5_mkey_seg *seg,  in set_sig_mkey_segment()  argument
2348 	memset(seg, 0, sizeof(*seg));  in set_sig_mkey_segment()
2350 	seg->flags = get_umr_flags(wr->wr.sig_handover.access_flags) |  in set_sig_mkey_segment()
2352 	seg->qpn_mkey7_0 = cpu_to_be32((sig_key & 0xff) | 0xffffff00);  in set_sig_mkey_segment()
2353 	seg->flags_pd = cpu_to_be32(MLX5_MKEY_REMOTE_INVAL | sigerr << 26 |  in set_sig_mkey_segment()
2355 	seg->len = cpu_to_be64(length);  in set_sig_mkey_segment()
2356 	seg->xlt_oct_size = cpu_to_be32(be16_to_cpu(get_klm_octo(nelements)));  in set_sig_mkey_segment()
2357 	seg->bsfs_octo_size = cpu_to_be32(MLX5_MKEY_BSF_OCTO_SIZE);  in set_sig_mkey_segment()
2373 			  void **seg, int *size)  in set_sig_umr_wr()  argument
2402 	set_sig_umr_segment(*seg, wr, klm_oct_size);  in set_sig_umr_wr()
2403 	*seg += sizeof(struct mlx5_wqe_umr_ctrl_seg);  in set_sig_umr_wr()
2405 	if (unlikely((*seg == qp->sq.qend)))  in set_sig_umr_wr()
2406 		*seg = mlx5_get_send_wqe(qp, 0);  in set_sig_umr_wr()
2408 	set_sig_mkey_segment(*seg, wr, klm_oct_size, region_len, pdn);  in set_sig_umr_wr()
2409 	*seg += sizeof(struct mlx5_mkey_seg);  in set_sig_umr_wr()
2411 	if (unlikely((*seg == qp->sq.qend)))  in set_sig_umr_wr()
2412 		*seg = mlx5_get_send_wqe(qp, 0);  in set_sig_umr_wr()
2414 	ret = set_sig_data_segment(wr, qp, seg, size);  in set_sig_umr_wr()
2423 		      u32 psv_idx, void **seg, int *size)  in set_psv_wr()  argument
2425 	struct mlx5_seg_set_psv *psv_seg = *seg;  in set_psv_wr()
2442 	*seg += sizeof(*psv_seg);  in set_psv_wr()
2448 static int set_frwr_li_wr(void **seg, struct ib_send_wr *wr, int *size,  in set_frwr_li_wr()  argument
2458 	set_frwr_umr_segment(*seg, wr, li);  in set_frwr_li_wr()
2459 	*seg += sizeof(struct mlx5_wqe_umr_ctrl_seg);  in set_frwr_li_wr()
2461 	if (unlikely((*seg == qp->sq.qend)))  in set_frwr_li_wr()
2462 		*seg = mlx5_get_send_wqe(qp, 0);  in set_frwr_li_wr()
2463 	set_mkey_segment(*seg, wr, li, &writ);  in set_frwr_li_wr()
2464 	*seg += sizeof(struct mlx5_mkey_seg);  in set_frwr_li_wr()
2466 	if (unlikely((*seg == qp->sq.qend)))  in set_frwr_li_wr()
2467 		*seg = mlx5_get_send_wqe(qp, 0);  in set_frwr_li_wr()
2473 		set_frwr_pages(*seg, wr, mdev, pd, writ);  in set_frwr_li_wr()
2474 		*seg += sizeof(struct mlx5_wqe_data_seg);  in set_frwr_li_wr()
2535 static int begin_wqe(struct mlx5_ib_qp *qp, void **seg,  in begin_wqe()  argument
2548 	*seg = mlx5_get_send_wqe(qp, *idx);  in begin_wqe()
2549 	*ctrl = *seg;  in begin_wqe()
2550 	*(uint32_t *)(*seg + 8) = 0;  in begin_wqe()
2558 	*seg += sizeof(**ctrl);  in begin_wqe()
2606 	void *seg;  in mlx5_ib_post_send()  local
2631 		err = begin_wqe(qp, &seg, &ctrl, wr, &idx, &size, nreq);  in mlx5_ib_post_send()
2641 			xrc = seg;  in mlx5_ib_post_send()
2643 			seg += sizeof(*xrc);  in mlx5_ib_post_send()
2651 				set_raddr_seg(seg, wr->wr.rdma.remote_addr,  in mlx5_ib_post_send()
2653 				seg += sizeof(struct mlx5_wqe_raddr_seg);  in mlx5_ib_post_send()
2669 				err = set_frwr_li_wr(&seg, wr, &size, mdev, to_mpd(ibqp->pd), qp);  in mlx5_ib_post_send()
2682 				err = set_frwr_li_wr(&seg, wr, &size, mdev, to_mpd(ibqp->pd), qp);  in mlx5_ib_post_send()
2696 				err = set_sig_umr_wr(wr, qp, &seg, &size);  in mlx5_ib_post_send()
2712 				err = begin_wqe(qp, &seg, &ctrl, wr,  in mlx5_ib_post_send()
2722 						 mr->sig->psv_memory.psv_idx, &seg,  in mlx5_ib_post_send()
2733 				err = begin_wqe(qp, &seg, &ctrl, wr,  in mlx5_ib_post_send()
2744 						 mr->sig->psv_wire.psv_idx, &seg,  in mlx5_ib_post_send()
2767 				set_raddr_seg(seg, wr->wr.rdma.remote_addr,  in mlx5_ib_post_send()
2769 				seg  += sizeof(struct mlx5_wqe_raddr_seg);  in mlx5_ib_post_send()
2781 			set_datagram_seg(seg, wr);  in mlx5_ib_post_send()
2782 			seg += sizeof(struct mlx5_wqe_datagram_seg);  in mlx5_ib_post_send()
2784 			if (unlikely((seg == qend)))  in mlx5_ib_post_send()
2785 				seg = mlx5_get_send_wqe(qp, 0);  in mlx5_ib_post_send()
2796 			set_reg_umr_segment(seg, wr);  in mlx5_ib_post_send()
2797 			seg += sizeof(struct mlx5_wqe_umr_ctrl_seg);  in mlx5_ib_post_send()
2799 			if (unlikely((seg == qend)))  in mlx5_ib_post_send()
2800 				seg = mlx5_get_send_wqe(qp, 0);  in mlx5_ib_post_send()
2801 			set_reg_mkey_segment(seg, wr);  in mlx5_ib_post_send()
2802 			seg += sizeof(struct mlx5_mkey_seg);  in mlx5_ib_post_send()
2804 			if (unlikely((seg == qend)))  in mlx5_ib_post_send()
2805 				seg = mlx5_get_send_wqe(qp, 0);  in mlx5_ib_post_send()
2815 			err = set_data_inl_seg(qp, wr, seg, &sz);  in mlx5_ib_post_send()
2824 			dpseg = seg;  in mlx5_ib_post_send()
2827 					seg = mlx5_get_send_wqe(qp, 0);  in mlx5_ib_post_send()
2828 					dpseg = seg;  in mlx5_ib_post_send()