210 	__be32 *wqe;  in stamp_send_wqe()  local
225 			wqe = buf + (i & ((1 << qp->sq.wqe_shift) - 1));  in stamp_send_wqe()
226 			*wqe = stamp;  in stamp_send_wqe()
232 			wqe = buf + i;  in stamp_send_wqe()
233 			*wqe = cpu_to_be32(0xffffffff);  in stamp_send_wqe()
242 	void *wqe;  in post_nop_wqe()  local
245 	ctrl = wqe = get_send_wqe(qp, n & (qp->sq.wqe_cnt - 1));  in post_nop_wqe()
249 		struct mlx4_wqe_datagram_seg *dgram = wqe + sizeof *ctrl;  in post_nop_wqe()
258 		inl = wqe + s;  in post_nop_wqe()
2144 				  void *wqe, unsigned *mlx_seg_len)  in build_sriov_qp0_header()  argument
2148 	struct mlx4_wqe_mlx_seg *mlx = wqe;  in build_sriov_qp0_header()
2149 	struct mlx4_wqe_inline_seg *inl = wqe + sizeof *mlx;  in build_sriov_qp0_header()
2266 			    void *wqe, unsigned *mlx_seg_len)  in build_mlx_header()  argument
2269 	struct mlx4_wqe_mlx_seg *mlx = wqe;  in build_mlx_header()
2270 	struct mlx4_wqe_ctrl_seg *ctrl = wqe;  in build_mlx_header()
2271 	struct mlx4_wqe_inline_seg *inl = wqe + sizeof *mlx;  in build_mlx_header()
2623 static void build_tunnel_header(struct ib_ud_wr *wr, void *wqe, unsigned *mlx_seg_len)  in build_tunnel_header()  argument
2625 	struct mlx4_wqe_inline_seg *inl = wqe;  in build_tunnel_header()
2706 static int build_lso_seg(struct mlx4_wqe_lso_seg *wqe, struct ib_ud_wr *wr,  in build_lso_seg()  argument
2710 	unsigned halign = ALIGN(sizeof *wqe + wr->hlen, 16);  in build_lso_seg()
2719 	memcpy(wqe->header, wr->header, wr->hlen);  in build_lso_seg()
2741 static void add_zero_len_inline(void *wqe)  in add_zero_len_inline()  argument
2743 	struct mlx4_wqe_inline_seg *inl = wqe;  in add_zero_len_inline()
2744 	memset(wqe, 0, 16);  in add_zero_len_inline()
2752 	void *wqe;  in mlx4_ib_post_send()  local
2795 		ctrl = wqe = get_send_wqe(qp, ind & (qp->sq.wqe_cnt - 1));  in mlx4_ib_post_send()
2810 		wqe += sizeof *ctrl;  in mlx4_ib_post_send()
2820 				set_raddr_seg(wqe, atomic_wr(wr)->remote_addr,  in mlx4_ib_post_send()
2822 				wqe  += sizeof (struct mlx4_wqe_raddr_seg);  in mlx4_ib_post_send()
2824 				set_atomic_seg(wqe, atomic_wr(wr));  in mlx4_ib_post_send()
2825 				wqe  += sizeof (struct mlx4_wqe_atomic_seg);  in mlx4_ib_post_send()
2833 				set_raddr_seg(wqe, atomic_wr(wr)->remote_addr,  in mlx4_ib_post_send()
2835 				wqe  += sizeof (struct mlx4_wqe_raddr_seg);  in mlx4_ib_post_send()
2837 				set_masked_atomic_seg(wqe, atomic_wr(wr));  in mlx4_ib_post_send()
2838 				wqe  += sizeof (struct mlx4_wqe_masked_atomic_seg);  in mlx4_ib_post_send()
2848 				set_raddr_seg(wqe, rdma_wr(wr)->remote_addr,  in mlx4_ib_post_send()
2850 				wqe  += sizeof (struct mlx4_wqe_raddr_seg);  in mlx4_ib_post_send()
2857 				set_local_inv_seg(wqe, wr->ex.invalidate_rkey);  in mlx4_ib_post_send()
2858 				wqe  += sizeof (struct mlx4_wqe_local_inval_seg);  in mlx4_ib_post_send()
2865 				set_reg_seg(wqe, reg_wr(wr));  in mlx4_ib_post_send()
2866 				wqe  += sizeof(struct mlx4_wqe_fmr_seg);  in mlx4_ib_post_send()
2873 				set_bind_seg(wqe, bind_mw_wr(wr));  in mlx4_ib_post_send()
2874 				wqe  += sizeof(struct mlx4_wqe_bind_seg);  in mlx4_ib_post_send()
2890 			wqe  += seglen;  in mlx4_ib_post_send()
2896 			set_datagram_seg(wqe, ud_wr(wr));  in mlx4_ib_post_send()
2898 			*(__be32 *) wqe |= cpu_to_be32(0x80000000);  in mlx4_ib_post_send()
2899 			wqe  += sizeof (struct mlx4_wqe_datagram_seg);  in mlx4_ib_post_send()
2903 			set_datagram_seg(wqe, ud_wr(wr));  in mlx4_ib_post_send()
2904 			wqe  += sizeof (struct mlx4_wqe_datagram_seg);  in mlx4_ib_post_send()
2908 				err = build_lso_seg(wqe, ud_wr(wr), qp, &seglen,  in mlx4_ib_post_send()
2914 				lso_wqe = (__be32 *) wqe;  in mlx4_ib_post_send()
2915 				wqe  += seglen;  in mlx4_ib_post_send()
2927 			wqe  += seglen;  in mlx4_ib_post_send()
2930 			add_zero_len_inline(wqe);  in mlx4_ib_post_send()
2931 			wqe += 16;  in mlx4_ib_post_send()
2933 			build_tunnel_header(ud_wr(wr), wqe, &seglen);  in mlx4_ib_post_send()
2934 			wqe  += seglen;  in mlx4_ib_post_send()
2943 			set_tunnel_datagram_seg(to_mdev(ibqp->device), wqe,  in mlx4_ib_post_send()
2946 			wqe  += sizeof (struct mlx4_wqe_datagram_seg);  in mlx4_ib_post_send()
2948 			build_tunnel_header(ud_wr(wr), wqe, &seglen);  in mlx4_ib_post_send()
2949 			wqe  += seglen;  in mlx4_ib_post_send()
2961 			wqe  += seglen;  in mlx4_ib_post_send()
2976 		dseg = wqe;  in mlx4_ib_post_send()